mirror of
https://github.com/papra-hq/papra.git
synced 2025-12-19 03:51:28 -06:00
Compare commits
1 Commits
@papra/app
...
storage-dr
| Author | SHA1 | Date | |
|---|---|---|---|
|
9ed9f34ee8 |
5
.changeset/brave-parrots-fold.md
Normal file
5
.changeset/brave-parrots-fold.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@papra/app-client": patch
|
||||
---
|
||||
|
||||
Lazy load the PDF viewer to reduce the main chunk size
|
||||
6
.changeset/brown-tigers-cut.md
Normal file
6
.changeset/brown-tigers-cut.md
Normal file
@@ -0,0 +1,6 @@
|
||||
---
|
||||
"@papra/app-client": patch
|
||||
"@papra/app-server": patch
|
||||
---
|
||||
|
||||
Allow for more complex intake-email origin adresses
|
||||
8
.changeset/curly-beds-dig.md
Normal file
8
.changeset/curly-beds-dig.md
Normal file
@@ -0,0 +1,8 @@
|
||||
---
|
||||
"@papra/webhooks": minor
|
||||
"@papra/api-sdk": minor
|
||||
"@papra/lecture": minor
|
||||
"@papra/cli": minor
|
||||
---
|
||||
|
||||
Ditched CommonJs build for packages
|
||||
5
.changeset/fuzzy-clocks-sell.md
Normal file
5
.changeset/fuzzy-clocks-sell.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@papra/app-server": patch
|
||||
---
|
||||
|
||||
Use node file streams in ingestion folder for smaller RAM footprint
|
||||
5
.changeset/hot-towns-draw.md
Normal file
5
.changeset/hot-towns-draw.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@papra/app-client": patch
|
||||
---
|
||||
|
||||
Simplified i18n tooling + improved performances
|
||||
5
.changeset/lazy-tables-cover.md
Normal file
5
.changeset/lazy-tables-cover.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@papra/app-server": patch
|
||||
---
|
||||
|
||||
Fixed an issue where tags assigned to only deleted documents won't show up in the tag list
|
||||
5
.changeset/purple-emus-smile.md
Normal file
5
.changeset/purple-emus-smile.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@papra/app-server": minor
|
||||
---
|
||||
|
||||
Dropped support for the dedicated backblaze b2 storage driver as b2 now fully support s3 client
|
||||
5
.changeset/rich-parents-flash.md
Normal file
5
.changeset/rich-parents-flash.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@papra/app-client": patch
|
||||
---
|
||||
|
||||
Prevent infinit loading in search modal when an error occure
|
||||
6
.changeset/rude-loops-lose.md
Normal file
6
.changeset/rude-loops-lose.md
Normal file
@@ -0,0 +1,6 @@
|
||||
---
|
||||
"@papra/app-server": minor
|
||||
"@papra/docs": minor
|
||||
---
|
||||
|
||||
Added documents encryption layer
|
||||
5
.changeset/shiny-aliens-wonder.md
Normal file
5
.changeset/shiny-aliens-wonder.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@papra/app-server": patch
|
||||
---
|
||||
|
||||
Properly handle missing files errors in storage drivers
|
||||
5
.changeset/sweet-cougars-bathe.md
Normal file
5
.changeset/sweet-cougars-bathe.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@papra/app-client": patch
|
||||
---
|
||||
|
||||
Improved the UX of the document content edition panel
|
||||
5
.changeset/twenty-grapes-brake.md
Normal file
5
.changeset/twenty-grapes-brake.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@papra/app-server": minor
|
||||
---
|
||||
|
||||
Stream file upload instead of full in-memory loading
|
||||
5
.changeset/young-laws-wish.md
Normal file
5
.changeset/young-laws-wish.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@papra/app-client": patch
|
||||
---
|
||||
|
||||
Added content edition support in demo mode
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -35,8 +35,6 @@ cache
|
||||
*.db-shm
|
||||
*.db-wal
|
||||
*.sqlite
|
||||
*.sqlite-shm
|
||||
*.sqlite-wal
|
||||
|
||||
local-documents
|
||||
ingestion
|
||||
|
||||
@@ -105,73 +105,6 @@ We recommend running the app locally for development. Follow these steps:
|
||||
|
||||
6. Open your browser and navigate to `http://localhost:3000`.
|
||||
|
||||
### IDE Setup
|
||||
|
||||
#### ESLint Extension
|
||||
|
||||
We recommend installing the [ESLint extension](https://marketplace.visualstudio.com/items?itemName=dbaeumer.vscode-eslint) for VS Code to get real-time linting feedback and automatic code fixing.
|
||||
The linting configuration is based on [@antfu/eslint-config](https://github.com/antfu/eslint-config), you can find specific IDE configurations in their repository.
|
||||
|
||||
<details>
|
||||
<summary>Recommended VS Code Settings</summary>
|
||||
|
||||
Create or update your `.vscode/settings.json` file with the following configuration:
|
||||
|
||||
```json
|
||||
{
|
||||
// Disable the default formatter, use eslint instead
|
||||
"prettier.enable": false,
|
||||
"editor.formatOnSave": false,
|
||||
|
||||
// Auto fix
|
||||
"editor.codeActionsOnSave": {
|
||||
"source.fixAll.eslint": "explicit",
|
||||
"source.organizeImports": "never"
|
||||
},
|
||||
|
||||
// Silent the stylistic rules in your IDE, but still auto fix them
|
||||
"eslint.rules.customizations": [
|
||||
{ "rule": "style/*", "severity": "off", "fixable": true },
|
||||
{ "rule": "format/*", "severity": "off", "fixable": true },
|
||||
{ "rule": "*-indent", "severity": "off", "fixable": true },
|
||||
{ "rule": "*-spacing", "severity": "off", "fixable": true },
|
||||
{ "rule": "*-spaces", "severity": "off", "fixable": true },
|
||||
{ "rule": "*-order", "severity": "off", "fixable": true },
|
||||
{ "rule": "*-dangle", "severity": "off", "fixable": true },
|
||||
{ "rule": "*-newline", "severity": "off", "fixable": true },
|
||||
{ "rule": "*quotes", "severity": "off", "fixable": true },
|
||||
{ "rule": "*semi", "severity": "off", "fixable": true }
|
||||
],
|
||||
|
||||
// Enable eslint for all supported languages
|
||||
"eslint.validate": [
|
||||
"javascript",
|
||||
"javascriptreact",
|
||||
"typescript",
|
||||
"typescriptreact",
|
||||
"vue",
|
||||
"html",
|
||||
"markdown",
|
||||
"json",
|
||||
"jsonc",
|
||||
"yaml",
|
||||
"toml",
|
||||
"xml",
|
||||
"gql",
|
||||
"graphql",
|
||||
"astro",
|
||||
"svelte",
|
||||
"css",
|
||||
"less",
|
||||
"scss",
|
||||
"pcss",
|
||||
"postcss"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
### Testing
|
||||
|
||||
We use **Vitest** for testing. Each package comes with its own testing commands.
|
||||
|
||||
@@ -1,11 +1,5 @@
|
||||
# @papra/docs
|
||||
|
||||
## 0.6.0
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- [#480](https://github.com/papra-hq/papra/pull/480) [`0a03f42`](https://github.com/papra-hq/papra/commit/0a03f42231f691d339c7ab5a5916c52385e31bd2) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Added documents encryption layer
|
||||
|
||||
## 0.5.3
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@papra/docs",
|
||||
"type": "module",
|
||||
"version": "0.6.0",
|
||||
"version": "0.5.3",
|
||||
"private": true,
|
||||
"packageManager": "pnpm@10.12.3",
|
||||
"description": "Papra documentation website",
|
||||
|
||||
@@ -1,41 +1,5 @@
|
||||
# @papra/app-client
|
||||
|
||||
## 0.9.2
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- [#501](https://github.com/papra-hq/papra/pull/501) [`b5bf0cc`](https://github.com/papra-hq/papra/commit/b5bf0cca4b571495329cb553da06e0d334ee8968) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Fix an issue preventing to disable the max upload size
|
||||
|
||||
- [#498](https://github.com/papra-hq/papra/pull/498) [`3da13f7`](https://github.com/papra-hq/papra/commit/3da13f759155df5d7c532160a7ea582385db63b6) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Removed the "open in new tab" button for security improvement (xss prevention)
|
||||
|
||||
## 0.9.1
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- [#492](https://github.com/papra-hq/papra/pull/492) [`54514e1`](https://github.com/papra-hq/papra/commit/54514e15db5deaffc59dcba34929b5e2e74282e1) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Added a client side guard for rejecting too-big files
|
||||
|
||||
- [#488](https://github.com/papra-hq/papra/pull/488) [`83e943c`](https://github.com/papra-hq/papra/commit/83e943c5b46432e55b6dfbaa587019a95ffab466) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Fix favicons display issues on firefox
|
||||
|
||||
- [#492](https://github.com/papra-hq/papra/pull/492) [`54514e1`](https://github.com/papra-hq/papra/commit/54514e15db5deaffc59dcba34929b5e2e74282e1) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Fix i18n messages when a file-too-big error happens
|
||||
|
||||
- [#492](https://github.com/papra-hq/papra/pull/492) [`54514e1`](https://github.com/papra-hq/papra/commit/54514e15db5deaffc59dcba34929b5e2e74282e1) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Clean all upload method to happen through the import status modal
|
||||
|
||||
## 0.9.0
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- [#471](https://github.com/papra-hq/papra/pull/471) [`e77a42f`](https://github.com/papra-hq/papra/commit/e77a42fbf14da011cd396426aa0bbea56c889740) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Lazy load the PDF viewer to reduce the main chunk size
|
||||
|
||||
- [#481](https://github.com/papra-hq/papra/pull/481) [`1606310`](https://github.com/papra-hq/papra/commit/1606310745e8edf405b527127078143481419e8c) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Allow for more complex intake-email origin adresses
|
||||
|
||||
- [#470](https://github.com/papra-hq/papra/pull/470) [`d488efe`](https://github.com/papra-hq/papra/commit/d488efe2cc4aa4f433cec4e9b8cc909b091eccc4) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Simplified i18n tooling + improved performances
|
||||
|
||||
- [#468](https://github.com/papra-hq/papra/pull/468) [`14c3587`](https://github.com/papra-hq/papra/commit/14c3587de07a605ec586bdc428d9e76956bf1c67) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Prevent infinit loading in search modal when an error occure
|
||||
|
||||
- [#468](https://github.com/papra-hq/papra/pull/468) [`14c3587`](https://github.com/papra-hq/papra/commit/14c3587de07a605ec586bdc428d9e76956bf1c67) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Improved the UX of the document content edition panel
|
||||
|
||||
- [#468](https://github.com/papra-hq/papra/pull/468) [`14c3587`](https://github.com/papra-hq/papra/commit/14c3587de07a605ec586bdc428d9e76956bf1c67) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Added content edition support in demo mode
|
||||
|
||||
## 0.8.2
|
||||
|
||||
## 0.8.1
|
||||
|
||||
@@ -6,7 +6,8 @@ export default antfu({
|
||||
},
|
||||
|
||||
ignores: [
|
||||
'public/manifest.json',
|
||||
// Generated file
|
||||
'src/modules/i18n/locales.types.ts',
|
||||
],
|
||||
|
||||
rules: {
|
||||
|
||||
@@ -27,23 +27,10 @@
|
||||
<meta property="twitter:image" content="https://papra.app/og-image.png">
|
||||
|
||||
<!-- Favicon and Icons -->
|
||||
<link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
|
||||
<link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
|
||||
<link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
|
||||
<link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
|
||||
<link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
|
||||
<link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
|
||||
<link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
|
||||
<link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
|
||||
<link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
|
||||
<link rel="icon" type="image/png" sizes="192x192" href="/android-icon-192x192.png">
|
||||
<link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png">
|
||||
<link rel="icon" type="image/png" sizes="96x96" href="/favicon-96x96.png">
|
||||
<link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png">
|
||||
<link rel="manifest" href="/manifest.json">
|
||||
<meta name="msapplication-TileColor" content="#ffffff">
|
||||
<meta name="msapplication-TileImage" content="/ms-icon-144x144.png">
|
||||
<meta name="theme-color" content="#ffffff">
|
||||
<link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96" />
|
||||
<link rel="shortcut icon" href="/favicon.ico" />
|
||||
<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png" />
|
||||
<link rel="manifest" href="/site.webmanifest" />
|
||||
|
||||
<!-- Structured Data (JSON-LD for rich snippets) -->
|
||||
<script type="application/ld+json">
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@papra/app-client",
|
||||
"type": "module",
|
||||
"version": "0.9.2",
|
||||
"version": "0.8.2",
|
||||
"private": true,
|
||||
"packageManager": "pnpm@10.12.3",
|
||||
"description": "Papra frontend client",
|
||||
@@ -21,10 +21,12 @@
|
||||
"serve": "vite preview",
|
||||
"lint": "eslint .",
|
||||
"lint:fix": "eslint --fix .",
|
||||
"test": "vitest run",
|
||||
"test": "pnpm check-i18n-types-outdated && vitest run",
|
||||
"test:watch": "vitest watch",
|
||||
"test:e2e": "playwright test",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"check-i18n-types-outdated": "pnpm script:generate-i18n-types && git diff --exit-code -- src/modules/i18n/locales.types.ts > /dev/null || (echo \"Locales types are outdated, please run 'pnpm script:generate-i18n-types' and commit the changes.\" && exit 1)",
|
||||
"script:get-missing-i18n-keys": "tsx src/scripts/get-missing-i18n-keys.script.ts",
|
||||
"script:sync-i18n-key-order": "tsx src/scripts/sync-i18n-key-order.script.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 2.6 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 2.5 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.1 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.2 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.5 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.8 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 2.1 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 2.2 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 2.6 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 2.8 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 3.4 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.3 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.4 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.5 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.6 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 3.0 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 3.0 KiB |
@@ -1,2 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<browserconfig><msapplication><tile><square70x70logo src="/ms-icon-70x70.png"/><square150x150logo src="/ms-icon-150x150.png"/><square310x310logo src="/ms-icon-310x310.png"/><TileColor>#ffffff</TileColor></tile></msapplication></browserconfig>
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 831 B |
Binary file not shown.
|
Before Width: | Height: | Size: 1.0 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.8 KiB After Width: | Height: | Size: 1.7 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.1 KiB After Width: | Height: | Size: 15 KiB |
@@ -1,41 +0,0 @@
|
||||
{
|
||||
"name": "Papra",
|
||||
"icons": [
|
||||
{
|
||||
"src": "\/android-icon-36x36.png",
|
||||
"sizes": "36x36",
|
||||
"type": "image\/png",
|
||||
"density": "0.75"
|
||||
},
|
||||
{
|
||||
"src": "\/android-icon-48x48.png",
|
||||
"sizes": "48x48",
|
||||
"type": "image\/png",
|
||||
"density": "1.0"
|
||||
},
|
||||
{
|
||||
"src": "\/android-icon-72x72.png",
|
||||
"sizes": "72x72",
|
||||
"type": "image\/png",
|
||||
"density": "1.5"
|
||||
},
|
||||
{
|
||||
"src": "\/android-icon-96x96.png",
|
||||
"sizes": "96x96",
|
||||
"type": "image\/png",
|
||||
"density": "2.0"
|
||||
},
|
||||
{
|
||||
"src": "\/android-icon-144x144.png",
|
||||
"sizes": "144x144",
|
||||
"type": "image\/png",
|
||||
"density": "3.0"
|
||||
},
|
||||
{
|
||||
"src": "\/android-icon-192x192.png",
|
||||
"sizes": "192x192",
|
||||
"type": "image\/png",
|
||||
"density": "4.0"
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 2.6 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 2.8 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 6.8 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.5 KiB |
@@ -540,7 +540,7 @@ export const translations: Partial<TranslationsDictionary> = {
|
||||
// API errors
|
||||
|
||||
'api-errors.document.already_exists': 'Das Dokument existiert bereits',
|
||||
'api-errors.document.size_too_large': 'Die Datei ist zu groß',
|
||||
'api-errors.document.file_too_big': 'Die Dokumentdatei ist zu groß',
|
||||
'api-errors.intake_email.limit_reached': 'Die maximale Anzahl an Eingangse-Mails für diese Organisation wurde erreicht. Bitte aktualisieren Sie Ihren Plan, um weitere Eingangse-Mails zu erstellen.',
|
||||
'api-errors.user.max_organization_count_reached': 'Sie haben die maximale Anzahl an Organisationen erreicht, die Sie erstellen können. Wenn Sie weitere erstellen möchten, kontaktieren Sie bitte den Support.',
|
||||
'api-errors.default': 'Beim Verarbeiten Ihrer Anfrage ist ein Fehler aufgetreten.',
|
||||
|
||||
@@ -538,7 +538,7 @@ export const translations = {
|
||||
// API errors
|
||||
|
||||
'api-errors.document.already_exists': 'The document already exists',
|
||||
'api-errors.document.size_too_large': 'The file size is too large',
|
||||
'api-errors.document.file_too_big': 'The document file is too big',
|
||||
'api-errors.intake_email.limit_reached': 'The maximum number of intake emails for this organization has been reached. Please upgrade your plan to create more intake emails.',
|
||||
'api-errors.user.max_organization_count_reached': 'You have reached the maximum number of organizations you can create, if you need to create more, please contact support.',
|
||||
'api-errors.default': 'An error occurred while processing your request.',
|
||||
|
||||
@@ -540,7 +540,7 @@ export const translations: Partial<TranslationsDictionary> = {
|
||||
// API errors
|
||||
|
||||
'api-errors.document.already_exists': 'El documento ya existe',
|
||||
'api-errors.document.size_too_large': 'El archivo es demasiado grande',
|
||||
'api-errors.document.file_too_big': 'El archivo del documento es demasiado grande',
|
||||
'api-errors.intake_email.limit_reached': 'Se ha alcanzado el número máximo de correos de ingreso para esta organización. Por favor, mejora tu plan para crear más correos de ingreso.',
|
||||
'api-errors.user.max_organization_count_reached': 'Has alcanzado el número máximo de organizaciones que puedes crear, si necesitas crear más, contacta al soporte.',
|
||||
'api-errors.default': 'Ocurrió un error al procesar tu solicitud.',
|
||||
|
||||
@@ -540,7 +540,7 @@ export const translations: Partial<TranslationsDictionary> = {
|
||||
// API errors
|
||||
|
||||
'api-errors.document.already_exists': 'Le document existe déjà',
|
||||
'api-errors.document.size_too_large': 'Le fichier est trop volumineux',
|
||||
'api-errors.document.file_too_big': 'Le fichier du document est trop grand',
|
||||
'api-errors.intake_email.limit_reached': 'Le nombre maximum d\'emails de réception pour cette organisation a été atteint. Veuillez mettre à niveau votre plan pour créer plus d\'emails de réception.',
|
||||
'api-errors.user.max_organization_count_reached': 'Vous avez atteint le nombre maximum d\'organisations que vous pouvez créer, si vous avez besoin de créer plus, veuillez contacter le support.',
|
||||
'api-errors.default': 'Une erreur est survenue lors du traitement de votre requête.',
|
||||
|
||||
@@ -540,7 +540,7 @@ export const translations: Partial<TranslationsDictionary> = {
|
||||
// API errors
|
||||
|
||||
'api-errors.document.already_exists': 'Il documento esiste già',
|
||||
'api-errors.document.size_too_large': 'Il file è troppo grande',
|
||||
'api-errors.document.file_too_big': 'Il file del documento è troppo grande',
|
||||
'api-errors.intake_email.limit_reached': 'È stato raggiunto il numero massimo di email di acquisizione per questa organizzazione. Aggiorna il tuo piano per crearne altre.',
|
||||
'api-errors.user.max_organization_count_reached': 'Hai raggiunto il numero massimo di organizzazioni che puoi creare, se hai bisogno di crearne altre contatta il supporto.',
|
||||
'api-errors.default': 'Si è verificato un errore durante l\'elaborazione della richiesta.',
|
||||
|
||||
@@ -540,7 +540,7 @@ export const translations: Partial<TranslationsDictionary> = {
|
||||
// API errors
|
||||
|
||||
'api-errors.document.already_exists': 'Dokument już istnieje',
|
||||
'api-errors.document.size_too_large': 'Plik jest zbyt duży',
|
||||
'api-errors.document.file_too_big': 'Plik dokumentu jest zbyt duży',
|
||||
'api-errors.intake_email.limit_reached': 'Osiągnięto maksymalną liczbę adresów e-mail do przyjęć dla tej organizacji. Aby utworzyć więcej adresów e-mail do przyjęć, zaktualizuj swój plan.',
|
||||
'api-errors.user.max_organization_count_reached': 'Osiągnięto maksymalną liczbę organizacji, które możesz utworzyć. Jeśli potrzebujesz utworzyć więcej, skontaktuj się z pomocą techniczną.',
|
||||
'api-errors.default': 'Wystąpił błąd podczas przetwarzania żądania.',
|
||||
|
||||
@@ -540,7 +540,7 @@ export const translations: Partial<TranslationsDictionary> = {
|
||||
// API errors
|
||||
|
||||
'api-errors.document.already_exists': 'O documento já existe',
|
||||
'api-errors.document.size_too_large': 'O arquivo é muito grande',
|
||||
'api-errors.document.file_too_big': 'O arquivo do documento é muito grande',
|
||||
'api-errors.intake_email.limit_reached': 'O número máximo de e-mails de entrada para esta organização foi atingido. Faça um upgrade no seu plano para criar mais e-mails de entrada.',
|
||||
'api-errors.user.max_organization_count_reached': 'Você atingiu o número máximo de organizações que pode criar. Se precisar criar mais, entre em contato com o suporte.',
|
||||
'api-errors.default': 'Ocorreu um erro ao processar sua solicitação.',
|
||||
|
||||
@@ -540,7 +540,7 @@ export const translations: Partial<TranslationsDictionary> = {
|
||||
// API errors
|
||||
|
||||
'api-errors.document.already_exists': 'O documento já existe',
|
||||
'api-errors.document.size_too_large': 'O arquivo é muito grande',
|
||||
'api-errors.document.file_too_big': 'O arquivo do documento é muito grande',
|
||||
'api-errors.intake_email.limit_reached': 'O número máximo de e-mails de entrada para esta organização foi atingido. Faça um upgrade no seu plano para criar mais e-mails de entrada.',
|
||||
'api-errors.user.max_organization_count_reached': 'Atingiu o número máximo de organizações que pode criar. Se precisar de criar mais, entre em contato com o suporte.',
|
||||
'api-errors.default': 'Ocorreu um erro ao processar a solicitação.',
|
||||
|
||||
@@ -540,7 +540,7 @@ export const translations: Partial<TranslationsDictionary> = {
|
||||
// API errors
|
||||
|
||||
'api-errors.document.already_exists': 'Documentul există deja',
|
||||
'api-errors.document.size_too_large': 'Fișierul este prea mare',
|
||||
'api-errors.document.file_too_big': 'Fișierul documentului este prea mare',
|
||||
'api-errors.intake_email.limit_reached': 'Numărul maxim de email-uri de primire pentru această organizație a fost atins. Te rugăm să-ți îmbunătățești planul pentru a crea mai multe email-uri de primire.',
|
||||
'api-errors.user.max_organization_count_reached': 'Ai atins numărul maxim de organizații pe care le poți crea. Dacă ai nevoie să creezi mai multe, te rugăm să contactezi asistența.',
|
||||
'api-errors.default': 'A apărut o eroare la procesarea cererii.',
|
||||
|
||||
@@ -38,9 +38,6 @@ export const buildTimeConfig = {
|
||||
isEnabled: asBoolean(import.meta.env.VITE_INTAKE_EMAILS_IS_ENABLED, false),
|
||||
},
|
||||
isSubscriptionsEnabled: asBoolean(import.meta.env.VITE_IS_SUBSCRIPTIONS_ENABLED, false),
|
||||
documentsStorage: {
|
||||
maxUploadSize: asNumber(import.meta.env.VITE_DOCUMENTS_STORAGE_MAX_UPLOAD_SIZE, 10 * 1024 * 1024),
|
||||
},
|
||||
} as const;
|
||||
|
||||
export type Config = typeof buildTimeConfig;
|
||||
|
||||
@@ -5,7 +5,6 @@ import { A } from '@solidjs/router';
|
||||
import { throttle } from 'lodash-es';
|
||||
import { createContext, createSignal, For, Match, Show, Switch, useContext } from 'solid-js';
|
||||
import { Portal } from 'solid-js/web';
|
||||
import { useConfig } from '@/modules/config/config.provider';
|
||||
import { useI18n } from '@/modules/i18n/i18n.provider';
|
||||
import { promptUploadFiles } from '@/modules/shared/files/upload';
|
||||
import { useI18nApiErrors } from '@/modules/shared/http/composables/i18n-api-errors';
|
||||
@@ -58,7 +57,6 @@ export const DocumentUploadProvider: ParentComponent = (props) => {
|
||||
const throttledInvalidateOrganizationDocumentsQuery = throttle(invalidateOrganizationDocumentsQuery, 500);
|
||||
const { getErrorMessage } = useI18nApiErrors();
|
||||
const { t } = useI18n();
|
||||
const { config } = useConfig();
|
||||
|
||||
const [getState, setState] = createSignal<'open' | 'closed' | 'collapsed'>('closed');
|
||||
const [getTasks, setTasks] = createSignal<Task[]>([]);
|
||||
@@ -72,14 +70,8 @@ export const DocumentUploadProvider: ParentComponent = (props) => {
|
||||
setState('open');
|
||||
|
||||
await Promise.all(files.map(async (file) => {
|
||||
const { maxUploadSize } = config.documentsStorage;
|
||||
updateTaskStatus({ file, status: 'uploading' });
|
||||
|
||||
if (maxUploadSize > 0 && file.size > maxUploadSize) {
|
||||
updateTaskStatus({ file, status: 'error', error: Object.assign(new Error('File too large'), { code: 'document.size_too_large' }) });
|
||||
return;
|
||||
}
|
||||
|
||||
const [result, error] = await safely(uploadDocument({ file, organizationId }));
|
||||
|
||||
if (error) {
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import type { Component } from 'solid-js';
|
||||
import { useParams } from '@solidjs/router';
|
||||
import { createSignal } from 'solid-js';
|
||||
import { promptUploadFiles } from '@/modules/shared/files/upload';
|
||||
import { queryClient } from '@/modules/shared/query/query-client';
|
||||
import { cn } from '@/modules/shared/style/cn';
|
||||
import { Button } from '@/modules/ui/components/button';
|
||||
import { useDocumentUpload } from './document-import-status.component';
|
||||
import { uploadDocument } from '../documents.services';
|
||||
|
||||
export const DocumentUploadArea: Component<{ organizationId?: string }> = (props) => {
|
||||
const [isDragging, setIsDragging] = createSignal(false);
|
||||
@@ -11,7 +13,21 @@ export const DocumentUploadArea: Component<{ organizationId?: string }> = (props
|
||||
|
||||
const getOrganizationId = () => props.organizationId ?? params.organizationId;
|
||||
|
||||
const { promptImport, uploadDocuments } = useDocumentUpload({ getOrganizationId });
|
||||
const uploadFiles = async ({ files }: { files: File[] }) => {
|
||||
for (const file of files) {
|
||||
await uploadDocument({ file, organizationId: getOrganizationId() });
|
||||
}
|
||||
|
||||
await queryClient.invalidateQueries({
|
||||
queryKey: ['organizations', getOrganizationId(), 'documents'],
|
||||
refetchType: 'all',
|
||||
});
|
||||
};
|
||||
|
||||
const promptImport = async () => {
|
||||
const { files } = await promptUploadFiles();
|
||||
await uploadFiles({ files });
|
||||
};
|
||||
|
||||
const handleDragOver = (event: DragEvent) => {
|
||||
event.preventDefault();
|
||||
@@ -30,7 +46,7 @@ export const DocumentUploadArea: Component<{ organizationId?: string }> = (props
|
||||
}
|
||||
|
||||
const files = [...event.dataTransfer.files].filter(file => file.type === 'application/pdf');
|
||||
await uploadDocuments({ files });
|
||||
await uploadFiles({ files });
|
||||
};
|
||||
|
||||
return (
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
import type { Document } from './documents.types';
|
||||
import { safely } from '@corentinth/chisels';
|
||||
import { throttle } from 'lodash-es';
|
||||
import { createSignal } from 'solid-js';
|
||||
import { useConfirmModal } from '../shared/confirm';
|
||||
import { promptUploadFiles } from '../shared/files/upload';
|
||||
import { isHttpErrorWithCode } from '../shared/http/http-errors';
|
||||
import { queryClient } from '../shared/query/query-client';
|
||||
import { createToast } from '../ui/components/sonner';
|
||||
import { deleteDocument, restoreDocument } from './documents.services';
|
||||
import { deleteDocument, restoreDocument, uploadDocument } from './documents.services';
|
||||
|
||||
export function invalidateOrganizationDocumentsQuery({ organizationId }: { organizationId: string }) {
|
||||
return queryClient.invalidateQueries({
|
||||
@@ -72,3 +76,57 @@ export function useRestoreDocument() {
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function toastUploadError({ error, file }: { error: Error; file: File }) {
|
||||
if (isHttpErrorWithCode({ error, code: 'document.already_exists' })) {
|
||||
createToast({
|
||||
type: 'error',
|
||||
message: 'Document already exists',
|
||||
description: `The document ${file.name} already exists, it has not been uploaded.`,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (isHttpErrorWithCode({ error, code: 'document.file_too_big' })) {
|
||||
createToast({
|
||||
type: 'error',
|
||||
message: 'Document too big',
|
||||
description: `The document ${file.name} is too big, it has not been uploaded.`,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
createToast({
|
||||
type: 'error',
|
||||
message: 'Failed to upload document',
|
||||
description: error.message,
|
||||
});
|
||||
}
|
||||
|
||||
export function useUploadDocuments({ organizationId }: { organizationId: string }) {
|
||||
const uploadDocuments = async ({ files }: { files: File[] }) => {
|
||||
const throttledInvalidateOrganizationDocumentsQuery = throttle(invalidateOrganizationDocumentsQuery, 500);
|
||||
|
||||
await Promise.all(files.map(async (file) => {
|
||||
const [, error] = await safely(uploadDocument({ file, organizationId }));
|
||||
|
||||
if (error) {
|
||||
toastUploadError({ error, file });
|
||||
}
|
||||
|
||||
await throttledInvalidateOrganizationDocumentsQuery({ organizationId });
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
||||
return {
|
||||
uploadDocuments,
|
||||
promptImport: async () => {
|
||||
const { files } = await promptUploadFiles();
|
||||
|
||||
await uploadDocuments({ files });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -214,6 +214,15 @@ export const DocumentPage: Component = () => {
|
||||
{t('documents.actions.download')}
|
||||
</Button>
|
||||
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={() => window.open(getDataUrl()!, '_blank')}
|
||||
size="sm"
|
||||
>
|
||||
<div class="i-tabler-eye size-4 mr-2"></div>
|
||||
{t('documents.actions.open-in-new-tab')}
|
||||
</Button>
|
||||
|
||||
{getDocument().isDeleted
|
||||
? (
|
||||
<Button
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { translations as defaultTranslations } from '@/locales/en.dictionary';
|
||||
import { translations as defaultTranslations } from '@/locales/en.dictionary';
|
||||
|
||||
export type TranslationKeys = keyof typeof defaultTranslations;
|
||||
export type TranslationsDictionary = Record<TranslationKeys, string>;
|
||||
|
||||
@@ -3,9 +3,9 @@ import { formatBytes } from '@corentinth/chisels';
|
||||
import { useParams } from '@solidjs/router';
|
||||
import { createQueries, keepPreviousData } from '@tanstack/solid-query';
|
||||
import { createSignal, Show, Suspense } from 'solid-js';
|
||||
import { useDocumentUpload } from '@/modules/documents/components/document-import-status.component';
|
||||
import { DocumentUploadArea } from '@/modules/documents/components/document-upload-area.component';
|
||||
import { createdAtColumn, DocumentsPaginatedList, standardActionsColumn, tagsColumn } from '@/modules/documents/components/documents-list.component';
|
||||
import { useUploadDocuments } from '@/modules/documents/documents.composables';
|
||||
import { fetchOrganizationDocuments, getOrganizationDocumentsStats } from '@/modules/documents/documents.services';
|
||||
import { useI18n } from '@/modules/i18n/i18n.provider';
|
||||
import { Button } from '@/modules/ui/components/button';
|
||||
@@ -32,7 +32,7 @@ export const OrganizationPage: Component = () => {
|
||||
],
|
||||
}));
|
||||
|
||||
const { promptImport } = useDocumentUpload({ getOrganizationId: () => params.organizationId });
|
||||
const { promptImport } = useUploadDocuments({ organizationId: params.organizationId });
|
||||
|
||||
return (
|
||||
<div class="p-6 mt-4 pb-32 max-w-5xl mx-auto">
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
env: {
|
||||
TZ: 'UTC',
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -1,49 +1,5 @@
|
||||
# @papra/app-server
|
||||
|
||||
## 0.9.2
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- [#493](https://github.com/papra-hq/papra/pull/493) [`ed4d7e4`](https://github.com/papra-hq/papra/commit/ed4d7e4a00b2ca2c7fe808201c322f957d6ed990) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Fix to allow cross docker volume file moving when consumption is done
|
||||
|
||||
- [#500](https://github.com/papra-hq/papra/pull/500) [`208a561`](https://github.com/papra-hq/papra/commit/208a561668ed2d1019430a9f4f5c5d3fd4cde603) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Added the possibility to define a Libsql/Sqlite driver for the tasks service
|
||||
|
||||
- [#499](https://github.com/papra-hq/papra/pull/499) [`40cb1d7`](https://github.com/papra-hq/papra/commit/40cb1d71d5e52c40aab7ea2c6bc222cea6d55b70) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Enhanced security by serving files as attachement and with an octet-stream content type
|
||||
|
||||
## 0.9.1
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- [#492](https://github.com/papra-hq/papra/pull/492) [`54514e1`](https://github.com/papra-hq/papra/commit/54514e15db5deaffc59dcba34929b5e2e74282e1) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Added a client side guard for rejecting too-big files
|
||||
|
||||
- [#491](https://github.com/papra-hq/papra/pull/491) [`bb9d555`](https://github.com/papra-hq/papra/commit/bb9d5556d3f16225ae40ca4d39600999e819b2c4) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Fix cleanup state when a too-big-file is uploaded
|
||||
|
||||
## 0.9.0
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- [#472](https://github.com/papra-hq/papra/pull/472) [`b08241f`](https://github.com/papra-hq/papra/commit/b08241f20fc326a65a8de0551a7bfa91d9e4c71d) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Dropped support for the dedicated backblaze b2 storage driver as b2 now fully support s3 client
|
||||
|
||||
- [#480](https://github.com/papra-hq/papra/pull/480) [`0a03f42`](https://github.com/papra-hq/papra/commit/0a03f42231f691d339c7ab5a5916c52385e31bd2) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Added documents encryption layer
|
||||
|
||||
- [#472](https://github.com/papra-hq/papra/pull/472) [`b08241f`](https://github.com/papra-hq/papra/commit/b08241f20fc326a65a8de0551a7bfa91d9e4c71d) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Stream file upload instead of full in-memory loading
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- [#481](https://github.com/papra-hq/papra/pull/481) [`1606310`](https://github.com/papra-hq/papra/commit/1606310745e8edf405b527127078143481419e8c) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Allow for more complex intake-email origin adresses
|
||||
|
||||
- [#483](https://github.com/papra-hq/papra/pull/483) [`ec0a437`](https://github.com/papra-hq/papra/commit/ec0a437d86b4c8c0979ba9d0c2ff7b39f054cec0) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Fix a bug where the ingestion folder was not working when the done or error destination folder path (INGESTION_FOLDER_POST_PROCESSING_MOVE_FOLDER_PATH and INGESTION_FOLDER_ERROR_FOLDER_PATH) were absolute.
|
||||
|
||||
- [#475](https://github.com/papra-hq/papra/pull/475) [`ea9d90d`](https://github.com/papra-hq/papra/commit/ea9d90d6cff6954297152b3ad16f99170e8cd0dc) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Use node file streams in ingestion folder for smaller RAM footprint
|
||||
|
||||
- [#477](https://github.com/papra-hq/papra/pull/477) [`a62d376`](https://github.com/papra-hq/papra/commit/a62d3767729ab02ae203a1ac7b7fd6eb6e011d98) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Fixed an issue where tags assigned to only deleted documents won't show up in the tag list
|
||||
|
||||
- [#472](https://github.com/papra-hq/papra/pull/472) [`b08241f`](https://github.com/papra-hq/papra/commit/b08241f20fc326a65a8de0551a7bfa91d9e4c71d) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Properly handle missing files errors in storage drivers
|
||||
|
||||
- Updated dependencies [[`14bc2b8`](https://github.com/papra-hq/papra/commit/14bc2b8f8d0d6605062f37188e7c57bbc61b2c1a)]:
|
||||
- @papra/webhooks@0.3.0
|
||||
- @papra/lecture@0.2.0
|
||||
|
||||
## 0.8.2
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@papra/app-server",
|
||||
"type": "module",
|
||||
"version": "0.9.2",
|
||||
"version": "0.8.2",
|
||||
"private": true,
|
||||
"packageManager": "pnpm@10.12.3",
|
||||
"description": "Papra app server",
|
||||
@@ -42,7 +42,6 @@
|
||||
"@aws-sdk/lib-storage": "^3.835.0",
|
||||
"@azure/storage-blob": "^12.27.0",
|
||||
"@cadence-mq/core": "^0.2.1",
|
||||
"@cadence-mq/driver-libsql": "^0.2.1",
|
||||
"@cadence-mq/driver-memory": "^0.2.0",
|
||||
"@corentinth/chisels": "^1.3.1",
|
||||
"@corentinth/friendly-ids": "^0.0.1",
|
||||
|
||||
@@ -21,8 +21,6 @@ const { db, client } = setupDatabase(config.database);
|
||||
const documentsStorageService = createDocumentStorageService({ documentStorageConfig: config.documentsStorage });
|
||||
|
||||
const taskServices = createTaskServices({ config });
|
||||
await taskServices.initialize();
|
||||
|
||||
const { app } = await createServer({ config, db, taskServices, documentsStorageService });
|
||||
|
||||
const server = serve(
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { Context, RouteDefinitionContext } from '../server.types';
|
||||
import type { Session } from './auth.types';
|
||||
import { get } from 'lodash-es';
|
||||
import { isDefined, isString } from '../../shared/utils';
|
||||
import { isDefined } from '../../shared/utils';
|
||||
|
||||
export function registerAuthRoutes({ app, auth, config }: RouteDefinitionContext) {
|
||||
app.on(
|
||||
@@ -26,7 +26,7 @@ export function registerAuthRoutes({ app, auth, config }: RouteDefinitionContext
|
||||
app.use('*', async (context: Context, next) => {
|
||||
const overrideUserId: unknown = get(context.env, 'loggedInUserId');
|
||||
|
||||
if (isDefined(overrideUserId) && isString(overrideUserId)) {
|
||||
if (isDefined(overrideUserId) && typeof overrideUserId === 'string') {
|
||||
context.set('userId', overrideUserId);
|
||||
context.set('session', {} as Session);
|
||||
context.set('authType', 'session');
|
||||
|
||||
@@ -69,9 +69,6 @@ describe('config models', () => {
|
||||
intakeEmails: {
|
||||
isEnabled: true,
|
||||
},
|
||||
documentsStorage: {
|
||||
maxUploadSize: 10485760,
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
@@ -13,7 +13,6 @@ export function getPublicConfig({ config }: { config: Config }) {
|
||||
'auth.providers.github.isEnabled',
|
||||
'auth.providers.google.isEnabled',
|
||||
'documents.deletedDocumentsRetentionDays',
|
||||
'documentsStorage.maxUploadSize',
|
||||
'intakeEmails.isEnabled',
|
||||
]),
|
||||
{
|
||||
|
||||
@@ -15,7 +15,6 @@ import { intakeEmailsConfig } from '../intake-emails/intake-emails.config';
|
||||
import { organizationsConfig } from '../organizations/organizations.config';
|
||||
import { organizationPlansConfig } from '../plans/plans.config';
|
||||
import { createLogger } from '../shared/logger/logger';
|
||||
import { isString } from '../shared/utils';
|
||||
import { subscriptionsConfig } from '../subscriptions/subscriptions.config';
|
||||
import { tasksConfig } from '../tasks/tasks.config';
|
||||
import { trackingConfig } from '../tracking/tracking.config';
|
||||
@@ -72,7 +71,7 @@ export const configDefinition = {
|
||||
schema: z.union([
|
||||
z.string(),
|
||||
z.array(z.string()),
|
||||
]).transform(value => (isString(value) ? value.split(',') : value)),
|
||||
]).transform(value => (typeof value === 'string' ? value.split(',') : value)),
|
||||
default: ['http://localhost:3000'],
|
||||
env: 'SERVER_CORS_ORIGINS',
|
||||
},
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
import { Buffer } from 'node:buffer';
|
||||
import { describe, expect, test } from 'vitest';
|
||||
import { MULTIPART_FORM_DATA_SINGLE_FILE_CONTENT_LENGTH_OVERHEAD } from './documents.constants';
|
||||
|
||||
const unusuallyLongFileName = 'an-unusually-long-file-name-in-order-to-test-the-content-length-header-with-the-metadata-that-are-included-in-the-form-data-so-lorem-ipsum-dolor-sit-amet-consectetur-adipiscing-elit-sed-do-eiusmod-tempor-incididunt-ut-labore-et-dolore-magna-aliqua-ut-enim-ad-minim-veniam-quis-nostrud-exercitation-ullamco-laboris-nisi-ut-aliquip-ex-ea-commodo-consequat-duis-aute-irure-dolor-in-reprehenderit-in-voluptate-velit-esse-cillum-dolore-eu-fugiat-nulla-pariatur-excepteur-sint-occaecat-proident-in-voluptate-velit-esse-cillum-dolore-eu-fugiat-nulla-pariatur-excepteur-sint-occaecat-proident-in-voluptate-velit-esse-cillum-dolore-eu-fugiat-nulla-pariatur-excepteur-sint-occaecat-proident.txt';
|
||||
|
||||
describe('documents constants', () => {
|
||||
// eslint-disable-next-line test/prefer-lowercase-title
|
||||
describe('MULTIPART_FORM_DATA_SINGLE_FILE_CONTENT_LENGTH_OVERHEAD', () => {
|
||||
test('when uploading a formdata multipart, the body has boundaries and other metadata, so the content length is greater than the file size', async () => {
|
||||
const fileSize = 100;
|
||||
const formData = new FormData();
|
||||
formData.append('file', new File(['a'.repeat(fileSize)], unusuallyLongFileName, { type: 'text/plain' }));
|
||||
const body = new Response(formData);
|
||||
const contentLength = Buffer.from(await body.arrayBuffer()).length;
|
||||
|
||||
expect(contentLength).to.be.greaterThan(fileSize);
|
||||
expect(contentLength).to.be.lessThan(fileSize + MULTIPART_FORM_DATA_SINGLE_FILE_CONTENT_LENGTH_OVERHEAD);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -11,6 +11,3 @@ export const ORIGINAL_DOCUMENTS_STORAGE_KEY = 'originals';
|
||||
// import { ocrLanguages } from '@papra/lecture';
|
||||
// console.log(JSON.stringify(ocrLanguages));
|
||||
export const OCR_LANGUAGES = ['afr', 'amh', 'ara', 'asm', 'aze', 'aze_cyrl', 'bel', 'ben', 'bod', 'bos', 'bul', 'cat', 'ceb', 'ces', 'chi_sim', 'chi_tra', 'chr', 'cym', 'dan', 'deu', 'dzo', 'ell', 'eng', 'enm', 'epo', 'est', 'eus', 'fas', 'fin', 'fra', 'frk', 'frm', 'gle', 'glg', 'grc', 'guj', 'hat', 'heb', 'hin', 'hrv', 'hun', 'iku', 'ind', 'isl', 'ita', 'ita_old', 'jav', 'jpn', 'kan', 'kat', 'kat_old', 'kaz', 'khm', 'kir', 'kor', 'kur', 'lao', 'lat', 'lav', 'lit', 'mal', 'mar', 'mkd', 'mlt', 'msa', 'mya', 'nep', 'nld', 'nor', 'ori', 'pan', 'pol', 'por', 'pus', 'ron', 'rus', 'san', 'sin', 'slk', 'slv', 'spa', 'spa_old', 'sqi', 'srp', 'srp_latn', 'swa', 'swe', 'syr', 'tam', 'tel', 'tgk', 'tgl', 'tha', 'tir', 'tur', 'uig', 'ukr', 'urd', 'uzb', 'uzb_cyrl', 'vie', 'yid'] as const;
|
||||
|
||||
// When uploading a formdata multipart, the body has boundaries and other metadata that need to be accounted for
|
||||
export const MULTIPART_FORM_DATA_SINGLE_FILE_CONTENT_LENGTH_OVERHEAD = 1024; // 1024 bytes
|
||||
|
||||
@@ -13,7 +13,7 @@ import { deferTriggerWebhooks } from '../webhooks/webhook.usecases';
|
||||
import { createDocumentActivityRepository } from './document-activity/document-activity.repository';
|
||||
import { deferRegisterDocumentActivityLog } from './document-activity/document-activity.usecases';
|
||||
import { createDocumentIsNotDeletedError } from './documents.errors';
|
||||
import { formatDocumentForApi, formatDocumentsForApi, isDocumentSizeLimitEnabled } from './documents.models';
|
||||
import { formatDocumentForApi, formatDocumentsForApi } from './documents.models';
|
||||
import { createDocumentsRepository } from './documents.repository';
|
||||
import { documentIdSchema } from './documents.schemas';
|
||||
import { createDocumentCreationUsecase, deleteAllTrashDocuments, deleteTrashDocument, ensureDocumentExists, getDocumentOrThrow } from './documents.usecases';
|
||||
@@ -34,8 +34,6 @@ export function registerDocumentsRoutes(context: RouteDefinitionContext) {
|
||||
}
|
||||
|
||||
function setupCreateDocumentRoute({ app, ...deps }: RouteDefinitionContext) {
|
||||
const { config } = deps;
|
||||
|
||||
app.post(
|
||||
'/api/organizations/:organizationId/documents',
|
||||
requireAuthentication({ apiKeyPermissions: ['documents:create'] }),
|
||||
@@ -46,15 +44,12 @@ function setupCreateDocumentRoute({ app, ...deps }: RouteDefinitionContext) {
|
||||
const { userId } = getUser({ context });
|
||||
const { organizationId } = context.req.valid('param');
|
||||
|
||||
const { maxUploadSize } = config.documentsStorage;
|
||||
|
||||
const { fileStream, fileName, mimeType } = await getFileStreamFromMultipartForm({
|
||||
body: context.req.raw.body,
|
||||
headers: context.req.header(),
|
||||
maxFileSize: isDocumentSizeLimitEnabled({ maxUploadSize }) ? maxUploadSize : undefined,
|
||||
});
|
||||
|
||||
const createDocument = createDocumentCreationUsecase({ ...deps });
|
||||
const createDocument = await createDocumentCreationUsecase({ ...deps });
|
||||
|
||||
const { document } = await createDocument({ fileStream, fileName, mimeType, userId, organizationId });
|
||||
|
||||
@@ -288,13 +283,9 @@ function setupGetDocumentFileRoute({ app, db, documentsStorageService }: RouteDe
|
||||
Readable.toWeb(fileStream),
|
||||
200,
|
||||
{
|
||||
// Prevent XSS by serving the file as an octet-stream
|
||||
'Content-Type': 'application/octet-stream',
|
||||
// Always use attachment for defense in depth - client uses blob API anyway
|
||||
'Content-Disposition': `attachment; filename*=UTF-8''${encodeURIComponent(document.name)}`,
|
||||
'Content-Type': document.mimeType,
|
||||
'Content-Disposition': `inline; filename*=UTF-8''${encodeURIComponent(document.name)}`,
|
||||
'Content-Length': String(document.originalSize),
|
||||
'X-Content-Type-Options': 'nosniff',
|
||||
'X-Frame-Options': 'DENY',
|
||||
},
|
||||
);
|
||||
},
|
||||
|
||||
@@ -35,7 +35,7 @@ describe('documents usecases', () => {
|
||||
});
|
||||
const documentsStorageService = createDocumentStorageService({ documentStorageConfig: config.documentsStorage });
|
||||
|
||||
const createDocument = createDocumentCreationUsecase({
|
||||
const createDocument = await createDocumentCreationUsecase({
|
||||
db,
|
||||
config,
|
||||
generateDocumentId: () => 'doc_1',
|
||||
@@ -96,7 +96,7 @@ describe('documents usecases', () => {
|
||||
const documentsStorageService = createDocumentStorageService({ documentStorageConfig: config.documentsStorage });
|
||||
|
||||
let documentIdIndex = 1;
|
||||
const createDocument = createDocumentCreationUsecase({
|
||||
const createDocument = await createDocumentCreationUsecase({
|
||||
db,
|
||||
config,
|
||||
generateDocumentId: () => `doc_${documentIdIndex++}`,
|
||||
@@ -201,7 +201,7 @@ describe('documents usecases', () => {
|
||||
organizationPlans: { isFreePlanUnlimited: true },
|
||||
});
|
||||
|
||||
const createDocument = createDocumentCreationUsecase({
|
||||
const createDocument = await createDocumentCreationUsecase({
|
||||
db,
|
||||
config,
|
||||
taskServices,
|
||||
@@ -256,7 +256,7 @@ describe('documents usecases', () => {
|
||||
const documentsRepository = createDocumentsRepository({ db });
|
||||
const documentsStorageService = createDocumentStorageService({ documentStorageConfig: config.documentsStorage });
|
||||
|
||||
const createDocument = createDocumentCreationUsecase({
|
||||
const createDocument = await createDocumentCreationUsecase({
|
||||
documentsStorageService,
|
||||
db,
|
||||
config,
|
||||
@@ -305,7 +305,7 @@ describe('documents usecases', () => {
|
||||
});
|
||||
|
||||
let documentIdIndex = 1;
|
||||
const createDocument = createDocumentCreationUsecase({
|
||||
const createDocument = await createDocumentCreationUsecase({
|
||||
db,
|
||||
config,
|
||||
generateDocumentId: () => `doc_${documentIdIndex++}`,
|
||||
@@ -369,7 +369,7 @@ describe('documents usecases', () => {
|
||||
}),
|
||||
} as PlansRepository;
|
||||
|
||||
const createDocument = createDocumentCreationUsecase({
|
||||
const createDocument = await createDocumentCreationUsecase({
|
||||
db,
|
||||
config: overrideConfig(),
|
||||
taskServices,
|
||||
@@ -434,7 +434,7 @@ describe('documents usecases', () => {
|
||||
}),
|
||||
} as PlansRepository;
|
||||
|
||||
const createDocument = createDocumentCreationUsecase({
|
||||
const createDocument = await createDocumentCreationUsecase({
|
||||
db,
|
||||
config: overrideConfig(),
|
||||
taskServices,
|
||||
@@ -492,7 +492,7 @@ describe('documents usecases', () => {
|
||||
}),
|
||||
} as PlansRepository;
|
||||
|
||||
const createDocument = createDocumentCreationUsecase({
|
||||
const createDocument = await createDocumentCreationUsecase({
|
||||
db,
|
||||
config: overrideConfig(),
|
||||
taskServices,
|
||||
|
||||
@@ -14,8 +14,6 @@ import type { DocumentsRepository } from './documents.repository';
|
||||
import type { Document } from './documents.types';
|
||||
import type { DocumentStorageService } from './storage/documents.storage.services';
|
||||
import type { EncryptionContext } from './storage/drivers/drivers.models';
|
||||
import { PassThrough } from 'node:stream';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { safely } from '@corentinth/chisels';
|
||||
import pLimit from 'p-limit';
|
||||
import { createOrganizationDocumentStorageLimitReachedError } from '../organizations/organizations.errors';
|
||||
@@ -103,27 +101,17 @@ export async function createDocument({
|
||||
},
|
||||
});
|
||||
|
||||
// Create a PassThrough stream that will be used for saving the file
|
||||
// This allows us to use pipeline for better error handling
|
||||
const outputStream = new PassThrough();
|
||||
|
||||
const streamProcessingPromise = pipeline(
|
||||
fileStream,
|
||||
hashStream,
|
||||
byteCountStream,
|
||||
outputStream,
|
||||
);
|
||||
const outputStream = fileStream
|
||||
.pipe(hashStream)
|
||||
.pipe(byteCountStream);
|
||||
|
||||
// We optimistically save the file to leverage streaming, if the file already exists, we will delete it
|
||||
const [newFileStorageContext] = await Promise.all([
|
||||
documentsStorageService.saveFile({
|
||||
fileStream: outputStream,
|
||||
storageKey: originalDocumentStorageKey,
|
||||
mimeType,
|
||||
fileName,
|
||||
}),
|
||||
streamProcessingPromise,
|
||||
]);
|
||||
const newFileStorageContext = await documentsStorageService.saveFile({
|
||||
fileStream: outputStream,
|
||||
storageKey: originalDocumentStorageKey,
|
||||
mimeType,
|
||||
fileName,
|
||||
});
|
||||
|
||||
const hash = getHash();
|
||||
const size = getByteCount();
|
||||
@@ -188,7 +176,7 @@ export async function createDocument({
|
||||
export type CreateDocumentUsecase = Awaited<ReturnType<typeof createDocumentCreationUsecase>>;
|
||||
export type DocumentUsecaseDependencies = Omit<Parameters<typeof createDocument>[0], 'fileStream' | 'fileName' | 'mimeType' | 'userId' | 'organizationId'>;
|
||||
|
||||
export function createDocumentCreationUsecase({
|
||||
export async function createDocumentCreationUsecase({
|
||||
db,
|
||||
config,
|
||||
taskServices,
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import { documentsTable } from "../documents.table";
|
||||
import { DocumentStorageService } from "./documents.storage.services";
|
||||
|
||||
export async function migrateDocumentsStorage({db, inputDocumentStorageService, outputDocumentStorageService, logger = createLogger({ namespace: 'migrateDocumentsStorage' })}: {
|
||||
db: Database;
|
||||
inputDocumentStorageService: DocumentStorageService;
|
||||
outputDocumentStorageService: DocumentStorageService;
|
||||
logger?: Logger;
|
||||
}) {
|
||||
|
||||
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import type { DocumentStorageConfig } from '../../documents.storage.types';
|
||||
import { AzuriteContainer } from '@testcontainers/azurite';
|
||||
import { describe } from 'vitest';
|
||||
import { TEST_CONTAINER_IMAGES } from '../../../../../../test/containers/images';
|
||||
import { overrideConfig } from '../../../../config/config.test-utils';
|
||||
import { runDriverTestSuites } from '../drivers.test-suite';
|
||||
import { azBlobStorageDriverFactory } from './az-blob.storage-driver';
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { Readable } from 'node:stream';
|
||||
import type { Config } from '../../../config/config.types';
|
||||
import type { ExtendNamedArguments, ExtendReturnPromise } from '../../../shared/types';
|
||||
import type { DocumentStorageConfig } from '../documents.storage.types';
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { Config } from '../../../../config/config.types';
|
||||
import type { DocumentStorageConfig } from '../../documents.storage.types';
|
||||
import fs from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
|
||||
@@ -37,14 +37,6 @@ export const fsStorageDriverFactory = defineStorageDriver(({ documentStorageConf
|
||||
writeStream.on('error', (error) => {
|
||||
reject(error);
|
||||
});
|
||||
|
||||
// Listen for errors on the input stream as well
|
||||
fileStream.on('error', (error) => {
|
||||
// Clean up the write stream and file
|
||||
writeStream.destroy();
|
||||
fs.unlink(storagePath, () => {}); // Ignore errors when cleaning up
|
||||
reject(error);
|
||||
});
|
||||
});
|
||||
},
|
||||
getFileStream: async ({ storageKey }) => {
|
||||
|
||||
@@ -3,6 +3,7 @@ import { CreateBucketCommand } from '@aws-sdk/client-s3';
|
||||
import { LocalstackContainer } from '@testcontainers/localstack';
|
||||
import { describe } from 'vitest';
|
||||
import { TEST_CONTAINER_IMAGES } from '../../../../../../test/containers/images';
|
||||
import { overrideConfig } from '../../../../config/config.test-utils';
|
||||
import { runDriverTestSuites } from '../drivers.test-suite';
|
||||
import { s3StorageDriverFactory } from './s3.storage-driver';
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@ import { DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, S3Client } fr
|
||||
|
||||
import { Upload } from '@aws-sdk/lib-storage';
|
||||
import { safely } from '@corentinth/chisels';
|
||||
import { isString } from '../../../../shared/utils';
|
||||
import { createFileNotFoundError } from '../../document-storage.errors';
|
||||
import { defineStorageDriver } from '../drivers.models';
|
||||
|
||||
@@ -13,7 +12,7 @@ function isS3NotFoundError(error: Error) {
|
||||
const codes = ['NoSuchKey', 'NotFound'];
|
||||
|
||||
return codes.includes(error.name)
|
||||
|| ('Code' in error && isString(error.Code) && codes.includes(error.Code));
|
||||
|| ('Code' in error && typeof error.Code === 'string' && codes.includes(error.Code));
|
||||
}
|
||||
|
||||
export const s3StorageDriverFactory = defineStorageDriver(({ documentStorageConfig }) => {
|
||||
|
||||
@@ -23,7 +23,7 @@ describe('document-encryption usecases', () => {
|
||||
|
||||
const storageDriver = inMemoryStorageDriverFactory();
|
||||
|
||||
const createDocumentWithoutEncryption = createDocumentCreationUsecase({
|
||||
const createDocumentWithoutEncryption = await createDocumentCreationUsecase({
|
||||
db,
|
||||
config: overrideConfig(),
|
||||
taskServices: noopTaskServices,
|
||||
@@ -61,7 +61,7 @@ describe('document-encryption usecases', () => {
|
||||
},
|
||||
});
|
||||
|
||||
const createDocumentWithEncryption = createDocumentCreationUsecase({
|
||||
const createDocumentWithEncryption = await createDocumentCreationUsecase({
|
||||
db,
|
||||
documentsStorageService: documentStorageServiceWithEncryption,
|
||||
config: overrideConfig(),
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
import type { Logger } from '@crowlog/logger';
|
||||
import type { Database } from '../../../app/database/database.types';
|
||||
import type { Config } from '../../../config/config.types';
|
||||
import type { DocumentStorageService } from '../documents.storage.services';
|
||||
import { eq, isNull } from 'drizzle-orm';
|
||||
import { eq, isNotNull, isNull } from 'drizzle-orm';
|
||||
import { createLogger } from '../../../shared/logger/logger';
|
||||
import { documentsTable } from '../../documents.table';
|
||||
import {
|
||||
createDocumentStorageService,
|
||||
|
||||
} from '../documents.storage.services';
|
||||
|
||||
export async function encryptAllUnencryptedDocuments({
|
||||
db,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import type { ConfigDefinition } from 'figue';
|
||||
import { z } from 'zod';
|
||||
import { booleanishSchema } from '../config/config.schemas';
|
||||
import { isString } from '../shared/utils';
|
||||
import { defaultIgnoredPatterns } from './ingestion-folders.constants';
|
||||
|
||||
export const ingestionFolderConfig = {
|
||||
@@ -62,7 +61,7 @@ export const ingestionFolderConfig = {
|
||||
schema: z.union([
|
||||
z.string(),
|
||||
z.array(z.string()),
|
||||
]).transform(value => (isString(value) ? value.split(',') : value)),
|
||||
]).transform(value => (typeof value === 'string' ? value.split(',') : value)),
|
||||
default: defaultIgnoredPatterns,
|
||||
env: 'INGESTION_FOLDER_IGNORED_PATTERNS',
|
||||
},
|
||||
|
||||
@@ -52,7 +52,7 @@ describe('ingestion-folders usecases', () => {
|
||||
organizationsRepository,
|
||||
logger,
|
||||
fs,
|
||||
createDocument: createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),
|
||||
createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),
|
||||
});
|
||||
|
||||
// Check database
|
||||
@@ -154,7 +154,7 @@ describe('ingestion-folders usecases', () => {
|
||||
organizationsRepository,
|
||||
logger,
|
||||
fs,
|
||||
createDocument: createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),
|
||||
createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),
|
||||
});
|
||||
|
||||
// Check database
|
||||
@@ -257,7 +257,7 @@ describe('ingestion-folders usecases', () => {
|
||||
organizationsRepository,
|
||||
logger,
|
||||
fs,
|
||||
createDocument: createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),
|
||||
createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),
|
||||
}));
|
||||
|
||||
expect(error).to.deep.equal(createInvalidPostProcessingStrategyError({ strategy: 'unknown' }));
|
||||
@@ -335,7 +335,7 @@ describe('ingestion-folders usecases', () => {
|
||||
throw new Error('File not found');
|
||||
},
|
||||
},
|
||||
createDocument: createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),
|
||||
createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),
|
||||
});
|
||||
|
||||
// Check logs
|
||||
@@ -475,7 +475,7 @@ describe('ingestion-folders usecases', () => {
|
||||
organizationsRepository,
|
||||
logger,
|
||||
fs,
|
||||
createDocument: createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),
|
||||
createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),
|
||||
});
|
||||
|
||||
// Check database
|
||||
|
||||
@@ -47,7 +47,7 @@ export function createIngestionFolderWatcher({
|
||||
return {
|
||||
startWatchingIngestionFolders: async () => {
|
||||
const organizationsRepository = createOrganizationsRepository({ db });
|
||||
const createDocument = createDocumentCreationUsecase({ db, config, logger, taskServices, documentsStorageService });
|
||||
const createDocument = await createDocumentCreationUsecase({ db, config, logger, taskServices, documentsStorageService });
|
||||
|
||||
const ignored = await buildPathIgnoreFunction({ config, cwd, organizationsRepository });
|
||||
|
||||
@@ -235,10 +235,10 @@ async function buildPathIgnoreFunction({
|
||||
const { organizationIds } = await organizationsRepository.getAllOrganizationIds();
|
||||
|
||||
const doneFolders = strategy === 'move'
|
||||
? isAbsolute(moveToFolderPath) ? [moveToFolderPath] : uniq(organizationIds.map(id => join(cwd, folderRootPath, id, moveToFolderPath)))
|
||||
? isAbsolute(moveToFolderPath) ? moveToFolderPath : uniq(organizationIds.map(id => join(cwd, folderRootPath, id, moveToFolderPath)))
|
||||
: [];
|
||||
|
||||
const errorFolders = isAbsolute(errorFolder) ? [errorFolder] : uniq(organizationIds.map(id => join(cwd, folderRootPath, id, errorFolder)));
|
||||
const errorFolders = isAbsolute(errorFolder) ? errorFolder : uniq(organizationIds.map(id => join(cwd, folderRootPath, id, errorFolder)));
|
||||
|
||||
const ignoredFolders = [...doneFolders, ...errorFolders];
|
||||
const matchExcludedPatterns = picomatch(ignoredPatterns);
|
||||
|
||||
@@ -189,7 +189,7 @@ function setupIngestIntakeEmailRoute({ app, db, config, trackingServices, taskSe
|
||||
|
||||
const intakeEmailsRepository = createIntakeEmailsRepository({ db });
|
||||
|
||||
const createDocument = createDocumentCreationUsecase({
|
||||
const createDocument = await createDocumentCreationUsecase({
|
||||
documentsStorageService,
|
||||
db,
|
||||
config,
|
||||
|
||||
@@ -29,7 +29,7 @@ describe('intake-emails usecases', () => {
|
||||
|
||||
const intakeEmailsRepository = createIntakeEmailsRepository({ db });
|
||||
|
||||
const createDocument = createDocumentCreationUsecase({
|
||||
const createDocument = await createDocumentCreationUsecase({
|
||||
db,
|
||||
taskServices,
|
||||
documentsStorageService: inMemoryStorageDriverFactory(),
|
||||
@@ -71,7 +71,7 @@ describe('intake-emails usecases', () => {
|
||||
|
||||
const intakeEmailsRepository = createIntakeEmailsRepository({ db });
|
||||
|
||||
const createDocument = createDocumentCreationUsecase({
|
||||
const createDocument = await createDocumentCreationUsecase({
|
||||
db,
|
||||
taskServices,
|
||||
documentsStorageService: inMemoryStorageDriverFactory(),
|
||||
@@ -104,7 +104,7 @@ describe('intake-emails usecases', () => {
|
||||
|
||||
const intakeEmailsRepository = createIntakeEmailsRepository({ db });
|
||||
|
||||
const createDocument = createDocumentCreationUsecase({
|
||||
const createDocument = await createDocumentCreationUsecase({
|
||||
db,
|
||||
taskServices,
|
||||
documentsStorageService: inMemoryStorageDriverFactory(),
|
||||
@@ -142,7 +142,7 @@ describe('intake-emails usecases', () => {
|
||||
|
||||
const intakeEmailsRepository = createIntakeEmailsRepository({ db });
|
||||
|
||||
const createDocument = createDocumentCreationUsecase({
|
||||
const createDocument = await createDocumentCreationUsecase({
|
||||
db,
|
||||
taskServices,
|
||||
documentsStorageService: inMemoryStorageDriverFactory(),
|
||||
@@ -191,7 +191,7 @@ describe('intake-emails usecases', () => {
|
||||
|
||||
const intakeEmailsRepository = createIntakeEmailsRepository({ db });
|
||||
|
||||
const createDocument = createDocumentCreationUsecase({
|
||||
const createDocument = await createDocumentCreationUsecase({
|
||||
db,
|
||||
taskServices,
|
||||
documentsStorageService: inMemoryStorageDriverFactory(),
|
||||
|
||||
@@ -3,7 +3,7 @@ import type { FsNative } from './fs.services';
|
||||
import { memfs } from 'memfs';
|
||||
import { createFsServices } from './fs.services';
|
||||
|
||||
export function buildInMemoryFs(volume: NestedDirectoryJSON) {
|
||||
export function createInMemoryFsServices(volume: NestedDirectoryJSON) {
|
||||
const { vol } = memfs(volume);
|
||||
|
||||
const fs = {
|
||||
@@ -12,16 +12,7 @@ export function buildInMemoryFs(volume: NestedDirectoryJSON) {
|
||||
} as FsNative;
|
||||
|
||||
return {
|
||||
fs,
|
||||
getFsState: () => vol.toJSON(),
|
||||
};
|
||||
}
|
||||
|
||||
export function createInMemoryFsServices(volume: NestedDirectoryJSON) {
|
||||
const { fs, getFsState } = buildInMemoryFs(volume);
|
||||
|
||||
return {
|
||||
getFsState,
|
||||
fs: createFsServices({ fs }),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
import { isNil, isString } from '../utils';
|
||||
|
||||
export function isCrossDeviceError({ error }: { error: Error & { code?: unknown } }) {
|
||||
if (isNil(error.code) || !isString(error.code)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return [
|
||||
'EXDEV', // Linux based OS (see `man rename`)
|
||||
'ERROR_NOT_SAME_DEVICE', // Windows
|
||||
].includes(error.code);
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
import { describe, expect, test } from 'vitest';
|
||||
import { buildInMemoryFs } from './fs.in-memory';
|
||||
import { moveFile } from './fs.services';
|
||||
|
||||
describe('fs services', () => {
|
||||
describe('moveFile', () => {
|
||||
test('moves a file from the source path to the destination path', async () => {
|
||||
const { fs, getFsState } = buildInMemoryFs({
|
||||
'/file.txt': 'test content',
|
||||
});
|
||||
|
||||
await moveFile({
|
||||
sourceFilePath: '/file.txt',
|
||||
destinationFilePath: '/renamed.txt',
|
||||
fs,
|
||||
});
|
||||
|
||||
expect(getFsState()).to.eql({
|
||||
'/renamed.txt': 'test content',
|
||||
});
|
||||
});
|
||||
|
||||
test('if the destination file is in a different partition or disk, or a different docker volume, the underlying rename operation fails with an EXDEV error, so we fallback to copy + delete the source file', async () => {
|
||||
const { fs, getFsState } = buildInMemoryFs({
|
||||
'/file.txt': 'test content',
|
||||
});
|
||||
|
||||
await moveFile({
|
||||
sourceFilePath: '/file.txt',
|
||||
destinationFilePath: '/renamed.txt',
|
||||
fs: {
|
||||
...fs,
|
||||
rename: async () => {
|
||||
// Simulate an EXDEV error
|
||||
throw Object.assign(new Error('EXDEV'), { code: 'EXDEV' });
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(getFsState()).to.eql({
|
||||
'/renamed.txt': 'test content',
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -2,9 +2,8 @@ import type { Readable } from 'node:stream';
|
||||
import { Buffer } from 'node:buffer';
|
||||
import fsSyncNative from 'node:fs';
|
||||
import fsPromisesNative from 'node:fs/promises';
|
||||
import { injectArguments, safely } from '@corentinth/chisels';
|
||||
import { injectArguments } from '@corentinth/chisels';
|
||||
import { pick } from 'lodash-es';
|
||||
import { isCrossDeviceError } from './fs.models';
|
||||
|
||||
// what we use from the native fs module
|
||||
export type FsNative = {
|
||||
@@ -14,13 +13,12 @@ export type FsNative = {
|
||||
stat: (path: string) => Promise<{ size: number }>;
|
||||
readFile: (path: string) => Promise<Buffer>;
|
||||
access: (path: string, mode: number) => Promise<void>;
|
||||
copyFile: (sourcePath: string, destinationPath: string) => Promise<void>;
|
||||
constants: { F_OK: number };
|
||||
createReadStream: (path: string) => Readable;
|
||||
};
|
||||
|
||||
const fsNative = {
|
||||
...pick(fsPromisesNative, 'mkdir', 'unlink', 'rename', 'readFile', 'access', 'constants', 'stat', 'copyFile'),
|
||||
...pick(fsPromisesNative, 'mkdir', 'unlink', 'rename', 'readFile', 'access', 'constants', 'stat'),
|
||||
createReadStream: fsSyncNative.createReadStream.bind(fsSyncNative) as (filePath: string) => Readable,
|
||||
} as FsNative;
|
||||
|
||||
@@ -68,19 +66,7 @@ export async function deleteFile({ filePath, fs = fsNative }: { filePath: string
|
||||
}
|
||||
|
||||
export async function moveFile({ sourceFilePath, destinationFilePath, fs = fsNative }: { sourceFilePath: string; destinationFilePath: string; fs?: FsNative }) {
|
||||
const [, error] = await safely(fs.rename(sourceFilePath, destinationFilePath));
|
||||
|
||||
// With different docker volumes, the rename operation fails with an EXDEV error,
|
||||
// so we fallback to copy and delete the source file
|
||||
if (error && isCrossDeviceError({ error })) {
|
||||
await fs.copyFile(sourceFilePath, destinationFilePath);
|
||||
await fs.unlink(sourceFilePath);
|
||||
return;
|
||||
}
|
||||
|
||||
if (error) {
|
||||
throw error;
|
||||
}
|
||||
await fs.rename(sourceFilePath, destinationFilePath);
|
||||
}
|
||||
|
||||
export async function readFile({ filePath, fs = fsNative }: { filePath: string; fs?: FsNative }) {
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import type { Context } from '../../app/server.types';
|
||||
import { isNil } from '../utils';
|
||||
|
||||
export function getHeader({ context, name }: { context: Context; name: string }) {
|
||||
return context.req.header(name);
|
||||
@@ -16,13 +15,3 @@ export function getImpersonatedUserIdFromHeader({ context }: { context: Context
|
||||
|
||||
return { impersonatedUserId };
|
||||
}
|
||||
|
||||
export function getContentLengthHeader({ headers }: { headers: Record<string, string> }): number | undefined {
|
||||
const contentLengthHeaderValue = headers['content-length'] ?? headers['Content-Length'];
|
||||
|
||||
if (isNil(contentLengthHeaderValue)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return Number(contentLengthHeaderValue);
|
||||
}
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
import { describe, expect, test } from 'vitest';
|
||||
import { isContentLengthPessimisticallyTooLarge } from './file-upload';
|
||||
|
||||
describe('file-upload', () => {
|
||||
describe('isContentLengthPessimisticallyTooLarge', () => {
|
||||
test(`a file upload request is considered pessimistically too large when
|
||||
- a content length header is present
|
||||
- a max file size limit is provided
|
||||
- the content length is greater than the max file size limit plus an over-estimated overhead due to the multipart form data (boundaries, metadata, etc)`, () => {
|
||||
expect(
|
||||
isContentLengthPessimisticallyTooLarge({
|
||||
contentLength: 1_000,
|
||||
maxFileSize: 1_000,
|
||||
overhead: 512,
|
||||
}),
|
||||
).to.eql(false);
|
||||
|
||||
expect(
|
||||
isContentLengthPessimisticallyTooLarge({
|
||||
contentLength: undefined,
|
||||
maxFileSize: 1_000,
|
||||
overhead: 512,
|
||||
}),
|
||||
).to.eql(false);
|
||||
|
||||
expect(
|
||||
isContentLengthPessimisticallyTooLarge({
|
||||
contentLength: 1_000,
|
||||
maxFileSize: undefined,
|
||||
overhead: 512,
|
||||
}),
|
||||
).to.eql(false);
|
||||
|
||||
expect(
|
||||
isContentLengthPessimisticallyTooLarge({
|
||||
contentLength: undefined,
|
||||
maxFileSize: undefined,
|
||||
overhead: 512,
|
||||
}),
|
||||
).to.eql(false);
|
||||
|
||||
expect(
|
||||
isContentLengthPessimisticallyTooLarge({
|
||||
contentLength: 1_513,
|
||||
maxFileSize: 1_000,
|
||||
overhead: 512,
|
||||
}),
|
||||
).to.eql(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,41 +1,18 @@
|
||||
import type { Logger } from '../logger/logger';
|
||||
import { Readable } from 'node:stream';
|
||||
import createBusboy from 'busboy';
|
||||
import { MULTIPART_FORM_DATA_SINGLE_FILE_CONTENT_LENGTH_OVERHEAD } from '../../documents/documents.constants';
|
||||
import { createDocumentSizeTooLargeError } from '../../documents/documents.errors';
|
||||
import { createError } from '../errors/errors';
|
||||
import { getContentLengthHeader } from '../headers/headers.models';
|
||||
import { createLogger } from '../logger/logger';
|
||||
import { isNil } from '../utils';
|
||||
|
||||
// Early check to avoid parsing the stream if the content length is set and too large
|
||||
export function isContentLengthPessimisticallyTooLarge({
|
||||
contentLength,
|
||||
maxFileSize,
|
||||
overhead = MULTIPART_FORM_DATA_SINGLE_FILE_CONTENT_LENGTH_OVERHEAD,
|
||||
}: {
|
||||
contentLength?: number;
|
||||
maxFileSize?: number;
|
||||
overhead?: number;
|
||||
}) {
|
||||
if (isNil(contentLength) || isNil(maxFileSize)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return contentLength > maxFileSize + overhead;
|
||||
}
|
||||
|
||||
export async function getFileStreamFromMultipartForm({
|
||||
body,
|
||||
headers,
|
||||
fieldName = 'file',
|
||||
maxFileSize,
|
||||
logger = createLogger({ namespace: 'file-upload' }),
|
||||
}: {
|
||||
body: ReadableStream | null | undefined;
|
||||
headers: Record<string, string>;
|
||||
fieldName?: string;
|
||||
maxFileSize?: number;
|
||||
logger?: Logger;
|
||||
}) {
|
||||
if (!body) {
|
||||
@@ -46,20 +23,12 @@ export async function getFileStreamFromMultipartForm({
|
||||
});
|
||||
}
|
||||
|
||||
const contentLength = getContentLengthHeader({ headers });
|
||||
if (isContentLengthPessimisticallyTooLarge({ contentLength, maxFileSize })) {
|
||||
logger.debug({ contentLength, maxFileSize }, 'Content length is pessimistically too large');
|
||||
|
||||
throw createDocumentSizeTooLargeError();
|
||||
}
|
||||
|
||||
const { promise, resolve, reject } = Promise.withResolvers<{ fileStream: Readable; fileName: string; mimeType: string }>();
|
||||
|
||||
const bb = createBusboy({
|
||||
headers,
|
||||
limits: {
|
||||
files: 1, // Only allow one file
|
||||
fileSize: maxFileSize,
|
||||
},
|
||||
})
|
||||
.on('file', (formFieldname, fileStream, info) => {
|
||||
@@ -75,11 +44,6 @@ export async function getFileStreamFromMultipartForm({
|
||||
}));
|
||||
}
|
||||
|
||||
fileStream.on('limit', () => {
|
||||
logger.info({ contentLength, maxFileSize }, 'File stream limit reached');
|
||||
fileStream.destroy(createDocumentSizeTooLargeError());
|
||||
});
|
||||
|
||||
resolve({
|
||||
fileStream,
|
||||
fileName: info.filename,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, test } from 'vitest';
|
||||
import { isDefined, isNil, isNonEmptyString, isString, omitUndefined } from './utils';
|
||||
import { isDefined, isNil, omitUndefined } from './utils';
|
||||
|
||||
describe('utils', () => {
|
||||
describe('omitUndefined', () => {
|
||||
@@ -47,38 +47,4 @@ describe('utils', () => {
|
||||
expect(isDefined({})).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isString', () => {
|
||||
test('returns true if the value is a string', () => {
|
||||
expect(isString('')).toBe(true);
|
||||
expect(isString('foo')).toBe(true);
|
||||
expect(isString(String(1))).toBe(true);
|
||||
});
|
||||
|
||||
test('returns false if the value is not a string', () => {
|
||||
expect(isString(undefined)).toBe(false);
|
||||
expect(isString(null)).toBe(false);
|
||||
expect(isString(0)).toBe(false);
|
||||
expect(isString(false)).toBe(false);
|
||||
expect(isString({})).toBe(false);
|
||||
expect(isString([])).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isNonEmptyString', () => {
|
||||
test('returns true if the value is a non-empty string', () => {
|
||||
expect(isNonEmptyString('')).toBe(false);
|
||||
expect(isNonEmptyString('foo')).toBe(true);
|
||||
expect(isNonEmptyString(String(1))).toBe(true);
|
||||
});
|
||||
|
||||
test('returns false if the value is not a non-empty string', () => {
|
||||
expect(isNonEmptyString(undefined)).toBe(false);
|
||||
expect(isNonEmptyString(null)).toBe(false);
|
||||
expect(isNonEmptyString(0)).toBe(false);
|
||||
expect(isNonEmptyString(false)).toBe(false);
|
||||
expect(isNonEmptyString({})).toBe(false);
|
||||
expect(isNonEmptyString([])).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -15,11 +15,3 @@ export function isNil(value: unknown): value is undefined | null {
|
||||
export function isDefined<T>(value: T): value is Exclude<T, undefined | null> {
|
||||
return !isNil(value);
|
||||
}
|
||||
|
||||
export function isString(value: unknown): value is string {
|
||||
return typeof value === 'string';
|
||||
}
|
||||
|
||||
export function isNonEmptyString(value: unknown): value is string {
|
||||
return isString(value) && value.length > 0;
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user