Merge branch 'develop' of https://github.com/TriliumNext/Notes into develop

src/services/import/single.spec.ts

@@ -1,4 +1,4 @@
-import { describe, expect, it } from "vitest";
+import { beforeAll, describe, expect, it } from "vitest";
 import fs from "fs";
 import path from "path";
 import { fileURLToPath } from "url";
@@ -10,40 +10,72 @@ import cls from "../cls.js";
 import sql_init from "../sql_init.js";
 import { initializeTranslations } from "../i18n.js";
 import single from "./single.js";
+import stripBom from "strip-bom";
 const scriptDir = dirname(fileURLToPath(import.meta.url));
 
-describe("processNoteContent", () => {
-    it("treats single MDX as Markdown", async () => {
-        const mdxSample = fs.readFileSync(path.join(scriptDir, "samples", "Text Note.mdx"));
-        const taskContext = TaskContext.getInstance("import-mdx", "import", {
-            textImportedAsText: true
-        });
+async function testImport(fileName: string, mimetype: string) {
+    const buffer = fs.readFileSync(path.join(scriptDir, "samples", fileName));
+    const taskContext = TaskContext.getInstance("import-mdx", "import", {
+        textImportedAsText: true,
+        codeImportedAsCode: true
+    });
 
-        await new Promise<void>((resolve, reject) => {
-            cls.init(async () => {
-                initializeTranslations();
-                sql_init.initializeDb();
-                await sql_init.dbReady;
+    return new Promise<{ buffer: Buffer, importedNote: BNote }>((resolve, reject) => {
+        cls.init(async () => {
+            const rootNote = becca.getNote("root");
+            if (!rootNote) {
+                reject("Missing root note.");
+            }
 
-                const rootNote = becca.getNote("root");
-                if (!rootNote) {
-                    reject("Missing root note.");
-                }
-
-                const importedNote = single.importSingleFile(taskContext, {
-                    originalname: "Text Note.mdx",
-                    mimetype: "text/mdx",
-                    buffer: mdxSample
-                }, rootNote as BNote);
-                try {
-                    expect(importedNote.mime).toBe("text/html");
-                    expect(importedNote.type).toBe("text");
-                    expect(importedNote.title).toBe("Text Note");
-                } catch (e) {
-                    reject(e);
-                }
-                resolve();
+            const importedNote = single.importSingleFile(taskContext, {
+                originalname: fileName,
+                mimetype,
+                buffer: buffer
+            }, rootNote as BNote);
+            resolve({
+                buffer,
+                importedNote
             });
         });
     });
+}
+
+describe("processNoteContent", () => {
+    beforeAll(async () => {
+        initializeTranslations();
+        sql_init.initializeDb();
+        await sql_init.dbReady;
+    });
+
+    it("treats single MDX as Markdown", async () => {
+        const { importedNote } = await testImport("Text Note.mdx", "text/mdx");
+        expect(importedNote.mime).toBe("text/html");
+        expect(importedNote.type).toBe("text");
+        expect(importedNote.title).toBe("Text Note");
+    });
+
+    it("supports HTML note with UTF-16 (w/ BOM) from Microsoft Outlook", async () => {
+        const { importedNote } = await testImport("IREN Reports Q2 FY25 Results.htm", "text/html");
+        expect(importedNote.mime).toBe("text/html");
+        expect(importedNote.title).toBe("IREN Reports Q2 FY25 Results");
+        expect(importedNote.getContent().toString().substring(0, 5)).toEqual("<html");
+    });
+
+    it("supports code note with UTF-16", async () => {
+        const { importedNote, buffer } = await testImport("UTF-16LE Code Note.json", "application/json");
+        expect(importedNote.mime).toBe("application/json");
+        expect(importedNote.getContent().toString()).toStrictEqual(stripBom(buffer.toString("utf-16le")));
+    });
+
+    it("supports plain text note with UTF-16", async () => {
+        const { importedNote } = await testImport("UTF-16LE Text Note.txt", "text/plain");
+        expect(importedNote.mime).toBe("text/html");
+        expect(importedNote.getContent().toString()).toBe("<p>Plain text goes here.<br></p>");
+    });
+
+    it("supports markdown note with UTF-16", async () => {
+        const { importedNote } = await testImport("UTF-16LE Text Note.md", "text/markdown");
+        expect(importedNote.mime).toBe("text/html");
+        expect(importedNote.getContent().toString()).toBe("<h2>Hello world</h2>\n<p>Plain text goes here.</p>\n");
+    });
 })

src/services/import/single.ts

@@ -8,7 +8,7 @@ import imageService from "../../services/image.js";
 import protectedSessionService from "../protected_session.js";
 import markdownService from "./markdown.js";
 import mimeService from "./mime.js";
-import { getNoteTitle } from "../../services/utils.js";
+import { getNoteTitle, processStringOrBuffer } from "../../services/utils.js";
 import importUtils from "./utils.js";
 import htmlSanitizer from "../html_sanitizer.js";
 import type { File } from "./common.js";
@@ -69,7 +69,7 @@ function importFile(taskContext: TaskContext, file: File, parentNote: BNote) {
 
 function importCodeNote(taskContext: TaskContext, file: File, parentNote: BNote) {
     const title = getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces);
-    const content = file.buffer.toString("utf-8");
+    const content = processStringOrBuffer(file.buffer);
     const detectedMime = mimeService.getMime(file.originalname) || file.mimetype;
     const mime = mimeService.normalizeMimeType(detectedMime);
 
@@ -89,7 +89,7 @@ function importCodeNote(taskContext: TaskContext, file: File, parentNote: BNote)
 
 function importPlainText(taskContext: TaskContext, file: File, parentNote: BNote) {
     const title = getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces);
-    const plainTextContent = file.buffer.toString("utf-8");
+    const plainTextContent = processStringOrBuffer(file.buffer);
     const htmlContent = convertTextToHtml(plainTextContent);
 
     const { note } = noteService.createNewNote({
@@ -125,7 +125,7 @@ function convertTextToHtml(text: string) {
 function importMarkdown(taskContext: TaskContext, file: File, parentNote: BNote) {
     const title = getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces);
 
-    const markdownContent = file.buffer.toString("utf-8");
+    const markdownContent = processStringOrBuffer(file.buffer);
     let htmlContent = markdownService.renderToHtml(markdownContent, title);
 
     if (taskContext.data?.safeImport) {
@@ -147,7 +147,7 @@ function importMarkdown(taskContext: TaskContext, file: File, parentNote: BNote)
 }
 
 function importHtml(taskContext: TaskContext, file: File, parentNote: BNote) {
-    let content = file.buffer.toString("utf-8");
+    let content = processStringOrBuffer(file.buffer);
 
     // Try to get title from HTML first, fall back to filename
     // We do this before sanitization since that turns all <h1>s into <h2>

src/services/import/zip.spec.ts

@@ -1,4 +1,4 @@
-import { describe, expect, it } from "vitest";
+import { beforeAll, describe, expect, it } from "vitest";
 import fs from "fs";
 import path from "path";
 import { fileURLToPath } from "url";
@@ -12,35 +12,46 @@ import sql_init from "../sql_init.js";
 import { initializeTranslations } from "../i18n.js";
 const scriptDir = dirname(fileURLToPath(import.meta.url));
 
-describe("processNoteContent", () => {
-    it("treats single MDX as Markdown in ZIP as text note", async () => {
-        const mdxSample = fs.readFileSync(path.join(scriptDir, "samples", "mdx.zip"));
-        const taskContext = TaskContext.getInstance("import-mdx", "import", {
-            textImportedAsText: true
-        });
+async function testImport(fileName: string) {
+    const mdxSample = fs.readFileSync(path.join(scriptDir, "samples", fileName));
+    const taskContext = TaskContext.getInstance("import-mdx", "import", {
+        textImportedAsText: true
+    });
 
-        await new Promise<void>((resolve, reject) => {
-            cls.init(async () => {
-                initializeTranslations();
-                sql_init.initializeDb();
-                await sql_init.dbReady;
+    return new Promise<{ importedNote: BNote; rootNote: BNote }>((resolve, reject) => {
+        cls.init(async () => {
+            const rootNote = becca.getNote("root");
+            if (!rootNote) {
+                expect(rootNote).toBeTruthy();
+                return;
+            }
 
-                const rootNote = becca.getNote("root");
-                if (!rootNote) {
-                    expect(rootNote).toBeTruthy();
-                    return;
-                }
-
-                const importedNote = await zip.importZip(taskContext, mdxSample, rootNote as BNote);
-                try {
-                    expect(importedNote.mime).toBe("text/mdx");
-                    expect(importedNote.type).toBe("text");
-                    expect(importedNote.title).toBe("Text Note");
-                } catch (e) {
-                    reject(e);
-                }
-                resolve();
+            const importedNote = await zip.importZip(taskContext, mdxSample, rootNote as BNote);
+            resolve({
+                importedNote,
+                rootNote
             });
         });
     });
+}
+
+describe("processNoteContent", () => {
+    beforeAll(async () => {
+        initializeTranslations();
+        sql_init.initializeDb();
+        await sql_init.dbReady;
+    });
+
+    it("treats single MDX as Markdown in ZIP as text note", async () => {
+        const { importedNote } = await testImport("mdx.zip");
+        expect(importedNote.mime).toBe("text/mdx");
+        expect(importedNote.type).toBe("text");
+        expect(importedNote.title).toBe("Text Note");
+    });
+
+    it("can import email from Microsoft Outlook with UTF-16 with BOM", async () => {
+        const { rootNote, importedNote } = await testImport("IREN.Reports.Q2.FY25.Results_files.zip");
+        const htmlNote = rootNote.children.find((ch) => ch.title === "IREN Reports Q2 FY25 Results");
+        expect(htmlNote?.getContent().toString().substring(0, 4)).toEqual("<div");
+    });
 })

src/services/import/zip.ts

@@ -1,7 +1,7 @@
 "use strict";
 
 import BAttribute from "../../becca/entities/battribute.js";
-import { removeTextFileExtension, newEntityId, getNoteTitle } from "../../services/utils.js";
+import { removeTextFileExtension, newEntityId, getNoteTitle, processStringOrBuffer } from "../../services/utils.js";
 import log from "../../services/log.js";
 import noteService from "../../services/notes.js";
 import attributeService from "../../services/attributes.js";
@@ -457,7 +457,7 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo
         }
 
         if (type !== "file" && type !== "image") {
-            content = content.toString("utf-8");
+            content = processStringOrBuffer(content);
         }
 
         const noteTitle = getNoteTitle(filePath, taskContext.data?.replaceUnderscoresWithSpaces || false, noteMeta);

src/services/utils.ts

@@ -1,5 +1,7 @@
 "use strict";
 
+import chardet from "chardet";
+import stripBom from "strip-bom";
 import crypto from "crypto";
 import { generator } from "rand-token";
 import unescape from "unescape";
@@ -330,6 +332,36 @@ function compareVersions(v1: string, v2: string): number {
     return 0;
 }
 
+/**
+ * For buffers, they are scanned for a supported encoding and decoded (UTF-8, UTF-16). In some cases, the BOM is also stripped.
+ *
+ * For strings, they are returned immediately without any transformation.
+ *
+ * For nullish values, an empty string is returned.
+ *
+ * @param data the string or buffer to process.
+ * @returns the string representation of the buffer, or the same string is it's a string.
+ */
+export function processStringOrBuffer(data: string | Buffer | null) {
+    if (!data) {
+        return "";
+    }
+
+    if (!Buffer.isBuffer(data)) {
+        return data;
+    }
+
+    const detectedEncoding = chardet.detect(data);
+    console.log("Detected as ", detectedEncoding);
+    switch (detectedEncoding) {
+        case "UTF-16LE":
+            return stripBom(data.toString("utf-16le"));
+        case "UTF-8":
+        default:
+            return data.toString("utf-8");
+    }
+}
+
 export default {
     compareVersions,
     crash,