From 3f3a70d9968638adf47a3fbbdda4d6d28e97221c Mon Sep 17 00:00:00 2001 From: Tom Moor Date: Sat, 8 Nov 2025 11:47:51 -0500 Subject: [PATCH] feat: Adjust line-height depending on script (#10565) * migration * Auto detect language and adjust line-height accordingly * Remove accidental commit * Remove unneccessary adjustment * test * mock --- .jestconfig.json | 1 + app/editor/index.tsx | 3 +- app/models/Document.ts | 3 + app/scenes/Document/components/Editor.tsx | 1 + package.json | 2 + server/__mocks__/franc.js | 10 ++++ server/__mocks__/iso-639-3.js | 18 ++++++ .../20251104013803-document-language.js | 15 +++++ server/models/Document.ts | 6 ++ server/presenters/document.ts | 1 + server/queues/tasks/DocumentUpdateTextTask.ts | 7 +++ shared/editor/components/Styles.ts | 58 +++++++++++++++++++ yarn.lock | 30 ++++++++++ 13 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 server/__mocks__/franc.js create mode 100644 server/__mocks__/iso-639-3.js create mode 100644 server/migrations/20251104013803-document-language.js diff --git a/.jestconfig.json b/.jestconfig.json index a53130629a..f7fcffebb1 100644 --- a/.jestconfig.json +++ b/.jestconfig.json @@ -1,6 +1,7 @@ { "workerIdleMemoryLimit": "0.75", "maxWorkers": "50%", + "transformIgnorePatterns": ["node_modules/(?!(franc|trigram-utils)/)"], "projects": [ { "displayName": "server", diff --git a/app/editor/index.tsx b/app/editor/index.tsx index 39307f2757..15c4248fe2 100644 --- a/app/editor/index.tsx +++ b/app/editor/index.tsx @@ -144,6 +144,7 @@ export type Props = { style?: React.CSSProperties; /** Optional style overrides for the contenteeditable */ editorStyle?: React.CSSProperties; + lang?: string; }; type State = { @@ -846,7 +847,7 @@ export class Editor extends React.PureComponent< editorStyle={this.props.editorStyle} commenting={!!this.props.onClickCommentMark} ref={this.elementRef} - lang="" + lang={this.props.lang ?? ""} /> {this.widgets && diff --git a/app/models/Document.ts b/app/models/Document.ts index 30cce9e593..5ecc852aa5 100644 --- a/app/models/Document.ts +++ b/app/models/Document.ts @@ -134,6 +134,9 @@ export default class Document extends ArchivableModel implements Searchable { @observable title: string; + /** The likely language of the document. */ + language: string | undefined; + /** * An icon (or) emoji to use as the document icon. */ diff --git a/app/scenes/Document/components/Editor.tsx b/app/scenes/Document/components/Editor.tsx index 461742b2b8..0783536908 100644 --- a/app/scenes/Document/components/Editor.tsx +++ b/app/scenes/Document/components/Editor.tsx @@ -229,6 +229,7 @@ function DocumentEditor(props: Props, ref: React.RefObject) { )} { + // Return 'eng' (English) by default, or 'und' (undetermined) for empty text + if (!text || text.trim().length === 0) { + return "und"; + } + return "eng"; +}); + +module.exports = { franc }; diff --git a/server/__mocks__/iso-639-3.js b/server/__mocks__/iso-639-3.js new file mode 100644 index 0000000000..6d18ef4f83 --- /dev/null +++ b/server/__mocks__/iso-639-3.js @@ -0,0 +1,18 @@ +// Mock for iso-639-3 language code conversion library +const iso6393To1 = { + eng: "en", + fra: "fr", + deu: "de", + spa: "es", + ita: "it", + por: "pt", + rus: "ru", + jpn: "ja", + zho: "zh", + ara: "ar", + hin: "hi", + ben: "bn", + und: undefined, // undetermined +}; + +module.exports = { iso6393To1 }; diff --git a/server/migrations/20251104013803-document-language.js b/server/migrations/20251104013803-document-language.js new file mode 100644 index 0000000000..421bd0695b --- /dev/null +++ b/server/migrations/20251104013803-document-language.js @@ -0,0 +1,15 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.addColumn("documents", "language", { + type: Sequelize.STRING(2), + allowNull: true, + }); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.removeColumn("documents", "language"); + }, +}; diff --git a/server/models/Document.ts b/server/models/Document.ts index 061661265d..b49fe2c4b4 100644 --- a/server/models/Document.ts +++ b/server/models/Document.ts @@ -70,6 +70,7 @@ import Fix from "./decorators/Fix"; import { DocumentHelper } from "./helpers/DocumentHelper"; import IsHexColor from "./validators/IsHexColor"; import Length from "./validators/Length"; +import { MaxLength } from "class-validator"; export const DOCUMENT_VERSION = 2; @@ -339,6 +340,11 @@ class Document extends ArchivableModel< @Column(DataType.TEXT) text: string; + /** The likely language of the content. */ + @Column(DataType.STRING(2)) + @MaxLength(2) + language: string; + /** * The content of the document as JSON, this is a snapshot at the last time the state was saved. */ diff --git a/server/presenters/document.ts b/server/presenters/document.ts index 090fdce651..8d358a5ffb 100644 --- a/server/presenters/document.ts +++ b/server/presenters/document.ts @@ -57,6 +57,7 @@ async function presentDocument( icon: document.icon, color: document.color, tasks: document.tasks, + language: document.language, createdAt: document.createdAt, createdBy: undefined, updatedAt: document.updatedAt, diff --git a/server/queues/tasks/DocumentUpdateTextTask.ts b/server/queues/tasks/DocumentUpdateTextTask.ts index 3a98e7b93f..f7e7ae36b1 100644 --- a/server/queues/tasks/DocumentUpdateTextTask.ts +++ b/server/queues/tasks/DocumentUpdateTextTask.ts @@ -1,7 +1,10 @@ +import { franc } from "franc"; +import { iso6393To1 } from "iso-639-3"; import { Node } from "prosemirror-model"; import { schema, serializer } from "@server/editor"; import { Document } from "@server/models"; import { DocumentEvent } from "@server/types"; +import { DocumentHelper } from "@server/models/helpers/DocumentHelper"; import BaseTask from "./BaseTask"; export default class DocumentUpdateTextTask extends BaseTask { @@ -13,6 +16,10 @@ export default class DocumentUpdateTextTask extends BaseTask { const node = Node.fromJSON(schema, document.content); document.text = serializer.serialize(node); + + const language = franc(DocumentHelper.toPlainText(document)); + document.language = iso6393To1[language]; + await document.save({ silent: true }); } } diff --git a/shared/editor/components/Styles.ts b/shared/editor/components/Styles.ts index ac3353d045..caba098fab 100644 --- a/shared/editor/components/Styles.ts +++ b/shared/editor/components/Styles.ts @@ -304,6 +304,63 @@ const emailStyle = (props: Props) => css` } `; +const textStyle = () => css` + /* Southeast Asian scripts */ + :lang(th), /* Thai */ + :lang(lo), /* Lao */ + :lang(km), /* Khmer */ + :lang(my) { + /* Burmese */ + p { + line-height: 1.8; + } + } + + /* South Asian scripts */ + :lang(hi), /* Hindi */ + :lang(mr), /* Marathi */ + :lang(ne), /* Nepali */ + :lang(bn), /* Bengali */ + :lang(gu), /* Gujarati */ + :lang(pa), /* Punjabi */ + :lang(te), /* Telugu */ + :lang(ta), /* Tamil */ + :lang(ml), /* Malayalam */ + :lang(si) { + /* Sinhala */ + p { + line-height: 1.7; + } + } + + /* Tibetan and related scripts */ + :lang(bo) { + p { + line-height: 1.8; + } + } + + /* Middle Eastern scripts */ + :lang(ar), /* Arabic */ + :lang(fa), /* Persian */ + :lang(ur), /* Urdu */ + :lang(he) { + /* Hebrew */ + p { + line-height: 1.6; + } + } + + /* Ethiopic and other complex scripts */ + :lang(am), /* Amharic */ + :lang(mn) { + /* Mongolian */ + p { + line-height: 1.7; + } + } +`; + const style = (props: Props) => css` flex-grow: ${props.grow ? 1 : 0}; justify-content: start; @@ -2017,6 +2074,7 @@ const EditorContainer = styled.div` ${codeBlockStyle} ${findAndReplaceStyle} ${emailStyle} + ${textStyle} `; export default EditorContainer; diff --git a/yarn.lock b/yarn.lock index 861dd5f032..cf7f5ce44b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6688,6 +6688,11 @@ co@^4.6.0: resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" integrity "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ= sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==" +collapse-white-space@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/collapse-white-space/-/collapse-white-space-2.1.0.tgz#640257174f9f42c740b40f3b55ee752924feefca" + integrity sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw== + collect-v8-coverage@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/collect-v8-coverage/-/collect-v8-coverage-1.0.1.tgz#cc2c8e94fc18bbdffe64d6534570c8a673b27f59" @@ -8460,6 +8465,13 @@ framesync@5.3.0: dependencies: tslib "^2.1.0" +franc@^6.2.0: + version "6.2.0" + resolved "https://registry.yarnpkg.com/franc/-/franc-6.2.0.tgz#2ced94b49b1df10bdebb8ab2cdfb57aa5551daa5" + integrity sha512-rcAewP7PSHvjq7Kgd7dhj82zE071kX5B4W1M4ewYMf/P+i6YsDQmj62Xz3VQm9zyUzUXwhIde/wHLGCMrM+yGg== + dependencies: + trigram-utils "^2.0.0" + fresh@~0.5.2: version "0.5.2" resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" @@ -9647,6 +9659,11 @@ isexe@^3.1.1: resolved "https://registry.yarnpkg.com/isexe/-/isexe-3.1.1.tgz#4a407e2bd78ddfb14bea0c27c6f7072dde775f0d" integrity sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ== +iso-639-3@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/iso-639-3/-/iso-639-3-3.0.1.tgz#4be56987c46fbda79da63a3d90d6552d7429dcea" + integrity sha512-SdljCYXOexv/JmbQ0tvigHN43yECoscVpe2y2hlEqy/CStXQlroPhZLj7zKLRiGqLJfw8k7B973UAMDoQczVgQ== + isomorphic-unfetch@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/isomorphic-unfetch/-/isomorphic-unfetch-3.1.0.tgz#87341d5f4f7b63843d468438128cb087b7c3e98f" @@ -11374,6 +11391,11 @@ mz@^2.7.0: object-assign "^4.0.1" thenify-all "^1.0.0" +n-gram@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/n-gram/-/n-gram-2.0.2.tgz#e544a7dffefc49c22d898b2f491e787941b3a2ba" + integrity sha512-S24aGsn+HLBxUGVAUFOwGpKs7LBcG4RudKU//eWzt/mQ97/NMKQxDWHyHx63UNWk/OOdihgmzoETn1tf5nQDzQ== + nanoid@^3.3.11: version "3.3.11" resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.11.tgz#4f4f112cefbe303202f2199838128936266d185b" @@ -14286,6 +14308,14 @@ tree-kill@^1.2.2: resolved "https://registry.yarnpkg.com/tree-kill/-/tree-kill-1.2.2.tgz#4ca09a9092c88b73a7cdc5e8a01b507b0790a0cc" integrity "sha1-TKCakJLIi3OnzcXooBtQeweQoMw= sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==" +trigram-utils@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/trigram-utils/-/trigram-utils-2.0.1.tgz#d22c08350f2cc7ae02ce6d497732db3ef43c8722" + integrity sha512-nfWIXHEaB+HdyslAfMxSqWKDdmqY9I32jS7GnqpdWQnLH89r6A5sdk3fDVYqGAZ0CrT8ovAFSAo6HRiWcWNIGQ== + dependencies: + collapse-white-space "^2.0.0" + n-gram "^2.0.0" + triple-beam@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9"