From 78b3f50bfd4bbe3de053d7b7d994ad779e3002a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sven=20M=C3=A4der?= Date: Sun, 20 Dec 2020 23:14:56 +0100 Subject: [PATCH 01/10] Use LaTeX delimiters by default, add /tex command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since parsing for $'s as maths delimiters is tricky, switch the default to \(...\) for inline and \[...\] for display maths as it is used in LaTeX. Add /tex command to explicitly parse in TeX mode, which uses $...$ for inline and $$...$$ for display maths. Signed-off-by: Sven Mäder --- src/SlashCommands.tsx | 18 +++++++ src/editor/deserialize.ts | 8 +-- src/editor/serialize.ts | 98 ++++++++++++++++++++++++++++--------- src/i18n/strings/en_EN.json | 1 + 4 files changed, 99 insertions(+), 26 deletions(-) diff --git a/src/SlashCommands.tsx b/src/SlashCommands.tsx index 79c21c4af5..e9bde933ec 100644 --- a/src/SlashCommands.tsx +++ b/src/SlashCommands.tsx @@ -48,6 +48,7 @@ import SettingsStore from "./settings/SettingsStore"; import {UIFeature} from "./settings/UIFeature"; import {CHAT_EFFECTS} from "./effects" import CallHandler from "./CallHandler"; +import {markdownSerializeIfNeeded} from './editor/serialize'; // XXX: workaround for https://github.com/microsoft/TypeScript/issues/31816 interface HTMLInputEvent extends Event { @@ -223,6 +224,23 @@ export const Commands = [ }, category: CommandCategories.messages, }), + new Command({ + command: 'tex', + args: '', + description: _td('Sends a message in TeX mode, using $ and $$ delimiters for maths'), + runFn: function(roomId, args) { + if (SettingsStore.getValue("feature_latex_maths")) { + if (args) { + let html = markdownSerializeIfNeeded(args, {forceHTML: false}, {forceTEX: true}); + return success(MatrixClientPeg.get().sendHtmlMessage(roomId, args, html)); + } + return reject(this.getUsage()); + } else { + return reject("Render LaTeX maths in messages needs to be enabled in Labs"); + } + }, + category: CommandCategories.messages, + }), new Command({ command: 'ddg', args: '', diff --git a/src/editor/deserialize.ts b/src/editor/deserialize.ts index 6336b4c46b..a1ee079af5 100644 --- a/src/editor/deserialize.ts +++ b/src/editor/deserialize.ts @@ -136,11 +136,11 @@ function parseElement(n: HTMLElement, partCreator: PartCreator, lastNode: HTMLEl // math nodes are translated back into delimited latex strings if (n.hasAttribute("data-mx-maths")) { const delimLeft = (n.nodeName == "SPAN") ? - (SdkConfig.get()['latex_maths_delims'] || {})['inline_left'] || "$" : - (SdkConfig.get()['latex_maths_delims'] || {})['display_left'] || "$$"; + (SdkConfig.get()['latex_maths_delims'] || {})['inline_left'] || "\\(" : + (SdkConfig.get()['latex_maths_delims'] || {})['display_left'] || "\\["; const delimRight = (n.nodeName == "SPAN") ? - (SdkConfig.get()['latex_maths_delims'] || {})['inline_right'] || "$" : - (SdkConfig.get()['latex_maths_delims'] || {})['display_right'] || "$$"; + (SdkConfig.get()['latex_maths_delims'] || {})['inline_right'] || "\\)" : + (SdkConfig.get()['latex_maths_delims'] || {})['display_right'] || "\\]"; const tex = n.getAttribute("data-mx-maths"); return partCreator.plain(delimLeft + tex + delimRight); } else if (!checkDescendInto(n)) { diff --git a/src/editor/serialize.ts b/src/editor/serialize.ts index c1f4da306b..ca798a324e 100644 --- a/src/editor/serialize.ts +++ b/src/editor/serialize.ts @@ -41,24 +41,57 @@ export function mdSerialize(model: EditorModel) { }, ""); } -export function htmlSerializeIfNeeded(model: EditorModel, {forceHTML = false} = {}) { - let md = mdSerialize(model); +export function markdownSerializeIfNeeded(md: string, {forceHTML = false} = {}, {forceTEX = false} = {}) { + // copy of raw input to remove unwanted math later + const orig = md; if (SettingsStore.getValue("feature_latex_maths")) { - const displayPattern = (SdkConfig.get()['latex_maths_delims'] || {})['display_pattern'] || - "\\$\\$(([^$]|\\\\\\$)*)\\$\\$"; - const inlinePattern = (SdkConfig.get()['latex_maths_delims'] || {})['inline_pattern'] || - "\\$(([^$]|\\\\\\$)*)\\$"; + if (forceTEX) { + // detect math with tex delimiters, inline: $...$, display $$...$$ + // preferably use negative lookbehinds, not supported in all major browsers: + // const displayPattern = "^(?\n\n\n\n`; - }); + // conditions for display math detection ($$...$$): + // - left delimiter ($$) is not escaped by a backslash + // - pattern starts at the beginning of a line + // - left delimiter is not followed by a space or tab character + // - pattern ends at the end of a line + const displayPattern = "^(?!\\\\)\\$\\$(?![ \\t])(([^$]|\\\\\\$)+?)\\$\\$$"; - md = md.replace(RegExp(inlinePattern, "gm"), function(m, p1) { - const p1e = AllHtmlEntities.encode(p1); - return ``; - }); + // conditions for inline math detection ($...$): + // - left and right delimiters ($) are not escaped by backslashes + // - pattern starts at the beginning of a line or follows a whitespace character + // - left delimiter is not followed by a whitespace character + // - right delimiter is not preseeded by a whitespace character + const inlinePattern = "(^|\\s)(?!\\\\)\\$(?!\\s)(([^$]|\\\\\\$)*[^\\\\\\s\\$](?:\\\\\\$)?)\\$"; + + md = md.replace(RegExp(displayPattern, "gm"), function(m, p1) { + const p1e = AllHtmlEntities.encode(p1); + return `
\n\n
\n\n`; + }); + + md = md.replace(RegExp(inlinePattern, "gm"), function(m, p1, p2) { + const p2e = AllHtmlEntities.encode(p2); + return `${p1}`; + }); + } else { + // detect math with latex delimiters, inline: \(...\), display \[...\] + const displayPattern = (SdkConfig.get()['latex_maths_delims'] || {})['display_pattern'] || + "^\\\\\\[(.*?)\\\\\\]$"; + const inlinePattern = (SdkConfig.get()['latex_maths_delims'] || {})['inline_pattern'] || + "(^|\\s)\\\\\\((.*?)\\\\\\)"; + + md = md.replace(RegExp(displayPattern, "gms"), function(m, p1) { + const p1e = AllHtmlEntities.encode(p1); + return `
\n\n
\n\n`; + }); + + md = md.replace(RegExp(inlinePattern, "gms"), function(m, p1, p2) { + const p2e = AllHtmlEntities.encode(p2); + return `${p1}`; + }); + } // make sure div tags always start on a new line, otherwise it will confuse // the markdown parser @@ -69,15 +102,30 @@ export function htmlSerializeIfNeeded(model: EditorModel, {forceHTML = false} = if (!parser.isPlainText() || forceHTML) { // feed Markdown output to HTML parser const phtml = cheerio.load(parser.toHTML(), - { _useHtmlParser2: true, decodeEntities: false }) + { _useHtmlParser2: true, decodeEntities: false }); - // add fallback output for latex math, which should not be interpreted as markdown - phtml('div, span').each(function(i, e) { - const tex = phtml(e).attr('data-mx-maths') - if (tex) { - phtml(e).html(`${tex}`) - } - }); + if (SettingsStore.getValue("feature_latex_maths")) { + // original Markdown without LaTeX replacements + const parserOrig = new Markdown(orig); + const phtmlOrig = cheerio.load(parserOrig.toHTML(), + { _useHtmlParser2: true, decodeEntities: false }); + + // since maths delimiters are handled before Markdown, + // code blocks could contain mangled content. + // replace code blocks with original content + phtml('code').contents('div, span').each(function(i) { + const origData = phtmlOrig('code').contents('div, span')[i].data; + phtml('code').contents('div, span')[i].data = origData; + }); + + // add fallback output for latex math, which should not be interpreted as markdown + phtml('div, span').each(function(i, e) { + const tex = phtml(e).attr('data-mx-maths') + if (tex) { + phtml(e).html(`${tex}`) + } + }); + } return phtml.html(); } // ensure removal of escape backslashes in non-Markdown messages @@ -86,6 +134,12 @@ export function htmlSerializeIfNeeded(model: EditorModel, {forceHTML = false} = } } +export function htmlSerializeIfNeeded(model: EditorModel, {forceHTML = false} = {}) { + let md = mdSerialize(model); + + return markdownSerializeIfNeeded(md, {forceHTML: forceHTML}); +} + export function textSerialize(model: EditorModel) { return model.parts.reduce((text, part) => { switch (part.type) { diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index 2fb70ecdb1..bf6c61414a 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -416,6 +416,7 @@ "Prepends ( ͡° ͜ʖ ͡°) to a plain-text message": "Prepends ( ͡° ͜ʖ ͡°) to a plain-text message", "Sends a message as plain text, without interpreting it as markdown": "Sends a message as plain text, without interpreting it as markdown", "Sends a message as html, without interpreting it as markdown": "Sends a message as html, without interpreting it as markdown", + "Sends a message in TeX mode, using $ and $$ delimiters for maths": "Sends a message in TeX mode, using $ and $$ delimiters for maths", "Searches DuckDuckGo for results": "Searches DuckDuckGo for results", "/ddg is not a command": "/ddg is not a command", "To use it, just wait for autocomplete results to load and tab through them.": "To use it, just wait for autocomplete results to load and tab through them.", From fb57123e25966912277ac65927304b088b9e7db8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sven=20M=C3=A4der?= Date: Tue, 22 Dec 2020 12:18:38 +0100 Subject: [PATCH 02/10] Improve inline latex regex matching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sven Mäder --- src/SlashCommands.tsx | 2 +- src/editor/serialize.ts | 29 +++++++++++++++++++---------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/SlashCommands.tsx b/src/SlashCommands.tsx index e9bde933ec..bf190fc450 100644 --- a/src/SlashCommands.tsx +++ b/src/SlashCommands.tsx @@ -231,7 +231,7 @@ export const Commands = [ runFn: function(roomId, args) { if (SettingsStore.getValue("feature_latex_maths")) { if (args) { - let html = markdownSerializeIfNeeded(args, {forceHTML: false}, {forceTEX: true}); + const html = markdownSerializeIfNeeded(args, {forceHTML: false}, {forceTEX: true}); return success(MatrixClientPeg.get().sendHtmlMessage(roomId, args, html)); } return reject(this.getUsage()); diff --git a/src/editor/serialize.ts b/src/editor/serialize.ts index ca798a324e..4ef722e334 100644 --- a/src/editor/serialize.ts +++ b/src/editor/serialize.ts @@ -52,18 +52,18 @@ export function markdownSerializeIfNeeded(md: string, {forceHTML = false} = {}, // const displayPattern = "^(? Date: Tue, 22 Dec 2020 13:31:58 +0100 Subject: [PATCH 03/10] Fix linting error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sven Mäder --- src/editor/serialize.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/editor/serialize.ts b/src/editor/serialize.ts index 4ef722e334..3aafa70fe8 100644 --- a/src/editor/serialize.ts +++ b/src/editor/serialize.ts @@ -144,7 +144,7 @@ export function markdownSerializeIfNeeded(md: string, {forceHTML = false} = {}, } export function htmlSerializeIfNeeded(model: EditorModel, {forceHTML = false} = {}) { - let md = mdSerialize(model); + const md = mdSerialize(model); return markdownSerializeIfNeeded(md, {forceHTML: forceHTML}); } From c4f0987487c58ed88dacc62ff155dfa140729fc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sven=20M=C3=A4der?= Date: Fri, 29 Jan 2021 00:11:06 +0100 Subject: [PATCH 04/10] Use TeX and LaTeX delimiters by default for serialize --- src/SlashCommands.tsx | 2 +- src/editor/serialize.ts | 71 +++++++++++++++++++++---------------- src/i18n/strings/en_EN.json | 2 +- 3 files changed, 43 insertions(+), 32 deletions(-) diff --git a/src/SlashCommands.tsx b/src/SlashCommands.tsx index bf190fc450..387b9991db 100644 --- a/src/SlashCommands.tsx +++ b/src/SlashCommands.tsx @@ -227,7 +227,7 @@ export const Commands = [ new Command({ command: 'tex', args: '', - description: _td('Sends a message in TeX mode, using $ and $$ delimiters for maths'), + description: _td('Sends a message in TeX mode, without restrictions'), runFn: function(roomId, args) { if (SettingsStore.getValue("feature_latex_maths")) { if (args) { diff --git a/src/editor/serialize.ts b/src/editor/serialize.ts index 3aafa70fe8..61d7878de5 100644 --- a/src/editor/serialize.ts +++ b/src/editor/serialize.ts @@ -47,28 +47,12 @@ export function markdownSerializeIfNeeded(md: string, {forceHTML = false} = {}, if (SettingsStore.getValue("feature_latex_maths")) { if (forceTEX) { - // detect math with tex delimiters, inline: $...$, display $$...$$ - // preferably use negative lookbehinds, not supported in all major browsers: - // const displayPattern = "^(?\n\n\n\n`; + md = md.replace(RegExp(displayPattern, "gm"), function(m, p1, p2) { + const p2e = AllHtmlEntities.encode(p2); + return `${p1}
\n\n
\n\n`; }); md = md.replace(RegExp(inlinePattern, "gm"), function(m, p1, p2) { @@ -76,24 +60,51 @@ export function markdownSerializeIfNeeded(md: string, {forceHTML = false} = {}, return `${p1}`; }); } else { + // detect math with tex delimiters, inline: $...$, display $$...$$ + // preferably use negative lookbehinds, not supported in all major browsers: + // const displayPattern = "^(?\n\n\n\n`; + }); + + md = md.replace(RegExp(inlinePatternDollar, "gm"), function(m, p1, p2) { + const p2e = AllHtmlEntities.encode(p2); + return `${p1}`; + }); + // detect math with latex delimiters, inline: \(...\), display \[...\] // conditions for display math detection \[...\]: - // - pattern starts at beginning of line - // - pattern ends at end of line + // - pattern starts at beginning of line or is not prefixed with backslash + // - pattern is not empty const displayPattern = (SdkConfig.get()['latex_maths_delims'] || {})['display_pattern'] || - "^\\\\\\[(.*?)\\\\\\]$"; + "(^|[^\\\\])\\\\\\[(?!\\\\\\])(.*?)\\\\\\]"; // conditions for inline math detection \(...\): // - pattern starts at beginning of line or is not prefixed with backslash - // (this allows escaping and requires that multiple consecutive - // patterns are separated by at least one character) + // - pattern is not empty const inlinePattern = (SdkConfig.get()['latex_maths_delims'] || {})['inline_pattern'] || - "(^|[^\\\\])\\\\\\((.*?)\\\\\\)"; + "(^|[^\\\\])\\\\\\((?!\\\\\\))(.*?)\\\\\\)"; - md = md.replace(RegExp(displayPattern, "gms"), function(m, p1) { - const p1e = AllHtmlEntities.encode(p1); - return `
\n\n
\n\n`; + md = md.replace(RegExp(displayPattern, "gms"), function(m, p1, p2) { + const p2e = AllHtmlEntities.encode(p2); + return `${p1}
\n\n
\n\n`; }); md = md.replace(RegExp(inlinePattern, "gms"), function(m, p1, p2) { diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index bf6c61414a..bcae343550 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -416,7 +416,7 @@ "Prepends ( ͡° ͜ʖ ͡°) to a plain-text message": "Prepends ( ͡° ͜ʖ ͡°) to a plain-text message", "Sends a message as plain text, without interpreting it as markdown": "Sends a message as plain text, without interpreting it as markdown", "Sends a message as html, without interpreting it as markdown": "Sends a message as html, without interpreting it as markdown", - "Sends a message in TeX mode, using $ and $$ delimiters for maths": "Sends a message in TeX mode, using $ and $$ delimiters for maths", + "Sends a message in TeX mode, without restrictions": "Sends a message in TeX mode, without restrictions", "Searches DuckDuckGo for results": "Searches DuckDuckGo for results", "/ddg is not a command": "/ddg is not a command", "To use it, just wait for autocomplete results to load and tab through them.": "To use it, just wait for autocomplete results to load and tab through them.", From bcc069771061b7559313ded1f453f91eaf0f283f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sven=20M=C3=A4der?= Date: Fri, 29 Jan 2021 13:05:49 +0100 Subject: [PATCH 05/10] Remove /tex command --- src/SlashCommands.tsx | 18 ------ src/editor/serialize.ts | 108 +++++++++++++++--------------------- src/i18n/strings/en_EN.json | 1 - 3 files changed, 44 insertions(+), 83 deletions(-) diff --git a/src/SlashCommands.tsx b/src/SlashCommands.tsx index 387b9991db..79c21c4af5 100644 --- a/src/SlashCommands.tsx +++ b/src/SlashCommands.tsx @@ -48,7 +48,6 @@ import SettingsStore from "./settings/SettingsStore"; import {UIFeature} from "./settings/UIFeature"; import {CHAT_EFFECTS} from "./effects" import CallHandler from "./CallHandler"; -import {markdownSerializeIfNeeded} from './editor/serialize'; // XXX: workaround for https://github.com/microsoft/TypeScript/issues/31816 interface HTMLInputEvent extends Event { @@ -224,23 +223,6 @@ export const Commands = [ }, category: CommandCategories.messages, }), - new Command({ - command: 'tex', - args: '', - description: _td('Sends a message in TeX mode, without restrictions'), - runFn: function(roomId, args) { - if (SettingsStore.getValue("feature_latex_maths")) { - if (args) { - const html = markdownSerializeIfNeeded(args, {forceHTML: false}, {forceTEX: true}); - return success(MatrixClientPeg.get().sendHtmlMessage(roomId, args, html)); - } - return reject(this.getUsage()); - } else { - return reject("Render LaTeX maths in messages needs to be enabled in Labs"); - } - }, - category: CommandCategories.messages, - }), new Command({ command: 'ddg', args: '', diff --git a/src/editor/serialize.ts b/src/editor/serialize.ts index 61d7878de5..6b1d97a0ef 100644 --- a/src/editor/serialize.ts +++ b/src/editor/serialize.ts @@ -41,77 +41,63 @@ export function mdSerialize(model: EditorModel) { }, ""); } -export function markdownSerializeIfNeeded(md: string, {forceHTML = false} = {}, {forceTEX = false} = {}) { +export function htmlSerializeIfNeeded(model: EditorModel, {forceHTML = false} = {}) { + let md = mdSerialize(model); // copy of raw input to remove unwanted math later const orig = md; if (SettingsStore.getValue("feature_latex_maths")) { - if (forceTEX) { - const displayPattern = "(^|[^\\\\$])\\$\\$(([^$]|\\\\\\$)+?)\\$\\$"; - const inlinePattern = "(^|[^\\\\$])\\$(([^$]|\\\\\\$)*([^\\\\\\$]|\\\\\\$))\\$"; + // detect math with tex delimiters, inline: $...$, display $$...$$ + // preferably use negative lookbehinds, not supported in all major browsers: + // const displayPattern = "^(?\n\n\n\n`; - }); + // conditions for display math detection $$...$$: + // - pattern starts at beginning of line or is not prefixed with backslash or dollar + // - left delimiter ($$) is not escaped by backslash + const displayPatternDollar = "(^|[^\\\\$])\\$\\$(([^$]|\\\\\\$)+?)\\$\\$"; - md = md.replace(RegExp(inlinePattern, "gm"), function(m, p1, p2) { - const p2e = AllHtmlEntities.encode(p2); - return `${p1}`; - }); - } else { - // detect math with tex delimiters, inline: $...$, display $$...$$ - // preferably use negative lookbehinds, not supported in all major browsers: - // const displayPattern = "^(?\n\n\n\n`; + }); - // conditions for inline math detection $...$: - // - pattern starts at beginning of line, follows whitespace character or punctuation - // - pattern is on a single line - // - left and right delimiters ($) are not escaped by backslashes - // - left delimiter is not followed by whitespace character - // - right delimiter is not prefixed with whitespace character - const inlinePatternDollar = "(^|\\s|[.,!?:;])(?!\\\\)\\$(?!\\s)(([^$\\n]|\\\\\\$)*([^\\\\\\s\\$]|\\\\\\$)(?:\\\\\\$)?)\\$"; + md = md.replace(RegExp(inlinePatternDollar, "gm"), function(m, p1, p2) { + const p2e = AllHtmlEntities.encode(p2); + return `${p1}`; + }); - md = md.replace(RegExp(displayPatternDollar, "gm"), function(m, p1, p2) { - const p2e = AllHtmlEntities.encode(p2); - return `${p1}
\n\n
\n\n`; - }); + // detect math with latex delimiters, inline: \(...\), display \[...\] - md = md.replace(RegExp(inlinePatternDollar, "gm"), function(m, p1, p2) { - const p2e = AllHtmlEntities.encode(p2); - return `${p1}`; - }); + // conditions for display math detection \[...\]: + // - pattern starts at beginning of line or is not prefixed with backslash + // - pattern is not empty + const displayPattern = (SdkConfig.get()['latex_maths_delims'] || {})['display_pattern'] || + "(^|[^\\\\])\\\\\\[(?!\\\\\\])(.*?)\\\\\\]"; - // detect math with latex delimiters, inline: \(...\), display \[...\] + // conditions for inline math detection \(...\): + // - pattern starts at beginning of line or is not prefixed with backslash + // - pattern is not empty + const inlinePattern = (SdkConfig.get()['latex_maths_delims'] || {})['inline_pattern'] || + "(^|[^\\\\])\\\\\\((?!\\\\\\))(.*?)\\\\\\)"; - // conditions for display math detection \[...\]: - // - pattern starts at beginning of line or is not prefixed with backslash - // - pattern is not empty - const displayPattern = (SdkConfig.get()['latex_maths_delims'] || {})['display_pattern'] || - "(^|[^\\\\])\\\\\\[(?!\\\\\\])(.*?)\\\\\\]"; + md = md.replace(RegExp(displayPattern, "gms"), function(m, p1, p2) { + const p2e = AllHtmlEntities.encode(p2); + return `${p1}
\n\n
\n\n`; + }); - // conditions for inline math detection \(...\): - // - pattern starts at beginning of line or is not prefixed with backslash - // - pattern is not empty - const inlinePattern = (SdkConfig.get()['latex_maths_delims'] || {})['inline_pattern'] || - "(^|[^\\\\])\\\\\\((?!\\\\\\))(.*?)\\\\\\)"; - - md = md.replace(RegExp(displayPattern, "gms"), function(m, p1, p2) { - const p2e = AllHtmlEntities.encode(p2); - return `${p1}
\n\n
\n\n`; - }); - - md = md.replace(RegExp(inlinePattern, "gms"), function(m, p1, p2) { - const p2e = AllHtmlEntities.encode(p2); - return `${p1}`; - }); - } + md = md.replace(RegExp(inlinePattern, "gms"), function(m, p1, p2) { + const p2e = AllHtmlEntities.encode(p2); + return `${p1}`; + }); // make sure div tags always start on a new line, otherwise it will confuse // the markdown parser @@ -154,12 +140,6 @@ export function markdownSerializeIfNeeded(md: string, {forceHTML = false} = {}, } } -export function htmlSerializeIfNeeded(model: EditorModel, {forceHTML = false} = {}) { - const md = mdSerialize(model); - - return markdownSerializeIfNeeded(md, {forceHTML: forceHTML}); -} - export function textSerialize(model: EditorModel) { return model.parts.reduce((text, part) => { switch (part.type) { diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index bcae343550..2fb70ecdb1 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -416,7 +416,6 @@ "Prepends ( ͡° ͜ʖ ͡°) to a plain-text message": "Prepends ( ͡° ͜ʖ ͡°) to a plain-text message", "Sends a message as plain text, without interpreting it as markdown": "Sends a message as plain text, without interpreting it as markdown", "Sends a message as html, without interpreting it as markdown": "Sends a message as html, without interpreting it as markdown", - "Sends a message in TeX mode, without restrictions": "Sends a message in TeX mode, without restrictions", "Searches DuckDuckGo for results": "Searches DuckDuckGo for results", "/ddg is not a command": "/ddg is not a command", "To use it, just wait for autocomplete results to load and tab through them.": "To use it, just wait for autocomplete results to load and tab through them.", From ac1f9b4247f7eefacc969bfcbdbf30984c8e15e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sven=20M=C3=A4der?= Date: Fri, 29 Jan 2021 15:49:20 +0100 Subject: [PATCH 06/10] Add config keys for alternative patterns --- src/editor/serialize.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/editor/serialize.ts b/src/editor/serialize.ts index 6b1d97a0ef..6655c64347 100644 --- a/src/editor/serialize.ts +++ b/src/editor/serialize.ts @@ -55,7 +55,9 @@ export function htmlSerializeIfNeeded(model: EditorModel, {forceHTML = false} = // conditions for display math detection $$...$$: // - pattern starts at beginning of line or is not prefixed with backslash or dollar // - left delimiter ($$) is not escaped by backslash - const displayPatternDollar = "(^|[^\\\\$])\\$\\$(([^$]|\\\\\\$)+?)\\$\\$"; + const displayPatternAlternative = (SdkConfig.get()['latex_maths_delims'] || + {})['display_pattern_alternative'] || + "(^|[^\\\\$])\\$\\$(([^$]|\\\\\\$)+?)\\$\\$"; // conditions for inline math detection $...$: // - pattern starts at beginning of line, follows whitespace character or punctuation @@ -63,14 +65,16 @@ export function htmlSerializeIfNeeded(model: EditorModel, {forceHTML = false} = // - left and right delimiters ($) are not escaped by backslashes // - left delimiter is not followed by whitespace character // - right delimiter is not prefixed with whitespace character - const inlinePatternDollar = "(^|\\s|[.,!?:;])(?!\\\\)\\$(?!\\s)(([^$\\n]|\\\\\\$)*([^\\\\\\s\\$]|\\\\\\$)(?:\\\\\\$)?)\\$"; + const inlinePatternAlternative = (SdkConfig.get()['latex_maths_delims'] || + {})['inline_pattern_alternative'] || + "(^|\\s|[.,!?:;])(?!\\\\)\\$(?!\\s)(([^$\\n]|\\\\\\$)*([^\\\\\\s\\$]|\\\\\\$)(?:\\\\\\$)?)\\$"; - md = md.replace(RegExp(displayPatternDollar, "gm"), function(m, p1, p2) { + md = md.replace(RegExp(displayPatternAlternative, "gm"), function(m, p1, p2) { const p2e = AllHtmlEntities.encode(p2); return `${p1}
\n\n
\n\n`; }); - md = md.replace(RegExp(inlinePatternDollar, "gm"), function(m, p1, p2) { + md = md.replace(RegExp(inlinePatternAlternative, "gm"), function(m, p1, p2) { const p2e = AllHtmlEntities.encode(p2); return `${p1}`; }); From 73130cad02b2e63b4941d12a47ccc92f696d003c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sven=20M=C3=A4der?= Date: Thu, 1 Apr 2021 12:09:51 +0200 Subject: [PATCH 07/10] Refactor latex replace code --- src/editor/serialize.ts | 97 +++++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 48 deletions(-) diff --git a/src/editor/serialize.ts b/src/editor/serialize.ts index 6655c64347..3eda28818a 100644 --- a/src/editor/serialize.ts +++ b/src/editor/serialize.ts @@ -47,60 +47,61 @@ export function htmlSerializeIfNeeded(model: EditorModel, {forceHTML = false} = const orig = md; if (SettingsStore.getValue("feature_latex_maths")) { - // detect math with tex delimiters, inline: $...$, display $$...$$ - // preferably use negative lookbehinds, not supported in all major browsers: - // const displayPattern = "^(?\n\n\n\n`; - }); + // conditions for display math detection \[...\]: + // - pattern starts at beginning of line or is not prefixed with backslash + // - pattern is not empty + "display": "(^|[^\\\\])\\\\\\[(?!\\\\\\])(.*?)\\\\\\]", - md = md.replace(RegExp(inlinePatternAlternative, "gm"), function(m, p1, p2) { - const p2e = AllHtmlEntities.encode(p2); - return `${p1}`; - }); + // conditions for inline math detection \(...\): + // - pattern starts at beginning of line or is not prefixed with backslash + // - pattern is not empty + "inline": "(^|[^\\\\])\\\\\\((?!\\\\\\))(.*?)\\\\\\)", + }, + }; - // detect math with latex delimiters, inline: \(...\), display \[...\] + patternNames.forEach(function(patternName) { + patternTypes.forEach(function(patternType) { + // get the regex replace pattern from config or use the default + const pattern = (SdkConfig.get()["latex_maths_delims"] || + {})[patternType + "_pattern_" + patternName] || + patternDefaults[patternName][patternType]; - // conditions for display math detection \[...\]: - // - pattern starts at beginning of line or is not prefixed with backslash - // - pattern is not empty - const displayPattern = (SdkConfig.get()['latex_maths_delims'] || {})['display_pattern'] || - "(^|[^\\\\])\\\\\\[(?!\\\\\\])(.*?)\\\\\\]"; - - // conditions for inline math detection \(...\): - // - pattern starts at beginning of line or is not prefixed with backslash - // - pattern is not empty - const inlinePattern = (SdkConfig.get()['latex_maths_delims'] || {})['inline_pattern'] || - "(^|[^\\\\])\\\\\\((?!\\\\\\))(.*?)\\\\\\)"; - - md = md.replace(RegExp(displayPattern, "gms"), function(m, p1, p2) { - const p2e = AllHtmlEntities.encode(p2); - return `${p1}
\n\n
\n\n`; - }); - - md = md.replace(RegExp(inlinePattern, "gms"), function(m, p1, p2) { - const p2e = AllHtmlEntities.encode(p2); - return `${p1}`; + md = md.replace(RegExp(pattern, "gms"), function(m, p1, p2) { + const p2e = AllHtmlEntities.encode(p2); + switch (patternType) { + case "display": + return `${p1}
\n\n
\n\n`; + case "inline": + return `${p1}`; + } + }); + }); }); // make sure div tags always start on a new line, otherwise it will confuse From 1d70045065614276d45569f959e020a524aa59eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sven=20M=C3=A4der?= Date: Tue, 6 Apr 2021 14:52:55 +0200 Subject: [PATCH 08/10] Refactor maths config options to nested structure --- src/editor/deserialize.ts | 8 ++++---- src/editor/serialize.ts | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/editor/deserialize.ts b/src/editor/deserialize.ts index 979c70140d..a0c4d74275 100644 --- a/src/editor/deserialize.ts +++ b/src/editor/deserialize.ts @@ -143,11 +143,11 @@ function parseElement(n: HTMLElement, partCreator: PartCreator, lastNode: HTMLEl // math nodes are translated back into delimited latex strings if (n.hasAttribute("data-mx-maths")) { const delimLeft = (n.nodeName == "SPAN") ? - (SdkConfig.get()['latex_maths_delims'] || {})['inline_left'] || "\\(" : - (SdkConfig.get()['latex_maths_delims'] || {})['display_left'] || "\\["; + ((SdkConfig.get()['latex_maths_delims'] || {})['inline'] || {})['left'] || "\\(" : + ((SdkConfig.get()['latex_maths_delims'] || {})['display'] || {})['left'] || "\\["; const delimRight = (n.nodeName == "SPAN") ? - (SdkConfig.get()['latex_maths_delims'] || {})['inline_right'] || "\\)" : - (SdkConfig.get()['latex_maths_delims'] || {})['display_right'] || "\\]"; + ((SdkConfig.get()['latex_maths_delims'] || {})['inline'] || {})['right'] || "\\)" : + ((SdkConfig.get()['latex_maths_delims'] || {})['display'] || {})['right'] || "\\]"; const tex = n.getAttribute("data-mx-maths"); return partCreator.plain(delimLeft + tex + delimRight); } else if (!checkDescendInto(n)) { diff --git a/src/editor/serialize.ts b/src/editor/serialize.ts index fc4a8d4314..3d7aff6720 100644 --- a/src/editor/serialize.ts +++ b/src/editor/serialize.ts @@ -92,8 +92,8 @@ export function htmlSerializeIfNeeded(model: EditorModel, {forceHTML = false} = patternNames.forEach(function(patternName) { patternTypes.forEach(function(patternType) { // get the regex replace pattern from config or use the default - const pattern = (SdkConfig.get()["latex_maths_delims"] || - {})[patternType + "_pattern_" + patternName] || + const pattern = (((SdkConfig.get()["latex_maths_delims"] || + {})[patternType] || {})["pattern"] || {})[patternName] || patternDefaults[patternName][patternType]; md = md.replace(RegExp(pattern, "gms"), function(m, p1, p2) { From 7205995deab6c40e7df4412a03d88e7a6413001d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sven=20M=C3=A4der?= Date: Wed, 7 Apr 2021 17:22:30 +0200 Subject: [PATCH 09/10] Remove unused function arguments --- src/editor/serialize.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/editor/serialize.ts b/src/editor/serialize.ts index 3d7aff6720..50bebf8f08 100644 --- a/src/editor/serialize.ts +++ b/src/editor/serialize.ts @@ -128,9 +128,9 @@ export function htmlSerializeIfNeeded(model: EditorModel, {forceHTML = false} = // since maths delimiters are handled before Markdown, // code blocks could contain mangled content. // replace code blocks with original content - phtml('code').contents('div, span').each(function(i) { - const origData = phtmlOrig('code').contents('div, span')[i].data; - phtml('code').contents('div, span')[i].data = origData; + phtml('code').contents().each(function(i) { + const origData = phtmlOrig('code').contents()[i].data; + phtml('code').contents()[i].data = origData; }); // add fallback output for latex math, which should not be interpreted as markdown From 31e85ec58000ce8256c3df893eeb784c39a89683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sven=20M=C3=A4der?= Date: Thu, 8 Apr 2021 11:38:05 +0200 Subject: [PATCH 10/10] Use cheerio api for code replacements --- src/editor/serialize.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/editor/serialize.ts b/src/editor/serialize.ts index 50bebf8f08..5167e3d376 100644 --- a/src/editor/serialize.ts +++ b/src/editor/serialize.ts @@ -128,9 +128,8 @@ export function htmlSerializeIfNeeded(model: EditorModel, {forceHTML = false} = // since maths delimiters are handled before Markdown, // code blocks could contain mangled content. // replace code blocks with original content - phtml('code').contents().each(function(i) { - const origData = phtmlOrig('code').contents()[i].data; - phtml('code').contents()[i].data = origData; + phtmlOrig('code').each(function(i) { + phtml('code').eq(i).text(phtmlOrig('code').eq(i).text()); }); // add fallback output for latex math, which should not be interpreted as markdown