From 718887dd272ca108b48385a65737124931d96fd4 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Sat, 10 Jul 2021 22:40:30 -0400 Subject: [PATCH] Update Emojibase and switch to IamCal (Slack-style) shortcodes for consistency with shortcodes commonly used by other platforms, as was decided in https://github.com/vector-im/element-web/issues/13857. One thing to be aware of is that the currently used version of Twemoji does not support a few of the newer emoji present in Emojibase, so these look a little out of place in the emoji picker. Optimally Twemoji would be updated at the same time, though I don't know how to do that. Signed-off-by: Robin Townsend --- package.json | 4 +- src/HtmlUtils.tsx | 18 +---- src/autocomplete/EmojiProvider.tsx | 69 ++++++++++--------- src/components/views/emojipicker/Preview.tsx | 12 ++-- .../views/emojipicker/QuickReactions.tsx | 7 +- src/emoji.ts | 25 ++++--- yarn.lock | 16 ++--- 7 files changed, 72 insertions(+), 79 deletions(-) diff --git a/package.json b/package.json index bb92ad11d8..4506579747 100644 --- a/package.json +++ b/package.json @@ -64,8 +64,8 @@ "counterpart": "^0.18.6", "diff-dom": "^4.2.2", "diff-match-patch": "^1.0.5", - "emojibase-data": "^5.1.1", - "emojibase-regex": "^4.1.1", + "emojibase-data": "^6.2.0", + "emojibase-regex": "^5.1.3", "escape-html": "^1.0.3", "file-saver": "^2.0.5", "filesize": "6.1.0", diff --git a/src/HtmlUtils.tsx b/src/HtmlUtils.tsx index 016b557477..26aeef9dd8 100644 --- a/src/HtmlUtils.tsx +++ b/src/HtmlUtils.tsx @@ -34,7 +34,7 @@ import { IExtendedSanitizeOptions } from './@types/sanitize-html'; import linkifyMatrix from './linkify-matrix'; import SettingsStore from './settings/SettingsStore'; import { tryTransformPermalinkToLocalHref } from "./utils/permalinks/Permalinks"; -import { SHORTCODE_TO_EMOJI, getEmojiFromUnicode } from "./emoji"; +import { getEmojiFromUnicode, getShortcodes } from "./emoji"; import ReplyThread from "./components/views/elements/ReplyThread"; import { mediaFromMxc } from "./customisations/Media"; @@ -78,20 +78,8 @@ function mightContainEmoji(str: string): boolean { * @return {String} The shortcode (such as :thumbup:) */ export function unicodeToShortcode(char: string): string { - const data = getEmojiFromUnicode(char); - return (data && data.shortcodes ? `:${data.shortcodes[0]}:` : ''); -} - -/** - * Returns the unicode character for an emoji shortcode - * - * @param {String} shortcode The shortcode (such as :thumbup:) - * @return {String} The emoji character; null if none exists - */ -export function shortcodeToUnicode(shortcode: string): string { - shortcode = shortcode.slice(1, shortcode.length - 1); - const data = SHORTCODE_TO_EMOJI.get(shortcode); - return data ? data.unicode : null; + const shortcodes = getShortcodes(getEmojiFromUnicode(char)); + return shortcodes.length > 0 ? `:${shortcodes[0]}:` : ''; } export function processHtmlForSending(html: string): string { diff --git a/src/autocomplete/EmojiProvider.tsx b/src/autocomplete/EmojiProvider.tsx index 2fc77e9a17..edf691e151 100644 --- a/src/autocomplete/EmojiProvider.tsx +++ b/src/autocomplete/EmojiProvider.tsx @@ -25,8 +25,7 @@ import { PillCompletion } from './Components'; import { ICompletion, ISelectionRange } from './Autocompleter'; import { uniq, sortBy } from 'lodash'; import SettingsStore from "../settings/SettingsStore"; -import { shortcodeToUnicode } from '../HtmlUtils'; -import { EMOJI, IEmoji } from '../emoji'; +import { EMOJI, IEmoji, getShortcodes } from '../emoji'; import EMOTICON_REGEX from 'emojibase-regex/emoticon'; @@ -38,21 +37,26 @@ const EMOJI_REGEX = new RegExp('(' + EMOTICON_REGEX.source + '|(?:^|\\s):[+-\\w] interface IEmojiShort { emoji: IEmoji; - shortname: string; + shortcode: string; + altShortcodes: string[]; _orderBy: number; } -const EMOJI_SHORTNAMES: IEmojiShort[] = EMOJI.sort((a, b) => { +const EMOJI_SHORTCODES: IEmojiShort[] = EMOJI.sort((a, b) => { if (a.group === b.group) { return a.order - b.order; } return a.group - b.group; -}).map((emoji, index) => ({ - emoji, - shortname: `:${emoji.shortcodes[0]}:`, - // Include the index so that we can preserve the original order - _orderBy: index, -})); +}).map((emoji, index) => { + const [shortcode, ...altShortcodes] = getShortcodes(emoji); + return { + emoji, + shortcode: shortcode ? `:${shortcode}:` : undefined, + altShortcodes: altShortcodes.map(s => `:${s}:`), + // Include the index so that we can preserve the original order + _orderBy: index, + }; +}).filter(emoji => emoji.shortcode); function score(query, space) { const index = space.indexOf(query); @@ -69,15 +73,15 @@ export default class EmojiProvider extends AutocompleteProvider { constructor() { super(EMOJI_REGEX); - this.matcher = new QueryMatcher(EMOJI_SHORTNAMES, { - keys: ['emoji.emoticon', 'shortname'], + this.matcher = new QueryMatcher(EMOJI_SHORTCODES, { + keys: ['emoji.emoticon', 'shortcode'], funcs: [ - (o) => o.emoji.shortcodes.length > 1 ? o.emoji.shortcodes.slice(1).map(s => `:${s}:`).join(" ") : "", // aliases + o => o.altShortcodes.join(" "), // aliases ], // For matching against ascii equivalents shouldMatchWordsOnly: false, }); - this.nameMatcher = new QueryMatcher(EMOJI_SHORTNAMES, { + this.nameMatcher = new QueryMatcher(EMOJI_SHORTCODES, { keys: ['emoji.annotation'], // For removing punctuation shouldMatchWordsOnly: true, @@ -105,34 +109,33 @@ export default class EmojiProvider extends AutocompleteProvider { const sorters = []; // make sure that emoticons come first - sorters.push((c) => score(matchedString, c.emoji.emoticon || "")); + sorters.push(c => score(matchedString, c.emoji.emoticon || "")); - // then sort by score (Infinity if matchedString not in shortname) - sorters.push((c) => score(matchedString, c.shortname)); + // then sort by score (Infinity if matchedString not in shortcode) + sorters.push(c => score(matchedString, c.shortcode)); // then sort by max score of all shortcodes, trim off the `:` - sorters.push((c) => Math.min(...c.emoji.shortcodes.map(s => score(matchedString.substring(1), s)))); - // If the matchedString is not empty, sort by length of shortname. Example: + sorters.push(c => Math.min( + ...[c.shortcode, ...c.altShortcodes].map(s => score(matchedString.substring(1), s)), + )); + // If the matchedString is not empty, sort by length of shortcode. Example: // matchedString = ":bookmark" // completions = [":bookmark:", ":bookmark_tabs:", ...] if (matchedString.length > 1) { - sorters.push((c) => c.shortname.length); + sorters.push(c => c.shortcode.length); } // Finally, sort by original ordering - sorters.push((c) => c._orderBy); + sorters.push(c => c._orderBy); completions = sortBy(uniq(completions), sorters); - completions = completions.map(({ shortname }) => { - const unicode = shortcodeToUnicode(shortname); - return { - completion: unicode, - component: ( - - { unicode } - - ), - range, - }; - }).slice(0, LIMIT); + completions = completions.map(c => ({ + completion: c.emoji.unicode, + component: ( + + { c.emoji.unicode } + + ), + range, + })).slice(0, LIMIT); } return completions; } diff --git a/src/components/views/emojipicker/Preview.tsx b/src/components/views/emojipicker/Preview.tsx index 9c2dbb9cbd..bd9982e50f 100644 --- a/src/components/views/emojipicker/Preview.tsx +++ b/src/components/views/emojipicker/Preview.tsx @@ -17,7 +17,7 @@ limitations under the License. import React from 'react'; -import { IEmoji } from "../../../emoji"; +import { IEmoji, getShortcodes } from "../../../emoji"; import { replaceableComponent } from "../../../utils/replaceableComponent"; interface IProps { @@ -30,8 +30,8 @@ class Preview extends React.PureComponent { const { unicode = "", annotation = "", - shortcodes: [shortcode = ""], - } = this.props.emoji || {}; + } = this.props.emoji; + const shortcode = getShortcodes(this.props.emoji)[0]; return (
@@ -42,9 +42,9 @@ class Preview extends React.PureComponent {
{annotation}
-
- {shortcode} -
+ { shortcode ? +
{shortcode}
: + null }
); diff --git a/src/components/views/emojipicker/QuickReactions.tsx b/src/components/views/emojipicker/QuickReactions.tsx index ffd3ce9760..2d78e3e4cf 100644 --- a/src/components/views/emojipicker/QuickReactions.tsx +++ b/src/components/views/emojipicker/QuickReactions.tsx @@ -18,7 +18,7 @@ limitations under the License. import React from 'react'; import { _t } from '../../../languageHandler'; -import { getEmojiFromUnicode, IEmoji } from "../../../emoji"; +import { getEmojiFromUnicode, getShortcodes, IEmoji } from "../../../emoji"; import Emoji from "./Emoji"; import { replaceableComponent } from "../../../utils/replaceableComponent"; @@ -62,6 +62,7 @@ class QuickReactions extends React.Component { }; render() { + const shortcode = this.state.hover ? getShortcodes(this.state.hover)[0] : undefined; return (

@@ -69,7 +70,9 @@ class QuickReactions extends React.Component { ? _t("Quick Reactions") : {this.state.hover.annotation} - {this.state.hover.shortcodes[0]} + { shortcode ? + {shortcode} : + null } }

diff --git a/src/emoji.ts b/src/emoji.ts index 7caeb06d21..ac4de654f7 100644 --- a/src/emoji.ts +++ b/src/emoji.ts @@ -15,14 +15,14 @@ limitations under the License. */ import EMOJIBASE from 'emojibase-data/en/compact.json'; +import SHORTCODES from 'emojibase-data/en/shortcodes/iamcal.json'; export interface IEmoji { annotation: string; - group: number; + group?: number; hexcode: string; - order: number; - shortcodes: string[]; - tags: string[]; + order?: number; + tags?: string[]; unicode: string; emoticon?: string; } @@ -34,10 +34,14 @@ interface IEmojiWithFilterString extends IEmoji { // The unicode is stored without the variant selector const UNICODE_TO_EMOJI = new Map(); // not exported as gets for it are handled by getEmojiFromUnicode export const EMOTICON_TO_EMOJI = new Map(); -export const SHORTCODE_TO_EMOJI = new Map(); export const getEmojiFromUnicode = unicode => UNICODE_TO_EMOJI.get(stripVariation(unicode)); +const toArray = (shortcodes?: string | string[]): string[] => + typeof shortcodes === "string" ? [shortcodes] : (shortcodes ?? []); +export const getShortcodes = (emoji: IEmoji): string[] => + toArray(SHORTCODES[emoji.hexcode]); + const EMOJIBASE_GROUP_ID_TO_CATEGORY = [ "people", // smileys "people", // actually people @@ -66,12 +70,14 @@ const ZERO_WIDTH_JOINER = "\u200D"; // Store various mappings from unicode/emoticon/shortcode to the Emoji objects EMOJIBASE.forEach((emoji: IEmojiWithFilterString) => { + const shortcodes = getShortcodes(emoji); const categoryId = EMOJIBASE_GROUP_ID_TO_CATEGORY[emoji.group]; if (DATA_BY_CATEGORY.hasOwnProperty(categoryId)) { DATA_BY_CATEGORY[categoryId].push(emoji); } + // This is used as the string to match the query against when filtering emojis - emoji.filterString = (`${emoji.annotation}\n${emoji.shortcodes.join('\n')}}\n${emoji.emoticon || ''}\n` + + emoji.filterString = (`${emoji.annotation}\n${shortcodes.join('\n')}}\n${emoji.emoticon || ''}\n` + `${emoji.unicode.split(ZERO_WIDTH_JOINER).join("\n")}`).toLowerCase(); // Add mapping from unicode to Emoji object @@ -87,13 +93,6 @@ EMOJIBASE.forEach((emoji: IEmojiWithFilterString) => { // Add mapping from emoticon to Emoji object EMOTICON_TO_EMOJI.set(emoji.emoticon, emoji); } - - if (emoji.shortcodes) { - // Add mapping from each shortcode to Emoji object - emoji.shortcodes.forEach(shortcode => { - SHORTCODE_TO_EMOJI.set(shortcode, emoji); - }); - } }); /** diff --git a/yarn.lock b/yarn.lock index 90f415673d..21dc0f8fd5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3022,15 +3022,15 @@ emoji-regex@^8.0.0: resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== -emojibase-data@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/emojibase-data/-/emojibase-data-5.1.1.tgz#0a0d63dd07ce1376b3d27642f28cafa46f651de6" - integrity sha512-za/ma5SfogHjwUmGFnDbTvSfm8GGFvFaPS27GPti16YZSp5EPgz+UDsZCATXvJGit+oRNBbG/FtybXHKi2UQgQ== +emojibase-data@^6.2.0: + version "6.2.0" + resolved "https://registry.yarnpkg.com/emojibase-data/-/emojibase-data-6.2.0.tgz#db6c75c36905284fa623f4aa5f468d2be6ed364a" + integrity sha512-SWKaXD2QeQs06IE7qfJftsI5924Dqzp+V9xaa5RzZIEWhmlrG6Jt2iKwfgOPHu+5S8MEtOI7GdpKsXj46chXOw== -emojibase-regex@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/emojibase-regex/-/emojibase-regex-4.1.1.tgz#6e781aca520281600fe7a177f1582c33cf1fc545" - integrity sha512-KSigB1zQkNKFygLZ5bAfHs87LJa1ni8QTQtq8lc53Y74NF3Dk2r7kfa8MpooTO8JBb5Xz660X4tSjDB+I+7elA== +emojibase-regex@^5.1.3: + version "5.1.3" + resolved "https://registry.yarnpkg.com/emojibase-regex/-/emojibase-regex-5.1.3.tgz#f0ef621ed6ec624becd2326f999fd4ea01b94554" + integrity sha512-gT8T9LxLA8VJdI+8KQtyykB9qKzd7WuUL3M2yw6y9tplFeufOUANg3UKVaKUvkMcRNvZsSElWhxcJrx8WPE12g== encoding@^0.1.11: version "0.1.13"