OpenCloud
/
element-web
mirror of https://github.com/vector-im/element-web
1
0
Fork 0
Code Issues Releases Wiki Activity

Update Emojibase and switch to IamCal (Slack-style) shortcodes 

for consistency with shortcodes commonly used by other platforms, as
was decided in https://github.com/vector-im/element-web/issues/13857.

One thing to be aware of is that the currently used version of Twemoji
does not support a few of the newer emoji present in Emojibase, so these
look a little out of place in the emoji picker. Optimally Twemoji would
be updated at the same time, though I don't know how to do that.

Signed-off-by: Robin Townsend <robin@robin.town>
pull/21833/head
Robin Townsend 2021-07-10 22:40:30 -04:00
parent 829169ec87
commit 718887dd27
7 changed files with 72 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
package.json
View File

@ -64,8 +64,8 @@
"counterpart": "^0.18.6", "counterpart": "^0.18.6",
"diff-dom": "^4.2.2", "diff-dom": "^4.2.2",
"diff-match-patch": "^1.0.5", "diff-match-patch": "^1.0.5",
"emojibase-data": "^5.1.1", "emojibase-data": "^6.2.0",
"emojibase-regex": "^4.1.1", "emojibase-regex": "^5.1.3",
"escape-html": "^1.0.3", "escape-html": "^1.0.3",
"file-saver": "^2.0.5", "file-saver": "^2.0.5",
"filesize": "6.1.0", "filesize": "6.1.0",

18
src/HtmlUtils.tsx
View File

@ -34,7 +34,7 @@ import { IExtendedSanitizeOptions } from './@types/sanitize-html';
import linkifyMatrix from './linkify-matrix'; import linkifyMatrix from './linkify-matrix';
import SettingsStore from './settings/SettingsStore'; import SettingsStore from './settings/SettingsStore';
import { tryTransformPermalinkToLocalHref } from "./utils/permalinks/Permalinks"; import { tryTransformPermalinkToLocalHref } from "./utils/permalinks/Permalinks";
import { SHORTCODE_TO_EMOJI, getEmojiFromUnicode } from "./emoji"; import { getEmojiFromUnicode, getShortcodes } from "./emoji";
import ReplyThread from "./components/views/elements/ReplyThread"; import ReplyThread from "./components/views/elements/ReplyThread";
import { mediaFromMxc } from "./customisations/Media"; import { mediaFromMxc } from "./customisations/Media";
@ -78,20 +78,8 @@ function mightContainEmoji(str: string): boolean {
* @return {String} The shortcode (such as :thumbup:) * @return {String} The shortcode (such as :thumbup:)
*/ */
export function unicodeToShortcode(char: string): string { export function unicodeToShortcode(char: string): string {
const data = getEmojiFromUnicode(char); const shortcodes = getShortcodes(getEmojiFromUnicode(char));
return (data && data.shortcodes ? `:${data.shortcodes[0]}:` : ''); return shortcodes.length > 0 ? `:${shortcodes[0]}:` : '';
}
/**
* Returns the unicode character for an emoji shortcode
*
* @param {String} shortcode The shortcode (such as :thumbup:)
* @return {String} The emoji character; null if none exists
*/
export function shortcodeToUnicode(shortcode: string): string {
shortcode = shortcode.slice(1, shortcode.length - 1);
const data = SHORTCODE_TO_EMOJI.get(shortcode);
return data ? data.unicode : null;
} }
export function processHtmlForSending(html: string): string { export function processHtmlForSending(html: string): string {

69
src/autocomplete/EmojiProvider.tsx
View File

@ -25,8 +25,7 @@ import { PillCompletion } from './Components';
import { ICompletion, ISelectionRange } from './Autocompleter'; import { ICompletion, ISelectionRange } from './Autocompleter';
import { uniq, sortBy } from 'lodash'; import { uniq, sortBy } from 'lodash';
import SettingsStore from "../settings/SettingsStore"; import SettingsStore from "../settings/SettingsStore";
import { shortcodeToUnicode } from '../HtmlUtils'; import { EMOJI, IEmoji, getShortcodes } from '../emoji';
import { EMOJI, IEmoji } from '../emoji';
import EMOTICON_REGEX from 'emojibase-regex/emoticon'; import EMOTICON_REGEX from 'emojibase-regex/emoticon';
@ -38,21 +37,26 @@ const EMOJI_REGEX = new RegExp('(' + EMOTICON_REGEX.source + '|(?:^|\\s):[+-\\w]
interface IEmojiShort { interface IEmojiShort {
emoji: IEmoji; emoji: IEmoji;
shortname: string; shortcode: string;
altShortcodes: string[];
_orderBy: number; _orderBy: number;
} }
const EMOJI_SHORTNAMES: IEmojiShort[] = EMOJI.sort((a, b) => { const EMOJI_SHORTCODES: IEmojiShort[] = EMOJI.sort((a, b) => {
if (a.group === b.group) { if (a.group === b.group) {
return a.order - b.order; return a.order - b.order;
} }
return a.group - b.group; return a.group - b.group;
}).map((emoji, index) => ({ }).map((emoji, index) => {
emoji, const [shortcode, ...altShortcodes] = getShortcodes(emoji);
shortname: `:${emoji.shortcodes[0]}:`, return {
// Include the index so that we can preserve the original order emoji,
_orderBy: index, shortcode: shortcode ? `:${shortcode}:` : undefined,
})); altShortcodes: altShortcodes.map(s => `:${s}:`),
// Include the index so that we can preserve the original order
_orderBy: index,
};
}).filter(emoji => emoji.shortcode);
function score(query, space) { function score(query, space) {
const index = space.indexOf(query); const index = space.indexOf(query);
@ -69,15 +73,15 @@ export default class EmojiProvider extends AutocompleteProvider {
constructor() { constructor() {
super(EMOJI_REGEX); super(EMOJI_REGEX);
this.matcher = new QueryMatcher<IEmojiShort>(EMOJI_SHORTNAMES, { this.matcher = new QueryMatcher<IEmojiShort>(EMOJI_SHORTCODES, {
keys: ['emoji.emoticon', 'shortname'], keys: ['emoji.emoticon', 'shortcode'],
funcs: [ funcs: [
(o) => o.emoji.shortcodes.length > 1 ? o.emoji.shortcodes.slice(1).map(s => `:${s}:`).join(" ") : "", // aliases o => o.altShortcodes.join(" "), // aliases
], ],
// For matching against ascii equivalents // For matching against ascii equivalents
shouldMatchWordsOnly: false, shouldMatchWordsOnly: false,
}); });
this.nameMatcher = new QueryMatcher(EMOJI_SHORTNAMES, { this.nameMatcher = new QueryMatcher(EMOJI_SHORTCODES, {
keys: ['emoji.annotation'], keys: ['emoji.annotation'],
// For removing punctuation // For removing punctuation
shouldMatchWordsOnly: true, shouldMatchWordsOnly: true,
@ -105,34 +109,33 @@ export default class EmojiProvider extends AutocompleteProvider {
const sorters = []; const sorters = [];
// make sure that emoticons come first // make sure that emoticons come first
sorters.push((c) => score(matchedString, c.emoji.emoticon || "")); sorters.push(c => score(matchedString, c.emoji.emoticon || ""));
// then sort by score (Infinity if matchedString not in shortname) // then sort by score (Infinity if matchedString not in shortcode)
sorters.push((c) => score(matchedString, c.shortname)); sorters.push(c => score(matchedString, c.shortcode));
// then sort by max score of all shortcodes, trim off the `:` // then sort by max score of all shortcodes, trim off the `:`
sorters.push((c) => Math.min(...c.emoji.shortcodes.map(s => score(matchedString.substring(1), s)))); sorters.push(c => Math.min(
// If the matchedString is not empty, sort by length of shortname. Example: ...[c.shortcode, ...c.altShortcodes].map(s => score(matchedString.substring(1), s)),
));
// If the matchedString is not empty, sort by length of shortcode. Example:
// matchedString = ":bookmark" // matchedString = ":bookmark"
// completions = [":bookmark:", ":bookmark_tabs:", ...] // completions = [":bookmark:", ":bookmark_tabs:", ...]
if (matchedString.length > 1) { if (matchedString.length > 1) {
sorters.push((c) => c.shortname.length); sorters.push(c => c.shortcode.length);
} }
// Finally, sort by original ordering // Finally, sort by original ordering
sorters.push((c) => c._orderBy); sorters.push(c => c._orderBy);
completions = sortBy(uniq(completions), sorters); completions = sortBy(uniq(completions), sorters);
completions = completions.map(({ shortname }) => { completions = completions.map(c => ({
const unicode = shortcodeToUnicode(shortname); completion: c.emoji.unicode,
return { component: (
completion: unicode, <PillCompletion title={c.shortcode} aria-label={c.emoji.unicode}>
component: ( <span>{ c.emoji.unicode }</span>
<PillCompletion title={shortname} aria-label={unicode}> </PillCompletion>
<span>{ unicode }</span> ),
</PillCompletion> range,
), })).slice(0, LIMIT);
range,
};
}).slice(0, LIMIT);
} }
return completions; return completions;
} }

12
src/components/views/emojipicker/Preview.tsx
View File

@ -17,7 +17,7 @@ limitations under the License.
import React from 'react'; import React from 'react';
import { IEmoji } from "../../../emoji"; import { IEmoji, getShortcodes } from "../../../emoji";
import { replaceableComponent } from "../../../utils/replaceableComponent"; import { replaceableComponent } from "../../../utils/replaceableComponent";
interface IProps { interface IProps {
@ -30,8 +30,8 @@ class Preview extends React.PureComponent<IProps> {
const { const {
unicode = "", unicode = "",
annotation = "", annotation = "",
shortcodes: [shortcode = ""], } = this.props.emoji;
} = this.props.emoji || {}; const shortcode = getShortcodes(this.props.emoji)[0];
return ( return (
<div className="mx_EmojiPicker_footer mx_EmojiPicker_preview"> <div className="mx_EmojiPicker_footer mx_EmojiPicker_preview">
@ -42,9 +42,9 @@ class Preview extends React.PureComponent<IProps> {
<div className="mx_EmojiPicker_name mx_EmojiPicker_preview_name"> <div className="mx_EmojiPicker_name mx_EmojiPicker_preview_name">
{annotation} {annotation}
</div> </div>
<div className="mx_EmojiPicker_shortcode"> { shortcode ?
{shortcode} <div className="mx_EmojiPicker_shortcode">{shortcode}</div> :
</div> null }
</div> </div>
</div> </div>
); );

7
src/components/views/emojipicker/QuickReactions.tsx
View File

@ -18,7 +18,7 @@ limitations under the License.
import React from 'react'; import React from 'react';
import { _t } from '../../../languageHandler'; import { _t } from '../../../languageHandler';
import { getEmojiFromUnicode, IEmoji } from "../../../emoji"; import { getEmojiFromUnicode, getShortcodes, IEmoji } from "../../../emoji";
import Emoji from "./Emoji"; import Emoji from "./Emoji";
import { replaceableComponent } from "../../../utils/replaceableComponent"; import { replaceableComponent } from "../../../utils/replaceableComponent";
@ -62,6 +62,7 @@ class QuickReactions extends React.Component<IProps, IState> {
}; };
render() { render() {
const shortcode = this.state.hover ? getShortcodes(this.state.hover)[0] : undefined;
return ( return (
<section className="mx_EmojiPicker_footer mx_EmojiPicker_quick mx_EmojiPicker_category"> <section className="mx_EmojiPicker_footer mx_EmojiPicker_quick mx_EmojiPicker_category">
<h2 className="mx_EmojiPicker_quick_header mx_EmojiPicker_category_label"> <h2 className="mx_EmojiPicker_quick_header mx_EmojiPicker_category_label">
@ -69,7 +70,9 @@ class QuickReactions extends React.Component<IProps, IState> {
? _t("Quick Reactions") ? _t("Quick Reactions")
: <React.Fragment> : <React.Fragment>
<span className="mx_EmojiPicker_name">{this.state.hover.annotation}</span> <span className="mx_EmojiPicker_name">{this.state.hover.annotation}</span>
<span className="mx_EmojiPicker_shortcode">{this.state.hover.shortcodes[0]}</span> { shortcode ?
<span className="mx_EmojiPicker_shortcode">{shortcode}</span> :
null }
</React.Fragment> </React.Fragment>
} }
</h2> </h2>

25
src/emoji.ts
View File

@ -15,14 +15,14 @@ limitations under the License.
*/ */
import EMOJIBASE from 'emojibase-data/en/compact.json'; import EMOJIBASE from 'emojibase-data/en/compact.json';
import SHORTCODES from 'emojibase-data/en/shortcodes/iamcal.json';
export interface IEmoji { export interface IEmoji {
annotation: string; annotation: string;
group: number; group?: number;
hexcode: string; hexcode: string;
order: number; order?: number;
shortcodes: string[]; tags?: string[];
tags: string[];
unicode: string; unicode: string;
emoticon?: string; emoticon?: string;
} }
@ -34,10 +34,14 @@ interface IEmojiWithFilterString extends IEmoji {
// The unicode is stored without the variant selector // The unicode is stored without the variant selector
const UNICODE_TO_EMOJI = new Map<string, IEmojiWithFilterString>(); // not exported as gets for it are handled by getEmojiFromUnicode const UNICODE_TO_EMOJI = new Map<string, IEmojiWithFilterString>(); // not exported as gets for it are handled by getEmojiFromUnicode
export const EMOTICON_TO_EMOJI = new Map<string, IEmojiWithFilterString>(); export const EMOTICON_TO_EMOJI = new Map<string, IEmojiWithFilterString>();
export const SHORTCODE_TO_EMOJI = new Map<string, IEmojiWithFilterString>();
export const getEmojiFromUnicode = unicode => UNICODE_TO_EMOJI.get(stripVariation(unicode)); export const getEmojiFromUnicode = unicode => UNICODE_TO_EMOJI.get(stripVariation(unicode));
const toArray = (shortcodes?: string | string[]): string[] =>
typeof shortcodes === "string" ? [shortcodes] : (shortcodes ?? []);
export const getShortcodes = (emoji: IEmoji): string[] =>
toArray(SHORTCODES[emoji.hexcode]);
const EMOJIBASE_GROUP_ID_TO_CATEGORY = [ const EMOJIBASE_GROUP_ID_TO_CATEGORY = [
"people", // smileys "people", // smileys
"people", // actually people "people", // actually people
@ -66,12 +70,14 @@ const ZERO_WIDTH_JOINER = "\u200D";
// Store various mappings from unicode/emoticon/shortcode to the Emoji objects // Store various mappings from unicode/emoticon/shortcode to the Emoji objects
EMOJIBASE.forEach((emoji: IEmojiWithFilterString) => { EMOJIBASE.forEach((emoji: IEmojiWithFilterString) => {
const shortcodes = getShortcodes(emoji);
const categoryId = EMOJIBASE_GROUP_ID_TO_CATEGORY[emoji.group]; const categoryId = EMOJIBASE_GROUP_ID_TO_CATEGORY[emoji.group];
if (DATA_BY_CATEGORY.hasOwnProperty(categoryId)) { if (DATA_BY_CATEGORY.hasOwnProperty(categoryId)) {
DATA_BY_CATEGORY[categoryId].push(emoji); DATA_BY_CATEGORY[categoryId].push(emoji);
} }
// This is used as the string to match the query against when filtering emojis // This is used as the string to match the query against when filtering emojis
emoji.filterString = (`${emoji.annotation}\n${emoji.shortcodes.join('\n')}}\n${emoji.emoticon || ''}\n` + emoji.filterString = (`${emoji.annotation}\n${shortcodes.join('\n')}}\n${emoji.emoticon || ''}\n` +
`${emoji.unicode.split(ZERO_WIDTH_JOINER).join("\n")}`).toLowerCase(); `${emoji.unicode.split(ZERO_WIDTH_JOINER).join("\n")}`).toLowerCase();
// Add mapping from unicode to Emoji object // Add mapping from unicode to Emoji object
@ -87,13 +93,6 @@ EMOJIBASE.forEach((emoji: IEmojiWithFilterString) => {
// Add mapping from emoticon to Emoji object // Add mapping from emoticon to Emoji object
EMOTICON_TO_EMOJI.set(emoji.emoticon, emoji); EMOTICON_TO_EMOJI.set(emoji.emoticon, emoji);
} }
if (emoji.shortcodes) {
// Add mapping from each shortcode to Emoji object
emoji.shortcodes.forEach(shortcode => {
SHORTCODE_TO_EMOJI.set(shortcode, emoji);
});
}
}); });
/** /**

16
yarn.lock
View File

@ -3022,15 +3022,15 @@ emoji-regex@^8.0.0:
resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37"
integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==
emojibase-data@^5.1.1: emojibase-data@^6.2.0:
version "5.1.1" version "6.2.0"
resolved "https://registry.yarnpkg.com/emojibase-data/-/emojibase-data-5.1.1.tgz#0a0d63dd07ce1376b3d27642f28cafa46f651de6" resolved "https://registry.yarnpkg.com/emojibase-data/-/emojibase-data-6.2.0.tgz#db6c75c36905284fa623f4aa5f468d2be6ed364a"
integrity sha512-za/ma5SfogHjwUmGFnDbTvSfm8GGFvFaPS27GPti16YZSp5EPgz+UDsZCATXvJGit+oRNBbG/FtybXHKi2UQgQ== integrity sha512-SWKaXD2QeQs06IE7qfJftsI5924Dqzp+V9xaa5RzZIEWhmlrG6Jt2iKwfgOPHu+5S8MEtOI7GdpKsXj46chXOw==
emojibase-regex@^4.1.1: emojibase-regex@^5.1.3:
version "4.1.1" version "5.1.3"
resolved "https://registry.yarnpkg.com/emojibase-regex/-/emojibase-regex-4.1.1.tgz#6e781aca520281600fe7a177f1582c33cf1fc545" resolved "https://registry.yarnpkg.com/emojibase-regex/-/emojibase-regex-5.1.3.tgz#f0ef621ed6ec624becd2326f999fd4ea01b94554"
integrity sha512-KSigB1zQkNKFygLZ5bAfHs87LJa1ni8QTQtq8lc53Y74NF3Dk2r7kfa8MpooTO8JBb5Xz660X4tSjDB+I+7elA== integrity sha512-gT8T9LxLA8VJdI+8KQtyykB9qKzd7WuUL3M2yw6y9tplFeufOUANg3UKVaKUvkMcRNvZsSElWhxcJrx8WPE12g==
encoding@^0.1.11: encoding@^0.1.11:
version "0.1.13" version "0.1.13"