From e32823e5fec61b069e9ddc57b731d6b346bad731 Mon Sep 17 00:00:00 2001 From: alunturner <56027671+alunturner@users.noreply.github.com> Date: Mon, 12 Jun 2023 12:28:00 +0100 Subject: [PATCH] Allow image pasting in plain mode in RTE (#11056) * get rough funcitonality working * try to tidy up types * fix merge error * fix signature change error * type wrangling * use onBeforeInput listener * add onBeforeInput handler, add logic to onPaste * fix type error * bring plain text listeners in line with useInputEventProcessor * extract common function to util file, move tests * tidy comment * tidy comments * fix typo * add util tests * add text paste test --- .../components/PlainTextComposer.tsx | 5 +- .../hooks/useInputEventProcessor.ts | 103 +--------------- .../hooks/usePlainTextListeners.ts | 34 +++++- .../rooms/wysiwyg_composer/hooks/utils.ts | 110 ++++++++++++++++++ .../components/PlainTextComposer-test.tsx | 12 ++ ...EventProcessor-test.tsx => utils-test.tsx} | 28 ++++- 6 files changed, 188 insertions(+), 104 deletions(-) rename test/components/views/rooms/wysiwyg_composer/hooks/{useInputEventProcessor-test.tsx => utils-test.tsx} (91%) diff --git a/src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx b/src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx index efc4971657..cf54fa7bef 100644 --- a/src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx +++ b/src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx @@ -50,10 +50,12 @@ export function PlainTextComposer({ initialContent, leftComponent, rightComponent, + eventRelation, }: PlainTextComposerProps): JSX.Element { const { ref: editorRef, autocompleteRef, + onBeforeInput, onInput, onPaste, onKeyDown, @@ -63,7 +65,7 @@ export function PlainTextComposer({ onSelect, handleCommand, handleMention, - } = usePlainTextListeners(initialContent, onChange, onSend); + } = usePlainTextListeners(initialContent, onChange, onSend, eventRelation); const composerFunctions = useComposerFunctions(editorRef, setContent); usePlainTextInitialization(initialContent, editorRef); @@ -77,6 +79,7 @@ export function PlainTextComposer({ className={classNames(className, { [`${className}-focused`]: isFocused })} onFocus={onFocus} onBlur={onFocus} + onBeforeInput={onBeforeInput} onInput={onInput} onPaste={onPaste} onKeyDown={onKeyDown} diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts b/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts index 27f4088014..a9cfa2966e 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts @@ -33,10 +33,7 @@ import { isCaretAtEnd, isCaretAtStart } from "../utils/selection"; import { getEventsFromEditorStateTransfer, getEventsFromRoom } from "../utils/event"; import { endEditing } from "../utils/editing"; import Autocomplete from "../../Autocomplete"; -import { handleEventWithAutocomplete } from "./utils"; -import ContentMessages from "../../../../../ContentMessages"; -import { getBlobSafeMimeType } from "../../../../../utils/blobs"; -import { isNotNull } from "../../../../../Typeguards"; +import { handleClipboardEvent, handleEventWithAutocomplete, isEventToHandleAsClipboardEvent } from "./utils"; export function useInputEventProcessor( onSend: () => void, @@ -61,17 +58,8 @@ export function useInputEventProcessor( onSend(); }; - // this is required to handle edge case image pasting in Safari, see - // https://github.com/vector-im/element-web/issues/25327 and it is caught by the - // `beforeinput` listener attached to the composer - const isInputEventForClipboard = - event instanceof InputEvent && event.inputType === "insertFromPaste" && isNotNull(event.dataTransfer); - const isClipboardEvent = event instanceof ClipboardEvent; - - const shouldHandleAsClipboardEvent = isClipboardEvent || isInputEventForClipboard; - - if (shouldHandleAsClipboardEvent) { - const data = isClipboardEvent ? event.clipboardData : event.dataTransfer; + if (isEventToHandleAsClipboardEvent(event)) { + const data = event instanceof ClipboardEvent ? event.clipboardData : event.dataTransfer; const handled = handleClipboardEvent(event, data, roomContext, mxClient, eventRelation); return handled ? null : event; } @@ -244,88 +232,3 @@ function handleInputEvent(event: InputEvent, send: Send, isCtrlEnterToSend: bool return event; } - -/** - * Takes an event and handles image pasting. Returns a boolean to indicate if it has handled - * the event or not. Must accept either clipboard or input events in order to prevent issue: - * https://github.com/vector-im/element-web/issues/25327 - * - * @param event - event to process - * @param roomContext - room in which the event occurs - * @param mxClient - current matrix client - * @param eventRelation - used to send the event to the correct place eg timeline vs thread - * @returns - boolean to show if the event was handled or not - */ -export function handleClipboardEvent( - event: ClipboardEvent | InputEvent, - data: DataTransfer | null, - roomContext: IRoomState, - mxClient: MatrixClient, - eventRelation?: IEventRelation, -): boolean { - // Logic in this function follows that of `SendMessageComposer.onPaste` - const { room, timelineRenderingType, replyToEvent } = roomContext; - - function handleError(error: unknown): void { - if (error instanceof Error) { - console.log(error.message); - } else if (typeof error === "string") { - console.log(error); - } - } - - if (event.type !== "paste" || data === null || room === undefined) { - return false; - } - - // Prioritize text on the clipboard over files if RTF is present as Office on macOS puts a bitmap - // in the clipboard as well as the content being copied. Modern versions of Office seem to not do this anymore. - // We check text/rtf instead of text/plain as when copy+pasting a file from Finder or Gnome Image Viewer - // it puts the filename in as text/plain which we want to ignore. - if (data.files.length && !data.types.includes("text/rtf")) { - ContentMessages.sharedInstance() - .sendContentListToRoom(Array.from(data.files), room.roomId, eventRelation, mxClient, timelineRenderingType) - .catch(handleError); - return true; - } - - // Safari `Insert from iPhone or iPad` - // data.getData("text/html") returns a string like: - if (data.types.includes("text/html")) { - const imgElementStr = data.getData("text/html"); - const parser = new DOMParser(); - const imgDoc = parser.parseFromString(imgElementStr, "text/html"); - - if ( - imgDoc.getElementsByTagName("img").length !== 1 || - !imgDoc.querySelector("img")?.src.startsWith("blob:") || - imgDoc.childNodes.length !== 1 - ) { - handleError("Failed to handle pasted content as Safari inserted content"); - return false; - } - const imgSrc = imgDoc.querySelector("img")!.src; - - fetch(imgSrc) - .then((response) => { - response - .blob() - .then((imgBlob) => { - const type = imgBlob.type; - const safetype = getBlobSafeMimeType(type); - const ext = type.split("/")[1]; - const parts = response.url.split("/"); - const filename = parts[parts.length - 1]; - const file = new File([imgBlob], filename + "." + ext, { type: safetype }); - ContentMessages.sharedInstance() - .sendContentToRoom(file, room.roomId, eventRelation, mxClient, replyToEvent) - .catch(handleError); - }) - .catch(handleError); - }) - .catch(handleError); - return true; - } - - return false; -} diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts b/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts index 2bccfc444a..21b43126bb 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts @@ -16,13 +16,16 @@ limitations under the License. import { KeyboardEvent, RefObject, SyntheticEvent, useCallback, useRef, useState } from "react"; import { Attributes, MappedSuggestion } from "@matrix-org/matrix-wysiwyg"; +import { IEventRelation } from "matrix-js-sdk/src/matrix"; import { useSettingValue } from "../../../../../hooks/useSettings"; import { IS_MAC, Key } from "../../../../../Keyboard"; import Autocomplete from "../../Autocomplete"; -import { handleEventWithAutocomplete } from "./utils"; +import { handleClipboardEvent, handleEventWithAutocomplete, isEventToHandleAsClipboardEvent } from "./utils"; import { useSuggestion } from "./useSuggestion"; import { isNotNull, isNotUndefined } from "../../../../../Typeguards"; +import { useRoomContext } from "../../../../../contexts/RoomContext"; +import { useMatrixClientContext } from "../../../../../contexts/MatrixClientContext"; function isDivElement(target: EventTarget): target is HTMLDivElement { return target instanceof HTMLDivElement; @@ -59,10 +62,12 @@ export function usePlainTextListeners( initialContent?: string, onChange?: (content: string) => void, onSend?: () => void, + eventRelation?: IEventRelation, ): { ref: RefObject; autocompleteRef: React.RefObject; content?: string; + onBeforeInput(event: SyntheticEvent): void; onInput(event: SyntheticEvent): void; onPaste(event: SyntheticEvent): void; onKeyDown(event: KeyboardEvent): void; @@ -72,6 +77,9 @@ export function usePlainTextListeners( onSelect: (event: SyntheticEvent) => void; suggestion: MappedSuggestion | null; } { + const roomContext = useRoomContext(); + const mxClient = useMatrixClientContext(); + const ref = useRef(null); const autocompleteRef = useRef(null); const [content, setContent] = useState(initialContent); @@ -115,6 +123,27 @@ export function usePlainTextListeners( [setText, enterShouldSend], ); + const onPaste = useCallback( + (event: SyntheticEvent) => { + const { nativeEvent } = event; + let imagePasteWasHandled = false; + + if (isEventToHandleAsClipboardEvent(nativeEvent)) { + const data = + nativeEvent instanceof ClipboardEvent ? nativeEvent.clipboardData : nativeEvent.dataTransfer; + imagePasteWasHandled = handleClipboardEvent(nativeEvent, data, roomContext, mxClient, eventRelation); + } + + // prevent default behaviour and skip call to onInput if the image paste event was handled + if (imagePasteWasHandled) { + event.preventDefault(); + } else { + onInput(event); + } + }, + [eventRelation, mxClient, onInput, roomContext], + ); + const onKeyDown = useCallback( (event: KeyboardEvent) => { // we need autocomplete to take priority when it is open for using enter to select @@ -149,8 +178,9 @@ export function usePlainTextListeners( return { ref, autocompleteRef, + onBeforeInput: onPaste, onInput, - onPaste: onInput, + onPaste, onKeyDown, content, setContent: setText, diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/utils.ts b/src/components/views/rooms/wysiwyg_composer/hooks/utils.ts index 636b5d2bf2..f95405c3bf 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/utils.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/utils.ts @@ -15,12 +15,17 @@ limitations under the License. */ import { MutableRefObject, RefObject } from "react"; +import { IEventRelation, MatrixClient } from "matrix-js-sdk/src/matrix"; +import { WysiwygEvent } from "@matrix-org/matrix-wysiwyg"; import { TimelineRenderingType } from "../../../../../contexts/RoomContext"; import { IRoomState } from "../../../../structures/RoomView"; import Autocomplete from "../../Autocomplete"; import { getKeyBindingsManager } from "../../../../../KeyBindingsManager"; import { KeyBindingAction } from "../../../../../accessibility/KeyboardShortcuts"; +import { getBlobSafeMimeType } from "../../../../../utils/blobs"; +import ContentMessages from "../../../../../ContentMessages"; +import { isNotNull } from "../../../../../Typeguards"; export function focusComposer( composerElement: MutableRefObject, @@ -110,3 +115,108 @@ export function handleEventWithAutocomplete( return handled; } + +/** + * Takes an event and handles image pasting. Returns a boolean to indicate if it has handled + * the event or not. Must accept either clipboard or input events in order to prevent issue: + * https://github.com/vector-im/element-web/issues/25327 + * + * @param event - event to process + * @param data - data from the event to process + * @param roomContext - room in which the event occurs + * @param mxClient - current matrix client + * @param eventRelation - used to send the event to the correct place eg timeline vs thread + * @returns - boolean to show if the event was handled or not + */ +export function handleClipboardEvent( + event: ClipboardEvent | InputEvent, + data: DataTransfer | null, + roomContext: IRoomState, + mxClient: MatrixClient, + eventRelation?: IEventRelation, +): boolean { + // Logic in this function follows that of `SendMessageComposer.onPaste` + const { room, timelineRenderingType, replyToEvent } = roomContext; + + function handleError(error: unknown): void { + if (error instanceof Error) { + console.log(error.message); + } else if (typeof error === "string") { + console.log(error); + } + } + + if (event.type !== "paste" || data === null || room === undefined) { + return false; + } + + // Prioritize text on the clipboard over files if RTF is present as Office on macOS puts a bitmap + // in the clipboard as well as the content being copied. Modern versions of Office seem to not do this anymore. + // We check text/rtf instead of text/plain as when copy+pasting a file from Finder or Gnome Image Viewer + // it puts the filename in as text/plain which we want to ignore. + if (data.files.length && !data.types.includes("text/rtf")) { + ContentMessages.sharedInstance() + .sendContentListToRoom(Array.from(data.files), room.roomId, eventRelation, mxClient, timelineRenderingType) + .catch(handleError); + return true; + } + + // Safari `Insert from iPhone or iPad` + // data.getData("text/html") returns a string like: + if (data.types.includes("text/html")) { + const imgElementStr = data.getData("text/html"); + const parser = new DOMParser(); + const imgDoc = parser.parseFromString(imgElementStr, "text/html"); + + if ( + imgDoc.getElementsByTagName("img").length !== 1 || + !imgDoc.querySelector("img")?.src.startsWith("blob:") || + imgDoc.childNodes.length !== 1 + ) { + handleError("Failed to handle pasted content as Safari inserted content"); + return false; + } + const imgSrc = imgDoc.querySelector("img")!.src; + + fetch(imgSrc) + .then((response) => { + response + .blob() + .then((imgBlob) => { + const type = imgBlob.type; + const safetype = getBlobSafeMimeType(type); + const ext = type.split("/")[1]; + const parts = response.url.split("/"); + const filename = parts[parts.length - 1]; + const file = new File([imgBlob], filename + "." + ext, { type: safetype }); + ContentMessages.sharedInstance() + .sendContentToRoom(file, room.roomId, eventRelation, mxClient, replyToEvent) + .catch(handleError); + }) + .catch(handleError); + }) + .catch(handleError); + return true; + } + + return false; +} + +/** + * Util to determine if an input event or clipboard event must be handled as a clipboard event. + * Due to https://github.com/vector-im/element-web/issues/25327, certain paste events + * must be listenened for with an onBeforeInput handler and so will be caught as input events. + * + * @param event - the event to test, can be a WysiwygEvent if it comes from the rich text editor, or + * input or clipboard events if from the plain text editor + * @returns - true if event should be handled as a clipboard event + */ +export function isEventToHandleAsClipboardEvent( + event: WysiwygEvent | InputEvent | ClipboardEvent, +): event is InputEvent | ClipboardEvent { + const isInputEventForClipboard = + event instanceof InputEvent && event.inputType === "insertFromPaste" && isNotNull(event.dataTransfer); + const isClipboardEvent = event instanceof ClipboardEvent; + + return isClipboardEvent || isInputEventForClipboard; +} diff --git a/test/components/views/rooms/wysiwyg_composer/components/PlainTextComposer-test.tsx b/test/components/views/rooms/wysiwyg_composer/components/PlainTextComposer-test.tsx index 9277ecb16c..cb7104f8e4 100644 --- a/test/components/views/rooms/wysiwyg_composer/components/PlainTextComposer-test.tsx +++ b/test/components/views/rooms/wysiwyg_composer/components/PlainTextComposer-test.tsx @@ -290,4 +290,16 @@ describe("PlainTextComposer", () => { expect(screen.getByTestId("autocomplete-wrapper")).toBeInTheDocument(); }); + + it("Should allow pasting of text values", async () => { + customRender(); + + const textBox = screen.getByRole("textbox"); + + await userEvent.click(textBox); + await userEvent.type(textBox, "hello"); + await userEvent.paste(" world"); + + expect(textBox).toHaveTextContent("hello world"); + }); }); diff --git a/test/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor-test.tsx b/test/components/views/rooms/wysiwyg_composer/hooks/utils-test.tsx similarity index 91% rename from test/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor-test.tsx rename to test/components/views/rooms/wysiwyg_composer/hooks/utils-test.tsx index 8d6f9d19cc..81489e3beb 100644 --- a/test/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor-test.tsx +++ b/test/components/views/rooms/wysiwyg_composer/hooks/utils-test.tsx @@ -16,11 +16,14 @@ limitations under the License. import { IEventRelation, MatrixEvent } from "matrix-js-sdk/src/matrix"; import { waitFor } from "@testing-library/react"; -import { handleClipboardEvent } from "../../../../../../src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor"; import { TimelineRenderingType } from "../../../../../../src/contexts/RoomContext"; import { mkStubRoom, stubClient } from "../../../../../test-utils"; import ContentMessages from "../../../../../../src/ContentMessages"; import { IRoomState } from "../../../../../../src/components/structures/RoomView"; +import { + handleClipboardEvent, + isEventToHandleAsClipboardEvent, +} from "../../../../../../src/components/views/rooms/wysiwyg_composer/hooks/utils"; const mockClient = stubClient(); const mockRoom = mkStubRoom("mock room", "mock room", mockClient); @@ -285,3 +288,26 @@ describe("handleClipboardEvent", () => { expect(output).toBe(true); }); }); + +describe("isEventToHandleAsClipboardEvent", () => { + it("returns true for ClipboardEvent", () => { + const input = new ClipboardEvent("clipboard"); + expect(isEventToHandleAsClipboardEvent(input)).toBe(true); + }); + + it("returns true for special case input", () => { + const input = new InputEvent("insertFromPaste", { inputType: "insertFromPaste" }); + Object.assign(input, { dataTransfer: "not null" }); + expect(isEventToHandleAsClipboardEvent(input)).toBe(true); + }); + + it("returns false for regular InputEvent", () => { + const input = new InputEvent("input"); + expect(isEventToHandleAsClipboardEvent(input)).toBe(false); + }); + + it("returns false for other input", () => { + const input = new KeyboardEvent("keyboard"); + expect(isEventToHandleAsClipboardEvent(input)).toBe(false); + }); +});