Allow image pasting in plain mode in RTE (#11056)

* get rough funcitonality working

* try to tidy up types

* fix merge error

* fix signature change error

* type wrangling

* use onBeforeInput listener

* add onBeforeInput handler, add logic to onPaste

* fix type error

* bring plain text listeners in line with useInputEventProcessor

* extract common function to util file, move tests

* tidy comment

* tidy comments

* fix typo

* add util tests

* add text paste test

src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx

@@ -50,10 +50,12 @@ export function PlainTextComposer({
     initialContent,
     leftComponent,
     rightComponent,
+    eventRelation,
 }: PlainTextComposerProps): JSX.Element {
     const {
         ref: editorRef,
         autocompleteRef,
+        onBeforeInput,
         onInput,
         onPaste,
         onKeyDown,
@@ -63,7 +65,7 @@ export function PlainTextComposer({
         onSelect,
         handleCommand,
         handleMention,
-    } = usePlainTextListeners(initialContent, onChange, onSend);
+    } = usePlainTextListeners(initialContent, onChange, onSend, eventRelation);
 
     const composerFunctions = useComposerFunctions(editorRef, setContent);
     usePlainTextInitialization(initialContent, editorRef);
@@ -77,6 +79,7 @@ export function PlainTextComposer({
             className={classNames(className, { [`${className}-focused`]: isFocused })}
             onFocus={onFocus}
             onBlur={onFocus}
+            onBeforeInput={onBeforeInput}
             onInput={onInput}
             onPaste={onPaste}
             onKeyDown={onKeyDown}

src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts

@@ -33,10 +33,7 @@ import { isCaretAtEnd, isCaretAtStart } from "../utils/selection";
 import { getEventsFromEditorStateTransfer, getEventsFromRoom } from "../utils/event";
 import { endEditing } from "../utils/editing";
 import Autocomplete from "../../Autocomplete";
-import { handleEventWithAutocomplete } from "./utils";
+import { handleClipboardEvent, handleEventWithAutocomplete, isEventToHandleAsClipboardEvent } from "./utils";
-import ContentMessages from "../../../../../ContentMessages";
-import { getBlobSafeMimeType } from "../../../../../utils/blobs";
-import { isNotNull } from "../../../../../Typeguards";
 
 export function useInputEventProcessor(
     onSend: () => void,
@@ -61,17 +58,8 @@ export function useInputEventProcessor(
                 onSend();
             };
 
-            // this is required to handle edge case image pasting in Safari, see
+            if (isEventToHandleAsClipboardEvent(event)) {
-            // https://github.com/vector-im/element-web/issues/25327 and it is caught by the
+                const data = event instanceof ClipboardEvent ? event.clipboardData : event.dataTransfer;
-            // `beforeinput` listener attached to the composer
-            const isInputEventForClipboard =
-                event instanceof InputEvent && event.inputType === "insertFromPaste" && isNotNull(event.dataTransfer);
-            const isClipboardEvent = event instanceof ClipboardEvent;
-
-            const shouldHandleAsClipboardEvent = isClipboardEvent || isInputEventForClipboard;
-
-            if (shouldHandleAsClipboardEvent) {
-                const data = isClipboardEvent ? event.clipboardData : event.dataTransfer;
                 const handled = handleClipboardEvent(event, data, roomContext, mxClient, eventRelation);
                 return handled ? null : event;
             }
@@ -244,88 +232,3 @@ function handleInputEvent(event: InputEvent, send: Send, isCtrlEnterToSend: bool
 
     return event;
 }
-
-/**
- * Takes an event and handles image pasting. Returns a boolean to indicate if it has handled
- * the event or not. Must accept either clipboard or input events in order to prevent issue:
- * https://github.com/vector-im/element-web/issues/25327
- *
- * @param event - event to process
- * @param roomContext - room in which the event occurs
- * @param mxClient - current matrix client
- * @param eventRelation - used to send the event to the correct place eg timeline vs thread
- * @returns - boolean to show if the event was handled or not
- */
-export function handleClipboardEvent(
-    event: ClipboardEvent | InputEvent,
-    data: DataTransfer | null,
-    roomContext: IRoomState,
-    mxClient: MatrixClient,
-    eventRelation?: IEventRelation,
-): boolean {
-    // Logic in this function follows that of `SendMessageComposer.onPaste`
-    const { room, timelineRenderingType, replyToEvent } = roomContext;
-
-    function handleError(error: unknown): void {
-        if (error instanceof Error) {
-            console.log(error.message);
-        } else if (typeof error === "string") {
-            console.log(error);
-        }
-    }
-
-    if (event.type !== "paste" || data === null || room === undefined) {
-        return false;
-    }
-
-    // Prioritize text on the clipboard over files if RTF is present as Office on macOS puts a bitmap
-    // in the clipboard as well as the content being copied. Modern versions of Office seem to not do this anymore.
-    // We check text/rtf instead of text/plain as when copy+pasting a file from Finder or Gnome Image Viewer
-    // it puts the filename in as text/plain which we want to ignore.
-    if (data.files.length && !data.types.includes("text/rtf")) {
-        ContentMessages.sharedInstance()
-            .sendContentListToRoom(Array.from(data.files), room.roomId, eventRelation, mxClient, timelineRenderingType)
-            .catch(handleError);
-        return true;
-    }
-
-    // Safari `Insert from iPhone or iPad`
-    // data.getData("text/html") returns a string like: <img src="blob:https://...">
-    if (data.types.includes("text/html")) {
-        const imgElementStr = data.getData("text/html");
-        const parser = new DOMParser();
-        const imgDoc = parser.parseFromString(imgElementStr, "text/html");
-
-        if (
-            imgDoc.getElementsByTagName("img").length !== 1 ||
-            !imgDoc.querySelector("img")?.src.startsWith("blob:") ||
-            imgDoc.childNodes.length !== 1
-        ) {
-            handleError("Failed to handle pasted content as Safari inserted content");
-            return false;
-        }
-        const imgSrc = imgDoc.querySelector("img")!.src;
-
-        fetch(imgSrc)
-            .then((response) => {
-                response
-                    .blob()
-                    .then((imgBlob) => {
-                        const type = imgBlob.type;
-                        const safetype = getBlobSafeMimeType(type);
-                        const ext = type.split("/")[1];
-                        const parts = response.url.split("/");
-                        const filename = parts[parts.length - 1];
-                        const file = new File([imgBlob], filename + "." + ext, { type: safetype });
-                        ContentMessages.sharedInstance()
-                            .sendContentToRoom(file, room.roomId, eventRelation, mxClient, replyToEvent)
-                            .catch(handleError);
-                    })
-                    .catch(handleError);
-            })
-            .catch(handleError);
-        return true;
-    }
-
-    return false;
-}

src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts

@@ -16,13 +16,16 @@ limitations under the License.
 
 import { KeyboardEvent, RefObject, SyntheticEvent, useCallback, useRef, useState } from "react";
 import { Attributes, MappedSuggestion } from "@matrix-org/matrix-wysiwyg";
+import { IEventRelation } from "matrix-js-sdk/src/matrix";
 
 import { useSettingValue } from "../../../../../hooks/useSettings";
 import { IS_MAC, Key } from "../../../../../Keyboard";
 import Autocomplete from "../../Autocomplete";
-import { handleEventWithAutocomplete } from "./utils";
+import { handleClipboardEvent, handleEventWithAutocomplete, isEventToHandleAsClipboardEvent } from "./utils";
 import { useSuggestion } from "./useSuggestion";
 import { isNotNull, isNotUndefined } from "../../../../../Typeguards";
+import { useRoomContext } from "../../../../../contexts/RoomContext";
+import { useMatrixClientContext } from "../../../../../contexts/MatrixClientContext";
 
 function isDivElement(target: EventTarget): target is HTMLDivElement {
     return target instanceof HTMLDivElement;
@@ -59,10 +62,12 @@ export function usePlainTextListeners(
     initialContent?: string,
     onChange?: (content: string) => void,
     onSend?: () => void,
+    eventRelation?: IEventRelation,
 ): {
     ref: RefObject<HTMLDivElement>;
     autocompleteRef: React.RefObject<Autocomplete>;
     content?: string;
+    onBeforeInput(event: SyntheticEvent<HTMLDivElement, InputEvent | ClipboardEvent>): void;
     onInput(event: SyntheticEvent<HTMLDivElement, InputEvent | ClipboardEvent>): void;
     onPaste(event: SyntheticEvent<HTMLDivElement, InputEvent | ClipboardEvent>): void;
     onKeyDown(event: KeyboardEvent<HTMLDivElement>): void;
@@ -72,6 +77,9 @@ export function usePlainTextListeners(
     onSelect: (event: SyntheticEvent<HTMLDivElement>) => void;
     suggestion: MappedSuggestion | null;
 } {
+    const roomContext = useRoomContext();
+    const mxClient = useMatrixClientContext();
+
     const ref = useRef<HTMLDivElement | null>(null);
     const autocompleteRef = useRef<Autocomplete | null>(null);
     const [content, setContent] = useState<string | undefined>(initialContent);
@@ -115,6 +123,27 @@ export function usePlainTextListeners(
         [setText, enterShouldSend],
     );
 
+    const onPaste = useCallback(
+        (event: SyntheticEvent<HTMLDivElement, InputEvent | ClipboardEvent>) => {
+            const { nativeEvent } = event;
+            let imagePasteWasHandled = false;
+
+            if (isEventToHandleAsClipboardEvent(nativeEvent)) {
+                const data =
+                    nativeEvent instanceof ClipboardEvent ? nativeEvent.clipboardData : nativeEvent.dataTransfer;
+                imagePasteWasHandled = handleClipboardEvent(nativeEvent, data, roomContext, mxClient, eventRelation);
+            }
+
+            // prevent default behaviour and skip call to onInput if the image paste event was handled
+            if (imagePasteWasHandled) {
+                event.preventDefault();
+            } else {
+                onInput(event);
+            }
+        },
+        [eventRelation, mxClient, onInput, roomContext],
+    );
+
     const onKeyDown = useCallback(
         (event: KeyboardEvent<HTMLDivElement>) => {
             // we need autocomplete to take priority when it is open for using enter to select
@@ -149,8 +178,9 @@ export function usePlainTextListeners(
     return {
         ref,
         autocompleteRef,
+        onBeforeInput: onPaste,
         onInput,
-        onPaste: onInput,
+        onPaste,
         onKeyDown,
         content,
         setContent: setText,

src/components/views/rooms/wysiwyg_composer/hooks/utils.ts

@@ -15,12 +15,17 @@ limitations under the License.
 */
 
 import { MutableRefObject, RefObject } from "react";
+import { IEventRelation, MatrixClient } from "matrix-js-sdk/src/matrix";
+import { WysiwygEvent } from "@matrix-org/matrix-wysiwyg";
 
 import { TimelineRenderingType } from "../../../../../contexts/RoomContext";
 import { IRoomState } from "../../../../structures/RoomView";
 import Autocomplete from "../../Autocomplete";
 import { getKeyBindingsManager } from "../../../../../KeyBindingsManager";
 import { KeyBindingAction } from "../../../../../accessibility/KeyboardShortcuts";
+import { getBlobSafeMimeType } from "../../../../../utils/blobs";
+import ContentMessages from "../../../../../ContentMessages";
+import { isNotNull } from "../../../../../Typeguards";
 
 export function focusComposer(
     composerElement: MutableRefObject<HTMLElement | null>,
@@ -110,3 +115,108 @@ export function handleEventWithAutocomplete(
 
     return handled;
 }
+
+/**
+ * Takes an event and handles image pasting. Returns a boolean to indicate if it has handled
+ * the event or not. Must accept either clipboard or input events in order to prevent issue:
+ * https://github.com/vector-im/element-web/issues/25327
+ *
+ * @param event - event to process
+ * @param data - data from the event to process
+ * @param roomContext - room in which the event occurs
+ * @param mxClient - current matrix client
+ * @param eventRelation - used to send the event to the correct place eg timeline vs thread
+ * @returns - boolean to show if the event was handled or not
+ */
+export function handleClipboardEvent(
+    event: ClipboardEvent | InputEvent,
+    data: DataTransfer | null,
+    roomContext: IRoomState,
+    mxClient: MatrixClient,
+    eventRelation?: IEventRelation,
+): boolean {
+    // Logic in this function follows that of `SendMessageComposer.onPaste`
+    const { room, timelineRenderingType, replyToEvent } = roomContext;
+
+    function handleError(error: unknown): void {
+        if (error instanceof Error) {
+            console.log(error.message);
+        } else if (typeof error === "string") {
+            console.log(error);
+        }
+    }
+
+    if (event.type !== "paste" || data === null || room === undefined) {
+        return false;
+    }
+
+    // Prioritize text on the clipboard over files if RTF is present as Office on macOS puts a bitmap
+    // in the clipboard as well as the content being copied. Modern versions of Office seem to not do this anymore.
+    // We check text/rtf instead of text/plain as when copy+pasting a file from Finder or Gnome Image Viewer
+    // it puts the filename in as text/plain which we want to ignore.
+    if (data.files.length && !data.types.includes("text/rtf")) {
+        ContentMessages.sharedInstance()
+            .sendContentListToRoom(Array.from(data.files), room.roomId, eventRelation, mxClient, timelineRenderingType)
+            .catch(handleError);
+        return true;
+    }
+
+    // Safari `Insert from iPhone or iPad`
+    // data.getData("text/html") returns a string like: <img src="blob:https://...">
+    if (data.types.includes("text/html")) {
+        const imgElementStr = data.getData("text/html");
+        const parser = new DOMParser();
+        const imgDoc = parser.parseFromString(imgElementStr, "text/html");
+
+        if (
+            imgDoc.getElementsByTagName("img").length !== 1 ||
+            !imgDoc.querySelector("img")?.src.startsWith("blob:") ||
+            imgDoc.childNodes.length !== 1
+        ) {
+            handleError("Failed to handle pasted content as Safari inserted content");
+            return false;
+        }
+        const imgSrc = imgDoc.querySelector("img")!.src;
+
+        fetch(imgSrc)
+            .then((response) => {
+                response
+                    .blob()
+                    .then((imgBlob) => {
+                        const type = imgBlob.type;
+                        const safetype = getBlobSafeMimeType(type);
+                        const ext = type.split("/")[1];
+                        const parts = response.url.split("/");
+                        const filename = parts[parts.length - 1];
+                        const file = new File([imgBlob], filename + "." + ext, { type: safetype });
+                        ContentMessages.sharedInstance()
+                            .sendContentToRoom(file, room.roomId, eventRelation, mxClient, replyToEvent)
+                            .catch(handleError);
+                    })
+                    .catch(handleError);
+            })
+            .catch(handleError);
+        return true;
+    }
+
+    return false;
+}
+
+/**
+ * Util to determine if an input event or clipboard event must be handled as a clipboard event.
+ * Due to https://github.com/vector-im/element-web/issues/25327, certain paste events
+ * must be listenened for with an onBeforeInput handler and so will be caught as input events.
+ *
+ * @param event - the event to test, can be a WysiwygEvent if it comes from the rich text editor, or
+ * input or clipboard events if from the plain text editor
+ * @returns - true if event should be handled as a clipboard event
+ */
+export function isEventToHandleAsClipboardEvent(
+    event: WysiwygEvent | InputEvent | ClipboardEvent,
+): event is InputEvent | ClipboardEvent {
+    const isInputEventForClipboard =
+        event instanceof InputEvent && event.inputType === "insertFromPaste" && isNotNull(event.dataTransfer);
+    const isClipboardEvent = event instanceof ClipboardEvent;
+
+    return isClipboardEvent || isInputEventForClipboard;
+}

test/components/views/rooms/wysiwyg_composer/components/PlainTextComposer-test.tsx

@@ -290,4 +290,16 @@ describe("PlainTextComposer", () => {
 
         expect(screen.getByTestId("autocomplete-wrapper")).toBeInTheDocument();
     });
+
+    it("Should allow pasting of text values", async () => {
+        customRender();
+
+        const textBox = screen.getByRole("textbox");
+
+        await userEvent.click(textBox);
+        await userEvent.type(textBox, "hello");
+        await userEvent.paste(" world");
+
+        expect(textBox).toHaveTextContent("hello world");
+    });
 });

test/components/views/rooms/wysiwyg_composer/hooks/utils-test.tsx

@@ -16,11 +16,14 @@ limitations under the License.
 import { IEventRelation, MatrixEvent } from "matrix-js-sdk/src/matrix";
 import { waitFor } from "@testing-library/react";
 
-import { handleClipboardEvent } from "../../../../../../src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor";
 import { TimelineRenderingType } from "../../../../../../src/contexts/RoomContext";
 import { mkStubRoom, stubClient } from "../../../../../test-utils";
 import ContentMessages from "../../../../../../src/ContentMessages";
 import { IRoomState } from "../../../../../../src/components/structures/RoomView";
+import {
+    handleClipboardEvent,
+    isEventToHandleAsClipboardEvent,
+} from "../../../../../../src/components/views/rooms/wysiwyg_composer/hooks/utils";
 
 const mockClient = stubClient();
 const mockRoom = mkStubRoom("mock room", "mock room", mockClient);
@@ -285,3 +288,26 @@ describe("handleClipboardEvent", () => {
         expect(output).toBe(true);
     });
 });
+
+describe("isEventToHandleAsClipboardEvent", () => {
+    it("returns true for ClipboardEvent", () => {
+        const input = new ClipboardEvent("clipboard");
+        expect(isEventToHandleAsClipboardEvent(input)).toBe(true);
+    });
+
+    it("returns true for special case input", () => {
+        const input = new InputEvent("insertFromPaste", { inputType: "insertFromPaste" });
+        Object.assign(input, { dataTransfer: "not null" });
+        expect(isEventToHandleAsClipboardEvent(input)).toBe(true);
+    });
+
+    it("returns false for regular InputEvent", () => {
+        const input = new InputEvent("input");
+        expect(isEventToHandleAsClipboardEvent(input)).toBe(false);
+    });
+
+    it("returns false for other input", () => {
+        const input = new KeyboardEvent("keyboard");
+        expect(isEventToHandleAsClipboardEvent(input)).toBe(false);
+    });
+});