From 103b60dfb5ca975a3d105f07dcda7cf4ac79011a Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 10 Aug 2022 08:57:56 -0400 Subject: [PATCH] Implement MSC3819: Allowing widgets to send/receive to-device messages (#8885) * Implement MSC3819: Allowing widgets to send/receive to-device messages * Don't change the room events and state events drivers * Update to latest matrix-widget-api changes * Support sending encrypted to-device messages * Use queueToDevice for better reliability * Update types for latest WidgetDriver changes * Upgrade matrix-widget-api * Add tests * Test StopGapWidget * Fix a potential memory leak --- package.json | 2 +- src/stores/widgets/StopGapWidget.ts | 37 ++++++--- src/stores/widgets/StopGapWidgetDriver.ts | 48 ++++++++++- src/widgets/CapabilityText.tsx | 9 +- test/stores/widgets/StopGapWidget-test.ts | 70 ++++++++++++++++ .../widgets/StopGapWidgetDriver-test.ts | 79 ++++++++++++++++++ .../StopGapWidgetDriver-test.ts.snap | 82 +++++++++++++++++++ test/test-utils/test-utils.ts | 11 ++- yarn.lock | 8 +- 9 files changed, 322 insertions(+), 24 deletions(-) create mode 100644 test/stores/widgets/StopGapWidget-test.ts create mode 100644 test/stores/widgets/StopGapWidgetDriver-test.ts create mode 100644 test/stores/widgets/__snapshots__/StopGapWidgetDriver-test.ts.snap diff --git a/package.json b/package.json index 933ccfd7ad..ca203360c6 100644 --- a/package.json +++ b/package.json @@ -94,7 +94,7 @@ "matrix-encrypt-attachment": "^1.0.3", "matrix-events-sdk": "^0.0.1-beta.7", "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#develop", - "matrix-widget-api": "^0.1.0-beta.18", + "matrix-widget-api": "^1.0.0", "minimist": "^1.2.5", "opus-recorder": "^8.0.3", "pako": "^2.0.3", diff --git a/src/stores/widgets/StopGapWidget.ts b/src/stores/widgets/StopGapWidget.ts index 7a1c5e0ba5..889a050ebf 100644 --- a/src/stores/widgets/StopGapWidget.ts +++ b/src/stores/widgets/StopGapWidget.ts @@ -33,6 +33,7 @@ import { WidgetKind, } from "matrix-widget-api"; import { EventEmitter } from "events"; +import { MatrixClient } from "matrix-js-sdk/src/client"; import { MatrixEvent, MatrixEventEvent } from "matrix-js-sdk/src/models/event"; import { logger } from "matrix-js-sdk/src/logger"; import { ClientEvent } from "matrix-js-sdk/src/client"; @@ -148,6 +149,7 @@ export class ElementWidget extends Widget { } export class StopGapWidget extends EventEmitter { + private client: MatrixClient; private messaging: ClientWidgetApi; private mockWidget: ElementWidget; private scalarToken: string; @@ -157,12 +159,13 @@ export class StopGapWidget extends EventEmitter { constructor(private appTileProps: IAppTileProps) { super(); - let app = appTileProps.app; + this.client = MatrixClientPeg.get(); + let app = appTileProps.app; // Backwards compatibility: not all old widgets have a creatorUserId if (!app.creatorUserId) { app = objectShallowClone(app); // clone to prevent accidental mutation - app.creatorUserId = MatrixClientPeg.get().getUserId(); + app.creatorUserId = this.client.getUserId(); } this.mockWidget = new ElementWidget(app); @@ -203,7 +206,7 @@ export class StopGapWidget extends EventEmitter { const fromCustomisation = WidgetVariableCustomisations?.provideVariables?.() ?? {}; const defaults: ITemplateParams = { widgetRoomId: this.roomId, - currentUserId: MatrixClientPeg.get().getUserId(), + currentUserId: this.client.getUserId(), userDisplayName: OwnProfileStore.instance.displayName, userHttpAvatarUrl: OwnProfileStore.instance.getHttpAvatarUrl(), clientId: ELEMENT_CLIENT_ID, @@ -260,8 +263,10 @@ export class StopGapWidget extends EventEmitter { */ public startMessaging(iframe: HTMLIFrameElement): any { if (this.started) return; + const allowedCapabilities = this.appTileProps.whitelistCapabilities || []; const driver = new StopGapWidgetDriver(allowedCapabilities, this.mockWidget, this.kind, this.roomId); + this.messaging = new ClientWidgetApi(this.mockWidget, iframe, driver); this.messaging.on("preparing", () => this.emit("preparing")); this.messaging.on("ready", () => this.emit("ready")); @@ -302,7 +307,7 @@ export class StopGapWidget extends EventEmitter { // Populate the map of "read up to" events for this widget with the current event in every room. // This is a bit inefficient, but should be okay. We do this for all rooms in case the widget // requests timeline capabilities in other rooms down the road. It's just easier to manage here. - for (const room of MatrixClientPeg.get().getRooms()) { + for (const room of this.client.getRooms()) { // Timelines are most recent last const events = room.getLiveTimeline()?.getEvents() || []; const roomEvent = events[events.length - 1]; @@ -311,8 +316,9 @@ export class StopGapWidget extends EventEmitter { } // Attach listeners for feeding events - the underlying widget classes handle permissions for us - MatrixClientPeg.get().on(ClientEvent.Event, this.onEvent); - MatrixClientPeg.get().on(MatrixEventEvent.Decrypted, this.onEventDecrypted); + this.client.on(ClientEvent.Event, this.onEvent); + this.client.on(MatrixEventEvent.Decrypted, this.onEventDecrypted); + this.client.on(ClientEvent.ToDeviceEvent, this.onToDeviceEvent); this.messaging.on(`action:${WidgetApiFromWidgetAction.UpdateAlwaysOnScreen}`, (ev: CustomEvent) => { @@ -363,7 +369,7 @@ export class StopGapWidget extends EventEmitter { // noinspection JSIgnoredPromiseFromCall IntegrationManagers.sharedInstance().getPrimaryManager().open( - MatrixClientPeg.get().getRoom(RoomViewStore.instance.getRoomId()), + this.client.getRoom(RoomViewStore.instance.getRoomId()), `type_${integType}`, integId, ); @@ -428,14 +434,13 @@ export class StopGapWidget extends EventEmitter { WidgetMessagingStore.instance.stopMessaging(this.mockWidget, this.roomId); this.messaging = null; - if (MatrixClientPeg.get()) { - MatrixClientPeg.get().off(ClientEvent.Event, this.onEvent); - MatrixClientPeg.get().off(MatrixEventEvent.Decrypted, this.onEventDecrypted); - } + this.client.off(ClientEvent.Event, this.onEvent); + this.client.off(MatrixEventEvent.Decrypted, this.onEventDecrypted); + this.client.off(ClientEvent.ToDeviceEvent, this.onToDeviceEvent); } private onEvent = (ev: MatrixEvent) => { - MatrixClientPeg.get().decryptEventIfNeeded(ev); + this.client.decryptEventIfNeeded(ev); if (ev.isBeingDecrypted() || ev.isDecryptionFailure()) return; this.feedEvent(ev); }; @@ -445,6 +450,12 @@ export class StopGapWidget extends EventEmitter { this.feedEvent(ev); }; + private onToDeviceEvent = async (ev: MatrixEvent) => { + await this.client.decryptEventIfNeeded(ev); + if (ev.isDecryptionFailure()) return; + await this.messaging.feedToDevice(ev.getEffectiveEvent(), ev.isEncrypted()); + }; + private feedEvent(ev: MatrixEvent) { if (!this.messaging) return; @@ -465,7 +476,7 @@ export class StopGapWidget extends EventEmitter { // Timelines are most recent last, so reverse the order and limit ourselves to 100 events // to avoid overusing the CPU. - const timeline = MatrixClientPeg.get().getRoom(ev.getRoomId()).getLiveTimeline(); + const timeline = this.client.getRoom(ev.getRoomId()).getLiveTimeline(); const events = arrayFastClone(timeline.getEvents()).reverse().slice(0, 100); for (const timelineEvent of events) { diff --git a/src/stores/widgets/StopGapWidgetDriver.ts b/src/stores/widgets/StopGapWidgetDriver.ts index 3b617e6f31..ee69f0ca9c 100644 --- a/src/stores/widgets/StopGapWidgetDriver.ts +++ b/src/stores/widgets/StopGapWidgetDriver.ts @@ -20,6 +20,7 @@ import { IOpenIDCredentials, IOpenIDUpdate, ISendEventDetails, + IRoomEvent, MatrixCapabilities, OpenIDRequestState, SimpleObservable, @@ -182,6 +183,49 @@ export class StopGapWidgetDriver extends WidgetDriver { return { roomId, eventId: r.event_id }; } + public async sendToDevice( + eventType: string, + encrypted: boolean, + contentMap: { [userId: string]: { [deviceId: string]: object } }, + ): Promise { + const client = MatrixClientPeg.get(); + + if (encrypted) { + const deviceInfoMap = await client.crypto.deviceList.downloadKeys(Object.keys(contentMap), false); + + await Promise.all( + Object.entries(contentMap).flatMap(([userId, userContentMap]) => + Object.entries(userContentMap).map(async ([deviceId, content]) => { + if (deviceId === "*") { + // Send the message to all devices we have keys for + await client.encryptAndSendToDevices( + Object.values(deviceInfoMap[userId]).map(deviceInfo => ({ + userId, deviceInfo, + })), + content, + ); + } else { + // Send the message to a specific device + await client.encryptAndSendToDevices( + [{ userId, deviceInfo: deviceInfoMap[userId][deviceId] }], + content, + ); + } + }), + ), + ); + } else { + await client.queueToDevice({ + eventType, + batch: Object.entries(contentMap).flatMap(([userId, userContentMap]) => + Object.entries(userContentMap).map(([deviceId, content]) => + ({ userId, deviceId, payload: content }), + ), + ), + }); + } + } + private pickRooms(roomIds: (string | Symbols.AnyRoom)[] = null): Room[] { const client = MatrixClientPeg.get(); if (!client) throw new Error("Not attached to a client"); @@ -197,7 +241,7 @@ export class StopGapWidgetDriver extends WidgetDriver { msgtype: string | undefined, limitPerRoom: number, roomIds: (string | Symbols.AnyRoom)[] = null, - ): Promise { + ): Promise { limitPerRoom = limitPerRoom > 0 ? Math.min(limitPerRoom, Number.MAX_SAFE_INTEGER) : Number.MAX_SAFE_INTEGER; // relatively arbitrary const rooms = this.pickRooms(roomIds); @@ -224,7 +268,7 @@ export class StopGapWidgetDriver extends WidgetDriver { stateKey: string | undefined, limitPerRoom: number, roomIds: (string | Symbols.AnyRoom)[] = null, - ): Promise { + ): Promise { limitPerRoom = limitPerRoom > 0 ? Math.min(limitPerRoom, Number.MAX_SAFE_INTEGER) : Number.MAX_SAFE_INTEGER; // relatively arbitrary const rooms = this.pickRooms(roomIds); diff --git a/src/widgets/CapabilityText.tsx b/src/widgets/CapabilityText.tsx index cd442f213b..e4790eaad2 100644 --- a/src/widgets/CapabilityText.tsx +++ b/src/widgets/CapabilityText.tsx @@ -17,6 +17,7 @@ limitations under the License. import { Capability, EventDirection, + EventKind, getTimelineRoomIDFromCapability, isTimelineCapability, isTimelineCapabilityFor, @@ -134,7 +135,7 @@ export class CapabilityText { }; private static bylineFor(eventCap: WidgetEventCapability): TranslatedString { - if (eventCap.isState) { + if (eventCap.kind === EventKind.State) { return !eventCap.keyStr ? _t("with an empty state key") : _t("with state key %(stateKey)s", { stateKey: eventCap.keyStr }); @@ -143,6 +144,8 @@ export class CapabilityText { } public static for(capability: Capability, kind: WidgetKind): TranslatedCapabilityText { + // TODO: Support MSC3819 (to-device capabilities) + // First see if we have a super simple line of text to provide back if (CapabilityText.simpleCaps[capability]) { const textForKind = CapabilityText.simpleCaps[capability]; @@ -184,13 +187,13 @@ export class CapabilityText { // Special case room messages so they show up a bit cleaner to the user. Result is // effectively "Send images" instead of "Send messages... of type images" if we were // to handle the msgtype nuances in this function. - if (!eventCap.isState && eventCap.eventType === EventType.RoomMessage) { + if (eventCap.kind === EventKind.Event && eventCap.eventType === EventType.RoomMessage) { return CapabilityText.forRoomMessageCap(eventCap, kind); } // See if we have a static line of text to provide for the given event type and // direction. The hope is that we do for common event types for friendlier copy. - const evSendRecv = eventCap.isState + const evSendRecv = eventCap.kind === EventKind.State ? CapabilityText.stateSendRecvCaps : CapabilityText.nonStateSendRecvCaps; if (evSendRecv[eventCap.eventType]) { diff --git a/test/stores/widgets/StopGapWidget-test.ts b/test/stores/widgets/StopGapWidget-test.ts new file mode 100644 index 0000000000..40292e451b --- /dev/null +++ b/test/stores/widgets/StopGapWidget-test.ts @@ -0,0 +1,70 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { mocked, MockedObject } from "jest-mock"; +import { MatrixClient, ClientEvent } from "matrix-js-sdk/src/client"; +import { ClientWidgetApi } from "matrix-widget-api"; + +import { stubClient, mkRoom, mkEvent } from "../../test-utils"; +import { MatrixClientPeg } from "../../../src/MatrixClientPeg"; +import { StopGapWidget } from "../../../src/stores/widgets/StopGapWidget"; + +jest.mock("matrix-widget-api/lib/ClientWidgetApi"); + +describe("StopGapWidget", () => { + let client: MockedObject; + let widget: StopGapWidget; + let messaging: MockedObject; + + beforeEach(() => { + stubClient(); + client = mocked(MatrixClientPeg.get()); + + widget = new StopGapWidget({ + app: { + id: "test", + creatorUserId: "@alice:example.org", + type: "example", + url: "https://example.org", + }, + room: mkRoom(client, "!1:example.org"), + userId: "@alice:example.org", + creatorUserId: "@alice:example.org", + waitForIframeLoad: true, + userWidget: false, + }); + // Start messaging without an iframe, since ClientWidgetApi is mocked + widget.startMessaging(null as unknown as HTMLIFrameElement); + messaging = mocked(mocked(ClientWidgetApi).mock.instances[0]); + }); + + afterEach(() => { + widget.stopMessaging(); + }); + + it("feeds incoming to-device messages to the widget", async () => { + const event = mkEvent({ + event: true, + type: "org.example.foo", + user: "@alice:example.org", + content: { hello: "world" }, + }); + + client.emit(ClientEvent.ToDeviceEvent, event); + await Promise.resolve(); // flush promises + expect(messaging.feedToDevice).toHaveBeenCalledWith(event.getEffectiveEvent(), false); + }); +}); diff --git a/test/stores/widgets/StopGapWidgetDriver-test.ts b/test/stores/widgets/StopGapWidgetDriver-test.ts new file mode 100644 index 0000000000..7dab35052b --- /dev/null +++ b/test/stores/widgets/StopGapWidgetDriver-test.ts @@ -0,0 +1,79 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { mocked, MockedObject } from "jest-mock"; +import { Widget, WidgetKind, WidgetDriver } from "matrix-widget-api"; +import { MatrixClient } from "matrix-js-sdk/src/client"; +import { DeviceInfo } from "matrix-js-sdk/src/crypto/deviceinfo"; + +import { MatrixClientPeg } from "../../../src/MatrixClientPeg"; +import { StopGapWidgetDriver } from "../../../src/stores/widgets/StopGapWidgetDriver"; +import { stubClient } from "../../test-utils"; + +describe("StopGapWidgetDriver", () => { + let client: MockedObject; + let driver: WidgetDriver; + + beforeEach(() => { + stubClient(); + client = mocked(MatrixClientPeg.get()); + + driver = new StopGapWidgetDriver( + [], + new Widget({ + id: "test", + creatorUserId: "@alice:example.org", + type: "example", + url: "https://example.org", + }), + WidgetKind.Room, + ); + }); + + describe("sendToDevice", () => { + const contentMap = { + "@alice:example.org": { + "*": { + hello: "alice", + }, + }, + "@bob:example.org": { + "bobDesktop": { + hello: "bob", + }, + }, + }; + + it("sends unencrypted messages", async () => { + await driver.sendToDevice("org.example.foo", false, contentMap); + expect(client.queueToDevice.mock.calls).toMatchSnapshot(); + }); + + it("sends encrypted messages", async () => { + const aliceWeb = new DeviceInfo("aliceWeb"); + const aliceMobile = new DeviceInfo("aliceMobile"); + const bobDesktop = new DeviceInfo("bobDesktop"); + + mocked(client.crypto.deviceList).downloadKeys.mockResolvedValue({ + "@alice:example.org": { aliceWeb, aliceMobile }, + "@bob:example.org": { bobDesktop }, + }); + + await driver.sendToDevice("org.example.foo", true, contentMap); + expect(client.encryptAndSendToDevices.mock.calls).toMatchSnapshot(); + }); + }); +}); diff --git a/test/stores/widgets/__snapshots__/StopGapWidgetDriver-test.ts.snap b/test/stores/widgets/__snapshots__/StopGapWidgetDriver-test.ts.snap new file mode 100644 index 0000000000..5f19dbb793 --- /dev/null +++ b/test/stores/widgets/__snapshots__/StopGapWidgetDriver-test.ts.snap @@ -0,0 +1,82 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`StopGapWidgetDriver sendToDevice sends encrypted messages 1`] = ` +Array [ + Array [ + Array [ + Object { + "deviceInfo": DeviceInfo { + "algorithms": undefined, + "deviceId": "aliceWeb", + "keys": Object {}, + "known": false, + "signatures": Object {}, + "unsigned": Object {}, + "verified": 0, + }, + "userId": "@alice:example.org", + }, + Object { + "deviceInfo": DeviceInfo { + "algorithms": undefined, + "deviceId": "aliceMobile", + "keys": Object {}, + "known": false, + "signatures": Object {}, + "unsigned": Object {}, + "verified": 0, + }, + "userId": "@alice:example.org", + }, + ], + Object { + "hello": "alice", + }, + ], + Array [ + Array [ + Object { + "deviceInfo": DeviceInfo { + "algorithms": undefined, + "deviceId": "bobDesktop", + "keys": Object {}, + "known": false, + "signatures": Object {}, + "unsigned": Object {}, + "verified": 0, + }, + "userId": "@bob:example.org", + }, + ], + Object { + "hello": "bob", + }, + ], +] +`; + +exports[`StopGapWidgetDriver sendToDevice sends unencrypted messages 1`] = ` +Array [ + Array [ + Object { + "batch": Array [ + Object { + "deviceId": "*", + "payload": Object { + "hello": "alice", + }, + "userId": "@alice:example.org", + }, + Object { + "deviceId": "bobDesktop", + "payload": Object { + "hello": "bob", + }, + "userId": "@bob:example.org", + }, + ], + "eventType": "org.example.foo", + }, + ], +] +`; diff --git a/test/test-utils/test-utils.ts b/test/test-utils/test-utils.ts index 39f8e8a7ec..391683d5d1 100644 --- a/test/test-utils/test-utils.ts +++ b/test/test-utils/test-utils.ts @@ -91,6 +91,12 @@ export function createTestClient(): MatrixClient { removeRoom: jest.fn(), }, + crypto: { + deviceList: { + downloadKeys: jest.fn(), + }, + }, + getPushActionsForEvent: jest.fn(), getRoom: jest.fn().mockImplementation(mkStubRoom), getRooms: jest.fn().mockReturnValue([]), @@ -163,6 +169,9 @@ export function createTestClient(): MatrixClient { downloadKeys: jest.fn(), fetchRoomEvent: jest.fn(), makeTxnId: jest.fn().mockImplementation(() => `t${txnId++}`), + sendToDevice: jest.fn().mockResolvedValue(undefined), + queueToDevice: jest.fn().mockResolvedValue(undefined), + encryptAndSendToDevices: jest.fn().mockResolvedValue(undefined), } as unknown as MatrixClient; } @@ -176,7 +185,7 @@ type MakeEventPassThruProps = { type MakeEventProps = MakeEventPassThruProps & { type: string; content: IContent; - room: Room["roomId"]; + room?: Room["roomId"]; // to-device messages are roomless // eslint-disable-next-line camelcase prev_content?: IContent; unsigned?: IUnsigned; diff --git a/yarn.lock b/yarn.lock index f2f623b7ed..56d63e5a17 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6813,10 +6813,10 @@ matrix-web-i18n@^1.3.0: "@babel/traverse" "^7.18.5" walk "^2.3.15" -matrix-widget-api@^0.1.0-beta.18: - version "0.1.0-beta.18" - resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-0.1.0-beta.18.tgz#4efd30edec3eeb4211285985464c062fcab59795" - integrity sha512-kCpcs6rrB94Mmr2/1gBJ+6auWyZ5UvOMOn5K2VFafz2/NDMzZg9OVWj9KFYnNAuwwBE5/tCztYEj6OQ+hgbwOQ== +matrix-widget-api@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.0.0.tgz#0cde6839cca66ad817ab12aca3490ccc8bac97d1" + integrity sha512-cy8p/8EteRPTFIAw7Q9EgPUJc2jD19ZahMR8bMKf2NkILDcjuPMC0UWnsJyB3fSnlGw+VbGepttRpULM31zX8Q== dependencies: "@types/events" "^3.0.0" events "^3.2.0"