OpenCloud
/
element-web
mirror of https://github.com/vector-im/element-web
1
0
Fork 0
Code Issues Releases Wiki Activity

Implement MSC3819: Allowing widgets to send/receive to-device messages (#8885) 

* Implement MSC3819: Allowing widgets to send/receive to-device messages

* Don't change the room events and state events drivers

* Update to latest matrix-widget-api changes

* Support sending encrypted to-device messages

* Use queueToDevice for better reliability

* Update types for latest WidgetDriver changes

* Upgrade matrix-widget-api

* Add tests

* Test StopGapWidget

* Fix a potential memory leak
pull/28217/head
Robin 2022-08-10 08:57:56 -04:00 committed by GitHub
parent 3d0982e9a6
commit 103b60dfb5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 322 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@ -94,7 +94,7 @@
"matrix-encrypt-attachment": "^1.0.3",
"matrix-events-sdk": "^0.0.1-beta.7",
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#develop",
"matrix-widget-api": "^0.1.0-beta.18",
"matrix-widget-api": "^1.0.0",
"minimist": "^1.2.5",
"opus-recorder": "^8.0.3",
"pako": "^2.0.3",

37
src/stores/widgets/StopGapWidget.ts
View File

@ -33,6 +33,7 @@ import {
WidgetKind,
} from "matrix-widget-api";
import { EventEmitter } from "events";
import { MatrixClient } from "matrix-js-sdk/src/client";
import { MatrixEvent, MatrixEventEvent } from "matrix-js-sdk/src/models/event";
import { logger } from "matrix-js-sdk/src/logger";
import { ClientEvent } from "matrix-js-sdk/src/client";
@ -148,6 +149,7 @@ export class ElementWidget extends Widget {
}
export class StopGapWidget extends EventEmitter {
private client: MatrixClient;
private messaging: ClientWidgetApi;
private mockWidget: ElementWidget;
private scalarToken: string;
@ -157,12 +159,13 @@ export class StopGapWidget extends EventEmitter {
constructor(private appTileProps: IAppTileProps) {
super();
let app = appTileProps.app;
this.client = MatrixClientPeg.get();
let app = appTileProps.app;
// Backwards compatibility: not all old widgets have a creatorUserId
if (!app.creatorUserId) {
app = objectShallowClone(app); // clone to prevent accidental mutation
app.creatorUserId = MatrixClientPeg.get().getUserId();
app.creatorUserId = this.client.getUserId();
}
this.mockWidget = new ElementWidget(app);
@ -203,7 +206,7 @@ export class StopGapWidget extends EventEmitter {
const fromCustomisation = WidgetVariableCustomisations?.provideVariables?.() ?? {};
const defaults: ITemplateParams = {
widgetRoomId: this.roomId,
currentUserId: MatrixClientPeg.get().getUserId(),
currentUserId: this.client.getUserId(),
userDisplayName: OwnProfileStore.instance.displayName,
userHttpAvatarUrl: OwnProfileStore.instance.getHttpAvatarUrl(),
clientId: ELEMENT_CLIENT_ID,
@ -260,8 +263,10 @@ export class StopGapWidget extends EventEmitter {
*/
public startMessaging(iframe: HTMLIFrameElement): any {
if (this.started) return;
const allowedCapabilities = this.appTileProps.whitelistCapabilities || [];
const driver = new StopGapWidgetDriver(allowedCapabilities, this.mockWidget, this.kind, this.roomId);
this.messaging = new ClientWidgetApi(this.mockWidget, iframe, driver);
this.messaging.on("preparing", () => this.emit("preparing"));
this.messaging.on("ready", () => this.emit("ready"));
@ -302,7 +307,7 @@ export class StopGapWidget extends EventEmitter {
// Populate the map of "read up to" events for this widget with the current event in every room.
// This is a bit inefficient, but should be okay. We do this for all rooms in case the widget
// requests timeline capabilities in other rooms down the road. It's just easier to manage here.
for (const room of MatrixClientPeg.get().getRooms()) {
for (const room of this.client.getRooms()) {
// Timelines are most recent last
const events = room.getLiveTimeline()?.getEvents() || [];
const roomEvent = events[events.length - 1];
@ -311,8 +316,9 @@ export class StopGapWidget extends EventEmitter {
}
// Attach listeners for feeding events - the underlying widget classes handle permissions for us
MatrixClientPeg.get().on(ClientEvent.Event, this.onEvent);
MatrixClientPeg.get().on(MatrixEventEvent.Decrypted, this.onEventDecrypted);
this.client.on(ClientEvent.Event, this.onEvent);
this.client.on(MatrixEventEvent.Decrypted, this.onEventDecrypted);
this.client.on(ClientEvent.ToDeviceEvent, this.onToDeviceEvent);
this.messaging.on(`action:${WidgetApiFromWidgetAction.UpdateAlwaysOnScreen}`,
(ev: CustomEvent<IStickyActionRequest>) => {
@ -363,7 +369,7 @@ export class StopGapWidget extends EventEmitter {
// noinspection JSIgnoredPromiseFromCall
IntegrationManagers.sharedInstance().getPrimaryManager().open(
MatrixClientPeg.get().getRoom(RoomViewStore.instance.getRoomId()),
this.client.getRoom(RoomViewStore.instance.getRoomId()),
`type_${integType}`,
integId,
);
@ -428,14 +434,13 @@ export class StopGapWidget extends EventEmitter {
WidgetMessagingStore.instance.stopMessaging(this.mockWidget, this.roomId);
this.messaging = null;
if (MatrixClientPeg.get()) {
MatrixClientPeg.get().off(ClientEvent.Event, this.onEvent);
MatrixClientPeg.get().off(MatrixEventEvent.Decrypted, this.onEventDecrypted);
}
this.client.off(ClientEvent.Event, this.onEvent);
this.client.off(MatrixEventEvent.Decrypted, this.onEventDecrypted);
this.client.off(ClientEvent.ToDeviceEvent, this.onToDeviceEvent);
}
private onEvent = (ev: MatrixEvent) => {
MatrixClientPeg.get().decryptEventIfNeeded(ev);
this.client.decryptEventIfNeeded(ev);
if (ev.isBeingDecrypted() || ev.isDecryptionFailure()) return;
this.feedEvent(ev);
};
@ -445,6 +450,12 @@ export class StopGapWidget extends EventEmitter {
this.feedEvent(ev);
};
private onToDeviceEvent = async (ev: MatrixEvent) => {
await this.client.decryptEventIfNeeded(ev);
if (ev.isDecryptionFailure()) return;
await this.messaging.feedToDevice(ev.getEffectiveEvent(), ev.isEncrypted());
};
private feedEvent(ev: MatrixEvent) {
if (!this.messaging) return;
@ -465,7 +476,7 @@ export class StopGapWidget extends EventEmitter {
// Timelines are most recent last, so reverse the order and limit ourselves to 100 events
// to avoid overusing the CPU.
const timeline = MatrixClientPeg.get().getRoom(ev.getRoomId()).getLiveTimeline();
const timeline = this.client.getRoom(ev.getRoomId()).getLiveTimeline();
const events = arrayFastClone(timeline.getEvents()).reverse().slice(0, 100);
for (const timelineEvent of events) {

48
src/stores/widgets/StopGapWidgetDriver.ts
View File

@ -20,6 +20,7 @@ import {
IOpenIDCredentials,
IOpenIDUpdate,
ISendEventDetails,
IRoomEvent,
MatrixCapabilities,
OpenIDRequestState,
SimpleObservable,
@ -182,6 +183,49 @@ export class StopGapWidgetDriver extends WidgetDriver {
return { roomId, eventId: r.event_id };
}
public async sendToDevice(
eventType: string,
encrypted: boolean,
contentMap: { [userId: string]: { [deviceId: string]: object } },
): Promise<void> {
const client = MatrixClientPeg.get();
if (encrypted) {
const deviceInfoMap = await client.crypto.deviceList.downloadKeys(Object.keys(contentMap), false);
await Promise.all(
Object.entries(contentMap).flatMap(([userId, userContentMap]) =>
Object.entries(userContentMap).map(async ([deviceId, content]) => {
if (deviceId === "*") {
// Send the message to all devices we have keys for
await client.encryptAndSendToDevices(
Object.values(deviceInfoMap[userId]).map(deviceInfo => ({
userId, deviceInfo,
})),
content,
);
} else {
// Send the message to a specific device
await client.encryptAndSendToDevices(
[{ userId, deviceInfo: deviceInfoMap[userId][deviceId] }],
content,
);
}
}),
),
);
} else {
await client.queueToDevice({
eventType,
batch: Object.entries(contentMap).flatMap(([userId, userContentMap]) =>
Object.entries(userContentMap).map(([deviceId, content]) =>
({ userId, deviceId, payload: content }),
),
),
});
}
}
private pickRooms(roomIds: (string | Symbols.AnyRoom)[] = null): Room[] {
const client = MatrixClientPeg.get();
if (!client) throw new Error("Not attached to a client");
@ -197,7 +241,7 @@ export class StopGapWidgetDriver extends WidgetDriver {
msgtype: string | undefined,
limitPerRoom: number,
roomIds: (string | Symbols.AnyRoom)[] = null,
): Promise<object[]> {
): Promise<IRoomEvent[]> {
limitPerRoom = limitPerRoom > 0 ? Math.min(limitPerRoom, Number.MAX_SAFE_INTEGER) : Number.MAX_SAFE_INTEGER; // relatively arbitrary
const rooms = this.pickRooms(roomIds);
@ -224,7 +268,7 @@ export class StopGapWidgetDriver extends WidgetDriver {
stateKey: string | undefined,
limitPerRoom: number,
roomIds: (string | Symbols.AnyRoom)[] = null,
): Promise<object[]> {
): Promise<IRoomEvent[]> {
limitPerRoom = limitPerRoom > 0 ? Math.min(limitPerRoom, Number.MAX_SAFE_INTEGER) : Number.MAX_SAFE_INTEGER; // relatively arbitrary
const rooms = this.pickRooms(roomIds);

9
src/widgets/CapabilityText.tsx
View File

@ -17,6 +17,7 @@ limitations under the License.
import {
Capability,
EventDirection,
EventKind,
getTimelineRoomIDFromCapability,
isTimelineCapability,
isTimelineCapabilityFor,
@ -134,7 +135,7 @@ export class CapabilityText {
};
private static bylineFor(eventCap: WidgetEventCapability): TranslatedString {
if (eventCap.isState) {
if (eventCap.kind === EventKind.State) {
return !eventCap.keyStr
? _t("with an empty state key")
: _t("with state key %(stateKey)s", { stateKey: eventCap.keyStr });
@ -143,6 +144,8 @@ export class CapabilityText {
}
public static for(capability: Capability, kind: WidgetKind): TranslatedCapabilityText {
// TODO: Support MSC3819 (to-device capabilities)
// First see if we have a super simple line of text to provide back
if (CapabilityText.simpleCaps[capability]) {
const textForKind = CapabilityText.simpleCaps[capability];
@ -184,13 +187,13 @@ export class CapabilityText {
// Special case room messages so they show up a bit cleaner to the user. Result is
// effectively "Send images" instead of "Send messages... of type images" if we were
// to handle the msgtype nuances in this function.
if (!eventCap.isState && eventCap.eventType === EventType.RoomMessage) {
if (eventCap.kind === EventKind.Event && eventCap.eventType === EventType.RoomMessage) {
return CapabilityText.forRoomMessageCap(eventCap, kind);
}
// See if we have a static line of text to provide for the given event type and
// direction. The hope is that we do for common event types for friendlier copy.
const evSendRecv = eventCap.isState
const evSendRecv = eventCap.kind === EventKind.State
? CapabilityText.stateSendRecvCaps
: CapabilityText.nonStateSendRecvCaps;
if (evSendRecv[eventCap.eventType]) {

70
test/stores/widgets/StopGapWidget-test.ts Normal file
View File

@ -0,0 +1,70 @@
/*
Copyright 2022 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { mocked, MockedObject } from "jest-mock";
import { MatrixClient, ClientEvent } from "matrix-js-sdk/src/client";
import { ClientWidgetApi } from "matrix-widget-api";
import { stubClient, mkRoom, mkEvent } from "../../test-utils";
import { MatrixClientPeg } from "../../../src/MatrixClientPeg";
import { StopGapWidget } from "../../../src/stores/widgets/StopGapWidget";
jest.mock("matrix-widget-api/lib/ClientWidgetApi");
describe("StopGapWidget", () => {
let client: MockedObject<MatrixClient>;
let widget: StopGapWidget;
let messaging: MockedObject<ClientWidgetApi>;
beforeEach(() => {
stubClient();
client = mocked(MatrixClientPeg.get());
widget = new StopGapWidget({
app: {
id: "test",
creatorUserId: "@alice:example.org",
type: "example",
url: "https://example.org",
},
room: mkRoom(client, "!1:example.org"),
userId: "@alice:example.org",
creatorUserId: "@alice:example.org",
waitForIframeLoad: true,
userWidget: false,
});
// Start messaging without an iframe, since ClientWidgetApi is mocked
widget.startMessaging(null as unknown as HTMLIFrameElement);
messaging = mocked(mocked(ClientWidgetApi).mock.instances[0]);
});
afterEach(() => {
widget.stopMessaging();
});
it("feeds incoming to-device messages to the widget", async () => {
const event = mkEvent({
event: true,
type: "org.example.foo",
user: "@alice:example.org",
content: { hello: "world" },
});
client.emit(ClientEvent.ToDeviceEvent, event);
await Promise.resolve(); // flush promises
expect(messaging.feedToDevice).toHaveBeenCalledWith(event.getEffectiveEvent(), false);
});
});

79
test/stores/widgets/StopGapWidgetDriver-test.ts Normal file
View File

@ -0,0 +1,79 @@
/*
Copyright 2022 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { mocked, MockedObject } from "jest-mock";
import { Widget, WidgetKind, WidgetDriver } from "matrix-widget-api";
import { MatrixClient } from "matrix-js-sdk/src/client";
import { DeviceInfo } from "matrix-js-sdk/src/crypto/deviceinfo";
import { MatrixClientPeg } from "../../../src/MatrixClientPeg";
import { StopGapWidgetDriver } from "../../../src/stores/widgets/StopGapWidgetDriver";
import { stubClient } from "../../test-utils";
describe("StopGapWidgetDriver", () => {
let client: MockedObject<MatrixClient>;
let driver: WidgetDriver;
beforeEach(() => {
stubClient();
client = mocked(MatrixClientPeg.get());
driver = new StopGapWidgetDriver(
[],
new Widget({
id: "test",
creatorUserId: "@alice:example.org",
type: "example",
url: "https://example.org",
}),
WidgetKind.Room,
);
});
describe("sendToDevice", () => {
const contentMap = {
"@alice:example.org": {
"*": {
hello: "alice",
},
},
"@bob:example.org": {
"bobDesktop": {
hello: "bob",
},
},
};
it("sends unencrypted messages", async () => {
await driver.sendToDevice("org.example.foo", false, contentMap);
expect(client.queueToDevice.mock.calls).toMatchSnapshot();
});
it("sends encrypted messages", async () => {
const aliceWeb = new DeviceInfo("aliceWeb");
const aliceMobile = new DeviceInfo("aliceMobile");
const bobDesktop = new DeviceInfo("bobDesktop");
mocked(client.crypto.deviceList).downloadKeys.mockResolvedValue({
"@alice:example.org": { aliceWeb, aliceMobile },
"@bob:example.org": { bobDesktop },
});
await driver.sendToDevice("org.example.foo", true, contentMap);
expect(client.encryptAndSendToDevices.mock.calls).toMatchSnapshot();
});
});
});

82
test/stores/widgets/__snapshots__/StopGapWidgetDriver-test.ts.snap Normal file
View File

@ -0,0 +1,82 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`StopGapWidgetDriver sendToDevice sends encrypted messages 1`] = `
Array [
Array [
Array [
Object {
"deviceInfo": DeviceInfo {
"algorithms": undefined,
"deviceId": "aliceWeb",
"keys": Object {},
"known": false,
"signatures": Object {},
"unsigned": Object {},
"verified": 0,
},
"userId": "@alice:example.org",
},
Object {
"deviceInfo": DeviceInfo {
"algorithms": undefined,
"deviceId": "aliceMobile",
"keys": Object {},
"known": false,
"signatures": Object {},
"unsigned": Object {},
"verified": 0,
},
"userId": "@alice:example.org",
},
],
Object {
"hello": "alice",
},
],
Array [
Array [
Object {
"deviceInfo": DeviceInfo {
"algorithms": undefined,
"deviceId": "bobDesktop",
"keys": Object {},
"known": false,
"signatures": Object {},
"unsigned": Object {},
"verified": 0,
},
"userId": "@bob:example.org",
},
],
Object {
"hello": "bob",
},
],
]
`;
exports[`StopGapWidgetDriver sendToDevice sends unencrypted messages 1`] = `
Array [
Array [
Object {
"batch": Array [
Object {
"deviceId": "*",
"payload": Object {
"hello": "alice",
},
"userId": "@alice:example.org",
},
Object {
"deviceId": "bobDesktop",
"payload": Object {
"hello": "bob",
},
"userId": "@bob:example.org",
},
],
"eventType": "org.example.foo",
},
],
]
`;

11
test/test-utils/test-utils.ts
View File

@ -91,6 +91,12 @@ export function createTestClient(): MatrixClient {
removeRoom: jest.fn(),
},
crypto: {
deviceList: {
downloadKeys: jest.fn(),
},
},
getPushActionsForEvent: jest.fn(),
getRoom: jest.fn().mockImplementation(mkStubRoom),
getRooms: jest.fn().mockReturnValue([]),
@ -163,6 +169,9 @@ export function createTestClient(): MatrixClient {
downloadKeys: jest.fn(),
fetchRoomEvent: jest.fn(),
makeTxnId: jest.fn().mockImplementation(() => `t${txnId++}`),
sendToDevice: jest.fn().mockResolvedValue(undefined),
queueToDevice: jest.fn().mockResolvedValue(undefined),
encryptAndSendToDevices: jest.fn().mockResolvedValue(undefined),
} as unknown as MatrixClient;
}
@ -176,7 +185,7 @@ type MakeEventPassThruProps = {
type MakeEventProps = MakeEventPassThruProps & {
type: string;
content: IContent;
room: Room["roomId"];
room?: Room["roomId"]; // to-device messages are roomless
// eslint-disable-next-line camelcase
prev_content?: IContent;
unsigned?: IUnsigned;

8
yarn.lock
View File

@ -6813,10 +6813,10 @@ matrix-web-i18n@^1.3.0:
"@babel/traverse" "^7.18.5"
walk "^2.3.15"
matrix-widget-api@^0.1.0-beta.18:
version "0.1.0-beta.18"
resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-0.1.0-beta.18.tgz#4efd30edec3eeb4211285985464c062fcab59795"
integrity sha512-kCpcs6rrB94Mmr2/1gBJ+6auWyZ5UvOMOn5K2VFafz2/NDMzZg9OVWj9KFYnNAuwwBE5/tCztYEj6OQ+hgbwOQ==
matrix-widget-api@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.0.0.tgz#0cde6839cca66ad817ab12aca3490ccc8bac97d1"
integrity sha512-cy8p/8EteRPTFIAw7Q9EgPUJc2jD19ZahMR8bMKf2NkILDcjuPMC0UWnsJyB3fSnlGw+VbGepttRpULM31zX8Q==
dependencies:
"@types/events" "^3.0.0"
events "^3.2.0"