From bac6e12946200f9599480a05213f5b1587da29d1 Mon Sep 17 00:00:00 2001 From: Michael Weimann Date: Wed, 12 Oct 2022 00:31:28 +0200 Subject: [PATCH] Implement Voice Broadcast recording (#9307) * Implement VoiceBroadcastRecording * Implement PR feedback * Add voice broadcast recording stores * Refactor startNewVoiceBroadcastRecording * Refactor VoiceBroadcastRecordingsStore to VoiceBroadcastRecording * Rename VoiceBroadcastRecording to VoiceBroadcastRecorder * Return remaining chunk on stop * Extract createVoiceMessageContent * Implement recording * Replace dev value with config * Fix clientInformation-test * Refactor VoiceBroadcastRecording * Fix VoiceBroadcastRecording types * Re-order getter * Mark voice_broadcast config as optional * Merge voice-broadcast modules * Remove underscore props * Add Optional types * Add return types everywhere * Remove test casts * Add magic comments * Trigger CI * Switch VoiceBroadcastRecorder to TypedEventEmitter * Trigger CI * Add voice broadcast chunk event content Co-authored-by: Travis Ralston --- src/IConfigOptions.ts | 5 + src/SdkConfig.ts | 3 + src/audio/VoiceRecording.ts | 11 +- .../audio/VoiceBroadcastRecorder.ts | 141 ++++++++++ .../components/VoiceBroadcastBody.tsx | 2 +- src/voice-broadcast/components/index.ts | 19 -- src/voice-broadcast/index.ts | 12 +- .../models/VoiceBroadcastRecording.ts | 117 ++++++++- src/voice-broadcast/models/index.ts | 17 -- .../stores/VoiceBroadcastRecordingsStore.ts | 16 +- src/voice-broadcast/stores/index.ts | 17 -- src/voice-broadcast/utils/index.ts | 18 -- .../utils/startNewVoiceBroadcastRecording.ts | 1 + test/SdkConfig-test.ts | 41 +++ .../audio/VoiceBroadcastRecorder-test.ts | 209 +++++++++++++++ .../components/VoiceBroadcastBody-test.tsx | 2 +- .../models/VoiceBroadcastRecording-test.ts | 242 +++++++++++++++++- .../VoiceBroadcastRecordingsStore-test.ts | 2 +- .../startNewVoiceBroadcastRecording-test.ts | 2 + 19 files changed, 773 insertions(+), 104 deletions(-) create mode 100644 src/voice-broadcast/audio/VoiceBroadcastRecorder.ts delete mode 100644 src/voice-broadcast/components/index.ts delete mode 100644 src/voice-broadcast/models/index.ts delete mode 100644 src/voice-broadcast/stores/index.ts delete mode 100644 src/voice-broadcast/utils/index.ts create mode 100644 test/SdkConfig-test.ts create mode 100644 test/voice-broadcast/audio/VoiceBroadcastRecorder-test.ts diff --git a/src/IConfigOptions.ts b/src/IConfigOptions.ts index 91391fc2a9..68d133ecd0 100644 --- a/src/IConfigOptions.ts +++ b/src/IConfigOptions.ts @@ -181,6 +181,11 @@ export interface IConfigOptions { sync_timeline_limit?: number; dangerously_allow_unsafe_and_insecure_passwords?: boolean; // developer option + + voice_broadcast?: { + // length per voice chunk in seconds + chunk_length?: number; + }; } export interface ISsoRedirectOptions { diff --git a/src/SdkConfig.ts b/src/SdkConfig.ts index 235ada7382..0d3400f4bb 100644 --- a/src/SdkConfig.ts +++ b/src/SdkConfig.ts @@ -46,6 +46,9 @@ export const DEFAULTS: IConfigOptions = { logo: require("../res/img/element-desktop-logo.svg").default, url: "https://element.io/get-started", }, + voice_broadcast: { + chunk_length: 60 * 1000, // one minute + }, }; export default class SdkConfig { diff --git a/src/audio/VoiceRecording.ts b/src/audio/VoiceRecording.ts index e98e85aba5..0e18756fe5 100644 --- a/src/audio/VoiceRecording.ts +++ b/src/audio/VoiceRecording.ts @@ -203,9 +203,7 @@ export class VoiceRecording extends EventEmitter implements IDestroyable { // In testing, recorder time and worker time lag by about 400ms, which is roughly the // time needed to encode a sample/frame. // - // Ref for recorderSeconds: https://github.com/chris-rudmin/opus-recorder#instance-fields - const recorderSeconds = this.recorder.encodedSamplePosition / 48000; - const secondsLeft = TARGET_MAX_LENGTH - recorderSeconds; + const secondsLeft = TARGET_MAX_LENGTH - this.recorderSeconds; if (secondsLeft < 0) { // go over to make sure we definitely capture that last frame // noinspection JSIgnoredPromiseFromCall - we aren't concerned with it overlapping this.stop(); @@ -217,6 +215,13 @@ export class VoiceRecording extends EventEmitter implements IDestroyable { } }; + /** + * {@link https://github.com/chris-rudmin/opus-recorder#instance-fields ref for recorderSeconds} + */ + public get recorderSeconds() { + return this.recorder.encodedSamplePosition / 48000; + } + public async start(): Promise { if (this.recording) { throw new Error("Recording already in progress"); diff --git a/src/voice-broadcast/audio/VoiceBroadcastRecorder.ts b/src/voice-broadcast/audio/VoiceBroadcastRecorder.ts new file mode 100644 index 0000000000..7f084f3f4a --- /dev/null +++ b/src/voice-broadcast/audio/VoiceBroadcastRecorder.ts @@ -0,0 +1,141 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { Optional } from "matrix-events-sdk"; +import { TypedEventEmitter } from "matrix-js-sdk/src/models/typed-event-emitter"; + +import { VoiceRecording } from "../../audio/VoiceRecording"; +import SdkConfig, { DEFAULTS } from "../../SdkConfig"; +import { concat } from "../../utils/arrays"; +import { IDestroyable } from "../../utils/IDestroyable"; + +export enum VoiceBroadcastRecorderEvent { + ChunkRecorded = "chunk_recorded", +} + +interface EventMap { + [VoiceBroadcastRecorderEvent.ChunkRecorded]: (chunk: ChunkRecordedPayload) => void; +} + +export interface ChunkRecordedPayload { + buffer: Uint8Array; + length: number; +} + +/** + * This class provides the function to seamlessly record fixed length chunks. + * Subscribe with on(VoiceBroadcastRecordingEvents.ChunkRecorded, (payload: ChunkRecordedPayload) => {}) + * to retrieve chunks while recording. + */ +export class VoiceBroadcastRecorder + extends TypedEventEmitter + implements IDestroyable { + private headers = new Uint8Array(0); + private chunkBuffer = new Uint8Array(0); + private previousChunkEndTimePosition = 0; + private pagesFromRecorderCount = 0; + + public constructor( + private voiceRecording: VoiceRecording, + public readonly targetChunkLength: number, + ) { + super(); + this.voiceRecording.onDataAvailable = this.onDataAvailable; + } + + public async start(): Promise { + return this.voiceRecording.start(); + } + + /** + * Stops the recording and returns the remaining chunk (if any). + */ + public async stop(): Promise> { + await this.voiceRecording.stop(); + return this.extractChunk(); + } + + public get contentType(): string { + return this.voiceRecording.contentType; + } + + private get chunkLength(): number { + return this.voiceRecording.recorderSeconds - this.previousChunkEndTimePosition; + } + + private onDataAvailable = (data: ArrayBuffer): void => { + const dataArray = new Uint8Array(data); + this.pagesFromRecorderCount++; + + if (this.pagesFromRecorderCount <= 2) { + // first two pages contain the headers + this.headers = concat(this.headers, dataArray); + return; + } + + this.handleData(dataArray); + }; + + private handleData(data: Uint8Array): void { + this.chunkBuffer = concat(this.chunkBuffer, data); + this.emitChunkIfTargetLengthReached(); + } + + private emitChunkIfTargetLengthReached(): void { + if (this.chunkLength >= this.targetChunkLength) { + this.emitAndResetChunk(); + } + } + + /** + * Extracts the current chunk and resets the buffer. + */ + private extractChunk(): Optional { + if (this.chunkBuffer.length === 0) { + return null; + } + + const currentRecorderTime = this.voiceRecording.recorderSeconds; + const payload: ChunkRecordedPayload = { + buffer: concat(this.headers, this.chunkBuffer), + length: this.chunkLength, + }; + this.chunkBuffer = new Uint8Array(0); + this.previousChunkEndTimePosition = currentRecorderTime; + return payload; + } + + private emitAndResetChunk(): void { + if (this.chunkBuffer.length === 0) { + return; + } + + this.emit( + VoiceBroadcastRecorderEvent.ChunkRecorded, + this.extractChunk(), + ); + } + + public destroy(): void { + this.removeAllListeners(); + this.voiceRecording.destroy(); + } +} + +export const createVoiceBroadcastRecorder = (): VoiceBroadcastRecorder => { + const targetChunkLength = SdkConfig.get("voice_broadcast")?.chunk_length || DEFAULTS.voice_broadcast!.chunk_length; + return new VoiceBroadcastRecorder(new VoiceRecording(), targetChunkLength); +}; diff --git a/src/voice-broadcast/components/VoiceBroadcastBody.tsx b/src/voice-broadcast/components/VoiceBroadcastBody.tsx index 1a57b5c019..3591325585 100644 --- a/src/voice-broadcast/components/VoiceBroadcastBody.tsx +++ b/src/voice-broadcast/components/VoiceBroadcastBody.tsx @@ -31,7 +31,7 @@ export const VoiceBroadcastBody: React.FC = ({ mxEvent }) => { const client = MatrixClientPeg.get(); const room = client.getRoom(mxEvent.getRoomId()); const recording = VoiceBroadcastRecordingsStore.instance().getByInfoEvent(mxEvent, client); - const [recordingState, setRecordingState] = useState(recording.state); + const [recordingState, setRecordingState] = useState(recording.getState()); useTypedEventEmitter( recording, diff --git a/src/voice-broadcast/components/index.ts b/src/voice-broadcast/components/index.ts deleted file mode 100644 index e98500a5d7..0000000000 --- a/src/voice-broadcast/components/index.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2022 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -export * from "./atoms/LiveBadge"; -export * from "./molecules/VoiceBroadcastRecordingBody"; -export * from "./VoiceBroadcastBody"; diff --git a/src/voice-broadcast/index.ts b/src/voice-broadcast/index.ts index 2ceca2d3ab..2f69b84918 100644 --- a/src/voice-broadcast/index.ts +++ b/src/voice-broadcast/index.ts @@ -21,10 +21,14 @@ limitations under the License. import { RelationType } from "matrix-js-sdk/src/matrix"; -export * from "./components"; -export * from "./models"; -export * from "./utils"; -export * from "./stores"; +export * from "./audio/VoiceBroadcastRecorder"; +export * from "./components/VoiceBroadcastBody"; +export * from "./components/atoms/LiveBadge"; +export * from "./components/molecules/VoiceBroadcastRecordingBody"; +export * from "./models/VoiceBroadcastRecording"; +export * from "./stores/VoiceBroadcastRecordingsStore"; +export * from "./utils/shouldDisplayAsVoiceBroadcastTile"; +export * from "./utils/startNewVoiceBroadcastRecording"; export const VoiceBroadcastInfoEventType = "io.element.voice_broadcast_info"; diff --git a/src/voice-broadcast/models/VoiceBroadcastRecording.ts b/src/voice-broadcast/models/VoiceBroadcastRecording.ts index e949644dee..97351b2e1b 100644 --- a/src/voice-broadcast/models/VoiceBroadcastRecording.ts +++ b/src/voice-broadcast/models/VoiceBroadcastRecording.ts @@ -14,10 +14,22 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { MatrixClient, MatrixEvent, RelationType } from "matrix-js-sdk/src/matrix"; +import { logger } from "matrix-js-sdk/src/logger"; +import { IAbortablePromise, MatrixClient, MatrixEvent, RelationType } from "matrix-js-sdk/src/matrix"; import { TypedEventEmitter } from "matrix-js-sdk/src/models/typed-event-emitter"; -import { VoiceBroadcastInfoEventType, VoiceBroadcastInfoState } from ".."; +import { + ChunkRecordedPayload, + createVoiceBroadcastRecorder, + VoiceBroadcastInfoEventType, + VoiceBroadcastInfoState, + VoiceBroadcastRecorder, + VoiceBroadcastRecorderEvent, +} from ".."; +import { uploadFile } from "../../ContentMessages"; +import { IEncryptedFile } from "../../customisations/models/IMediaEventContent"; +import { createVoiceMessageContent } from "../../utils/createVoiceMessageContent"; +import { IDestroyable } from "../../utils/IDestroyable"; export enum VoiceBroadcastRecordingEvent { StateChanged = "liveness_changed", @@ -27,8 +39,12 @@ interface EventMap { [VoiceBroadcastRecordingEvent.StateChanged]: (state: VoiceBroadcastInfoState) => void; } -export class VoiceBroadcastRecording extends TypedEventEmitter { - private _state: VoiceBroadcastInfoState; +export class VoiceBroadcastRecording + extends TypedEventEmitter + implements IDestroyable { + private state: VoiceBroadcastInfoState; + private recorder: VoiceBroadcastRecorder; + private sequence = 1; public constructor( public readonly infoEvent: MatrixEvent, @@ -43,21 +59,89 @@ export class VoiceBroadcastRecording extends TypedEventEmitter { + this.state = !relatedEvents?.find((event: MatrixEvent) => { return event.getContent()?.state === VoiceBroadcastInfoState.Stopped; }) ? VoiceBroadcastInfoState.Started : VoiceBroadcastInfoState.Stopped; // TODO Michael W: add listening for updates } + public async start(): Promise { + return this.getRecorder().start(); + } + + public async stop(): Promise { + this.setState(VoiceBroadcastInfoState.Stopped); + await this.stopRecorder(); + await this.sendStoppedStateEvent(); + } + + public getState(): VoiceBroadcastInfoState { + return this.state; + } + + private getRecorder(): VoiceBroadcastRecorder { + if (!this.recorder) { + this.recorder = createVoiceBroadcastRecorder(); + this.recorder.on(VoiceBroadcastRecorderEvent.ChunkRecorded, this.onChunkRecorded); + } + + return this.recorder; + } + + public destroy(): void { + if (this.recorder) { + this.recorder.off(VoiceBroadcastRecorderEvent.ChunkRecorded, this.onChunkRecorded); + this.recorder.stop(); + } + + this.removeAllListeners(); + } + private setState(state: VoiceBroadcastInfoState): void { - this._state = state; + this.state = state; this.emit(VoiceBroadcastRecordingEvent.StateChanged, this.state); } - public async stop() { - this.setState(VoiceBroadcastInfoState.Stopped); - // TODO Michael W: add error handling + private onChunkRecorded = async (chunk: ChunkRecordedPayload): Promise => { + const { url, file } = await this.uploadFile(chunk); + await this.sendVoiceMessage(chunk, url, file); + }; + + private uploadFile(chunk: ChunkRecordedPayload): IAbortablePromise<{ url?: string, file?: IEncryptedFile }> { + return uploadFile( + this.client, + this.infoEvent.getRoomId(), + new Blob( + [chunk.buffer], + { + type: this.getRecorder().contentType, + }, + ), + ); + } + + private async sendVoiceMessage(chunk: ChunkRecordedPayload, url: string, file: IEncryptedFile): Promise { + const content = createVoiceMessageContent( + url, + this.getRecorder().contentType, + Math.round(chunk.length * 1000), + chunk.buffer.length, + file, + ); + content["m.relates_to"] = { + rel_type: RelationType.Reference, + event_id: this.infoEvent.getId(), + }; + content["io.element.voice_broadcast_chunk"] = { + sequence: this.sequence++, + }; + + await this.client.sendMessage(this.infoEvent.getRoomId(), content); + } + + private async sendStoppedStateEvent(): Promise { + // TODO Michael W: add error handling for state event await this.client.sendStateEvent( this.infoEvent.getRoomId(), VoiceBroadcastInfoEventType, @@ -72,7 +156,18 @@ export class VoiceBroadcastRecording extends TypedEventEmitter { + if (!this.recorder) { + return; + } + + try { + const lastChunk = await this.recorder.stop(); + if (lastChunk) { + await this.onChunkRecorded(lastChunk); + } + } catch (err) { + logger.warn("error stopping voice broadcast recorder", err); + } } } diff --git a/src/voice-broadcast/models/index.ts b/src/voice-broadcast/models/index.ts deleted file mode 100644 index 053c032156..0000000000 --- a/src/voice-broadcast/models/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* -Copyright 2022 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -export * from "./VoiceBroadcastRecording"; diff --git a/src/voice-broadcast/stores/VoiceBroadcastRecordingsStore.ts b/src/voice-broadcast/stores/VoiceBroadcastRecordingsStore.ts index a8fb681873..380fd1d318 100644 --- a/src/voice-broadcast/stores/VoiceBroadcastRecordingsStore.ts +++ b/src/voice-broadcast/stores/VoiceBroadcastRecordingsStore.ts @@ -31,7 +31,7 @@ interface EventMap { * This store provides access to the current and specific Voice Broadcast recordings. */ export class VoiceBroadcastRecordingsStore extends TypedEventEmitter { - private _current: VoiceBroadcastRecording | null; + private current: VoiceBroadcastRecording | null; private recordings = new Map(); public constructor() { @@ -39,15 +39,15 @@ export class VoiceBroadcastRecordingsStore extends TypedEventEmitter { + describe("with default values", () => { + it("should return the default config", () => { + expect(SdkConfig.get()).toEqual(DEFAULTS); + }); + }); + + describe("with custom values", () => { + beforeEach(() => { + SdkConfig.put({ + voice_broadcast: { + chunk_length: 1337, + }, + }); + }); + + it("should return the custom config", () => { + const customConfig = JSON.parse(JSON.stringify(DEFAULTS)); + customConfig.voice_broadcast.chunk_length = 1337; + expect(SdkConfig.get()).toEqual(customConfig); + }); + }); +}); diff --git a/test/voice-broadcast/audio/VoiceBroadcastRecorder-test.ts b/test/voice-broadcast/audio/VoiceBroadcastRecorder-test.ts new file mode 100644 index 0000000000..e60d7e2d96 --- /dev/null +++ b/test/voice-broadcast/audio/VoiceBroadcastRecorder-test.ts @@ -0,0 +1,209 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { mocked } from "jest-mock"; + +import { VoiceRecording } from "../../../src/audio/VoiceRecording"; +import SdkConfig from "../../../src/SdkConfig"; +import { concat } from "../../../src/utils/arrays"; +import { + ChunkRecordedPayload, + createVoiceBroadcastRecorder, + VoiceBroadcastRecorder, + VoiceBroadcastRecorderEvent, +} from "../../../src/voice-broadcast"; + +describe("VoiceBroadcastRecorder", () => { + describe("createVoiceBroadcastRecorder", () => { + beforeEach(() => { + jest.spyOn(SdkConfig, "get").mockImplementation((key: string) => { + if (key === "voice_broadcast") { + return { + chunk_length: 1337, + }; + } + }); + }); + + afterEach(() => { + mocked(SdkConfig.get).mockRestore(); + }); + + it("should return a VoiceBroadcastRecorder instance with targetChunkLength from config", () => { + const voiceBroadcastRecorder = createVoiceBroadcastRecorder(); + expect(voiceBroadcastRecorder).toBeInstanceOf(VoiceBroadcastRecorder); + expect(voiceBroadcastRecorder.targetChunkLength).toBe(1337); + }); + }); + + describe("instance", () => { + const chunkLength = 30; + const headers1 = new Uint8Array([1, 2]); + const headers2 = new Uint8Array([3, 4]); + const chunk1 = new Uint8Array([5, 6]); + const chunk2a = new Uint8Array([7, 8]); + const chunk2b = new Uint8Array([9, 10]); + const contentType = "test content type"; + + let voiceRecording: VoiceRecording; + let voiceBroadcastRecorder: VoiceBroadcastRecorder; + let onChunkRecorded: (chunk: ChunkRecordedPayload) => void; + + const itShouldNotEmitAChunkRecordedEvent = () => { + it("should not emit a ChunkRecorded event", () => { + expect(voiceRecording.emit).not.toHaveBeenCalledWith( + VoiceBroadcastRecorderEvent.ChunkRecorded, + expect.anything(), + ); + }); + }; + + beforeEach(() => { + voiceRecording = { + contentType, + start: jest.fn().mockResolvedValue(undefined), + stop: jest.fn().mockResolvedValue(undefined), + on: jest.fn(), + off: jest.fn(), + emit: jest.fn(), + destroy: jest.fn(), + recorderSeconds: 23, + } as unknown as VoiceRecording; + voiceBroadcastRecorder = new VoiceBroadcastRecorder(voiceRecording, chunkLength); + jest.spyOn(voiceBroadcastRecorder, "removeAllListeners"); + onChunkRecorded = jest.fn(); + voiceBroadcastRecorder.on(VoiceBroadcastRecorderEvent.ChunkRecorded, onChunkRecorded); + }); + + afterEach(() => { + voiceBroadcastRecorder.destroy(); + }); + + it("start should forward the call to VoiceRecording.start", async () => { + await voiceBroadcastRecorder.start(); + expect(voiceRecording.start).toHaveBeenCalled(); + }); + + describe("stop", () => { + beforeEach(async () => { + await voiceBroadcastRecorder.stop(); + }); + + it("should forward the call to VoiceRecording.stop", async () => { + expect(voiceRecording.stop).toHaveBeenCalled(); + }); + + itShouldNotEmitAChunkRecordedEvent(); + }); + + describe("when calling destroy", () => { + beforeEach(() => { + voiceBroadcastRecorder.destroy(); + }); + + it("should call VoiceRecording.destroy", () => { + expect(voiceRecording.destroy).toHaveBeenCalled(); + }); + + it("should remove all listeners", () => { + expect(voiceBroadcastRecorder.removeAllListeners).toHaveBeenCalled(); + }); + }); + + it("contentType should return the value from VoiceRecording", () => { + expect(voiceBroadcastRecorder.contentType).toBe(contentType); + }); + + describe("when the first page from recorder has been received", () => { + beforeEach(() => { + voiceRecording.onDataAvailable(headers1); + }); + + itShouldNotEmitAChunkRecordedEvent(); + }); + + describe("when a second page from recorder has been received", () => { + beforeEach(() => { + voiceRecording.onDataAvailable(headers1); + voiceRecording.onDataAvailable(headers2); + }); + + itShouldNotEmitAChunkRecordedEvent(); + }); + + describe("when a third page from recorder has been received", () => { + beforeEach(() => { + voiceRecording.onDataAvailable(headers1); + voiceRecording.onDataAvailable(headers2); + voiceRecording.onDataAvailable(chunk1); + }); + + itShouldNotEmitAChunkRecordedEvent(); + + describe("stop", () => { + let stopPayload: ChunkRecordedPayload; + + beforeEach(async () => { + stopPayload = await voiceBroadcastRecorder.stop(); + }); + + it("should return the remaining chunk", () => { + expect(stopPayload).toEqual({ + buffer: concat(headers1, headers2, chunk1), + length: 23, + }); + }); + }); + }); + + describe("when some chunks have been received", () => { + beforeEach(() => { + // simulate first chunk + voiceRecording.onDataAvailable(headers1); + voiceRecording.onDataAvailable(headers2); + // set recorder seconds to something greater than the test chunk length of 30 + // @ts-ignore + voiceRecording.recorderSeconds = 42; + voiceRecording.onDataAvailable(chunk1); + + // simulate a second chunk + voiceRecording.onDataAvailable(chunk2a); + // add another 30 seconds for the next chunk + // @ts-ignore + voiceRecording.recorderSeconds = 72; + voiceRecording.onDataAvailable(chunk2b); + }); + + it("should emit ChunkRecorded events", () => { + expect(onChunkRecorded).toHaveBeenNthCalledWith( + 1, + { + buffer: concat(headers1, headers2, chunk1), + length: 42, + }, + ); + + expect(onChunkRecorded).toHaveBeenNthCalledWith( + 2, + { + buffer: concat(headers1, headers2, chunk2a, chunk2b), + length: 72 - 42, // 72 (position at second chunk) - 42 (position of first chunk) + }, + ); + }); + }); + }); +}); diff --git a/test/voice-broadcast/components/VoiceBroadcastBody-test.tsx b/test/voice-broadcast/components/VoiceBroadcastBody-test.tsx index fed396a683..2c94ea6d0b 100644 --- a/test/voice-broadcast/components/VoiceBroadcastBody-test.tsx +++ b/test/voice-broadcast/components/VoiceBroadcastBody-test.tsx @@ -155,7 +155,7 @@ describe("VoiceBroadcastBody", () => { itShouldRenderANonLiveVoiceBroadcast(); it("should call stop on the recording", () => { - expect(recording.state).toBe(VoiceBroadcastInfoState.Stopped); + expect(recording.getState()).toBe(VoiceBroadcastInfoState.Stopped); expect(onRecordingStateChanged).toHaveBeenCalledWith(VoiceBroadcastInfoState.Stopped); }); }); diff --git a/test/voice-broadcast/models/VoiceBroadcastRecording-test.ts b/test/voice-broadcast/models/VoiceBroadcastRecording-test.ts index b0e9939164..357180c700 100644 --- a/test/voice-broadcast/models/VoiceBroadcastRecording-test.ts +++ b/test/voice-broadcast/models/VoiceBroadcastRecording-test.ts @@ -15,25 +15,57 @@ limitations under the License. */ import { mocked } from "jest-mock"; -import { EventTimelineSet, EventType, MatrixClient, MatrixEvent, RelationType, Room } from "matrix-js-sdk/src/matrix"; +import { + EventTimelineSet, + EventType, + MatrixClient, + MatrixEvent, + MsgType, + RelationType, + Room, +} from "matrix-js-sdk/src/matrix"; import { Relations } from "matrix-js-sdk/src/models/relations"; +import { uploadFile } from "../../../src/ContentMessages"; +import { IEncryptedFile } from "../../../src/customisations/models/IMediaEventContent"; +import { createVoiceMessageContent } from "../../../src/utils/createVoiceMessageContent"; import { + ChunkRecordedPayload, + createVoiceBroadcastRecorder, VoiceBroadcastInfoEventContent, VoiceBroadcastInfoEventType, VoiceBroadcastInfoState, + VoiceBroadcastRecorder, + VoiceBroadcastRecorderEvent, VoiceBroadcastRecording, VoiceBroadcastRecordingEvent, } from "../../../src/voice-broadcast"; import { mkEvent, mkStubRoom, stubClient } from "../../test-utils"; +jest.mock("../../../src/voice-broadcast/audio/VoiceBroadcastRecorder", () => ({ + ...jest.requireActual("../../../src/voice-broadcast/audio/VoiceBroadcastRecorder") as object, + createVoiceBroadcastRecorder: jest.fn(), +})); + +jest.mock("../../../src/ContentMessages", () => ({ + uploadFile: jest.fn(), +})); + +jest.mock("../../../src/utils/createVoiceMessageContent", () => ({ + createVoiceMessageContent: jest.fn(), +})); + describe("VoiceBroadcastRecording", () => { const roomId = "!room:example.com"; + const uploadedUrl = "mxc://example.com/vb"; + const uploadedFile = { file: true } as unknown as IEncryptedFile; let room: Room; let client: MatrixClient; let infoEvent: MatrixEvent; let voiceBroadcastRecording: VoiceBroadcastRecording; let onStateChanged: (state: VoiceBroadcastInfoState) => void; + let voiceBroadcastRecorder: VoiceBroadcastRecorder; + let onChunkRecorded: (chunk: ChunkRecordedPayload) => Promise; const mkVoiceBroadcastInfoEvent = (content: VoiceBroadcastInfoEventContent) => { return mkEvent({ @@ -48,6 +80,7 @@ describe("VoiceBroadcastRecording", () => { const setUpVoiceBroadcastRecording = () => { voiceBroadcastRecording = new VoiceBroadcastRecording(infoEvent, client); voiceBroadcastRecording.on(VoiceBroadcastRecordingEvent.StateChanged, onStateChanged); + jest.spyOn(voiceBroadcastRecording, "removeAllListeners"); }; beforeEach(() => { @@ -59,6 +92,65 @@ describe("VoiceBroadcastRecording", () => { } }); onStateChanged = jest.fn(); + voiceBroadcastRecorder = { + contentType: "audio/ogg", + on: jest.fn(), + off: jest.fn(), + start: jest.fn(), + stop: jest.fn(), + } as unknown as VoiceBroadcastRecorder; + mocked(createVoiceBroadcastRecorder).mockReturnValue(voiceBroadcastRecorder); + onChunkRecorded = jest.fn(); + + mocked(voiceBroadcastRecorder.on).mockImplementation( + (event: VoiceBroadcastRecorderEvent, listener: any): VoiceBroadcastRecorder => { + if (event === VoiceBroadcastRecorderEvent.ChunkRecorded) { + onChunkRecorded = listener; + } + + return voiceBroadcastRecorder; + }, + ); + + mocked(uploadFile).mockResolvedValue({ + url: uploadedUrl, + file: uploadedFile, + }); + + mocked(createVoiceMessageContent).mockImplementation(( + mxc: string, + mimetype: string, + duration: number, + size: number, + file?: IEncryptedFile, + waveform?: number[], + ) => { + return { + body: "Voice message", + msgtype: MsgType.Audio, + url: mxc, + file, + info: { + duration, + mimetype, + size, + }, + ["org.matrix.msc1767.text"]: "Voice message", + ["org.matrix.msc1767.file"]: { + url: mxc, + file, + name: "Voice message.ogg", + mimetype, + size, + }, + ["org.matrix.msc1767.audio"]: { + duration, + // https://github.com/matrix-org/matrix-doc/pull/3246 + waveform, + }, + ["org.matrix.msc3245.voice"]: {}, // No content, this is a rendering hint + }; + }); }); afterEach(() => { @@ -74,7 +166,7 @@ describe("VoiceBroadcastRecording", () => { }); it("should be in Started state", () => { - expect(voiceBroadcastRecording.state).toBe(VoiceBroadcastInfoState.Started); + expect(voiceBroadcastRecording.getState()).toBe(VoiceBroadcastInfoState.Started); }); describe("and calling stop()", () => { @@ -98,13 +190,155 @@ describe("VoiceBroadcastRecording", () => { }); it("should be in state stopped", () => { - expect(voiceBroadcastRecording.state).toBe(VoiceBroadcastInfoState.Stopped); + expect(voiceBroadcastRecording.getState()).toBe(VoiceBroadcastInfoState.Stopped); }); it("should emit a stopped state changed event", () => { expect(onStateChanged).toHaveBeenCalledWith(VoiceBroadcastInfoState.Stopped); }); }); + + describe("and calling start", () => { + beforeEach(async () => { + await voiceBroadcastRecording.start(); + }); + + it("should start the recorder", () => { + expect(voiceBroadcastRecorder.start).toHaveBeenCalled(); + }); + + describe("and a chunk has been recorded", () => { + beforeEach(async () => { + await onChunkRecorded({ + buffer: new Uint8Array([1, 2, 3]), + length: 23, + }); + }); + + it("should send a voice message", () => { + expect(uploadFile).toHaveBeenCalledWith( + client, + roomId, + new Blob([new Uint8Array([1, 2, 3])], { type: voiceBroadcastRecorder.contentType }), + ); + + expect(mocked(client.sendMessage)).toHaveBeenCalledWith( + roomId, + { + body: "Voice message", + file: { + file: true, + }, + info: { + duration: 23000, + mimetype: "audio/ogg", + size: 3, + }, + ["m.relates_to"]: { + event_id: infoEvent.getId(), + rel_type: "m.reference", + }, + msgtype: "m.audio", + ["org.matrix.msc1767.audio"]: { + duration: 23000, + waveform: undefined, + }, + ["org.matrix.msc1767.file"]: { + file: { + file: true, + }, + mimetype: "audio/ogg", + name: "Voice message.ogg", + size: 3, + url: "mxc://example.com/vb", + }, + ["org.matrix.msc1767.text"]: "Voice message", + ["org.matrix.msc3245.voice"]: {}, + url: "mxc://example.com/vb", + ["io.element.voice_broadcast_chunk"]: { + sequence: 1, + }, + }, + ); + }); + }); + + describe("and calling stop", () => { + beforeEach(async () => { + await onChunkRecorded({ + buffer: new Uint8Array([1, 2, 3]), + length: 23, + }); + mocked(voiceBroadcastRecorder.stop).mockResolvedValue({ + buffer: new Uint8Array([4, 5, 6]), + length: 42, + }); + await voiceBroadcastRecording.stop(); + }); + + it("should send the last chunk", () => { + expect(uploadFile).toHaveBeenCalledWith( + client, + roomId, + new Blob([new Uint8Array([4, 5, 6])], { type: voiceBroadcastRecorder.contentType }), + ); + + expect(mocked(client.sendMessage)).toHaveBeenCalledWith( + roomId, + { + body: "Voice message", + file: { + file: true, + }, + info: { + duration: 42000, + mimetype: "audio/ogg", + size: 3, + }, + ["m.relates_to"]: { + event_id: infoEvent.getId(), + rel_type: "m.reference", + }, + msgtype: "m.audio", + ["org.matrix.msc1767.audio"]: { + duration: 42000, + waveform: undefined, + }, + ["org.matrix.msc1767.file"]: { + file: { + file: true, + }, + mimetype: "audio/ogg", + name: "Voice message.ogg", + size: 3, + url: "mxc://example.com/vb", + }, + ["org.matrix.msc1767.text"]: "Voice message", + ["org.matrix.msc3245.voice"]: {}, + url: "mxc://example.com/vb", + ["io.element.voice_broadcast_chunk"]: { + sequence: 2, + }, + }, + ); + }); + }); + + describe("and calling destroy", () => { + beforeEach(() => { + voiceBroadcastRecording.destroy(); + }); + + it("should stop the recorder and remove all listeners", () => { + expect(mocked(voiceBroadcastRecorder.stop)).toHaveBeenCalled(); + expect(mocked(voiceBroadcastRecorder.off)).toHaveBeenCalledWith( + VoiceBroadcastRecorderEvent.ChunkRecorded, + onChunkRecorded, + ); + expect(mocked(voiceBroadcastRecording.removeAllListeners)).toHaveBeenCalled(); + }); + }); + }); }); describe("when created for a Voice Broadcast Info with a Stopped relation", () => { @@ -152,7 +386,7 @@ describe("VoiceBroadcastRecording", () => { }); it("should be in Stopped state", () => { - expect(voiceBroadcastRecording.state).toBe(VoiceBroadcastInfoState.Stopped); + expect(voiceBroadcastRecording.getState()).toBe(VoiceBroadcastInfoState.Stopped); }); }); }); diff --git a/test/voice-broadcast/stores/VoiceBroadcastRecordingsStore-test.ts b/test/voice-broadcast/stores/VoiceBroadcastRecordingsStore-test.ts index 9a3fa1ca96..a6a5c4ab23 100644 --- a/test/voice-broadcast/stores/VoiceBroadcastRecordingsStore-test.ts +++ b/test/voice-broadcast/stores/VoiceBroadcastRecordingsStore-test.ts @@ -76,7 +76,7 @@ describe("VoiceBroadcastRecordingsStore", () => { }); it("should return it as current", () => { - expect(recordings.current).toBe(recording); + expect(recordings.getCurrent()).toBe(recording); }); it("should return it by id", () => { diff --git a/test/voice-broadcast/utils/startNewVoiceBroadcastRecording-test.ts b/test/voice-broadcast/utils/startNewVoiceBroadcastRecording-test.ts index f1f630abc7..0096817b3f 100644 --- a/test/voice-broadcast/utils/startNewVoiceBroadcastRecording-test.ts +++ b/test/voice-broadcast/utils/startNewVoiceBroadcastRecording-test.ts @@ -109,6 +109,7 @@ describe("startNewVoiceBroadcastRecording", () => { return { infoEvent, client, + start: jest.fn(), } as unknown as VoiceBroadcastRecording; }); }); @@ -120,6 +121,7 @@ describe("startNewVoiceBroadcastRecording", () => { expect(ok).toBe(true); expect(mocked(room.off)).toHaveBeenCalledWith(RoomStateEvent.Events, roomOnStateEventsCallback); expect(recording.infoEvent).toBe(infoEvent); + expect(recording.start).toHaveBeenCalled(); done(); });