OpenCloud
/
element-web
mirror of https://github.com/vector-im/element-web
1
0
Fork 0
Code Issues Releases Wiki Activity

Implement voice broadcast playback buffering (#9435) 

Co-authored-by: Kerry <kerrya@element.io>
pull/28217/head
Michael Weimann 2022-10-17 17:35:13 +02:00 committed by GitHub
parent 877c95df8f
commit 788dd904b7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 201 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/voice-broadcast/components/molecules/VoiceBroadcastPlaybackBody.tsx
View File

@ -20,7 +20,9 @@ import {
PlaybackControlButton,
VoiceBroadcastHeader,
VoiceBroadcastPlayback,
VoiceBroadcastPlaybackState,
} from "../..";
import Spinner from "../../../components/views/elements/Spinner";
import { useVoiceBroadcastPlayback } from "../../hooks/useVoiceBroadcastPlayback";
interface VoiceBroadcastPlaybackBodyProps {
@ -38,6 +40,10 @@ export const VoiceBroadcastPlaybackBody: React.FC<VoiceBroadcastPlaybackBodyProp
playbackState,
} = useVoiceBroadcastPlayback(playback);
const control = playbackState === VoiceBroadcastPlaybackState.Buffering
? <Spinner />
: <PlaybackControlButton onClick={toggle} state={playbackState} />;
return (
<div className="mx_VoiceBroadcastPlaybackBody">
<VoiceBroadcastHeader
@ -47,10 +53,7 @@ export const VoiceBroadcastPlaybackBody: React.FC<VoiceBroadcastPlaybackBodyProp
showBroadcast={true}
/>
<div className="mx_VoiceBroadcastPlaybackBody_controls">
<PlaybackControlButton
onClick={toggle}
state={playbackState}
/>
{ control }
</div>
</div>
);

42
src/voice-broadcast/models/VoiceBroadcastPlayback.ts
View File

@ -36,6 +36,7 @@ export enum VoiceBroadcastPlaybackState {
Paused,
Playing,
Stopped,
Buffering,
}
export enum VoiceBroadcastPlaybackEvent {
@ -91,7 +92,7 @@ export class VoiceBroadcastPlayback
this.chunkRelationHelper.emitCurrent();
}
private addChunkEvent(event: MatrixEvent): boolean {
private addChunkEvent = async (event: MatrixEvent): Promise<boolean> => {
const eventId = event.getId();
if (!eventId
@ -102,8 +103,17 @@ export class VoiceBroadcastPlayback
}
this.chunkEvents.set(eventId, event);
if (this.getState() !== VoiceBroadcastPlaybackState.Stopped) {
await this.enqueueChunk(event);
}
if (this.getState() === VoiceBroadcastPlaybackState.Buffering) {
await this.start();
}
return true;
}
};
private addInfoEvent = (event: MatrixEvent): void => {
if (this.lastInfoEvent && this.lastInfoEvent.getTs() >= event.getTs()) {
@ -149,20 +159,30 @@ export class VoiceBroadcastPlayback
playback.on(UPDATE_EVENT, (state) => this.onPlaybackStateChange(playback, state));
}
private onPlaybackStateChange(playback: Playback, newState: PlaybackState) {
private async onPlaybackStateChange(playback: Playback, newState: PlaybackState) {
if (newState !== PlaybackState.Stopped) {
return;
}
const next = this.queue[this.queue.indexOf(playback) + 1];
await this.playNext(playback);
}
private async playNext(current: Playback): Promise<void> {
const next = this.queue[this.queue.indexOf(current) + 1];
if (next) {
this.setState(VoiceBroadcastPlaybackState.Playing);
this.currentlyPlaying = next;
next.play();
await next.play();
return;
}
this.setState(VoiceBroadcastPlaybackState.Stopped);
if (this.getInfoState() === VoiceBroadcastInfoState.Stopped) {
this.setState(VoiceBroadcastPlaybackState.Stopped);
} else {
// No more chunks available, although the broadcast is not finished → enter buffering state.
this.setState(VoiceBroadcastPlaybackState.Buffering);
}
}
public async start(): Promise<void> {
@ -174,14 +194,14 @@ export class VoiceBroadcastPlayback
? 0 // start at the beginning for an ended voice broadcast
: this.queue.length - 1; // start at the current chunk for an ongoing voice broadcast
if (this.queue.length === 0 || !this.queue[toPlayIndex]) {
this.setState(VoiceBroadcastPlaybackState.Stopped);
if (this.queue[toPlayIndex]) {
this.setState(VoiceBroadcastPlaybackState.Playing);
this.currentlyPlaying = this.queue[toPlayIndex];
await this.currentlyPlaying.play();
return;
}
this.setState(VoiceBroadcastPlaybackState.Playing);
this.currentlyPlaying = this.queue[toPlayIndex];
await this.currentlyPlaying.play();
this.setState(VoiceBroadcastPlaybackState.Buffering);
}
public get length(): number {

23
test/voice-broadcast/components/molecules/VoiceBroadcastPlaybackBody-test.tsx
View File

@ -18,11 +18,13 @@ import React from "react";
import { MatrixClient, MatrixEvent } from "matrix-js-sdk/src/matrix";
import { render, RenderResult } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { mocked } from "jest-mock";
import {
VoiceBroadcastInfoEventType,
VoiceBroadcastPlayback,
VoiceBroadcastPlaybackBody,
VoiceBroadcastPlaybackState,
} from "../../../../src/voice-broadcast";
import { mkEvent, stubClient } from "../../../test-utils";
@ -40,6 +42,7 @@ describe("VoiceBroadcastPlaybackBody", () => {
let client: MatrixClient;
let infoEvent: MatrixEvent;
let playback: VoiceBroadcastPlayback;
let renderResult: RenderResult;
beforeAll(() => {
client = stubClient();
@ -50,13 +53,29 @@ describe("VoiceBroadcastPlaybackBody", () => {
room: roomId,
user: userId,
});
});
beforeEach(() => {
playback = new VoiceBroadcastPlayback(infoEvent, client);
jest.spyOn(playback, "toggle");
jest.spyOn(playback, "getState");
});
describe("when rendering a buffering voice broadcast", () => {
beforeEach(() => {
mocked(playback.getState).mockReturnValue(VoiceBroadcastPlaybackState.Buffering);
});
beforeEach(() => {
renderResult = render(<VoiceBroadcastPlaybackBody playback={playback} />);
});
it("should render as expected", () => {
expect(renderResult.container).toMatchSnapshot();
});
});
describe("when rendering a broadcast", () => {
let renderResult: RenderResult;
beforeEach(() => {
renderResult = render(<VoiceBroadcastPlaybackBody playback={playback} />);
});

75
test/voice-broadcast/components/molecules/__snapshots__/VoiceBroadcastPlaybackBody-test.tsx.snap
View File

@ -77,3 +77,78 @@ exports[`VoiceBroadcastPlaybackBody when rendering a broadcast should render as
</div>
</div>
`;
exports[`VoiceBroadcastPlaybackBody when rendering a buffering voice broadcast should render as expected 1`] = `
<div>
<div
class="mx_VoiceBroadcastPlaybackBody"
>
<div
class="mx_VoiceBroadcastHeader"
>
<div
data-testid="room-avatar"
>
room avatar:
My room
</div>
<div
class="mx_VoiceBroadcastHeader_content"
>
<div
class="mx_VoiceBroadcastHeader_room"
>
My room
</div>
<div
class="mx_VoiceBroadcastHeader_line"
>
<i
aria-hidden="true"
class="mx_Icon mx_Icon_16 mx_Icon_compound-secondary-content"
role="presentation"
style="mask-image: url(\\"image-file-stub\\");"
/>
@user:example.com
</div>
<div
class="mx_VoiceBroadcastHeader_line"
>
<i
aria-hidden="true"
class="mx_Icon mx_Icon_16 mx_Icon_compound-secondary-content"
role="presentation"
style="mask-image: url(\\"image-file-stub\\");"
/>
Voice broadcast
</div>
</div>
<div
class="mx_LiveBadge"
>
<i
aria-hidden="true"
class="mx_Icon mx_Icon_16 mx_Icon_live-badge"
role="presentation"
style="mask-image: url(\\"image-file-stub\\");"
/>
Live
</div>
</div>
<div
class="mx_VoiceBroadcastPlaybackBody_controls"
>
<div
class="mx_Spinner"
>
<div
aria-label="Loading..."
class="mx_Spinner_icon"
role="progressbar"
style="width: 32px; height: 32px;"
/>
</div>
</div>
</div>
</div>
`;

72
test/voice-broadcast/models/VoiceBroadcastPlayback-test.ts
View File

@ -21,6 +21,7 @@ import { Relations } from "matrix-js-sdk/src/models/relations";
import { Playback, PlaybackState } from "../../../src/audio/Playback";
import { PlaybackManager } from "../../../src/audio/PlaybackManager";
import { getReferenceRelationsForEvent } from "../../../src/events";
import { RelationsHelperEvent } from "../../../src/events/RelationsHelper";
import { MediaEventHelper } from "../../../src/utils/MediaEventHelper";
import {
VoiceBroadcastChunkEventType,
@ -51,15 +52,19 @@ describe("VoiceBroadcastPlayback", () => {
let chunk0Event: MatrixEvent;
let chunk1Event: MatrixEvent;
let chunk2Event: MatrixEvent;
let chunk3Event: MatrixEvent;
const chunk0Data = new ArrayBuffer(1);
const chunk1Data = new ArrayBuffer(2);
const chunk2Data = new ArrayBuffer(3);
const chunk3Data = new ArrayBuffer(3);
let chunk0Helper: MediaEventHelper;
let chunk1Helper: MediaEventHelper;
let chunk2Helper: MediaEventHelper;
let chunk3Helper: MediaEventHelper;
let chunk0Playback: Playback;
let chunk1Playback: Playback;
let chunk2Playback: Playback;
let chunk3Playback: Playback;
const itShouldSetTheStateTo = (state: VoiceBroadcastPlaybackState) => {
it(`should set the state to ${state}`, () => {
@ -133,20 +138,24 @@ describe("VoiceBroadcastPlayback", () => {
chunk0Event = mkChunkEvent(0);
chunk1Event = mkChunkEvent(1);
chunk2Event = mkChunkEvent(2);
chunk3Event = mkChunkEvent(3);
chunk0Helper = mkChunkHelper(chunk0Data);
chunk1Helper = mkChunkHelper(chunk1Data);
chunk2Helper = mkChunkHelper(chunk2Data);
chunk3Helper = mkChunkHelper(chunk3Data);
chunk0Playback = createTestPlayback();
chunk1Playback = createTestPlayback();
chunk2Playback = createTestPlayback();
chunk3Playback = createTestPlayback();
jest.spyOn(PlaybackManager.instance, "createPlaybackInstance").mockImplementation(
(buffer: ArrayBuffer, _waveForm?: number[]) => {
if (buffer === chunk0Data) return chunk0Playback;
if (buffer === chunk1Data) return chunk1Playback;
if (buffer === chunk2Data) return chunk2Playback;
if (buffer === chunk3Data) return chunk3Playback;
},
);
@ -154,6 +163,7 @@ describe("VoiceBroadcastPlayback", () => {
if (event === chunk0Event) return chunk0Helper;
if (event === chunk1Event) return chunk1Helper;
if (event === chunk2Event) return chunk2Helper;
if (event === chunk3Event) return chunk3Helper;
});
});
@ -162,6 +172,38 @@ describe("VoiceBroadcastPlayback", () => {
onStateChanged = jest.fn();
});
describe("when there is a running broadcast without chunks yet", () => {
beforeEach(() => {
infoEvent = mkInfoEvent(VoiceBroadcastInfoState.Running);
playback = mkPlayback();
setUpChunkEvents([]);
});
describe("and calling start", () => {
beforeEach(async () => {
await playback.start();
});
it("should be in buffering state", () => {
expect(playback.getState()).toBe(VoiceBroadcastPlaybackState.Buffering);
});
describe("and receiving the first chunk", () => {
beforeEach(() => {
// TODO Michael W: Use RelationsHelper
// @ts-ignore
playback.chunkRelationHelper.emit(RelationsHelperEvent.Add, chunk1Event);
});
itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Playing);
it("should play the first chunk", () => {
expect(chunk1Playback.play).toHaveBeenCalled();
});
});
});
});
describe("when there is a running voice broadcast with some chunks", () => {
beforeEach(() => {
infoEvent = mkInfoEvent(VoiceBroadcastInfoState.Running);
@ -175,10 +217,32 @@ describe("VoiceBroadcastPlayback", () => {
});
it("should play the last chunk", () => {
// assert that the first chunk is being played
// assert that the last chunk is played first
expect(chunk2Playback.play).toHaveBeenCalled();
expect(chunk1Playback.play).not.toHaveBeenCalled();
});
describe("and the playback of the last chunk ended", () => {
beforeEach(() => {
chunk2Playback.emit(PlaybackState.Stopped);
});
itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Buffering);
describe("and the next chunk arrived", () => {
beforeEach(() => {
// TODO Michael W: Use RelationsHelper
// @ts-ignore
playback.chunkRelationHelper.emit(RelationsHelperEvent.Add, chunk3Event);
});
itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Playing);
it("should play the next chunk", () => {
expect(chunk3Playback.play).toHaveBeenCalled();
});
});
});
});
});
@ -198,7 +262,7 @@ describe("VoiceBroadcastPlayback", () => {
await playback.start();
});
itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Stopped);
itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Buffering);
});
});
@ -211,9 +275,7 @@ describe("VoiceBroadcastPlayback", () => {
expect(playback.infoEvent).toBe(infoEvent);
});
it("should be in state Stopped", () => {
expect(playback.getState()).toBe(VoiceBroadcastPlaybackState.Stopped);
});
itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Stopped);
describe("and calling start", () => {
beforeEach(async () => {