From 788dd904b75668237972c9a272c754ba283a1708 Mon Sep 17 00:00:00 2001
From: Michael Weimann <michaelw@matrix.org>
Date: Mon, 17 Oct 2022 17:35:13 +0200
Subject: [PATCH] Implement voice broadcast playback buffering (#9435)

Co-authored-by: Kerry <kerrya@element.io>
---
 .../molecules/VoiceBroadcastPlaybackBody.tsx  | 11 ++-
 .../models/VoiceBroadcastPlayback.ts          | 42 ++++++++---
 .../VoiceBroadcastPlaybackBody-test.tsx       | 23 +++++-
 .../VoiceBroadcastPlaybackBody-test.tsx.snap  | 75 +++++++++++++++++++
 .../models/VoiceBroadcastPlayback-test.ts     | 72 ++++++++++++++++--
 5 files changed, 201 insertions(+), 22 deletions(-)

diff --git a/src/voice-broadcast/components/molecules/VoiceBroadcastPlaybackBody.tsx b/src/voice-broadcast/components/molecules/VoiceBroadcastPlaybackBody.tsx
index 8edc5d0d9a..035b3ce6e5 100644
--- a/src/voice-broadcast/components/molecules/VoiceBroadcastPlaybackBody.tsx
+++ b/src/voice-broadcast/components/molecules/VoiceBroadcastPlaybackBody.tsx
@@ -20,7 +20,9 @@ import {
     PlaybackControlButton,
     VoiceBroadcastHeader,
     VoiceBroadcastPlayback,
+    VoiceBroadcastPlaybackState,
 } from "../..";
+import Spinner from "../../../components/views/elements/Spinner";
 import { useVoiceBroadcastPlayback } from "../../hooks/useVoiceBroadcastPlayback";
 
 interface VoiceBroadcastPlaybackBodyProps {
@@ -38,6 +40,10 @@ export const VoiceBroadcastPlaybackBody: React.FC<VoiceBroadcastPlaybackBodyProp
         playbackState,
     } = useVoiceBroadcastPlayback(playback);
 
+    const control = playbackState === VoiceBroadcastPlaybackState.Buffering
+        ? <Spinner />
+        : <PlaybackControlButton onClick={toggle} state={playbackState} />;
+
     return (
         <div className="mx_VoiceBroadcastPlaybackBody">
             <VoiceBroadcastHeader
@@ -47,10 +53,7 @@ export const VoiceBroadcastPlaybackBody: React.FC<VoiceBroadcastPlaybackBodyProp
                 showBroadcast={true}
             />
             <div className="mx_VoiceBroadcastPlaybackBody_controls">
-                <PlaybackControlButton
-                    onClick={toggle}
-                    state={playbackState}
-                />
+                { control }
             </div>
         </div>
     );
diff --git a/src/voice-broadcast/models/VoiceBroadcastPlayback.ts b/src/voice-broadcast/models/VoiceBroadcastPlayback.ts
index 16ae9317e0..641deb66ad 100644
--- a/src/voice-broadcast/models/VoiceBroadcastPlayback.ts
+++ b/src/voice-broadcast/models/VoiceBroadcastPlayback.ts
@@ -36,6 +36,7 @@ export enum VoiceBroadcastPlaybackState {
     Paused,
     Playing,
     Stopped,
+    Buffering,
 }
 
 export enum VoiceBroadcastPlaybackEvent {
@@ -91,7 +92,7 @@ export class VoiceBroadcastPlayback
         this.chunkRelationHelper.emitCurrent();
     }
 
-    private addChunkEvent(event: MatrixEvent): boolean {
+    private addChunkEvent = async (event: MatrixEvent): Promise<boolean> => {
         const eventId = event.getId();
 
         if (!eventId
@@ -102,8 +103,17 @@ export class VoiceBroadcastPlayback
         }
 
         this.chunkEvents.set(eventId, event);
+
+        if (this.getState() !== VoiceBroadcastPlaybackState.Stopped) {
+            await this.enqueueChunk(event);
+        }
+
+        if (this.getState() === VoiceBroadcastPlaybackState.Buffering) {
+            await this.start();
+        }
+
         return true;
-    }
+    };
 
     private addInfoEvent = (event: MatrixEvent): void => {
         if (this.lastInfoEvent && this.lastInfoEvent.getTs() >= event.getTs()) {
@@ -149,20 +159,30 @@ export class VoiceBroadcastPlayback
         playback.on(UPDATE_EVENT, (state) => this.onPlaybackStateChange(playback, state));
     }
 
-    private onPlaybackStateChange(playback: Playback, newState: PlaybackState) {
+    private async onPlaybackStateChange(playback: Playback, newState: PlaybackState) {
         if (newState !== PlaybackState.Stopped) {
             return;
         }
 
-        const next = this.queue[this.queue.indexOf(playback) + 1];
+        await this.playNext(playback);
+    }
+
+    private async playNext(current: Playback): Promise<void> {
+        const next = this.queue[this.queue.indexOf(current) + 1];
 
         if (next) {
+            this.setState(VoiceBroadcastPlaybackState.Playing);
             this.currentlyPlaying = next;
-            next.play();
+            await next.play();
             return;
         }
 
-        this.setState(VoiceBroadcastPlaybackState.Stopped);
+        if (this.getInfoState() === VoiceBroadcastInfoState.Stopped) {
+            this.setState(VoiceBroadcastPlaybackState.Stopped);
+        } else {
+            // No more chunks available, although the broadcast is not finished → enter buffering state.
+            this.setState(VoiceBroadcastPlaybackState.Buffering);
+        }
     }
 
     public async start(): Promise<void> {
@@ -174,14 +194,14 @@ export class VoiceBroadcastPlayback
             ? 0 // start at the beginning for an ended voice broadcast
             : this.queue.length - 1; // start at the current chunk for an ongoing voice broadcast
 
-        if (this.queue.length === 0 || !this.queue[toPlayIndex]) {
-            this.setState(VoiceBroadcastPlaybackState.Stopped);
+        if (this.queue[toPlayIndex]) {
+            this.setState(VoiceBroadcastPlaybackState.Playing);
+            this.currentlyPlaying = this.queue[toPlayIndex];
+            await this.currentlyPlaying.play();
             return;
         }
 
-        this.setState(VoiceBroadcastPlaybackState.Playing);
-        this.currentlyPlaying = this.queue[toPlayIndex];
-        await this.currentlyPlaying.play();
+        this.setState(VoiceBroadcastPlaybackState.Buffering);
     }
 
     public get length(): number {
diff --git a/test/voice-broadcast/components/molecules/VoiceBroadcastPlaybackBody-test.tsx b/test/voice-broadcast/components/molecules/VoiceBroadcastPlaybackBody-test.tsx
index a4269533c3..63c4b76fbd 100644
--- a/test/voice-broadcast/components/molecules/VoiceBroadcastPlaybackBody-test.tsx
+++ b/test/voice-broadcast/components/molecules/VoiceBroadcastPlaybackBody-test.tsx
@@ -18,11 +18,13 @@ import React from "react";
 import { MatrixClient, MatrixEvent } from "matrix-js-sdk/src/matrix";
 import { render, RenderResult } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
+import { mocked } from "jest-mock";
 
 import {
     VoiceBroadcastInfoEventType,
     VoiceBroadcastPlayback,
     VoiceBroadcastPlaybackBody,
+    VoiceBroadcastPlaybackState,
 } from "../../../../src/voice-broadcast";
 import { mkEvent, stubClient } from "../../../test-utils";
 
@@ -40,6 +42,7 @@ describe("VoiceBroadcastPlaybackBody", () => {
     let client: MatrixClient;
     let infoEvent: MatrixEvent;
     let playback: VoiceBroadcastPlayback;
+    let renderResult: RenderResult;
 
     beforeAll(() => {
         client = stubClient();
@@ -50,13 +53,29 @@ describe("VoiceBroadcastPlaybackBody", () => {
             room: roomId,
             user: userId,
         });
+    });
+
+    beforeEach(() => {
         playback = new VoiceBroadcastPlayback(infoEvent, client);
         jest.spyOn(playback, "toggle");
+        jest.spyOn(playback, "getState");
+    });
+
+    describe("when rendering a buffering voice broadcast", () => {
+        beforeEach(() => {
+            mocked(playback.getState).mockReturnValue(VoiceBroadcastPlaybackState.Buffering);
+        });
+
+        beforeEach(() => {
+            renderResult = render(<VoiceBroadcastPlaybackBody playback={playback} />);
+        });
+
+        it("should render as expected", () => {
+            expect(renderResult.container).toMatchSnapshot();
+        });
     });
 
     describe("when rendering a broadcast", () => {
-        let renderResult: RenderResult;
-
         beforeEach(() => {
             renderResult = render(<VoiceBroadcastPlaybackBody playback={playback} />);
         });
diff --git a/test/voice-broadcast/components/molecules/__snapshots__/VoiceBroadcastPlaybackBody-test.tsx.snap b/test/voice-broadcast/components/molecules/__snapshots__/VoiceBroadcastPlaybackBody-test.tsx.snap
index b2515a78c2..beeb3f9ee3 100644
--- a/test/voice-broadcast/components/molecules/__snapshots__/VoiceBroadcastPlaybackBody-test.tsx.snap
+++ b/test/voice-broadcast/components/molecules/__snapshots__/VoiceBroadcastPlaybackBody-test.tsx.snap
@@ -77,3 +77,78 @@ exports[`VoiceBroadcastPlaybackBody when rendering a broadcast should render as
   </div>
 </div>
 `;
+
+exports[`VoiceBroadcastPlaybackBody when rendering a buffering voice broadcast should render as expected 1`] = `
+<div>
+  <div
+    class="mx_VoiceBroadcastPlaybackBody"
+  >
+    <div
+      class="mx_VoiceBroadcastHeader"
+    >
+      <div
+        data-testid="room-avatar"
+      >
+        room avatar: 
+        My room
+      </div>
+      <div
+        class="mx_VoiceBroadcastHeader_content"
+      >
+        <div
+          class="mx_VoiceBroadcastHeader_room"
+        >
+          My room
+        </div>
+        <div
+          class="mx_VoiceBroadcastHeader_line"
+        >
+          <i
+            aria-hidden="true"
+            class="mx_Icon mx_Icon_16 mx_Icon_compound-secondary-content"
+            role="presentation"
+            style="mask-image: url(\\"image-file-stub\\");"
+          />
+          @user:example.com
+        </div>
+        <div
+          class="mx_VoiceBroadcastHeader_line"
+        >
+          <i
+            aria-hidden="true"
+            class="mx_Icon mx_Icon_16 mx_Icon_compound-secondary-content"
+            role="presentation"
+            style="mask-image: url(\\"image-file-stub\\");"
+          />
+          Voice broadcast
+        </div>
+      </div>
+      <div
+        class="mx_LiveBadge"
+      >
+        <i
+          aria-hidden="true"
+          class="mx_Icon mx_Icon_16 mx_Icon_live-badge"
+          role="presentation"
+          style="mask-image: url(\\"image-file-stub\\");"
+        />
+        Live
+      </div>
+    </div>
+    <div
+      class="mx_VoiceBroadcastPlaybackBody_controls"
+    >
+      <div
+        class="mx_Spinner"
+      >
+        <div
+          aria-label="Loading..."
+          class="mx_Spinner_icon"
+          role="progressbar"
+          style="width: 32px; height: 32px;"
+        />
+      </div>
+    </div>
+  </div>
+</div>
+`;
diff --git a/test/voice-broadcast/models/VoiceBroadcastPlayback-test.ts b/test/voice-broadcast/models/VoiceBroadcastPlayback-test.ts
index 15a3416226..7e7722321c 100644
--- a/test/voice-broadcast/models/VoiceBroadcastPlayback-test.ts
+++ b/test/voice-broadcast/models/VoiceBroadcastPlayback-test.ts
@@ -21,6 +21,7 @@ import { Relations } from "matrix-js-sdk/src/models/relations";
 import { Playback, PlaybackState } from "../../../src/audio/Playback";
 import { PlaybackManager } from "../../../src/audio/PlaybackManager";
 import { getReferenceRelationsForEvent } from "../../../src/events";
+import { RelationsHelperEvent } from "../../../src/events/RelationsHelper";
 import { MediaEventHelper } from "../../../src/utils/MediaEventHelper";
 import {
     VoiceBroadcastChunkEventType,
@@ -51,15 +52,19 @@ describe("VoiceBroadcastPlayback", () => {
     let chunk0Event: MatrixEvent;
     let chunk1Event: MatrixEvent;
     let chunk2Event: MatrixEvent;
+    let chunk3Event: MatrixEvent;
     const chunk0Data = new ArrayBuffer(1);
     const chunk1Data = new ArrayBuffer(2);
     const chunk2Data = new ArrayBuffer(3);
+    const chunk3Data = new ArrayBuffer(3);
     let chunk0Helper: MediaEventHelper;
     let chunk1Helper: MediaEventHelper;
     let chunk2Helper: MediaEventHelper;
+    let chunk3Helper: MediaEventHelper;
     let chunk0Playback: Playback;
     let chunk1Playback: Playback;
     let chunk2Playback: Playback;
+    let chunk3Playback: Playback;
 
     const itShouldSetTheStateTo = (state: VoiceBroadcastPlaybackState) => {
         it(`should set the state to ${state}`, () => {
@@ -133,20 +138,24 @@ describe("VoiceBroadcastPlayback", () => {
         chunk0Event = mkChunkEvent(0);
         chunk1Event = mkChunkEvent(1);
         chunk2Event = mkChunkEvent(2);
+        chunk3Event = mkChunkEvent(3);
 
         chunk0Helper = mkChunkHelper(chunk0Data);
         chunk1Helper = mkChunkHelper(chunk1Data);
         chunk2Helper = mkChunkHelper(chunk2Data);
+        chunk3Helper = mkChunkHelper(chunk3Data);
 
         chunk0Playback = createTestPlayback();
         chunk1Playback = createTestPlayback();
         chunk2Playback = createTestPlayback();
+        chunk3Playback = createTestPlayback();
 
         jest.spyOn(PlaybackManager.instance, "createPlaybackInstance").mockImplementation(
             (buffer: ArrayBuffer, _waveForm?: number[]) => {
                 if (buffer === chunk0Data) return chunk0Playback;
                 if (buffer === chunk1Data) return chunk1Playback;
                 if (buffer === chunk2Data) return chunk2Playback;
+                if (buffer === chunk3Data) return chunk3Playback;
             },
         );
 
@@ -154,6 +163,7 @@ describe("VoiceBroadcastPlayback", () => {
             if (event === chunk0Event) return chunk0Helper;
             if (event === chunk1Event) return chunk1Helper;
             if (event === chunk2Event) return chunk2Helper;
+            if (event === chunk3Event) return chunk3Helper;
         });
     });
 
@@ -162,6 +172,38 @@ describe("VoiceBroadcastPlayback", () => {
         onStateChanged = jest.fn();
     });
 
+    describe("when there is a running broadcast without chunks yet", () => {
+        beforeEach(() => {
+            infoEvent = mkInfoEvent(VoiceBroadcastInfoState.Running);
+            playback = mkPlayback();
+            setUpChunkEvents([]);
+        });
+
+        describe("and calling start", () => {
+            beforeEach(async () => {
+                await playback.start();
+            });
+
+            it("should be in buffering state", () => {
+                expect(playback.getState()).toBe(VoiceBroadcastPlaybackState.Buffering);
+            });
+
+            describe("and receiving the first chunk", () => {
+                beforeEach(() => {
+                    // TODO Michael W: Use RelationsHelper
+                    // @ts-ignore
+                    playback.chunkRelationHelper.emit(RelationsHelperEvent.Add, chunk1Event);
+                });
+
+                itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Playing);
+
+                it("should play the first chunk", () => {
+                    expect(chunk1Playback.play).toHaveBeenCalled();
+                });
+            });
+        });
+    });
+
     describe("when there is a running voice broadcast with some chunks", () => {
         beforeEach(() => {
             infoEvent = mkInfoEvent(VoiceBroadcastInfoState.Running);
@@ -175,10 +217,32 @@ describe("VoiceBroadcastPlayback", () => {
             });
 
             it("should play the last chunk", () => {
-                // assert that the first chunk is being played
+                // assert that the last chunk is played first
                 expect(chunk2Playback.play).toHaveBeenCalled();
                 expect(chunk1Playback.play).not.toHaveBeenCalled();
             });
+
+            describe("and the playback of the last chunk ended", () => {
+                beforeEach(() => {
+                    chunk2Playback.emit(PlaybackState.Stopped);
+                });
+
+                itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Buffering);
+
+                describe("and the next chunk arrived", () => {
+                    beforeEach(() => {
+                        // TODO Michael W: Use RelationsHelper
+                        // @ts-ignore
+                        playback.chunkRelationHelper.emit(RelationsHelperEvent.Add, chunk3Event);
+                    });
+
+                    itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Playing);
+
+                    it("should play the next chunk", () => {
+                        expect(chunk3Playback.play).toHaveBeenCalled();
+                    });
+                });
+            });
         });
     });
 
@@ -198,7 +262,7 @@ describe("VoiceBroadcastPlayback", () => {
                     await playback.start();
                 });
 
-                itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Stopped);
+                itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Buffering);
             });
         });
 
@@ -211,9 +275,7 @@ describe("VoiceBroadcastPlayback", () => {
                 expect(playback.infoEvent).toBe(infoEvent);
             });
 
-            it("should be in state Stopped", () => {
-                expect(playback.getState()).toBe(VoiceBroadcastPlaybackState.Stopped);
-            });
+            itShouldSetTheStateTo(VoiceBroadcastPlaybackState.Stopped);
 
             describe("and calling start", () => {
                 beforeEach(async () => {