From 4bf42babc740b5a7f4e5010a94de81a91eddc1e1 Mon Sep 17 00:00:00 2001 From: Kerry Date: Tue, 1 Mar 2022 09:43:32 +0100 Subject: [PATCH] dont re-prepare voice messages (#7897) * dont reprepare voice messages Signed-off-by: Kerry Archibald * remove debug Signed-off-by: Kerry Archibald * test Playback Signed-off-by: Kerry Archibald * test RecordingPlayback Signed-off-by: Kerry Archibald * forgotten copyright Signed-off-by: Kerry Archibald * add comments Signed-off-by: Kerry Archibald --- src/audio/Playback.ts | 7 + .../views/audio_messages/AudioPlayerBase.tsx | 3 +- .../views/audio_messages/PlayPauseButton.tsx | 2 + .../audio_messages/RecordingPlayback.tsx | 1 + test/audio/Playback-test.ts | 165 ++++++++++++++++++ .../audio_messages/RecordingPlayback-test.tsx | 161 +++++++++++++++++ 6 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 test/audio/Playback-test.ts create mode 100644 test/components/views/audio_messages/RecordingPlayback-test.tsx diff --git a/src/audio/Playback.ts b/src/audio/Playback.ts index 633eb0ef78..a25794b59e 100644 --- a/src/audio/Playback.ts +++ b/src/audio/Playback.ts @@ -133,6 +133,13 @@ export class Playback extends EventEmitter implements IDestroyable { } public async prepare() { + // don't attempt to decode the media again + // AudioContext.decodeAudioData detaches the array buffer `this.buf` + // meaning it cannot be re-read + if (this.state !== PlaybackState.Decoding) { + return; + } + // The point where we use an audio element is fairly arbitrary, though we don't want // it to be too low. As of writing, voice messages want to show a waveform but audio // messages do not. Using an audio element means we can't show a waveform preview, so diff --git a/src/components/views/audio_messages/AudioPlayerBase.tsx b/src/components/views/audio_messages/AudioPlayerBase.tsx index 321fdd6ecf..1ce0ed1e95 100644 --- a/src/components/views/audio_messages/AudioPlayerBase.tsx +++ b/src/components/views/audio_messages/AudioPlayerBase.tsx @@ -40,8 +40,9 @@ export default abstract class AudioPlayerBase extends React.PureComponent { 'mx_PlayPauseButton_pause': isPlaying, 'mx_PlayPauseButton_disabled': isDisabled, }); + return diff --git a/test/audio/Playback-test.ts b/test/audio/Playback-test.ts new file mode 100644 index 0000000000..7d9a7633b2 --- /dev/null +++ b/test/audio/Playback-test.ts @@ -0,0 +1,165 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { mocked } from 'jest-mock'; +import { logger } from 'matrix-js-sdk/src/logger'; + +import '../skinned-sdk'; // Must be first for skinning to work + +import { createAudioContext, decodeOgg } from '../../src/audio/compat'; +import { Playback, PlaybackState } from "../../src/audio/Playback"; + +jest.mock('../../src/audio/compat', () => ({ + createAudioContext: jest.fn(), + decodeOgg: jest.fn(), +})); + +describe('Playback', () => { + const mockAudioBufferSourceNode = { + addEventListener: jest.fn(), + connect: jest.fn(), + start: jest.fn(), + }; + const mockAudioContext = { + decodeAudioData: jest.fn(), + suspend: jest.fn(), + resume: jest.fn(), + createBufferSource: jest.fn().mockReturnValue(mockAudioBufferSourceNode), + }; + + const mockAudioBuffer = { + duration: 99, + getChannelData: jest.fn(), + }; + + const mockChannelData = new Float32Array(); + + beforeEach(() => { + jest.spyOn(logger, 'error').mockRestore(); + mockAudioBuffer.getChannelData.mockClear().mockReturnValue(mockChannelData); + mockAudioContext.decodeAudioData.mockReset().mockImplementation( + (_b, callback) => callback(mockAudioBuffer), + ); + mockAudioContext.resume.mockClear().mockResolvedValue(undefined); + mockAudioContext.suspend.mockClear().mockResolvedValue(undefined); + mocked(decodeOgg).mockClear().mockResolvedValue(new ArrayBuffer(1)); + mocked(createAudioContext).mockReturnValue(mockAudioContext as unknown as AudioContext); + }); + + it('initialises correctly', () => { + const buffer = new ArrayBuffer(8); + + const playback = new Playback(buffer); + + expect(playback.sizeBytes).toEqual(8); + expect(playback.clockInfo).toBeTruthy(); + expect(playback.currentState).toEqual(PlaybackState.Decoding); + }); + + it('toggles playback on from stopped state', async () => { + const buffer = new ArrayBuffer(8); + const playback = new Playback(buffer); + await playback.prepare(); + // state is Stopped + await playback.toggle(); + + expect(mockAudioBufferSourceNode.start).toHaveBeenCalled(); + expect(mockAudioContext.resume).toHaveBeenCalled(); + expect(playback.currentState).toEqual(PlaybackState.Playing); + }); + + it('toggles playback to paused from playing state', async () => { + const buffer = new ArrayBuffer(8); + const playback = new Playback(buffer); + await playback.prepare(); + await playback.toggle(); + expect(playback.currentState).toEqual(PlaybackState.Playing); + + await playback.toggle(); + + expect(mockAudioContext.suspend).toHaveBeenCalled(); + expect(playback.currentState).toEqual(PlaybackState.Paused); + }); + + it('stop playbacks', async () => { + const buffer = new ArrayBuffer(8); + const playback = new Playback(buffer); + await playback.prepare(); + await playback.toggle(); + expect(playback.currentState).toEqual(PlaybackState.Playing); + + await playback.stop(); + + expect(mockAudioContext.suspend).toHaveBeenCalled(); + expect(playback.currentState).toEqual(PlaybackState.Stopped); + }); + + describe('prepare()', () => { + it('decodes audio data when not greater than 5mb', async () => { + const buffer = new ArrayBuffer(8); + + const playback = new Playback(buffer); + + await playback.prepare(); + + expect(mockAudioContext.decodeAudioData).toHaveBeenCalledTimes(1); + expect(mockAudioBuffer.getChannelData).toHaveBeenCalledWith(0); + + // clock was updated + expect(playback.clockInfo.durationSeconds).toEqual(mockAudioBuffer.duration); + + expect(playback.currentState).toEqual(PlaybackState.Stopped); + }); + + it('tries to decode ogg when decodeAudioData fails', async () => { + // stub logger to keep console clean from expected error + jest.spyOn(logger, 'error').mockReturnValue(undefined); + jest.spyOn(logger, 'warn').mockReturnValue(undefined); + + const buffer = new ArrayBuffer(8); + const decodingError = new Error('test'); + mockAudioContext.decodeAudioData.mockImplementationOnce( + (_b, _callback, error) => error(decodingError), + ).mockImplementationOnce( + (_b, callback) => callback(mockAudioBuffer), + ); + + const playback = new Playback(buffer); + + await playback.prepare(); + + expect(mockAudioContext.decodeAudioData).toHaveBeenCalledTimes(2); + expect(decodeOgg).toHaveBeenCalled(); + + // clock was updated + expect(playback.clockInfo.durationSeconds).toEqual(mockAudioBuffer.duration); + + expect(playback.currentState).toEqual(PlaybackState.Stopped); + }); + + it('does not try to re-decode audio', async () => { + const buffer = new ArrayBuffer(8); + const playback = new Playback(buffer); + await playback.prepare(); + expect(playback.currentState).toEqual(PlaybackState.Stopped); + + await playback.prepare(); + + // only called once in first prepare + expect(mockAudioContext.decodeAudioData).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/test/components/views/audio_messages/RecordingPlayback-test.tsx b/test/components/views/audio_messages/RecordingPlayback-test.tsx new file mode 100644 index 0000000000..f8a6c0ef92 --- /dev/null +++ b/test/components/views/audio_messages/RecordingPlayback-test.tsx @@ -0,0 +1,161 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import React from 'react'; +import { mount } from 'enzyme'; +import { mocked } from 'jest-mock'; +import { logger } from 'matrix-js-sdk/src/logger'; +import { act } from 'react-dom/test-utils'; + +import '../../../skinned-sdk'; +import RecordingPlayback from '../../../../src/components/views/audio_messages/RecordingPlayback'; +import { Playback } from '../../../../src/audio/Playback'; +import RoomContext, { TimelineRenderingType } from '../../../../src/contexts/RoomContext'; +import { createAudioContext } from '../../../../src/audio/compat'; +import { findByTestId, flushPromises } from '../../../test-utils'; +import PlaybackWaveform from '../../../../src/components/views/audio_messages/PlaybackWaveform'; + +jest.mock('../../../../src/audio/compat', () => ({ + createAudioContext: jest.fn(), + decodeOgg: jest.fn().mockResolvedValue({}), +})); + +describe('', () => { + const mockAudioBufferSourceNode = { + addEventListener: jest.fn(), + connect: jest.fn(), + start: jest.fn(), + }; + + const mockAudioContext = { + decodeAudioData: jest.fn(), + suspend: jest.fn(), + resume: jest.fn(), + currentTime: 0, + createBufferSource: jest.fn().mockReturnValue(mockAudioBufferSourceNode), + }; + + const mockAudioBuffer = { + duration: 99, + getChannelData: jest.fn(), + }; + + const mockChannelData = new Float32Array(); + + const defaultRoom = { roomId: '!room:server.org', timelineRenderingType: TimelineRenderingType.File }; + const getComponent = (props: { playback: Playback }, room = defaultRoom) => + mount(, { + wrappingComponent: RoomContext.Provider, + wrappingComponentProps: { value: room }, + }); + + beforeEach(() => { + jest.spyOn(logger, 'error').mockRestore(); + mockAudioBuffer.getChannelData.mockClear().mockReturnValue(mockChannelData); + mockAudioContext.decodeAudioData.mockReset().mockImplementation( + (_b, callback) => callback(mockAudioBuffer), + ); + mocked(createAudioContext).mockReturnValue(mockAudioContext as unknown as AudioContext); + }); + + const getPlayButton = component => findByTestId(component, 'play-pause-button').at(0); + + it('renders recording playback', () => { + const playback = new Playback(new ArrayBuffer(8)); + const component = getComponent({ playback }); + expect(component).toBeTruthy(); + }); + + it('disables play button while playback is decoding', async () => { + const playback = new Playback(new ArrayBuffer(8)); + const component = getComponent({ playback }); + expect(getPlayButton(component).props().disabled).toBeTruthy(); + }); + + it('enables play button when playback is finished decoding', async () => { + const playback = new Playback(new ArrayBuffer(8)); + const component = getComponent({ playback }); + await flushPromises(); + component.setProps({}); + expect(getPlayButton(component).props().disabled).toBeFalsy(); + }); + + it('displays error when playback decoding fails', async () => { + // stub logger to keep console clean from expected error + jest.spyOn(logger, 'error').mockReturnValue(undefined); + jest.spyOn(logger, 'warn').mockReturnValue(undefined); + mockAudioContext.decodeAudioData.mockImplementation( + (_b, _cb, error) => error(new Error('oh no')), + ); + const playback = new Playback(new ArrayBuffer(8)); + const component = getComponent({ playback }); + await flushPromises(); + expect(component.find('.text-warning').length).toBeFalsy(); + }); + + it('displays pre-prepared playback with correct playback phase', async () => { + const playback = new Playback(new ArrayBuffer(8)); + await playback.prepare(); + const component = getComponent({ playback }); + // playback already decoded, button is not disabled + expect(getPlayButton(component).props().disabled).toBeFalsy(); + expect(component.find('.text-warning').length).toBeFalsy(); + }); + + it('toggles playback on play pause button click', async () => { + const playback = new Playback(new ArrayBuffer(8)); + jest.spyOn(playback, 'toggle').mockResolvedValue(undefined); + await playback.prepare(); + const component = getComponent({ playback }); + + act(() => { + getPlayButton(component).simulate('click'); + }); + + expect(playback.toggle).toHaveBeenCalled(); + }); + + it.each([ + [TimelineRenderingType.Notification], + [TimelineRenderingType.File], + [TimelineRenderingType.Pinned], + ])('does not render waveform when timeline rendering type for room is %s', (timelineRenderingType) => { + const playback = new Playback(new ArrayBuffer(8)); + const room = { + ...defaultRoom, + timelineRenderingType, + }; + const component = getComponent({ playback }, room); + + expect(component.find(PlaybackWaveform).length).toBeFalsy(); + }); + + it.each([ + [TimelineRenderingType.Room], + [TimelineRenderingType.Thread], + [TimelineRenderingType.ThreadsList], + [TimelineRenderingType.Search], + ])('renders waveform when timeline rendering type for room is %s', (timelineRenderingType) => { + const playback = new Playback(new ArrayBuffer(8)); + const room = { + ...defaultRoom, + timelineRenderingType, + }; + const component = getComponent({ playback }, room); + + expect(component.find(PlaybackWaveform).length).toBeTruthy(); + }); +});