diff --git a/res/css/views/rooms/_VoiceRecordComposerTile.scss b/res/css/views/rooms/_VoiceRecordComposerTile.scss index 8100a03890..dc4835c4dd 100644 --- a/res/css/views/rooms/_VoiceRecordComposerTile.scss +++ b/res/css/views/rooms/_VoiceRecordComposerTile.scss @@ -36,11 +36,12 @@ limitations under the License. } .mx_VoiceRecordComposerTile_waveformContainer { - padding: 5px; + padding: 8px; // makes us 4px taller than the send/stop button padding-right: 4px; // there's 1px from the waveform itself, so account for that padding-left: 15px; // +10px for the live circle, +5px for regular padding background-color: $voice-record-waveform-bg-color; border-radius: 12px; + margin: 6px; // force the composer area to put a gutter around us margin-right: 12px; // isolate from stop button // Cheat at alignment a bit @@ -52,7 +53,7 @@ limitations under the License. color: $voice-record-waveform-fg-color; font-size: $font-14px; - &::before { + &.mx_VoiceRecordComposerTile_recording::before { animation: recording-pulse 2s infinite; content: ''; @@ -61,7 +62,7 @@ limitations under the License. height: 10px; position: absolute; left: 8px; - top: 16px; // vertically center + top: 18px; // vertically center border-radius: 10px; } diff --git a/src/components/views/rooms/VoiceRecordComposerTile.tsx b/src/components/views/rooms/VoiceRecordComposerTile.tsx index 9b7f0da472..3fcafafd05 100644 --- a/src/components/views/rooms/VoiceRecordComposerTile.tsx +++ b/src/components/views/rooms/VoiceRecordComposerTile.tsx @@ -17,7 +17,7 @@ limitations under the License. import AccessibleTooltipButton from "../elements/AccessibleTooltipButton"; import {_t} from "../../../languageHandler"; import React from "react"; -import {VoiceRecording} from "../../../voice/VoiceRecording"; +import {RecordingState, VoiceRecording} from "../../../voice/VoiceRecording"; import {Room} from "matrix-js-sdk/src/models/room"; import {MatrixClientPeg} from "../../../MatrixClientPeg"; import classNames from "classnames"; @@ -25,6 +25,8 @@ import LiveRecordingWaveform from "../voice_messages/LiveRecordingWaveform"; import {replaceableComponent} from "../../../utils/replaceableComponent"; import LiveRecordingClock from "../voice_messages/LiveRecordingClock"; import {VoiceRecordingStore} from "../../../stores/VoiceRecordingStore"; +import {UPDATE_EVENT} from "../../../stores/AsyncStore"; +import PlaybackWaveform from "../voice_messages/PlaybackWaveform"; interface IProps { room: Room; @@ -32,6 +34,7 @@ interface IProps { interface IState { recorder?: VoiceRecording; + recordingPhase?: RecordingState; } /** @@ -43,87 +46,126 @@ export default class VoiceRecordComposerTile extends React.PureComponent { - // TODO: @@ TravisR: We do not want to auto-send on stop. + public async componentWillUnmount() { + await VoiceRecordingStore.instance.disposeRecording(); + } + + // called by composer + public async send() { + if (!this.state.recorder) { + throw new Error("No recording started - cannot send anything"); + } + + await this.state.recorder.stop(); + const mxc = await this.state.recorder.upload(); + MatrixClientPeg.get().sendMessage(this.props.room.roomId, { + "body": "Voice message", + "msgtype": "org.matrix.msc2516.voice", + //"msgtype": MsgType.Audio, + "url": mxc, + "info": { + duration: Math.round(this.state.recorder.durationSeconds * 1000), + mimetype: this.state.recorder.contentType, + size: this.state.recorder.contentLength, + }, + + // MSC1767 experiment + "org.matrix.msc1767.text": "Voice message", + "org.matrix.msc1767.file": { + url: mxc, + name: "Voice message.ogg", + mimetype: this.state.recorder.contentType, + size: this.state.recorder.contentLength, + }, + "org.matrix.msc1767.audio": { + duration: Math.round(this.state.recorder.durationSeconds * 1000), + // TODO: @@ TravisR: Waveform? (MSC1767 decision) + }, + "org.matrix.experimental.msc2516.voice": { // MSC2516+MSC1767 experiment + duration: Math.round(this.state.recorder.durationSeconds * 1000), + + // Events can't have floats, so we try to maintain resolution by using 1024 + // as a maximum value. The waveform contains values between zero and 1, so this + // should come out largely sane. + // + // We're expecting about one data point per second of audio. + waveform: this.state.recorder.finalWaveform.map(v => Math.round(v * 1024)), + }, + }); + await VoiceRecordingStore.instance.disposeRecording(); + this.setState({recorder: null}); + } + + private onRecordStartEndClick = async () => { if (this.state.recorder) { await this.state.recorder.stop(); - const mxc = await this.state.recorder.upload(); - MatrixClientPeg.get().sendMessage(this.props.room.roomId, { - "body": "Voice message", - "msgtype": "org.matrix.msc2516.voice", - //"msgtype": MsgType.Audio, - "url": mxc, - "info": { - duration: Math.round(this.state.recorder.durationSeconds * 1000), - mimetype: this.state.recorder.contentType, - size: this.state.recorder.contentLength, - }, - - // MSC1767 experiment - "org.matrix.msc1767.text": "Voice message", - "org.matrix.msc1767.file": { - url: mxc, - name: "Voice message.ogg", - mimetype: this.state.recorder.contentType, - size: this.state.recorder.contentLength, - }, - "org.matrix.msc1767.audio": { - duration: Math.round(this.state.recorder.durationSeconds * 1000), - // TODO: @@ TravisR: Waveform? (MSC1767 decision) - }, - "org.matrix.experimental.msc2516.voice": { // MSC2516+MSC1767 experiment - duration: Math.round(this.state.recorder.durationSeconds * 1000), - - // Events can't have floats, so we try to maintain resolution by using 1024 - // as a maximum value. The waveform contains values between zero and 1, so this - // should come out largely sane. - // - // We're expecting about one data point per second of audio. - waveform: this.state.recorder.finalWaveform.map(v => Math.round(v * 1024)), - }, - }); - await VoiceRecordingStore.instance.disposeRecording(); - this.setState({recorder: null}); return; } const recorder = VoiceRecordingStore.instance.startRecording(); await recorder.start(); - this.setState({recorder}); + + // We don't need to remove the listener: the recorder will clean that up for us. + recorder.on(UPDATE_EVENT, (ev: RecordingState) => { + if (ev === RecordingState.EndingSoon) return; // ignore this state: it has no UI purpose here + this.setState({recordingPhase: ev}); + }); + + this.setState({recorder, recordingPhase: RecordingState.Started}); }; private renderWaveformArea() { if (!this.state.recorder) return null; - return
- - + const classes = classNames({ + 'mx_VoiceRecordComposerTile_waveformContainer': true, + 'mx_VoiceRecordComposerTile_recording': this.state.recordingPhase === RecordingState.Started, + }); + + const clock = ; + let waveform = ; + if (this.state.recordingPhase !== RecordingState.Started) { + waveform = ; + } + + return
+ {clock} + {waveform}
; } public render() { - const classes = classNames({ - 'mx_MessageComposer_button': !this.state.recorder, - 'mx_MessageComposer_voiceMessage': !this.state.recorder, - 'mx_VoiceRecordComposerTile_stop': !!this.state.recorder, - }); + let recordingInfo; + if (!this.state.recordingPhase || this.state.recordingPhase === RecordingState.Started) { + const classes = classNames({ + 'mx_MessageComposer_button': !this.state.recorder, + 'mx_MessageComposer_voiceMessage': !this.state.recorder, + 'mx_VoiceRecordComposerTile_stop': this.state.recorder?.isRecording, + }); - let tooltip = _t("Record a voice message"); - if (!!this.state.recorder) { - // TODO: @@ TravisR: Change to match behaviour - tooltip = _t("Stop & send recording"); + let tooltip = _t("Record a voice message"); + if (!!this.state.recorder) { + tooltip = _t("Stop the recording"); + } + + let stopOrRecordBtn = ; + if (this.state.recorder && !this.state.recorder?.isRecording) { + stopOrRecordBtn = null; + } + + recordingInfo = stopOrRecordBtn; } return (<> {this.renderWaveformArea()} - + {recordingInfo} ); } } diff --git a/src/components/views/voice_messages/IRecordingWaveformStateProps.ts b/src/components/views/voice_messages/IRecordingWaveformStateProps.ts new file mode 100644 index 0000000000..fcdbf3e3b1 --- /dev/null +++ b/src/components/views/voice_messages/IRecordingWaveformStateProps.ts @@ -0,0 +1,26 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import {VoiceRecording} from "../../../voice/VoiceRecording"; + +export interface IRecordingWaveformProps { + recorder: VoiceRecording; +} + +export interface IRecordingWaveformState { + heights: number[]; +} + +export const DOWNSAMPLE_TARGET = 35; // number of bars we want diff --git a/src/components/views/voice_messages/LiveRecordingWaveform.tsx b/src/components/views/voice_messages/LiveRecordingWaveform.tsx index c1f5e97fff..e9b3fea629 100644 --- a/src/components/views/voice_messages/LiveRecordingWaveform.tsx +++ b/src/components/views/voice_messages/LiveRecordingWaveform.tsx @@ -20,22 +20,13 @@ import {replaceableComponent} from "../../../utils/replaceableComponent"; import {arrayFastResample, arraySeed} from "../../../utils/arrays"; import {percentageOf} from "../../../utils/numbers"; import Waveform from "./Waveform"; - -interface IProps { - recorder: VoiceRecording; -} - -interface IState { - heights: number[]; -} - -const DOWNSAMPLE_TARGET = 35; // number of bars we want +import {DOWNSAMPLE_TARGET, IRecordingWaveformProps, IRecordingWaveformState} from "./IRecordingWaveformStateProps"; /** * A waveform which shows the waveform of a live recording */ @replaceableComponent("views.voice_messages.LiveRecordingWaveform") -export default class LiveRecordingWaveform extends React.PureComponent { +export default class LiveRecordingWaveform extends React.PureComponent { public constructor(props) { super(props); diff --git a/src/components/views/voice_messages/PlaybackWaveform.tsx b/src/components/views/voice_messages/PlaybackWaveform.tsx new file mode 100644 index 0000000000..02647aa3ee --- /dev/null +++ b/src/components/views/voice_messages/PlaybackWaveform.tsx @@ -0,0 +1,43 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import React from "react"; +import {IRecordingUpdate, VoiceRecording} from "../../../voice/VoiceRecording"; +import {replaceableComponent} from "../../../utils/replaceableComponent"; +import {arrayFastResample, arraySeed, arrayTrimFill} from "../../../utils/arrays"; +import {percentageOf} from "../../../utils/numbers"; +import Waveform from "./Waveform"; +import {DOWNSAMPLE_TARGET, IRecordingWaveformProps, IRecordingWaveformState} from "./IRecordingWaveformStateProps"; + +/** + * A waveform which shows the waveform of a previously recorded recording + */ +@replaceableComponent("views.voice_messages.LiveRecordingWaveform") +export default class PlaybackWaveform extends React.PureComponent { + public constructor(props) { + super(props); + + // Like the live recording waveform + const bars = arrayFastResample(this.props.recorder.finalWaveform, DOWNSAMPLE_TARGET); + const seed = arraySeed(0, DOWNSAMPLE_TARGET); + const heights = arrayTrimFill(bars, DOWNSAMPLE_TARGET, seed).map(b => percentageOf(b, 0, 0.5)); + this.state = {heights}; + } + + public render() { + return ; + } +} diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index c6f7a8d25e..f592cc19cf 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -1645,7 +1645,7 @@ "Jump to first unread message.": "Jump to first unread message.", "Mark all as read": "Mark all as read", "Record a voice message": "Record a voice message", - "Stop & send recording": "Stop & send recording", + "Stop the recording": "Stop the recording", "Error updating main address": "Error updating main address", "There was an error updating the room's main address. It may not be allowed by the server or a temporary failure occurred.": "There was an error updating the room's main address. It may not be allowed by the server or a temporary failure occurred.", "There was an error updating the room's alternative addresses. It may not be allowed by the server or a temporary failure occurred.": "There was an error updating the room's alternative addresses. It may not be allowed by the server or a temporary failure occurred.", diff --git a/src/utils/arrays.ts b/src/utils/arrays.ts index cea377bfe9..f7e693452b 100644 --- a/src/utils/arrays.ts +++ b/src/utils/arrays.ts @@ -73,6 +73,26 @@ export function arraySeed(val: T, length: number): T[] { return a; } +/** + * Trims or fills the array to ensure it meets the desired length. The seed array + * given is pulled from to fill any missing slots - it is recommended that this be + * at least `len` long. The resulting array will be exactly `len` long, either + * trimmed from the source or filled with the some/all of the seed array. + * @param {T[]} a The array to trim/fill. + * @param {number} len The length to trim or fill to, as needed. + * @param {T[]} seed Values to pull from if the array needs filling. + * @returns {T[]} The resulting array of `len` length. + */ +export function arrayTrimFill(a: T[], len: number, seed: T[]): T[] { + // Dev note: we do length checks because the spread operator can result in some + // performance penalties in more critical code paths. As a utility, it should be + // as fast as possible to not cause a problem for the call stack, no matter how + // critical that stack is. + if (a.length === len) return a; + if (a.length > len) return a.slice(0, len); + return a.concat(seed.slice(0, len - a.length)); +} + /** * Clones an array as fast as possible, retaining references of the array's values. * @param a The array to clone. Must be defined. diff --git a/src/voice/VoiceRecording.ts b/src/voice/VoiceRecording.ts index b0cc3cd407..68a944da51 100644 --- a/src/voice/VoiceRecording.ts +++ b/src/voice/VoiceRecording.ts @@ -25,6 +25,7 @@ import {IDestroyable} from "../utils/IDestroyable"; import {Singleflight} from "../utils/Singleflight"; import {PayloadEvent, WORKLET_NAME} from "./consts"; import {arrayFastClone} from "../utils/arrays"; +import {UPDATE_EVENT} from "../stores/AsyncStore"; const CHANNELS = 1; // stereo isn't important const SAMPLE_RATE = 48000; // 48khz is what WebRTC uses. 12khz is where we lose quality. @@ -79,6 +80,16 @@ export class VoiceRecording extends EventEmitter implements IDestroyable { return this.recorderContext.currentTime; } + public get isRecording(): boolean { + return this.recording; + } + + public emit(event: string, ...args: any[]): boolean { + super.emit(event, ...args); + super.emit(UPDATE_EVENT, event, ...args); + return true; // we don't ever care if the event had listeners, so just return "yes" + } + private async makeRecorder() { this.recorderStream = await navigator.mediaDevices.getUserMedia({ audio: { diff --git a/test/utils/arrays-test.ts b/test/utils/arrays-test.ts index ececd274b2..c5be59ab43 100644 --- a/test/utils/arrays-test.ts +++ b/test/utils/arrays-test.ts @@ -22,6 +22,7 @@ import { arrayHasOrderChange, arrayMerge, arraySeed, + arrayTrimFill, arrayUnion, ArrayUtil, GroupedArray, @@ -64,6 +65,38 @@ describe('arrays', () => { }); }); + describe('arrayTrimFill', () => { + it('should shrink arrays', () => { + const input = [1, 2, 3]; + const output = [1, 2]; + const seed = [4, 5, 6]; + const result = arrayTrimFill(input, output.length, seed); + expect(result).toBeDefined(); + expect(result).toHaveLength(output.length); + expect(result).toEqual(output); + }); + + it('should expand arrays', () => { + const input = [1, 2, 3]; + const output = [1, 2, 3, 4, 5]; + const seed = [4, 5, 6]; + const result = arrayTrimFill(input, output.length, seed); + expect(result).toBeDefined(); + expect(result).toHaveLength(output.length); + expect(result).toEqual(output); + }); + + it('should keep arrays the same', () => { + const input = [1, 2, 3]; + const output = [1, 2, 3]; + const seed = [4, 5, 6]; + const result = arrayTrimFill(input, output.length, seed); + expect(result).toBeDefined(); + expect(result).toHaveLength(output.length); + expect(result).toEqual(output); + }); + }); + describe('arraySeed', () => { it('should create an array of given length', () => { const val = 1;