+ if (this.state.recordingPhase !== RecordingState.Started) {
+ // TODO: @@ TR: Should we disable this during upload? What does a failed upload look like?
+ return
;
+ }
+
+ // only other UI is the recording-in-progress UI
+ return
;
}
- public render() {
- const classes = classNames({
- 'mx_MessageComposer_button': !this.state.recorder,
- 'mx_MessageComposer_voiceMessage': !this.state.recorder,
- 'mx_VoiceRecordComposerTile_stop': !!this.state.recorder,
- });
+ public render(): ReactNode {
+ let recordingInfo;
+ let deleteButton;
+ if (!this.state.recordingPhase || this.state.recordingPhase === RecordingState.Started) {
+ const classes = classNames({
+ 'mx_MessageComposer_button': !this.state.recorder,
+ 'mx_MessageComposer_voiceMessage': !this.state.recorder,
+ 'mx_VoiceRecordComposerTile_stop': this.state.recorder?.isRecording,
+ });
- let tooltip = _t("Record a voice message");
- if (!!this.state.recorder) {
- // TODO: @@ TravisR: Change to match behaviour
- tooltip = _t("Stop & send recording");
+ let tooltip = _t("Record a voice message");
+ if (!!this.state.recorder) {
+ tooltip = _t("Stop the recording");
+ }
+
+ let stopOrRecordBtn =
;
+ if (this.state.recorder && !this.state.recorder?.isRecording) {
+ stopOrRecordBtn = null;
+ }
+
+ recordingInfo = stopOrRecordBtn;
+ }
+
+ if (this.state.recorder && this.state.recordingPhase !== RecordingState.Uploading) {
+ deleteButton =
;
}
return (<>
+ {deleteButton}
{this.renderWaveformArea()}
-
+ {recordingInfo}
);
}
}
diff --git a/src/components/views/voice_messages/Clock.tsx b/src/components/views/voice_messages/Clock.tsx
index 6c256957e9..23e6762c52 100644
--- a/src/components/views/voice_messages/Clock.tsx
+++ b/src/components/views/voice_messages/Clock.tsx
@@ -29,14 +29,20 @@ interface IState {
* displayed, making it possible to see "82:29".
*/
@replaceableComponent("views.voice_messages.Clock")
-export default class Clock extends React.PureComponent
{
+export default class Clock extends React.Component {
public constructor(props) {
super(props);
}
+ shouldComponentUpdate(nextProps: Readonly, nextState: Readonly, nextContext: any): boolean {
+ const currentFloor = Math.floor(this.props.seconds);
+ const nextFloor = Math.floor(nextProps.seconds);
+ return currentFloor !== nextFloor;
+ }
+
public render() {
const minutes = Math.floor(this.props.seconds / 60).toFixed(0).padStart(2, '0');
- const seconds = Math.round(this.props.seconds % 60).toFixed(0).padStart(2, '0'); // hide millis
+ const seconds = Math.floor(this.props.seconds % 60).toFixed(0).padStart(2, '0'); // hide millis
return {minutes}:{seconds};
}
}
diff --git a/src/components/views/voice_messages/LiveRecordingClock.tsx b/src/components/views/voice_messages/LiveRecordingClock.tsx
index 5e9006c6ab..b82539eb16 100644
--- a/src/components/views/voice_messages/LiveRecordingClock.tsx
+++ b/src/components/views/voice_messages/LiveRecordingClock.tsx
@@ -31,7 +31,7 @@ interface IState {
* A clock for a live recording.
*/
@replaceableComponent("views.voice_messages.LiveRecordingClock")
-export default class LiveRecordingClock extends React.Component {
+export default class LiveRecordingClock extends React.PureComponent {
public constructor(props) {
super(props);
@@ -39,12 +39,6 @@ export default class LiveRecordingClock extends React.Component
this.props.recorder.liveData.onUpdate(this.onRecordingUpdate);
}
- shouldComponentUpdate(nextProps: Readonly, nextState: Readonly, nextContext: any): boolean {
- const currentFloor = Math.floor(this.state.seconds);
- const nextFloor = Math.floor(nextState.seconds);
- return currentFloor !== nextFloor;
- }
-
private onRecordingUpdate = (update: IRecordingUpdate) => {
this.setState({seconds: update.timeSeconds});
};
diff --git a/src/components/views/voice_messages/LiveRecordingWaveform.tsx b/src/components/views/voice_messages/LiveRecordingWaveform.tsx
index c1f5e97fff..e7c34c9177 100644
--- a/src/components/views/voice_messages/LiveRecordingWaveform.tsx
+++ b/src/components/views/voice_messages/LiveRecordingWaveform.tsx
@@ -20,6 +20,7 @@ import {replaceableComponent} from "../../../utils/replaceableComponent";
import {arrayFastResample, arraySeed} from "../../../utils/arrays";
import {percentageOf} from "../../../utils/numbers";
import Waveform from "./Waveform";
+import {PLAYBACK_WAVEFORM_SAMPLES} from "../../../voice/Playback";
interface IProps {
recorder: VoiceRecording;
@@ -29,8 +30,6 @@ interface IState {
heights: number[];
}
-const DOWNSAMPLE_TARGET = 35; // number of bars we want
-
/**
* A waveform which shows the waveform of a live recording
*/
@@ -39,14 +38,14 @@ export default class LiveRecordingWaveform extends React.PureComponent {
// The waveform and the downsample target are pretty close, so we should be fine to
// do this, despite the docs on arrayFastResample.
- const bars = arrayFastResample(Array.from(update.waveform), DOWNSAMPLE_TARGET);
+ const bars = arrayFastResample(Array.from(update.waveform), PLAYBACK_WAVEFORM_SAMPLES);
this.setState({
// The incoming data is between zero and one, but typically even screaming into a
// microphone won't send you over 0.6, so we artificially adjust the gain for the
diff --git a/src/components/views/voice_messages/PlayPauseButton.tsx b/src/components/views/voice_messages/PlayPauseButton.tsx
new file mode 100644
index 0000000000..1f87eb012d
--- /dev/null
+++ b/src/components/views/voice_messages/PlayPauseButton.tsx
@@ -0,0 +1,61 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React, {ReactNode} from "react";
+import {replaceableComponent} from "../../../utils/replaceableComponent";
+import AccessibleTooltipButton from "../elements/AccessibleTooltipButton";
+import {_t} from "../../../languageHandler";
+import {Playback, PlaybackState} from "../../../voice/Playback";
+import classNames from "classnames";
+
+interface IProps {
+ // Playback instance to manipulate. Cannot change during the component lifecycle.
+ playback: Playback;
+
+ // The playback phase to render. Able to change during the component lifecycle.
+ playbackPhase: PlaybackState;
+}
+
+/**
+ * Displays a play/pause button (activating the play/pause function of the recorder)
+ * to be displayed in reference to a recording.
+ */
+@replaceableComponent("views.voice_messages.PlayPauseButton")
+export default class PlayPauseButton extends React.PureComponent {
+ public constructor(props) {
+ super(props);
+ }
+
+ private onClick = async () => {
+ await this.props.playback.toggle();
+ };
+
+ public render(): ReactNode {
+ const isPlaying = this.props.playback.isPlaying;
+ const isDisabled = this.props.playbackPhase === PlaybackState.Decoding;
+ const classes = classNames('mx_PlayPauseButton', {
+ 'mx_PlayPauseButton_play': !isPlaying,
+ 'mx_PlayPauseButton_pause': isPlaying,
+ 'mx_PlayPauseButton_disabled': isDisabled,
+ });
+ return ;
+ }
+}
diff --git a/src/components/views/voice_messages/PlaybackClock.tsx b/src/components/views/voice_messages/PlaybackClock.tsx
new file mode 100644
index 0000000000..2e8ec9a3e7
--- /dev/null
+++ b/src/components/views/voice_messages/PlaybackClock.tsx
@@ -0,0 +1,71 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React from "react";
+import {replaceableComponent} from "../../../utils/replaceableComponent";
+import Clock from "./Clock";
+import {Playback, PlaybackState} from "../../../voice/Playback";
+import {UPDATE_EVENT} from "../../../stores/AsyncStore";
+
+interface IProps {
+ playback: Playback;
+}
+
+interface IState {
+ seconds: number;
+ durationSeconds: number;
+ playbackPhase: PlaybackState;
+}
+
+/**
+ * A clock for a playback of a recording.
+ */
+@replaceableComponent("views.voice_messages.PlaybackClock")
+export default class PlaybackClock extends React.PureComponent {
+ public constructor(props) {
+ super(props);
+
+ this.state = {
+ seconds: this.props.playback.clockInfo.timeSeconds,
+ // we track the duration on state because we won't really know what the clip duration
+ // is until the first time update, and as a PureComponent we are trying to dedupe state
+ // updates as much as possible. This is just the easiest way to avoid a forceUpdate() or
+ // member property to track "did we get a duration".
+ durationSeconds: this.props.playback.clockInfo.durationSeconds,
+ playbackPhase: PlaybackState.Stopped, // assume not started, so full clock
+ };
+ this.props.playback.on(UPDATE_EVENT, this.onPlaybackUpdate);
+ this.props.playback.clockInfo.liveData.onUpdate(this.onTimeUpdate);
+ }
+
+ private onPlaybackUpdate = (ev: PlaybackState) => {
+ // Convert Decoding -> Stopped because we don't care about the distinction here
+ if (ev === PlaybackState.Decoding) ev = PlaybackState.Stopped;
+ this.setState({playbackPhase: ev});
+ };
+
+ private onTimeUpdate = (time: number[]) => {
+ this.setState({seconds: time[0], durationSeconds: time[1]});
+ };
+
+ public render() {
+ let seconds = this.state.seconds;
+ if (this.state.playbackPhase === PlaybackState.Stopped) {
+ seconds = this.state.durationSeconds;
+ }
+ return ;
+ }
+}
diff --git a/src/components/views/voice_messages/PlaybackWaveform.tsx b/src/components/views/voice_messages/PlaybackWaveform.tsx
new file mode 100644
index 0000000000..123af5dfa5
--- /dev/null
+++ b/src/components/views/voice_messages/PlaybackWaveform.tsx
@@ -0,0 +1,68 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React from "react";
+import {replaceableComponent} from "../../../utils/replaceableComponent";
+import {arraySeed, arrayTrimFill} from "../../../utils/arrays";
+import Waveform from "./Waveform";
+import {Playback, PLAYBACK_WAVEFORM_SAMPLES} from "../../../voice/Playback";
+import {percentageOf} from "../../../utils/numbers";
+
+interface IProps {
+ playback: Playback;
+}
+
+interface IState {
+ heights: number[];
+ progress: number;
+}
+
+/**
+ * A waveform which shows the waveform of a previously recorded recording
+ */
+@replaceableComponent("views.voice_messages.PlaybackWaveform")
+export default class PlaybackWaveform extends React.PureComponent {
+ public constructor(props) {
+ super(props);
+
+ this.state = {
+ heights: this.toHeights(this.props.playback.waveform),
+ progress: 0, // default no progress
+ };
+
+ this.props.playback.waveformData.onUpdate(this.onWaveformUpdate);
+ this.props.playback.clockInfo.liveData.onUpdate(this.onTimeUpdate);
+ }
+
+ private toHeights(waveform: number[]) {
+ const seed = arraySeed(0, PLAYBACK_WAVEFORM_SAMPLES);
+ return arrayTrimFill(waveform, PLAYBACK_WAVEFORM_SAMPLES, seed);
+ }
+
+ private onWaveformUpdate = (waveform: number[]) => {
+ this.setState({heights: this.toHeights(waveform)});
+ };
+
+ private onTimeUpdate = (time: number[]) => {
+ // Track percentages to very coarse precision, otherwise 0.002 ends up highlighting a bar.
+ const progress = Number(percentageOf(time[0], 0, time[1]).toFixed(1));
+ this.setState({progress});
+ };
+
+ public render() {
+ return ;
+ }
+}
diff --git a/src/components/views/voice_messages/RecordingPlayback.tsx b/src/components/views/voice_messages/RecordingPlayback.tsx
new file mode 100644
index 0000000000..776997cec2
--- /dev/null
+++ b/src/components/views/voice_messages/RecordingPlayback.tsx
@@ -0,0 +1,62 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {Playback, PlaybackState} from "../../../voice/Playback";
+import React, {ReactNode} from "react";
+import {UPDATE_EVENT} from "../../../stores/AsyncStore";
+import PlaybackWaveform from "./PlaybackWaveform";
+import PlayPauseButton from "./PlayPauseButton";
+import PlaybackClock from "./PlaybackClock";
+
+interface IProps {
+ // Playback instance to render. Cannot change during component lifecycle: create
+ // an all-new component instead.
+ playback: Playback;
+}
+
+interface IState {
+ playbackPhase: PlaybackState;
+}
+
+export default class RecordingPlayback extends React.PureComponent {
+ constructor(props: IProps) {
+ super(props);
+
+ this.state = {
+ playbackPhase: PlaybackState.Decoding, // default assumption
+ };
+
+ // We don't need to de-register: the class handles this for us internally
+ this.props.playback.on(UPDATE_EVENT, this.onPlaybackUpdate);
+
+ // Don't wait for the promise to complete - it will emit a progress update when it
+ // is done, and it's not meant to take long anyhow.
+ // noinspection JSIgnoredPromiseFromCall
+ this.props.playback.prepare();
+ }
+
+ private onPlaybackUpdate = (ev: PlaybackState) => {
+ this.setState({playbackPhase: ev});
+ };
+
+ public render(): ReactNode {
+ return
+ }
+}
diff --git a/src/components/views/voice_messages/Waveform.tsx b/src/components/views/voice_messages/Waveform.tsx
index 5fa68dcadc..840a5a12b3 100644
--- a/src/components/views/voice_messages/Waveform.tsx
+++ b/src/components/views/voice_messages/Waveform.tsx
@@ -16,9 +16,11 @@ limitations under the License.
import React from "react";
import {replaceableComponent} from "../../../utils/replaceableComponent";
+import classNames from "classnames";
interface IProps {
relHeights: number[]; // relative heights (0-1)
+ progress: number; // percent complete, 0-1, default 100%
}
interface IState {
@@ -28,9 +30,16 @@ interface IState {
* A simple waveform component. This renders bars (centered vertically) for each
* height provided in the component properties. Updating the properties will update
* the rendered waveform.
+ *
+ * For CSS purposes, a mx_Waveform_bar_100pct class is added when the bar should be
+ * "filled", as a demonstration of the progress property.
*/
@replaceableComponent("views.voice_messages.Waveform")
export default class Waveform extends React.PureComponent {
+ public static defaultProps = {
+ progress: 1,
+ };
+
public constructor(props) {
super(props);
}
@@ -38,7 +47,13 @@ export default class Waveform extends React.PureComponent {
public render() {
return
{this.props.relHeights.map((h, i) => {
- return ;
+ const progress = this.props.progress;
+ const isCompleteBar = (i / this.props.relHeights.length) <= progress && progress > 0;
+ const classes = classNames({
+ 'mx_Waveform_bar': true,
+ 'mx_Waveform_bar_100pct': isCompleteBar,
+ });
+ return ;
})}
;
}
diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json
index c6f7a8d25e..264c35144e 100644
--- a/src/i18n/strings/en_EN.json
+++ b/src/i18n/strings/en_EN.json
@@ -899,6 +899,8 @@
"Incoming call": "Incoming call",
"Decline": "Decline",
"Accept": "Accept",
+ "Pause": "Pause",
+ "Play": "Play",
"The other party cancelled the verification.": "The other party cancelled the verification.",
"Verified!": "Verified!",
"You've successfully verified this user.": "You've successfully verified this user.",
@@ -1645,7 +1647,8 @@
"Jump to first unread message.": "Jump to first unread message.",
"Mark all as read": "Mark all as read",
"Record a voice message": "Record a voice message",
- "Stop & send recording": "Stop & send recording",
+ "Stop the recording": "Stop the recording",
+ "Delete recording": "Delete recording",
"Error updating main address": "Error updating main address",
"There was an error updating the room's main address. It may not be allowed by the server or a temporary failure occurred.": "There was an error updating the room's main address. It may not be allowed by the server or a temporary failure occurred.",
"There was an error updating the room's alternative addresses. It may not be allowed by the server or a temporary failure occurred.": "There was an error updating the room's alternative addresses. It may not be allowed by the server or a temporary failure occurred.",
diff --git a/src/stores/VoiceRecordingStore.ts b/src/stores/VoiceRecordingStore.ts
index cc999f23f8..8ee44359fb 100644
--- a/src/stores/VoiceRecordingStore.ts
+++ b/src/stores/VoiceRecordingStore.ts
@@ -78,3 +78,5 @@ export class VoiceRecordingStore extends AsyncStoreWithClient {
return this.updateState({recording: null});
}
}
+
+window.mxVoiceRecordingStore = VoiceRecordingStore.instance;
diff --git a/src/utils/arrays.ts b/src/utils/arrays.ts
index cea377bfe9..f7e693452b 100644
--- a/src/utils/arrays.ts
+++ b/src/utils/arrays.ts
@@ -73,6 +73,26 @@ export function arraySeed(val: T, length: number): T[] {
return a;
}
+/**
+ * Trims or fills the array to ensure it meets the desired length. The seed array
+ * given is pulled from to fill any missing slots - it is recommended that this be
+ * at least `len` long. The resulting array will be exactly `len` long, either
+ * trimmed from the source or filled with the some/all of the seed array.
+ * @param {T[]} a The array to trim/fill.
+ * @param {number} len The length to trim or fill to, as needed.
+ * @param {T[]} seed Values to pull from if the array needs filling.
+ * @returns {T[]} The resulting array of `len` length.
+ */
+export function arrayTrimFill(a: T[], len: number, seed: T[]): T[] {
+ // Dev note: we do length checks because the spread operator can result in some
+ // performance penalties in more critical code paths. As a utility, it should be
+ // as fast as possible to not cause a problem for the call stack, no matter how
+ // critical that stack is.
+ if (a.length === len) return a;
+ if (a.length > len) return a.slice(0, len);
+ return a.concat(seed.slice(0, len - a.length));
+}
+
/**
* Clones an array as fast as possible, retaining references of the array's values.
* @param a The array to clone. Must be defined.
diff --git a/src/voice/Playback.ts b/src/voice/Playback.ts
new file mode 100644
index 0000000000..99b1f62866
--- /dev/null
+++ b/src/voice/Playback.ts
@@ -0,0 +1,141 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import EventEmitter from "events";
+import {UPDATE_EVENT} from "../stores/AsyncStore";
+import {arrayFastResample, arraySeed} from "../utils/arrays";
+import {SimpleObservable} from "matrix-widget-api";
+import {IDestroyable} from "../utils/IDestroyable";
+import {PlaybackClock} from "./PlaybackClock";
+
+export enum PlaybackState {
+ Decoding = "decoding",
+ Stopped = "stopped", // no progress on timeline
+ Paused = "paused", // some progress on timeline
+ Playing = "playing", // active progress through timeline
+}
+
+export const PLAYBACK_WAVEFORM_SAMPLES = 35;
+const DEFAULT_WAVEFORM = arraySeed(0, PLAYBACK_WAVEFORM_SAMPLES);
+
+export class Playback extends EventEmitter implements IDestroyable {
+ private readonly context: AudioContext;
+ private source: AudioBufferSourceNode;
+ private state = PlaybackState.Decoding;
+ private audioBuf: AudioBuffer;
+ private resampledWaveform: number[];
+ private waveformObservable = new SimpleObservable();
+ private readonly clock: PlaybackClock;
+
+ /**
+ * Creates a new playback instance from a buffer.
+ * @param {ArrayBuffer} buf The buffer containing the sound sample.
+ * @param {number[]} seedWaveform Optional seed waveform to present until the proper waveform
+ * can be calculated. Contains values between zero and one, inclusive.
+ */
+ constructor(private buf: ArrayBuffer, seedWaveform = DEFAULT_WAVEFORM) {
+ super();
+ this.context = new AudioContext();
+ this.resampledWaveform = arrayFastResample(seedWaveform, PLAYBACK_WAVEFORM_SAMPLES);
+ this.waveformObservable.update(this.resampledWaveform);
+ this.clock = new PlaybackClock(this.context);
+
+ // TODO: @@ TR: Calculate real waveform
+ }
+
+ public get waveform(): number[] {
+ return this.resampledWaveform;
+ }
+
+ public get waveformData(): SimpleObservable {
+ return this.waveformObservable;
+ }
+
+ public get clockInfo(): PlaybackClock {
+ return this.clock;
+ }
+
+ public get currentState(): PlaybackState {
+ return this.state;
+ }
+
+ public get isPlaying(): boolean {
+ return this.currentState === PlaybackState.Playing;
+ }
+
+ public emit(event: PlaybackState, ...args: any[]): boolean {
+ this.state = event;
+ super.emit(event, ...args);
+ super.emit(UPDATE_EVENT, event, ...args);
+ return true; // we don't ever care if the event had listeners, so just return "yes"
+ }
+
+ public destroy() {
+ // noinspection JSIgnoredPromiseFromCall - not concerned about being called async here
+ this.stop();
+ this.removeAllListeners();
+ this.clock.destroy();
+ this.waveformObservable.close();
+ }
+
+ public async prepare() {
+ this.audioBuf = await this.context.decodeAudioData(this.buf);
+ this.emit(PlaybackState.Stopped); // signal that we're not decoding anymore
+ this.clock.durationSeconds = this.audioBuf.duration;
+ }
+
+ private onPlaybackEnd = async () => {
+ await this.context.suspend();
+ this.emit(PlaybackState.Stopped);
+ };
+
+ public async play() {
+ // We can't restart a buffer source, so we need to create a new one if we hit the end
+ if (this.state === PlaybackState.Stopped) {
+ if (this.source) {
+ this.source.disconnect();
+ this.source.removeEventListener("ended", this.onPlaybackEnd);
+ }
+
+ this.source = this.context.createBufferSource();
+ this.source.connect(this.context.destination);
+ this.source.buffer = this.audioBuf;
+ this.source.start(); // start immediately
+ this.source.addEventListener("ended", this.onPlaybackEnd);
+ }
+
+ // We use the context suspend/resume functions because it allows us to pause a source
+ // node, but that still doesn't help us when the source node runs out (see above).
+ await this.context.resume();
+ this.clock.flagStart();
+ this.emit(PlaybackState.Playing);
+ }
+
+ public async pause() {
+ await this.context.suspend();
+ this.emit(PlaybackState.Paused);
+ }
+
+ public async stop() {
+ await this.onPlaybackEnd();
+ this.clock.flagStop();
+ }
+
+ public async toggle() {
+ if (this.isPlaying) await this.pause();
+ else await this.play();
+ }
+}
diff --git a/src/voice/PlaybackClock.ts b/src/voice/PlaybackClock.ts
new file mode 100644
index 0000000000..06d6381691
--- /dev/null
+++ b/src/voice/PlaybackClock.ts
@@ -0,0 +1,78 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {SimpleObservable} from "matrix-widget-api";
+import {IDestroyable} from "../utils/IDestroyable";
+
+// Because keeping track of time is sufficiently complicated...
+export class PlaybackClock implements IDestroyable {
+ private clipStart = 0;
+ private stopped = true;
+ private lastCheck = 0;
+ private observable = new SimpleObservable();
+ private timerId: number;
+ private clipDuration = 0;
+
+ public constructor(private context: AudioContext) {
+ }
+
+ public get durationSeconds(): number {
+ return this.clipDuration;
+ }
+
+ public set durationSeconds(val: number) {
+ this.clipDuration = val;
+ this.observable.update([this.timeSeconds, this.clipDuration]);
+ }
+
+ public get timeSeconds(): number {
+ return (this.context.currentTime - this.clipStart) % this.clipDuration;
+ }
+
+ public get liveData(): SimpleObservable {
+ return this.observable;
+ }
+
+ private checkTime = () => {
+ const now = this.timeSeconds;
+ if (this.lastCheck !== now) {
+ this.observable.update([now, this.durationSeconds]);
+ this.lastCheck = now;
+ }
+ };
+
+ public flagStart() {
+ if (this.stopped) {
+ this.clipStart = this.context.currentTime;
+ this.stopped = false;
+ }
+
+ if (!this.timerId) {
+ // case to number because the types are wrong
+ // 100ms interval to make sure the time is as accurate as possible
+ this.timerId = setInterval(this.checkTime, 100);
+ }
+ }
+
+ public flagStop() {
+ this.stopped = true;
+ }
+
+ public destroy() {
+ this.observable.close();
+ if (this.timerId) clearInterval(this.timerId);
+ }
+}
diff --git a/src/voice/VoiceRecording.ts b/src/voice/VoiceRecording.ts
index b0cc3cd407..eb705200ca 100644
--- a/src/voice/VoiceRecording.ts
+++ b/src/voice/VoiceRecording.ts
@@ -24,7 +24,8 @@ import EventEmitter from "events";
import {IDestroyable} from "../utils/IDestroyable";
import {Singleflight} from "../utils/Singleflight";
import {PayloadEvent, WORKLET_NAME} from "./consts";
-import {arrayFastClone} from "../utils/arrays";
+import {UPDATE_EVENT} from "../stores/AsyncStore";
+import {Playback} from "./Playback";
const CHANNELS = 1; // stereo isn't important
const SAMPLE_RATE = 48000; // 48khz is what WebRTC uses. 12khz is where we lose quality.
@@ -52,20 +53,17 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
private recorderStream: MediaStream;
private recorderFFT: AnalyserNode;
private recorderWorklet: AudioWorkletNode;
- private buffer = new Uint8Array(0);
+ private buffer = new Uint8Array(0); // use this.audioBuffer to access
private mxc: string;
private recording = false;
private observable: SimpleObservable;
private amplitudes: number[] = []; // at each second mark, generated
+ private playback: Playback;
public constructor(private client: MatrixClient) {
super();
}
- public get finalWaveform(): number[] {
- return arrayFastClone(this.amplitudes);
- }
-
public get contentType(): string {
return "audio/ogg";
}
@@ -79,6 +77,16 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
return this.recorderContext.currentTime;
}
+ public get isRecording(): boolean {
+ return this.recording;
+ }
+
+ public emit(event: string, ...args: any[]): boolean {
+ super.emit(event, ...args);
+ super.emit(UPDATE_EVENT, event, ...args);
+ return true; // we don't ever care if the event had listeners, so just return "yes"
+ }
+
private async makeRecorder() {
this.recorderStream = await navigator.mediaDevices.getUserMedia({
audio: {
@@ -154,6 +162,12 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
};
}
+ private get audioBuffer(): Uint8Array {
+ // We need a clone of the buffer to avoid accidentally changing the position
+ // on the real thing.
+ return this.buffer.slice(0);
+ }
+
public get liveData(): SimpleObservable {
if (!this.recording) throw new Error("No observable when not recording");
return this.observable;
@@ -203,8 +217,19 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
// Now that we've updated the data/waveform, let's do a time check. We don't want to
// go horribly over the limit. We also emit a warning state if needed.
- const secondsLeft = TARGET_MAX_LENGTH - timeSeconds;
- if (secondsLeft <= 0) {
+ //
+ // We use the recorder's perspective of time to make sure we don't cut off the last
+ // frame of audio, otherwise we end up with a 1:59 clip (119.68 seconds). This extra
+ // safety can allow us to overshoot the target a bit, but at least when we say 2min
+ // maximum we actually mean it.
+ //
+ // In testing, recorder time and worker time lag by about 400ms, which is roughly the
+ // time needed to encode a sample/frame.
+ //
+ // Ref for recorderSeconds: https://github.com/chris-rudmin/opus-recorder#instance-fields
+ const recorderSeconds = this.recorder.encodedSamplePosition / 48000;
+ const secondsLeft = TARGET_MAX_LENGTH - recorderSeconds;
+ if (secondsLeft < 0) { // go over to make sure we definitely capture that last frame
// noinspection JSIgnoredPromiseFromCall - we aren't concerned with it overlapping
this.stop();
} else if (secondsLeft <= TARGET_WARN_TIME_LEFT) {
@@ -239,9 +264,9 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
}
// Disconnect the source early to start shutting down resources
+ await this.recorder.stop(); // stop first to flush the last frame
this.recorderSource.disconnect();
this.recorderWorklet.disconnect();
- await this.recorder.stop();
// close the context after the recorder so the recorder doesn't try to
// connect anything to the context (this would generate a warning)
@@ -255,15 +280,33 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
await this.recorder.close();
this.emit(RecordingState.Ended);
- return this.buffer;
+ return this.audioBuffer;
});
}
+ /**
+ * Gets a playback instance for this voice recording. Note that the playback will not
+ * have been prepared fully, meaning the `prepare()` function needs to be called on it.
+ *
+ * The same playback instance is returned each time.
+ *
+ * @returns {Playback} The playback instance.
+ */
+ public getPlayback(): Playback {
+ this.playback = Singleflight.for(this, "playback").do(() => {
+ return new Playback(this.audioBuffer.buffer, this.amplitudes); // cast to ArrayBuffer proper;
+ });
+ return this.playback;
+ }
+
public destroy() {
// noinspection JSIgnoredPromiseFromCall - not concerned about stop() being called async here
this.stop();
this.removeAllListeners();
Singleflight.forgetAllFor(this);
+ // noinspection JSIgnoredPromiseFromCall - not concerned about being called async here
+ this.playback?.destroy();
+ this.observable.close();
}
public async upload(): Promise {
@@ -274,7 +317,7 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
if (this.mxc) return this.mxc;
this.emit(RecordingState.Uploading);
- this.mxc = await this.client.uploadContent(new Blob([this.buffer], {
+ this.mxc = await this.client.uploadContent(new Blob([this.audioBuffer], {
type: this.contentType,
}), {
onlyContentUri: false, // to stop the warnings in the console
@@ -283,5 +326,3 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
return this.mxc;
}
}
-
-window.mxVoiceRecorder = VoiceRecording;
diff --git a/test/utils/arrays-test.ts b/test/utils/arrays-test.ts
index ececd274b2..c5be59ab43 100644
--- a/test/utils/arrays-test.ts
+++ b/test/utils/arrays-test.ts
@@ -22,6 +22,7 @@ import {
arrayHasOrderChange,
arrayMerge,
arraySeed,
+ arrayTrimFill,
arrayUnion,
ArrayUtil,
GroupedArray,
@@ -64,6 +65,38 @@ describe('arrays', () => {
});
});
+ describe('arrayTrimFill', () => {
+ it('should shrink arrays', () => {
+ const input = [1, 2, 3];
+ const output = [1, 2];
+ const seed = [4, 5, 6];
+ const result = arrayTrimFill(input, output.length, seed);
+ expect(result).toBeDefined();
+ expect(result).toHaveLength(output.length);
+ expect(result).toEqual(output);
+ });
+
+ it('should expand arrays', () => {
+ const input = [1, 2, 3];
+ const output = [1, 2, 3, 4, 5];
+ const seed = [4, 5, 6];
+ const result = arrayTrimFill(input, output.length, seed);
+ expect(result).toBeDefined();
+ expect(result).toHaveLength(output.length);
+ expect(result).toEqual(output);
+ });
+
+ it('should keep arrays the same', () => {
+ const input = [1, 2, 3];
+ const output = [1, 2, 3];
+ const seed = [4, 5, 6];
+ const result = arrayTrimFill(input, output.length, seed);
+ expect(result).toBeDefined();
+ expect(result).toHaveLength(output.length);
+ expect(result).toEqual(output);
+ });
+ });
+
describe('arraySeed', () => {
it('should create an array of given length', () => {
const val = 1;