OpenCloud
/
riot-web
mirror of https://github.com/vector-im/riot-web
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #5935 from matrix-org/travis/voice/control-bar 

Voice messages: Composer controls
pull/21833/head
Travis Ralston 2021-04-28 09:38:28 -06:00 committed by GitHub
parent c95c1aeffd 6764b8d645
commit 8d8525cdeb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 897 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
res/css/_components.scss
View File

@ -248,6 +248,8 @@
@import "./views/toasts/_AnalyticsToast.scss";
@import "./views/toasts/_NonUrgentEchoFailureToast.scss";
@import "./views/verification/_VerificationShowSas.scss";
@import "./views/voice_messages/_PlayPauseButton.scss";
@import "./views/voice_messages/_PlaybackContainer.scss";
@import "./views/voice_messages/_Waveform.scss";
@import "./views/voip/_CallContainer.scss";
@import "./views/voip/_CallView.scss";

60
res/css/views/rooms/_VoiceRecordComposerTile.scss
View File

@ -35,44 +35,40 @@ limitations under the License.
}
}
.mx_VoiceRecordComposerTile_waveformContainer {
padding: 5px;
padding-right: 4px; // there's 1px from the waveform itself, so account for that
padding-left: 15px; // +10px for the live circle, +5px for regular padding
background-color: $voice-record-waveform-bg-color;
border-radius: 12px;
margin-right: 12px; // isolate from stop button
.mx_VoiceRecordComposerTile_delete {
width: 14px; // w&h are size of icon
height: 18px;
vertical-align: middle;
margin-right: 7px; // distance from left edge of waveform container (container has some margin too)
background-color: $muted-fg-color;
mask-repeat: no-repeat;
mask-size: contain;
mask-image: url('$(res)/img/element-icons/trashcan.svg');
}
// Cheat at alignment a bit
display: flex;
align-items: center;
.mx_VoiceMessagePrimaryContainer {
// Note: remaining class properties are in the PlayerContainer CSS.
margin: 6px; // force the composer area to put a gutter around us
margin-right: 12px; // isolate from stop button
position: relative; // important for the live circle
color: $voice-record-waveform-fg-color;
font-size: $font-14px;
&.mx_VoiceRecordComposerTile_recording {
padding-left: 16px; // +10px for the live circle, +6px for regular padding
&::before {
animation: recording-pulse 2s infinite;
&::before {
animation: recording-pulse 2s infinite;
content: '';
background-color: $voice-record-live-circle-color;
width: 10px;
height: 10px;
position: absolute;
left: 8px;
top: 16px; // vertically center
border-radius: 10px;
}
.mx_Waveform_bar {
background-color: $voice-record-waveform-fg-color;
}
.mx_Clock {
padding-right: 8px; // isolate from waveform
padding-left: 10px; // isolate from live circle
width: 42px; // we're not using a monospace font, so fake it
content: '';
background-color: $voice-record-live-circle-color;
width: 10px;
height: 10px;
position: absolute;
left: 8px;
top: 16px; // vertically center
border-radius: 10px;
}
}
}

51
res/css/views/voice_messages/_PlayPauseButton.scss Normal file
View File

@ -0,0 +1,51 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
.mx_PlayPauseButton {
position: relative;
width: 32px;
height: 32px;
border-radius: 32px;
background-color: $primary-bg-color;
&::before {
content: '';
position: absolute; // sizing varies by icon
background-color: $muted-fg-color;
mask-repeat: no-repeat;
mask-size: contain;
}
&.mx_PlayPauseButton_disabled::before {
opacity: 0.5;
}
&.mx_PlayPauseButton_play::before {
width: 13px;
height: 16px;
top: 8px; // center
left: 12px; // center
mask-image: url('$(res)/img/element-icons/play.svg');
}
&.mx_PlayPauseButton_pause::before {
width: 10px;
height: 12px;
top: 10px; // center
left: 11px; // center
mask-image: url('$(res)/img/element-icons/pause.svg');
}
}

54
res/css/views/voice_messages/_PlaybackContainer.scss Normal file
View File

@ -0,0 +1,54 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Dev note: there's no actual component called <PlaybackContainer />. These classes
// are shared amongst multiple voice message components.
// Container for live recording and playback controls
.mx_VoiceMessagePrimaryContainer {
padding: 6px; // makes us 4px taller than the send/stop button
padding-right: 5px; // there's 1px from the waveform itself, so account for that
background-color: $voice-record-waveform-bg-color;
border-radius: 12px;
// Cheat at alignment a bit
display: flex;
align-items: center;
color: $voice-record-waveform-fg-color;
font-size: $font-14px;
.mx_Waveform {
// We want the bars to be 2px shorter than the play/pause button in the waveform control
height: 28px; // default is 30px, so we're subtracting the 2px border off the bars
.mx_Waveform_bar {
background-color: $voice-record-waveform-incomplete-fg-color;
&.mx_Waveform_bar_100pct {
// Small animation to remove the mechanical feel of progress
transition: background-color 250ms ease;
background-color: $voice-record-waveform-fg-color;
}
}
}
.mx_Clock {
padding-right: 4px; // isolate from waveform
padding-left: 8px; // isolate from live circle
width: 40px; // we're not using a monospace font, so fake it
}
}

4
res/img/element-icons/pause.svg Normal file
View File

@ -0,0 +1,4 @@
<svg width="10" height="12" viewBox="0 0 10 12" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M0 1C0 0.447715 0.447715 0 1 0H2C2.55228 0 3 0.447715 3 1V11C3 11.5523 2.55228 12 2 12H1C0.447715 12 0 11.5523 0 11V1Z" fill="#737D8C"/>
<path d="M7 1C7 0.447715 7.44772 0 8 0H9C9.55228 0 10 0.447715 10 1V11C10 11.5523 9.55228 12 9 12H8C7.44772 12 7 11.5523 7 11V1Z" fill="#737D8C"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 396 B

3
res/img/element-icons/play.svg Normal file
View File

@ -0,0 +1,3 @@
<svg width="13" height="16" viewBox="0 0 13 16" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M0 14.2104V1.78956C0 1.00724 0.857827 0.527894 1.5241 0.937906L11.6161 7.14834C12.2506 7.53883 12.2506 8.46117 11.6161 8.85166L1.5241 15.0621C0.857828 15.4721 0 14.9928 0 14.2104Z" fill="#737D8C"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 310 B

1
res/themes/legacy-light/css/_legacy-light.scss
View File

@ -196,6 +196,7 @@ $voice-record-stop-border-color: #E3E8F0;
$voice-record-stop-symbol-color: #ff4b55;
$voice-record-waveform-bg-color: #E3E8F0;
$voice-record-waveform-fg-color: $muted-fg-color;
$voice-record-waveform-incomplete-fg-color: #C1C6CD;
$voice-record-live-circle-color: #ff4b55;
$roomtile-preview-color: #9e9e9e;

1
res/themes/light/css/_light.scss
View File

@ -186,6 +186,7 @@ $voice-record-stop-border-color: #E3E8F0;
$voice-record-stop-symbol-color: #ff4b55; // $warning-color, but without letting people change it in themes
$voice-record-waveform-bg-color: #E3E8F0;
$voice-record-waveform-fg-color: $muted-fg-color;
$voice-record-waveform-incomplete-fg-color: #C1C6CD;
$voice-record-live-circle-color: #ff4b55; // $warning-color, but without letting people change it in themes
$roomtile-preview-color: $secondary-fg-color;

4
src/@types/global.d.ts vendored
View File

@ -39,9 +39,9 @@ import {ModalWidgetStore} from "../stores/ModalWidgetStore";
import { WidgetLayoutStore } from "../stores/widgets/WidgetLayoutStore";
import VoipUserMapper from "../VoipUserMapper";
import {SpaceStoreClass} from "../stores/SpaceStore";
import {VoiceRecording} from "../voice/VoiceRecording";
import TypingStore from "../stores/TypingStore";
import { EventIndexPeg } from "../indexing/EventIndexPeg";
import {VoiceRecordingStore} from "../stores/VoiceRecordingStore";
declare global {
interface Window {
@ -73,7 +73,7 @@ declare global {
mxModalWidgetStore: ModalWidgetStore;
mxVoipUserMapper: VoipUserMapper;
mxSpaceStore: SpaceStoreClass;
mxVoiceRecorder: typeof VoiceRecording;
mxVoiceRecordingStore: VoiceRecordingStore;
mxTypingStore: TypingStore;
mxEventIndexPeg: EventIndexPeg;
}

12
src/components/views/rooms/MessageComposer.tsx
View File

@ -198,6 +198,7 @@ interface IState {
export default class MessageComposer extends React.Component<IProps, IState> {
private dispatcherRef: string;
private messageComposerInput: SendMessageComposer;
private voiceRecordingButton: VoiceRecordComposerTile;
constructor(props) {
super(props);
@ -322,7 +323,15 @@ export default class MessageComposer extends React.Component<IProps, IState> {
});
}
sendMessage = () => {
sendMessage = async () => {
if (this.state.haveRecording && this.voiceRecordingButton) {
// There shouldn't be any text message to send when a voice recording is active, so
// just send out the voice recording.
await this.voiceRecordingButton.send();
return;
}
// XXX: Private function access
this.messageComposerInput._sendMessage();
}
@ -387,6 +396,7 @@ export default class MessageComposer extends React.Component<IProps, IState> {
if (SettingsStore.getValue("feature_voice_messages")) {
controls.push(<VoiceRecordComposerTile
key="controls_voice_record"
ref={c => this.voiceRecordingButton = c}
room={this.props.room} />);
}

179
src/components/views/rooms/VoiceRecordComposerTile.tsx
View File

@ -16,8 +16,8 @@ limitations under the License.
import AccessibleTooltipButton from "../elements/AccessibleTooltipButton";
import {_t} from "../../../languageHandler";
import React from "react";
import {VoiceRecording} from "../../../voice/VoiceRecording";
import React, {ReactNode} from "react";
import {RecordingState, VoiceRecording} from "../../../voice/VoiceRecording";
import {Room} from "matrix-js-sdk/src/models/room";
import {MatrixClientPeg} from "../../../MatrixClientPeg";
import classNames from "classnames";
@ -25,6 +25,8 @@ import LiveRecordingWaveform from "../voice_messages/LiveRecordingWaveform";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import LiveRecordingClock from "../voice_messages/LiveRecordingClock";
import {VoiceRecordingStore} from "../../../stores/VoiceRecordingStore";
import {UPDATE_EVENT} from "../../../stores/AsyncStore";
import RecordingPlayback from "../voice_messages/RecordingPlayback";
interface IProps {
room: Room;
@ -32,6 +34,7 @@ interface IProps {
interface IState {
recorder?: VoiceRecording;
recordingPhase?: RecordingState;
}
/**
@ -43,87 +46,141 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
super(props);
this.state = {
recorder: null, // not recording by default
recorder: null, // no recording started by default
};
}
private onStartStopVoiceMessage = async () => {
// TODO: @@ TravisR: We do not want to auto-send on stop.
public async componentWillUnmount() {
await VoiceRecordingStore.instance.disposeRecording();
}
// called by composer
public async send() {
if (!this.state.recorder) {
throw new Error("No recording started - cannot send anything");
}
await this.state.recorder.stop();
const mxc = await this.state.recorder.upload();
MatrixClientPeg.get().sendMessage(this.props.room.roomId, {
"body": "Voice message",
"msgtype": "org.matrix.msc2516.voice",
//"msgtype": MsgType.Audio,
"url": mxc,
"info": {
duration: Math.round(this.state.recorder.durationSeconds * 1000),
mimetype: this.state.recorder.contentType,
size: this.state.recorder.contentLength,
},
// MSC1767 experiment
"org.matrix.msc1767.text": "Voice message",
"org.matrix.msc1767.file": {
url: mxc,
name: "Voice message.ogg",
mimetype: this.state.recorder.contentType,
size: this.state.recorder.contentLength,
},
"org.matrix.msc1767.audio": {
duration: Math.round(this.state.recorder.durationSeconds * 1000),
// TODO: @@ TravisR: Waveform? (MSC1767 decision)
},
"org.matrix.experimental.msc2516.voice": { // MSC2516+MSC1767 experiment
duration: Math.round(this.state.recorder.durationSeconds * 1000),
// Events can't have floats, so we try to maintain resolution by using 1024
// as a maximum value. The waveform contains values between zero and 1, so this
// should come out largely sane.
//
// We're expecting about one data point per second of audio.
waveform: this.state.recorder.getPlayback().waveform.map(v => Math.round(v * 1024)),
},
});
await this.disposeRecording();
}
private async disposeRecording() {
await VoiceRecordingStore.instance.disposeRecording();
// Reset back to no recording, which means no phase (ie: restart component entirely)
this.setState({recorder: null, recordingPhase: null});
}
private onCancel = async () => {
await this.disposeRecording();
};
private onRecordStartEndClick = async () => {
if (this.state.recorder) {
await this.state.recorder.stop();
const mxc = await this.state.recorder.upload();
MatrixClientPeg.get().sendMessage(this.props.room.roomId, {
"body": "Voice message",
"msgtype": "org.matrix.msc2516.voice",
//"msgtype": MsgType.Audio,
"url": mxc,
"info": {
duration: Math.round(this.state.recorder.durationSeconds * 1000),
mimetype: this.state.recorder.contentType,
size: this.state.recorder.contentLength,
},
// MSC1767 experiment
"org.matrix.msc1767.text": "Voice message",
"org.matrix.msc1767.file": {
url: mxc,
name: "Voice message.ogg",
mimetype: this.state.recorder.contentType,
size: this.state.recorder.contentLength,
},
"org.matrix.msc1767.audio": {
duration: Math.round(this.state.recorder.durationSeconds * 1000),
// TODO: @@ TravisR: Waveform? (MSC1767 decision)
},
"org.matrix.experimental.msc2516.voice": { // MSC2516+MSC1767 experiment
duration: Math.round(this.state.recorder.durationSeconds * 1000),
// Events can't have floats, so we try to maintain resolution by using 1024
// as a maximum value. The waveform contains values between zero and 1, so this
// should come out largely sane.
//
// We're expecting about one data point per second of audio.
waveform: this.state.recorder.finalWaveform.map(v => Math.round(v * 1024)),
},
});
await VoiceRecordingStore.instance.disposeRecording();
this.setState({recorder: null});
return;
}
const recorder = VoiceRecordingStore.instance.startRecording();
await recorder.start();
this.setState({recorder});
// We don't need to remove the listener: the recorder will clean that up for us.
recorder.on(UPDATE_EVENT, (ev: RecordingState) => {
if (ev === RecordingState.EndingSoon) return; // ignore this state: it has no UI purpose here
this.setState({recordingPhase: ev});
});
this.setState({recorder, recordingPhase: RecordingState.Started});
};
private renderWaveformArea() {
if (!this.state.recorder) return null;
private renderWaveformArea(): ReactNode {
if (!this.state.recorder) return null; // no recorder means we're not recording: no waveform
return <div className='mx_VoiceRecordComposerTile_waveformContainer'>
if (this.state.recordingPhase !== RecordingState.Started) {
// TODO: @@ TR: Should we disable this during upload? What does a failed upload look like?
return <RecordingPlayback playback={this.state.recorder.getPlayback()} />;
}
// only other UI is the recording-in-progress UI
return <div className="mx_VoiceMessagePrimaryContainer mx_VoiceRecordComposerTile_recording">
<LiveRecordingClock recorder={this.state.recorder} />
<LiveRecordingWaveform recorder={this.state.recorder} />
</div>;
}
public render() {
const classes = classNames({
'mx_MessageComposer_button': !this.state.recorder,
'mx_MessageComposer_voiceMessage': !this.state.recorder,
'mx_VoiceRecordComposerTile_stop': !!this.state.recorder,
});
public render(): ReactNode {
let recordingInfo;
let deleteButton;
if (!this.state.recordingPhase || this.state.recordingPhase === RecordingState.Started) {
const classes = classNames({
'mx_MessageComposer_button': !this.state.recorder,
'mx_MessageComposer_voiceMessage': !this.state.recorder,
'mx_VoiceRecordComposerTile_stop': this.state.recorder?.isRecording,
});
let tooltip = _t("Record a voice message");
if (!!this.state.recorder) {
// TODO: @@ TravisR: Change to match behaviour
tooltip = _t("Stop & send recording");
let tooltip = _t("Record a voice message");
if (!!this.state.recorder) {
tooltip = _t("Stop the recording");
}
let stopOrRecordBtn = <AccessibleTooltipButton
className={classes}
onClick={this.onRecordStartEndClick}
title={tooltip}
/>;
if (this.state.recorder && !this.state.recorder?.isRecording) {
stopOrRecordBtn = null;
}
recordingInfo = stopOrRecordBtn;
}
if (this.state.recorder && this.state.recordingPhase !== RecordingState.Uploading) {
deleteButton = <AccessibleTooltipButton
className='mx_VoiceRecordComposerTile_delete'
title={_t("Delete recording")}
onClick={this.onCancel}
/>;
}
return (<>
{deleteButton}
{this.renderWaveformArea()}
<AccessibleTooltipButton
className={classes}
onClick={this.onStartStopVoiceMessage}
title={tooltip}
/>
{recordingInfo}
</>);
}
}

10
src/components/views/voice_messages/Clock.tsx
View File

@ -29,14 +29,20 @@ interface IState {
* displayed, making it possible to see "82:29".
*/
@replaceableComponent("views.voice_messages.Clock")
export default class Clock extends React.PureComponent<IProps, IState> {
export default class Clock extends React.Component<IProps, IState> {
public constructor(props) {
super(props);
}
shouldComponentUpdate(nextProps: Readonly<IProps>, nextState: Readonly<IState>, nextContext: any): boolean {
const currentFloor = Math.floor(this.props.seconds);
const nextFloor = Math.floor(nextProps.seconds);
return currentFloor !== nextFloor;
}
public render() {
const minutes = Math.floor(this.props.seconds / 60).toFixed(0).padStart(2, '0');
const seconds = Math.round(this.props.seconds % 60).toFixed(0).padStart(2, '0'); // hide millis
const seconds = Math.floor(this.props.seconds % 60).toFixed(0).padStart(2, '0'); // hide millis
return <span className='mx_Clock'>{minutes}:{seconds}</span>;
}
}

8
src/components/views/voice_messages/LiveRecordingClock.tsx
View File

@ -31,7 +31,7 @@ interface IState {
* A clock for a live recording.
*/
@replaceableComponent("views.voice_messages.LiveRecordingClock")
export default class LiveRecordingClock extends React.Component<IProps, IState> {
export default class LiveRecordingClock extends React.PureComponent<IProps, IState> {
public constructor(props) {
super(props);
@ -39,12 +39,6 @@ export default class LiveRecordingClock extends React.Component<IProps, IState>
this.props.recorder.liveData.onUpdate(this.onRecordingUpdate);
}
shouldComponentUpdate(nextProps: Readonly<IProps>, nextState: Readonly<IState>, nextContext: any): boolean {
const currentFloor = Math.floor(this.state.seconds);
const nextFloor = Math.floor(nextState.seconds);
return currentFloor !== nextFloor;
}
private onRecordingUpdate = (update: IRecordingUpdate) => {
this.setState({seconds: update.timeSeconds});
};

7
src/components/views/voice_messages/LiveRecordingWaveform.tsx
View File

@ -20,6 +20,7 @@ import {replaceableComponent} from "../../../utils/replaceableComponent";
import {arrayFastResample, arraySeed} from "../../../utils/arrays";
import {percentageOf} from "../../../utils/numbers";
import Waveform from "./Waveform";
import {PLAYBACK_WAVEFORM_SAMPLES} from "../../../voice/Playback";
interface IProps {
recorder: VoiceRecording;
@ -29,8 +30,6 @@ interface IState {
heights: number[];
}
const DOWNSAMPLE_TARGET = 35; // number of bars we want
/**
* A waveform which shows the waveform of a live recording
*/
@ -39,14 +38,14 @@ export default class LiveRecordingWaveform extends React.PureComponent<IProps, I
public constructor(props) {
super(props);
this.state = {heights: arraySeed(0, DOWNSAMPLE_TARGET)};
this.state = {heights: arraySeed(0, PLAYBACK_WAVEFORM_SAMPLES)};
this.props.recorder.liveData.onUpdate(this.onRecordingUpdate);
}
private onRecordingUpdate = (update: IRecordingUpdate) => {
// The waveform and the downsample target are pretty close, so we should be fine to
// do this, despite the docs on arrayFastResample.
const bars = arrayFastResample(Array.from(update.waveform), DOWNSAMPLE_TARGET);
const bars = arrayFastResample(Array.from(update.waveform), PLAYBACK_WAVEFORM_SAMPLES);
this.setState({
// The incoming data is between zero and one, but typically even screaming into a
// microphone won't send you over 0.6, so we artificially adjust the gain for the

61
src/components/views/voice_messages/PlayPauseButton.tsx Normal file
View File

@ -0,0 +1,61 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React, {ReactNode} from "react";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import AccessibleTooltipButton from "../elements/AccessibleTooltipButton";
import {_t} from "../../../languageHandler";
import {Playback, PlaybackState} from "../../../voice/Playback";
import classNames from "classnames";
interface IProps {
// Playback instance to manipulate. Cannot change during the component lifecycle.
playback: Playback;
// The playback phase to render. Able to change during the component lifecycle.
playbackPhase: PlaybackState;
}
/**
* Displays a play/pause button (activating the play/pause function of the recorder)
* to be displayed in reference to a recording.
*/
@replaceableComponent("views.voice_messages.PlayPauseButton")
export default class PlayPauseButton extends React.PureComponent<IProps> {
public constructor(props) {
super(props);
}
private onClick = async () => {
await this.props.playback.toggle();
};
public render(): ReactNode {
const isPlaying = this.props.playback.isPlaying;
const isDisabled = this.props.playbackPhase === PlaybackState.Decoding;
const classes = classNames('mx_PlayPauseButton', {
'mx_PlayPauseButton_play': !isPlaying,
'mx_PlayPauseButton_pause': isPlaying,
'mx_PlayPauseButton_disabled': isDisabled,
});
return <AccessibleTooltipButton
className={classes}
title={isPlaying ? _t("Pause") : _t("Play")}
onClick={this.onClick}
disabled={isDisabled}
/>;
}
}

71
src/components/views/voice_messages/PlaybackClock.tsx Normal file
View File

@ -0,0 +1,71 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React from "react";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import Clock from "./Clock";
import {Playback, PlaybackState} from "../../../voice/Playback";
import {UPDATE_EVENT} from "../../../stores/AsyncStore";
interface IProps {
playback: Playback;
}
interface IState {
seconds: number;
durationSeconds: number;
playbackPhase: PlaybackState;
}
/**
* A clock for a playback of a recording.
*/
@replaceableComponent("views.voice_messages.PlaybackClock")
export default class PlaybackClock extends React.PureComponent<IProps, IState> {
public constructor(props) {
super(props);
this.state = {
seconds: this.props.playback.clockInfo.timeSeconds,
// we track the duration on state because we won't really know what the clip duration
// is until the first time update, and as a PureComponent we are trying to dedupe state
// updates as much as possible. This is just the easiest way to avoid a forceUpdate() or
// member property to track "did we get a duration".
durationSeconds: this.props.playback.clockInfo.durationSeconds,
playbackPhase: PlaybackState.Stopped, // assume not started, so full clock
};
this.props.playback.on(UPDATE_EVENT, this.onPlaybackUpdate);
this.props.playback.clockInfo.liveData.onUpdate(this.onTimeUpdate);
}
private onPlaybackUpdate = (ev: PlaybackState) => {
// Convert Decoding -> Stopped because we don't care about the distinction here
if (ev === PlaybackState.Decoding) ev = PlaybackState.Stopped;
this.setState({playbackPhase: ev});
};
private onTimeUpdate = (time: number[]) => {
this.setState({seconds: time[0], durationSeconds: time[1]});
};
public render() {
let seconds = this.state.seconds;
if (this.state.playbackPhase === PlaybackState.Stopped) {
seconds = this.state.durationSeconds;
}
return <Clock seconds={seconds} />;
}
}

68
src/components/views/voice_messages/PlaybackWaveform.tsx Normal file
View File

@ -0,0 +1,68 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React from "react";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import {arraySeed, arrayTrimFill} from "../../../utils/arrays";
import Waveform from "./Waveform";
import {Playback, PLAYBACK_WAVEFORM_SAMPLES} from "../../../voice/Playback";
import {percentageOf} from "../../../utils/numbers";
interface IProps {
playback: Playback;
}
interface IState {
heights: number[];
progress: number;
}
/**
* A waveform which shows the waveform of a previously recorded recording
*/
@replaceableComponent("views.voice_messages.PlaybackWaveform")
export default class PlaybackWaveform extends React.PureComponent<IProps, IState> {
public constructor(props) {
super(props);
this.state = {
heights: this.toHeights(this.props.playback.waveform),
progress: 0, // default no progress
};
this.props.playback.waveformData.onUpdate(this.onWaveformUpdate);
this.props.playback.clockInfo.liveData.onUpdate(this.onTimeUpdate);
}
private toHeights(waveform: number[]) {
const seed = arraySeed(0, PLAYBACK_WAVEFORM_SAMPLES);
return arrayTrimFill(waveform, PLAYBACK_WAVEFORM_SAMPLES, seed);
}
private onWaveformUpdate = (waveform: number[]) => {
this.setState({heights: this.toHeights(waveform)});
};
private onTimeUpdate = (time: number[]) => {
// Track percentages to very coarse precision, otherwise 0.002 ends up highlighting a bar.
const progress = Number(percentageOf(time[0], 0, time[1]).toFixed(1));
this.setState({progress});
};
public render() {
return <Waveform relHeights={this.state.heights} progress={this.state.progress} />;
}
}

62
src/components/views/voice_messages/RecordingPlayback.tsx Normal file
View File

@ -0,0 +1,62 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import {Playback, PlaybackState} from "../../../voice/Playback";
import React, {ReactNode} from "react";
import {UPDATE_EVENT} from "../../../stores/AsyncStore";
import PlaybackWaveform from "./PlaybackWaveform";
import PlayPauseButton from "./PlayPauseButton";
import PlaybackClock from "./PlaybackClock";
interface IProps {
// Playback instance to render. Cannot change during component lifecycle: create
// an all-new component instead.
playback: Playback;
}
interface IState {
playbackPhase: PlaybackState;
}
export default class RecordingPlayback extends React.PureComponent<IProps, IState> {
constructor(props: IProps) {
super(props);
this.state = {
playbackPhase: PlaybackState.Decoding, // default assumption
};
// We don't need to de-register: the class handles this for us internally
this.props.playback.on(UPDATE_EVENT, this.onPlaybackUpdate);
// Don't wait for the promise to complete - it will emit a progress update when it
// is done, and it's not meant to take long anyhow.
// noinspection JSIgnoredPromiseFromCall
this.props.playback.prepare();
}
private onPlaybackUpdate = (ev: PlaybackState) => {
this.setState({playbackPhase: ev});
};
public render(): ReactNode {
return <div className='mx_VoiceMessagePrimaryContainer'>
<PlayPauseButton playback={this.props.playback} playbackPhase={this.state.playbackPhase} />
<PlaybackClock playback={this.props.playback} />
<PlaybackWaveform playback={this.props.playback} />
</div>
}
}

17
src/components/views/voice_messages/Waveform.tsx
View File

@ -16,9 +16,11 @@ limitations under the License.
import React from "react";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import classNames from "classnames";
interface IProps {
relHeights: number[]; // relative heights (0-1)
progress: number; // percent complete, 0-1, default 100%
}
interface IState {
@ -28,9 +30,16 @@ interface IState {
* A simple waveform component. This renders bars (centered vertically) for each
* height provided in the component properties. Updating the properties will update
* the rendered waveform.
*
* For CSS purposes, a mx_Waveform_bar_100pct class is added when the bar should be
* "filled", as a demonstration of the progress property.
*/
@replaceableComponent("views.voice_messages.Waveform")
export default class Waveform extends React.PureComponent<IProps, IState> {
public static defaultProps = {
progress: 1,
};
public constructor(props) {
super(props);
}
@ -38,7 +47,13 @@ export default class Waveform extends React.PureComponent<IProps, IState> {
public render() {
return <div className='mx_Waveform'>
{this.props.relHeights.map((h, i) => {
return <span key={i} style={{height: (h * 100) + '%'}} className='mx_Waveform_bar' />;
const progress = this.props.progress;
const isCompleteBar = (i / this.props.relHeights.length) <= progress && progress > 0;
const classes = classNames({
'mx_Waveform_bar': true,
'mx_Waveform_bar_100pct': isCompleteBar,
});
return <span key={i} style={{height: (h * 100) + '%'}} className={classes} />;
})}
</div>;
}

5
src/i18n/strings/en_EN.json
View File

@ -899,6 +899,8 @@
"Incoming call": "Incoming call",
"Decline": "Decline",
"Accept": "Accept",
"Pause": "Pause",
"Play": "Play",
"The other party cancelled the verification.": "The other party cancelled the verification.",
"Verified!": "Verified!",
"You've successfully verified this user.": "You've successfully verified this user.",
@ -1645,7 +1647,8 @@
"Jump to first unread message.": "Jump to first unread message.",
"Mark all as read": "Mark all as read",
"Record a voice message": "Record a voice message",
"Stop & send recording": "Stop & send recording",
"Stop the recording": "Stop the recording",
"Delete recording": "Delete recording",
"Error updating main address": "Error updating main address",
"There was an error updating the room's main address. It may not be allowed by the server or a temporary failure occurred.": "There was an error updating the room's main address. It may not be allowed by the server or a temporary failure occurred.",
"There was an error updating the room's alternative addresses. It may not be allowed by the server or a temporary failure occurred.": "There was an error updating the room's alternative addresses. It may not be allowed by the server or a temporary failure occurred.",

2
src/stores/VoiceRecordingStore.ts
View File

@ -78,3 +78,5 @@ export class VoiceRecordingStore extends AsyncStoreWithClient<IState> {
return this.updateState({recording: null});
}
}
window.mxVoiceRecordingStore = VoiceRecordingStore.instance;

20
src/utils/arrays.ts
View File

@ -73,6 +73,26 @@ export function arraySeed<T>(val: T, length: number): T[] {
return a;
}
/**
* Trims or fills the array to ensure it meets the desired length. The seed array
* given is pulled from to fill any missing slots - it is recommended that this be
* at least `len` long. The resulting array will be exactly `len` long, either
* trimmed from the source or filled with the some/all of the seed array.
* @param {T[]} a The array to trim/fill.
* @param {number} len The length to trim or fill to, as needed.
* @param {T[]} seed Values to pull from if the array needs filling.
* @returns {T[]} The resulting array of `len` length.
*/
export function arrayTrimFill<T>(a: T[], len: number, seed: T[]): T[] {
// Dev note: we do length checks because the spread operator can result in some
// performance penalties in more critical code paths. As a utility, it should be
// as fast as possible to not cause a problem for the call stack, no matter how
// critical that stack is.
if (a.length === len) return a;
if (a.length > len) return a.slice(0, len);
return a.concat(seed.slice(0, len - a.length));
}
/**
* Clones an array as fast as possible, retaining references of the array's values.
* @param a The array to clone. Must be defined.

141
src/voice/Playback.ts Normal file
View File

@ -0,0 +1,141 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import EventEmitter from "events";
import {UPDATE_EVENT} from "../stores/AsyncStore";
import {arrayFastResample, arraySeed} from "../utils/arrays";
import {SimpleObservable} from "matrix-widget-api";
import {IDestroyable} from "../utils/IDestroyable";
import {PlaybackClock} from "./PlaybackClock";
export enum PlaybackState {
Decoding = "decoding",
Stopped = "stopped", // no progress on timeline
Paused = "paused", // some progress on timeline
Playing = "playing", // active progress through timeline
}
export const PLAYBACK_WAVEFORM_SAMPLES = 35;
const DEFAULT_WAVEFORM = arraySeed(0, PLAYBACK_WAVEFORM_SAMPLES);
export class Playback extends EventEmitter implements IDestroyable {
private readonly context: AudioContext;
private source: AudioBufferSourceNode;
private state = PlaybackState.Decoding;
private audioBuf: AudioBuffer;
private resampledWaveform: number[];
private waveformObservable = new SimpleObservable<number[]>();
private readonly clock: PlaybackClock;
/**
* Creates a new playback instance from a buffer.
* @param {ArrayBuffer} buf The buffer containing the sound sample.
* @param {number[]} seedWaveform Optional seed waveform to present until the proper waveform
* can be calculated. Contains values between zero and one, inclusive.
*/
constructor(private buf: ArrayBuffer, seedWaveform = DEFAULT_WAVEFORM) {
super();
this.context = new AudioContext();
this.resampledWaveform = arrayFastResample(seedWaveform, PLAYBACK_WAVEFORM_SAMPLES);
this.waveformObservable.update(this.resampledWaveform);
this.clock = new PlaybackClock(this.context);
// TODO: @@ TR: Calculate real waveform
}
public get waveform(): number[] {
return this.resampledWaveform;
}
public get waveformData(): SimpleObservable<number[]> {
return this.waveformObservable;
}
public get clockInfo(): PlaybackClock {
return this.clock;
}
public get currentState(): PlaybackState {
return this.state;
}
public get isPlaying(): boolean {
return this.currentState === PlaybackState.Playing;
}
public emit(event: PlaybackState, ...args: any[]): boolean {
this.state = event;
super.emit(event, ...args);
super.emit(UPDATE_EVENT, event, ...args);
return true; // we don't ever care if the event had listeners, so just return "yes"
}
public destroy() {
// noinspection JSIgnoredPromiseFromCall - not concerned about being called async here
this.stop();
this.removeAllListeners();
this.clock.destroy();
this.waveformObservable.close();
}
public async prepare() {
this.audioBuf = await this.context.decodeAudioData(this.buf);
this.emit(PlaybackState.Stopped); // signal that we're not decoding anymore
this.clock.durationSeconds = this.audioBuf.duration;
}
private onPlaybackEnd = async () => {
await this.context.suspend();
this.emit(PlaybackState.Stopped);
};
public async play() {
// We can't restart a buffer source, so we need to create a new one if we hit the end
if (this.state === PlaybackState.Stopped) {
if (this.source) {
this.source.disconnect();
this.source.removeEventListener("ended", this.onPlaybackEnd);
}
this.source = this.context.createBufferSource();
this.source.connect(this.context.destination);
this.source.buffer = this.audioBuf;
this.source.start(); // start immediately
this.source.addEventListener("ended", this.onPlaybackEnd);
}
// We use the context suspend/resume functions because it allows us to pause a source
// node, but that still doesn't help us when the source node runs out (see above).
await this.context.resume();
this.clock.flagStart();
this.emit(PlaybackState.Playing);
}
public async pause() {
await this.context.suspend();
this.emit(PlaybackState.Paused);
}
public async stop() {
await this.onPlaybackEnd();
this.clock.flagStop();
}
public async toggle() {
if (this.isPlaying) await this.pause();
else await this.play();
}
}

78
src/voice/PlaybackClock.ts Normal file
View File

@ -0,0 +1,78 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import {SimpleObservable} from "matrix-widget-api";
import {IDestroyable} from "../utils/IDestroyable";
// Because keeping track of time is sufficiently complicated...
export class PlaybackClock implements IDestroyable {
private clipStart = 0;
private stopped = true;
private lastCheck = 0;
private observable = new SimpleObservable<number[]>();
private timerId: number;
private clipDuration = 0;
public constructor(private context: AudioContext) {
}
public get durationSeconds(): number {
return this.clipDuration;
}
public set durationSeconds(val: number) {
this.clipDuration = val;
this.observable.update([this.timeSeconds, this.clipDuration]);
}
public get timeSeconds(): number {
return (this.context.currentTime - this.clipStart) % this.clipDuration;
}
public get liveData(): SimpleObservable<number[]> {
return this.observable;
}
private checkTime = () => {
const now = this.timeSeconds;
if (this.lastCheck !== now) {
this.observable.update([now, this.durationSeconds]);
this.lastCheck = now;
}
};
public flagStart() {
if (this.stopped) {
this.clipStart = this.context.currentTime;
this.stopped = false;
}
if (!this.timerId) {
// case to number because the types are wrong
// 100ms interval to make sure the time is as accurate as possible
this.timerId = <number><any>setInterval(this.checkTime, 100);
}
}
public flagStop() {
this.stopped = true;
}
public destroy() {
this.observable.close();
if (this.timerId) clearInterval(this.timerId);
}
}

67
src/voice/VoiceRecording.ts
View File

@ -24,7 +24,8 @@ import EventEmitter from "events";
import {IDestroyable} from "../utils/IDestroyable";
import {Singleflight} from "../utils/Singleflight";
import {PayloadEvent, WORKLET_NAME} from "./consts";
import {arrayFastClone} from "../utils/arrays";
import {UPDATE_EVENT} from "../stores/AsyncStore";
import {Playback} from "./Playback";
const CHANNELS = 1; // stereo isn't important
const SAMPLE_RATE = 48000; // 48khz is what WebRTC uses. 12khz is where we lose quality.
@ -52,20 +53,17 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
private recorderStream: MediaStream;
private recorderFFT: AnalyserNode;
private recorderWorklet: AudioWorkletNode;
private buffer = new Uint8Array(0);
private buffer = new Uint8Array(0); // use this.audioBuffer to access
private mxc: string;
private recording = false;
private observable: SimpleObservable<IRecordingUpdate>;
private amplitudes: number[] = []; // at each second mark, generated
private playback: Playback;
public constructor(private client: MatrixClient) {
super();
}
public get finalWaveform(): number[] {
return arrayFastClone(this.amplitudes);
}
public get contentType(): string {
return "audio/ogg";
}
@ -79,6 +77,16 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
return this.recorderContext.currentTime;
}
public get isRecording(): boolean {
return this.recording;
}
public emit(event: string, ...args: any[]): boolean {
super.emit(event, ...args);
super.emit(UPDATE_EVENT, event, ...args);
return true; // we don't ever care if the event had listeners, so just return "yes"
}
private async makeRecorder() {
this.recorderStream = await navigator.mediaDevices.getUserMedia({
audio: {
@ -154,6 +162,12 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
};
}
private get audioBuffer(): Uint8Array {
// We need a clone of the buffer to avoid accidentally changing the position
// on the real thing.
return this.buffer.slice(0);
}
public get liveData(): SimpleObservable<IRecordingUpdate> {
if (!this.recording) throw new Error("No observable when not recording");
return this.observable;
@ -203,8 +217,19 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
// Now that we've updated the data/waveform, let's do a time check. We don't want to
// go horribly over the limit. We also emit a warning state if needed.
const secondsLeft = TARGET_MAX_LENGTH - timeSeconds;
if (secondsLeft <= 0) {
//
// We use the recorder's perspective of time to make sure we don't cut off the last
// frame of audio, otherwise we end up with a 1:59 clip (119.68 seconds). This extra
// safety can allow us to overshoot the target a bit, but at least when we say 2min
// maximum we actually mean it.
//
// In testing, recorder time and worker time lag by about 400ms, which is roughly the
// time needed to encode a sample/frame.
//
// Ref for recorderSeconds: https://github.com/chris-rudmin/opus-recorder#instance-fields
const recorderSeconds = this.recorder.encodedSamplePosition / 48000;
const secondsLeft = TARGET_MAX_LENGTH - recorderSeconds;
if (secondsLeft < 0) { // go over to make sure we definitely capture that last frame
// noinspection JSIgnoredPromiseFromCall - we aren't concerned with it overlapping
this.stop();
} else if (secondsLeft <= TARGET_WARN_TIME_LEFT) {
@ -239,9 +264,9 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
}
// Disconnect the source early to start shutting down resources
await this.recorder.stop(); // stop first to flush the last frame
this.recorderSource.disconnect();
this.recorderWorklet.disconnect();
await this.recorder.stop();
// close the context after the recorder so the recorder doesn't try to
// connect anything to the context (this would generate a warning)
@ -255,15 +280,33 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
await this.recorder.close();
this.emit(RecordingState.Ended);
return this.buffer;
return this.audioBuffer;
});
}
/**
* Gets a playback instance for this voice recording. Note that the playback will not
* have been prepared fully, meaning the `prepare()` function needs to be called on it.
*
* The same playback instance is returned each time.
*
* @returns {Playback} The playback instance.
*/
public getPlayback(): Playback {
this.playback = Singleflight.for(this, "playback").do(() => {
return new Playback(this.audioBuffer.buffer, this.amplitudes); // cast to ArrayBuffer proper;
});
return this.playback;
}
public destroy() {
// noinspection JSIgnoredPromiseFromCall - not concerned about stop() being called async here
this.stop();
this.removeAllListeners();
Singleflight.forgetAllFor(this);
// noinspection JSIgnoredPromiseFromCall - not concerned about being called async here
this.playback?.destroy();
this.observable.close();
}
public async upload(): Promise<string> {
@ -274,7 +317,7 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
if (this.mxc) return this.mxc;
this.emit(RecordingState.Uploading);
this.mxc = await this.client.uploadContent(new Blob([this.buffer], {
this.mxc = await this.client.uploadContent(new Blob([this.audioBuffer], {
type: this.contentType,
}), {
onlyContentUri: false, // to stop the warnings in the console
@ -283,5 +326,3 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
return this.mxc;
}
}
window.mxVoiceRecorder = VoiceRecording;

33
test/utils/arrays-test.ts
View File

@ -22,6 +22,7 @@ import {
arrayHasOrderChange,
arrayMerge,
arraySeed,
arrayTrimFill,
arrayUnion,
ArrayUtil,
GroupedArray,
@ -64,6 +65,38 @@ describe('arrays', () => {
});
});
describe('arrayTrimFill', () => {
it('should shrink arrays', () => {
const input = [1, 2, 3];
const output = [1, 2];
const seed = [4, 5, 6];
const result = arrayTrimFill(input, output.length, seed);
expect(result).toBeDefined();
expect(result).toHaveLength(output.length);
expect(result).toEqual(output);
});
it('should expand arrays', () => {
const input = [1, 2, 3];
const output = [1, 2, 3, 4, 5];
const seed = [4, 5, 6];
const result = arrayTrimFill(input, output.length, seed);
expect(result).toBeDefined();
expect(result).toHaveLength(output.length);
expect(result).toEqual(output);
});
it('should keep arrays the same', () => {
const input = [1, 2, 3];
const output = [1, 2, 3];
const seed = [4, 5, 6];
const result = arrayTrimFill(input, output.length, seed);
expect(result).toBeDefined();
expect(result).toHaveLength(output.length);
expect(result).toEqual(output);
});
});
describe('arraySeed', () => {
it('should create an array of given length', () => {
const val = 1;