OpenCloud
/
riot-web
mirror of https://github.com/vector-im/riot-web
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #5801 from matrix-org/travis/voice-messages/waveform 

Show waveform and timer in voice messages
pull/21833/head
Travis Ralston 2021-03-30 12:16:19 -06:00 committed by GitHub
parent b089f3d2f9 b15412056e
commit b68fabb44b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 407 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
res/css/_components.scss
View File

@ -247,6 +247,7 @@
@import "./views/toasts/_AnalyticsToast.scss";
@import "./views/toasts/_NonUrgentEchoFailureToast.scss";
@import "./views/verification/_VerificationShowSas.scss";
@import "./views/voice_messages/_Waveform.scss";
@import "./views/voip/_CallContainer.scss";
@import "./views/voip/_CallView.scss";
@import "./views/voip/_DialPad.scss";

40
res/css/views/rooms/_VoiceRecordComposerTile.scss
View File

@ -34,3 +34,43 @@ limitations under the License.
background-color: $voice-record-stop-symbol-color;
}
}
.mx_VoiceRecordComposerTile_waveformContainer {
padding: 5px;
padding-right: 4px; // there's 1px from the waveform itself, so account for that
padding-left: 15px; // +10px for the live circle, +5px for regular padding
background-color: $voice-record-waveform-bg-color;
border-radius: 12px;
margin-right: 12px; // isolate from stop button
// Cheat at alignment a bit
display: flex;
align-items: center;
position: relative; // important for the live circle
color: $voice-record-waveform-fg-color;
font-size: $font-14px;
&::before {
// TODO: @@ TravisR: Animate
content: '';
background-color: $voice-record-live-circle-color;
width: 10px;
height: 10px;
position: absolute;
left: 8px;
top: 16px; // vertically center
border-radius: 10px;
}
.mx_Waveform_bar {
background-color: $voice-record-waveform-fg-color;
}
.mx_Clock {
padding-right: 8px; // isolate from waveform
padding-left: 10px; // isolate from live circle
width: 42px; // we're not using a monospace font, so fake it
}
}

40
res/css/views/voice_messages/_Waveform.scss Normal file
View File

@ -0,0 +1,40 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
.mx_Waveform {
position: relative;
height: 30px; // tallest bar can only be 30px
top: 1px; // because of our border trick (see below), we're off by 1px of aligntment
display: flex;
align-items: center; // so the bars grow from the middle
overflow: hidden; // this is cheaper than a `max-height: calc(100% - 4px)` in the bar's CSS.
// A bar is meant to be a 2x2 circle when at zero height, and otherwise a 2px wide line
// with rounded caps.
.mx_Waveform_bar {
width: 0; // 0px width means we'll end up using the border as our width
border: 1px solid transparent; // transparent means we'll use the background colour
border-radius: 2px; // rounded end caps, based on the border
min-height: 0; // like the width, we'll rely on the border to give us height
max-height: 100%; // this makes the `height: 42%` work on the element
margin-left: 1px; // we want 2px between each bar, so 1px on either side for balance
margin-right: 1px;
// background color is handled by the parent components
}
}

3
res/themes/legacy-light/css/_legacy-light.scss
View File

@ -190,6 +190,9 @@ $groupFilterPanel-divider-color: $roomlist-header-color;
$voice-record-stop-border-color: #E3E8F0;
$voice-record-stop-symbol-color: $warning-color;
$voice-record-waveform-bg-color: #E3E8F0;
$voice-record-waveform-fg-color: $muted-fg-color;
$voice-record-live-circle-color: $warning-color;
$roomtile-preview-color: #9e9e9e;
$roomtile-default-badge-bg-color: #61708b;

3
res/themes/light/css/_light.scss
View File

@ -181,6 +181,9 @@ $groupFilterPanel-divider-color: $roomlist-header-color;
$voice-record-stop-border-color: #E3E8F0;
$voice-record-stop-symbol-color: $warning-color;
$voice-record-waveform-bg-color: #E3E8F0;
$voice-record-waveform-fg-color: $muted-fg-color;
$voice-record-live-circle-color: $warning-color;
$roomtile-preview-color: $secondary-fg-color;
$roomtile-default-badge-bg-color: #61708b;

25
src/components/views/rooms/VoiceRecordComposerTile.tsx
View File

@ -21,6 +21,9 @@ import {VoiceRecorder} from "../../../voice/VoiceRecorder";
import {Room} from "matrix-js-sdk/src/models/room";
import {MatrixClientPeg} from "../../../MatrixClientPeg";
import classNames from "classnames";
import LiveRecordingWaveform from "../voice_messages/LiveRecordingWaveform";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import LiveRecordingClock from "../voice_messages/LiveRecordingClock";
interface IProps {
room: Room;
@ -31,6 +34,10 @@ interface IState {
recorder?: VoiceRecorder;
}
/**
* Container tile for rendering the voice message recorder in the composer.
*/
@replaceableComponent("views.rooms.VoiceRecordComposerTile")
export default class VoiceRecordComposerTile extends React.PureComponent<IProps, IState> {
public constructor(props) {
super(props);
@ -57,13 +64,18 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
const recorder = new VoiceRecorder(MatrixClientPeg.get());
await recorder.start();
this.props.onRecording(true);
// TODO: @@ TravisR: Run through EQ component
// recorder.frequencyData.onUpdate((freq) => {
// console.log('@@ UPDATE', freq);
// });
this.setState({recorder});
};
private renderWaveformArea() {
if (!this.state.recorder) return null;
return <div className='mx_VoiceRecordComposerTile_waveformContainer'>
<LiveRecordingClock recorder={this.state.recorder} />
<LiveRecordingWaveform recorder={this.state.recorder} />
</div>;
}
public render() {
const classes = classNames({
'mx_MessageComposer_button': !this.state.recorder,
@ -77,12 +89,13 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
tooltip = _t("Stop & send recording");
}
return (
return (<>
{this.renderWaveformArea()}
<AccessibleTooltipButton
className={classes}
onClick={this.onStartStopVoiceMessage}
title={tooltip}
/>
);
</>);
}
}

42
src/components/views/voice_messages/Clock.tsx Normal file
View File

@ -0,0 +1,42 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React from "react";
import {replaceableComponent} from "../../../utils/replaceableComponent";
interface IProps {
seconds: number;
}
interface IState {
}
/**
* Simply converts seconds into minutes and seconds. Note that hours will not be
* displayed, making it possible to see "82:29".
*/
@replaceableComponent("views.voice_messages.Clock")
export default class Clock extends React.PureComponent<IProps, IState> {
public constructor(props) {
super(props);
}
public render() {
const minutes = Math.floor(this.props.seconds / 60).toFixed(0).padStart(2, '0');
const seconds = Math.round(this.props.seconds % 60).toFixed(0).padStart(2, '0'); // hide millis
return <span className='mx_Clock'>{minutes}:{seconds}</span>;
}
}

55
src/components/views/voice_messages/LiveRecordingClock.tsx Normal file
View File

@ -0,0 +1,55 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React from "react";
import {IRecordingUpdate, VoiceRecorder} from "../../../voice/VoiceRecorder";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import Clock from "./Clock";
interface IProps {
recorder: VoiceRecorder;
}
interface IState {
seconds: number;
}
/**
* A clock for a live recording.
*/
@replaceableComponent("views.voice_messages.LiveRecordingClock")
export default class LiveRecordingClock extends React.Component<IProps, IState> {
public constructor(props) {
super(props);
this.state = {seconds: 0};
this.props.recorder.liveData.onUpdate(this.onRecordingUpdate);
}
shouldComponentUpdate(nextProps: Readonly<IProps>, nextState: Readonly<IState>, nextContext: any): boolean {
const currentFloor = Math.floor(this.state.seconds);
const nextFloor = Math.floor(nextState.seconds);
return currentFloor !== nextFloor;
}
private onRecordingUpdate = (update: IRecordingUpdate) => {
this.setState({seconds: update.timeSeconds});
};
public render() {
return <Clock seconds={this.state.seconds} />;
}
}

62
src/components/views/voice_messages/LiveRecordingWaveform.tsx Normal file
View File

@ -0,0 +1,62 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React from "react";
import {IRecordingUpdate, VoiceRecorder} from "../../../voice/VoiceRecorder";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import {arrayFastResample, arraySeed} from "../../../utils/arrays";
import {percentageOf} from "../../../utils/numbers";
import Waveform from "./Waveform";
interface IProps {
recorder: VoiceRecorder;
}
interface IState {
heights: number[];
}
const DOWNSAMPLE_TARGET = 35; // number of bars we want
/**
* A waveform which shows the waveform of a live recording
*/
@replaceableComponent("views.voice_messages.LiveRecordingWaveform")
export default class LiveRecordingWaveform extends React.PureComponent<IProps, IState> {
public constructor(props) {
super(props);
this.state = {heights: arraySeed(0, DOWNSAMPLE_TARGET)};
this.props.recorder.liveData.onUpdate(this.onRecordingUpdate);
}
private onRecordingUpdate = (update: IRecordingUpdate) => {
// The waveform and the downsample target are pretty close, so we should be fine to
// do this, despite the docs on arrayFastResample.
const bars = arrayFastResample(Array.from(update.waveform), DOWNSAMPLE_TARGET);
this.setState({
// The incoming data is between zero and one, but typically even screaming into a
// microphone won't send you over 0.6, so we artificially adjust the gain for the
// waveform. This results in a slightly more cinematic/animated waveform for the
// user.
heights: bars.map(b => percentageOf(b, 0, 0.50)),
});
};
public render() {
return <Waveform relHeights={this.state.heights} />;
}
}

45
src/components/views/voice_messages/Waveform.tsx Normal file
View File

@ -0,0 +1,45 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React from "react";
import {replaceableComponent} from "../../../utils/replaceableComponent";
interface IProps {
relHeights: number[]; // relative heights (0-1)
}
interface IState {
}
/**
* A simple waveform component. This renders bars (centered vertically) for each
* height provided in the component properties. Updating the properties will update
* the rendered waveform.
*/
@replaceableComponent("views.voice_messages.Waveform")
export default class Waveform extends React.PureComponent<IProps, IState> {
public constructor(props) {
super(props);
}
public render() {
return <div className='mx_Waveform'>
{this.props.relHeights.map((h, i) => {
return <span key={i} style={{height: (h * 100) + '%'}} className='mx_Waveform_bar' />;
})}
</div>;
}
}

35
src/utils/arrays.ts
View File

@ -14,6 +14,41 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
/**
* Quickly resample an array to have less data points. This isn't a perfect representation,
* though this does work best if given a large array to downsample to a much smaller array.
* @param {number[]} input The input array to downsample.
* @param {number} points The number of samples to end up with.
* @returns {number[]} The downsampled array.
*/
export function arrayFastResample(input: number[], points: number): number[] {
// Heavily inpired by matrix-media-repo (used with permission)
// https://github.com/turt2live/matrix-media-repo/blob/abe72c87d2e29/util/util_audio/fastsample.go#L10
const everyNth = Math.round(input.length / points);
const samples: number[] = [];
for (let i = 0; i < input.length; i += everyNth) {
samples.push(input[i]);
}
while (samples.length < points) {
samples.push(input[input.length - 1]);
}
return samples;
}
/**
* Creates an array of the given length, seeded with the given value.
* @param {T} val The value to seed the array with.
* @param {number} length The length of the array to create.
* @returns {T[]} The array.
*/
export function arraySeed<T>(val: T, length: number): T[] {
const a: T[] = [];
for (let i = 0; i < length; i++) {
a.push(val);
}
return a;
}
/**
* Clones an array as fast as possible, retaining references of the array's values.
* @param a The array to clone. Must be defined.

88
src/voice/VoiceRecorder.ts
View File

@ -19,18 +19,15 @@ import encoderPath from 'opus-recorder/dist/encoderWorker.min.js';
import {MatrixClient} from "matrix-js-sdk/src/client";
import CallMediaHandler from "../CallMediaHandler";
import {SimpleObservable} from "matrix-widget-api";
import {clamp} from "../utils/numbers";
const CHANNELS = 1; // stereo isn't important
const SAMPLE_RATE = 48000; // 48khz is what WebRTC uses. 12khz is where we lose quality.
const BITRATE = 24000; // 24kbps is pretty high quality for our use case in opus.
const FREQ_SAMPLE_RATE = 4; // Target rate of frequency data (samples / sec). We don't need this super often.
export interface IFrequencyPackage {
dbBars: Float32Array;
dbMin: number;
dbMax: number;
// TODO: @@ TravisR: Generalize this for a timing package?
export interface IRecordingUpdate {
waveform: number[]; // floating points between 0 (low) and 1 (high).
timeSeconds: number; // float
}
export class VoiceRecorder {
@ -38,12 +35,12 @@ export class VoiceRecorder {
private recorderContext: AudioContext;
private recorderSource: MediaStreamAudioSourceNode;
private recorderStream: MediaStream;
private recorderFreqNode: AnalyserNode;
private recorderFFT: AnalyserNode;
private recorderProcessor: ScriptProcessorNode;
private buffer = new Uint8Array(0);
private mxc: string;
private recording = false;
private observable: SimpleObservable<IFrequencyPackage>;
private freqTimerId: number;
private observable: SimpleObservable<IRecordingUpdate>;
public constructor(private client: MatrixClient) {
}
@ -57,11 +54,31 @@ export class VoiceRecorder {
},
});
this.recorderContext = new AudioContext({
latencyHint: "interactive",
// latencyHint: "interactive", // we don't want a latency hint (this causes data smoothing)
});
this.recorderSource = this.recorderContext.createMediaStreamSource(this.recorderStream);
this.recorderFreqNode = this.recorderContext.createAnalyser();
this.recorderSource.connect(this.recorderFreqNode);
this.recorderFFT = this.recorderContext.createAnalyser();
// Bring the FFT time domain down a bit. The default is 2048, and this must be a power
// of two. We use 64 points because we happen to know down the line we need less than
// that, but 32 would be too few. Large numbers are not helpful here and do not add
// precision: they introduce higher precision outputs of the FFT (frequency data), but
// it makes the time domain less than helpful.
this.recorderFFT.fftSize = 64;
// We use an audio processor to get accurate timing information.
// The size of the audio buffer largely decides how quickly we push timing/waveform data
// out of this class. Smaller buffers mean we update more frequently as we can't hold as
// many bytes. Larger buffers mean slower updates. For scale, 1024 gives us about 30Hz of
// updates and 2048 gives us about 20Hz. We use 1024 to get as close to perceived realtime
// as possible. Must be a power of 2.
this.recorderProcessor = this.recorderContext.createScriptProcessor(1024, CHANNELS, CHANNELS);
// Connect our inputs and outputs
this.recorderSource.connect(this.recorderFFT);
this.recorderSource.connect(this.recorderProcessor);
this.recorderProcessor.connect(this.recorderContext.destination);
this.recorder = new Recorder({
encoderPath, // magic from webpack
encoderSampleRate: SAMPLE_RATE,
@ -87,7 +104,7 @@ export class VoiceRecorder {
};
}
public get frequencyData(): SimpleObservable<IFrequencyPackage> {
public get liveData(): SimpleObservable<IRecordingUpdate> {
if (!this.recording) throw new Error("No observable when not recording");
return this.observable;
}
@ -107,6 +124,34 @@ export class VoiceRecorder {
return this.mxc;
}
private tryUpdateLiveData = (ev: AudioProcessingEvent) => {
if (!this.recording) return;
// The time domain is the input to the FFT, which means we use an array of the same
// size. The time domain is also known as the audio waveform. We're ignoring the
// output of the FFT here (frequency data) because we're not interested in it.
const data = new Float32Array(this.recorderFFT.fftSize);
this.recorderFFT.getFloatTimeDomainData(data);
// We can't just `Array.from()` the array because we're dealing with 32bit floats
// and the built-in function won't consider that when converting between numbers.
// However, the runtime will convert the float32 to a float64 during the math operations
// which is why the loop works below. Note that a `.map()` call also doesn't work
// and will instead return a Float32Array still.
const translatedData: number[] = [];
for (let i = 0; i < data.length; i++) {
// We're clamping the values so we can do that math operation mentioned above,
// and to ensure that we produce consistent data (it's possible for the array
// to exceed the specified range with some audio input devices).
translatedData.push(clamp(data[i], 0, 1));
}
this.observable.update({
waveform: translatedData,
timeSeconds: ev.playbackTime,
});
};
public async start(): Promise<void> {
if (this.mxc || this.hasRecording) {
throw new Error("Recording already prepared");
@ -117,18 +162,9 @@ export class VoiceRecorder {
if (this.observable) {
this.observable.close();
}
this.observable = new SimpleObservable<IFrequencyPackage>();
this.observable = new SimpleObservable<IRecordingUpdate>();
await this.makeRecorder();
this.freqTimerId = setInterval(() => {
if (!this.recording) return;
const data = new Float32Array(this.recorderFreqNode.frequencyBinCount);
this.recorderFreqNode.getFloatFrequencyData(data);
this.observable.update({
dbBars: data,
dbMin: this.recorderFreqNode.minDecibels,
dbMax: this.recorderFreqNode.maxDecibels,
});
}, 1000 / FREQ_SAMPLE_RATE) as any as number; // XXX: Linter doesn't understand timer environment
this.recorderProcessor.addEventListener("audioprocess", this.tryUpdateLiveData);
await this.recorder.start();
this.recording = true;
}
@ -150,8 +186,8 @@ export class VoiceRecorder {
this.recorderStream.getTracks().forEach(t => t.stop());
// Finally do our post-processing and clean up
clearInterval(this.freqTimerId);
this.recording = false;
this.recorderProcessor.removeEventListener("audioprocess", this.tryUpdateLiveData);
await this.recorder.close();
return this.buffer;