diff --git a/res/css/_components.scss b/res/css/_components.scss index 742bb0e795..215b6605a5 100644 --- a/res/css/_components.scss +++ b/res/css/_components.scss @@ -247,6 +247,7 @@ @import "./views/toasts/_AnalyticsToast.scss"; @import "./views/toasts/_NonUrgentEchoFailureToast.scss"; @import "./views/verification/_VerificationShowSas.scss"; +@import "./views/voice_messages/_Waveform.scss"; @import "./views/voip/_CallContainer.scss"; @import "./views/voip/_CallView.scss"; @import "./views/voip/_DialPad.scss"; diff --git a/res/css/views/rooms/_VoiceRecordComposerTile.scss b/res/css/views/rooms/_VoiceRecordComposerTile.scss index bb36991b4f..2fb112a38c 100644 --- a/res/css/views/rooms/_VoiceRecordComposerTile.scss +++ b/res/css/views/rooms/_VoiceRecordComposerTile.scss @@ -34,3 +34,43 @@ limitations under the License. background-color: $voice-record-stop-symbol-color; } } + +.mx_VoiceRecordComposerTile_waveformContainer { + padding: 5px; + padding-right: 4px; // there's 1px from the waveform itself, so account for that + padding-left: 15px; // +10px for the live circle, +5px for regular padding + background-color: $voice-record-waveform-bg-color; + border-radius: 12px; + margin-right: 12px; // isolate from stop button + + // Cheat at alignment a bit + display: flex; + align-items: center; + + position: relative; // important for the live circle + + color: $voice-record-waveform-fg-color; + font-size: $font-14px; + + &::before { + // TODO: @@ TravisR: Animate + content: ''; + background-color: $voice-record-live-circle-color; + width: 10px; + height: 10px; + position: absolute; + left: 8px; + top: 16px; // vertically center + border-radius: 10px; + } + + .mx_Waveform_bar { + background-color: $voice-record-waveform-fg-color; + } + + .mx_Clock { + padding-right: 8px; // isolate from waveform + padding-left: 10px; // isolate from live circle + width: 42px; // we're not using a monospace font, so fake it + } +} diff --git a/res/css/views/voice_messages/_Waveform.scss b/res/css/views/voice_messages/_Waveform.scss new file mode 100644 index 0000000000..cf03c84601 --- /dev/null +++ b/res/css/views/voice_messages/_Waveform.scss @@ -0,0 +1,40 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.mx_Waveform { + position: relative; + height: 30px; // tallest bar can only be 30px + top: 1px; // because of our border trick (see below), we're off by 1px of aligntment + + display: flex; + align-items: center; // so the bars grow from the middle + + overflow: hidden; // this is cheaper than a `max-height: calc(100% - 4px)` in the bar's CSS. + + // A bar is meant to be a 2x2 circle when at zero height, and otherwise a 2px wide line + // with rounded caps. + .mx_Waveform_bar { + width: 0; // 0px width means we'll end up using the border as our width + border: 1px solid transparent; // transparent means we'll use the background colour + border-radius: 2px; // rounded end caps, based on the border + min-height: 0; // like the width, we'll rely on the border to give us height + max-height: 100%; // this makes the `height: 42%` work on the element + margin-left: 1px; // we want 2px between each bar, so 1px on either side for balance + margin-right: 1px; + + // background color is handled by the parent components + } +} diff --git a/res/themes/legacy-light/css/_legacy-light.scss b/res/themes/legacy-light/css/_legacy-light.scss index 8938efb792..121366decb 100644 --- a/res/themes/legacy-light/css/_legacy-light.scss +++ b/res/themes/legacy-light/css/_legacy-light.scss @@ -190,6 +190,9 @@ $groupFilterPanel-divider-color: $roomlist-header-color; $voice-record-stop-border-color: #E3E8F0; $voice-record-stop-symbol-color: $warning-color; +$voice-record-waveform-bg-color: #E3E8F0; +$voice-record-waveform-fg-color: $muted-fg-color; +$voice-record-live-circle-color: $warning-color; $roomtile-preview-color: #9e9e9e; $roomtile-default-badge-bg-color: #61708b; diff --git a/res/themes/light/css/_light.scss b/res/themes/light/css/_light.scss index 0db291fc41..f082247754 100644 --- a/res/themes/light/css/_light.scss +++ b/res/themes/light/css/_light.scss @@ -181,6 +181,9 @@ $groupFilterPanel-divider-color: $roomlist-header-color; $voice-record-stop-border-color: #E3E8F0; $voice-record-stop-symbol-color: $warning-color; +$voice-record-waveform-bg-color: #E3E8F0; +$voice-record-waveform-fg-color: $muted-fg-color; +$voice-record-live-circle-color: $warning-color; $roomtile-preview-color: $secondary-fg-color; $roomtile-default-badge-bg-color: #61708b; diff --git a/src/components/views/rooms/VoiceRecordComposerTile.tsx b/src/components/views/rooms/VoiceRecordComposerTile.tsx index 0d381001a1..b4999ac0df 100644 --- a/src/components/views/rooms/VoiceRecordComposerTile.tsx +++ b/src/components/views/rooms/VoiceRecordComposerTile.tsx @@ -21,6 +21,9 @@ import {VoiceRecorder} from "../../../voice/VoiceRecorder"; import {Room} from "matrix-js-sdk/src/models/room"; import {MatrixClientPeg} from "../../../MatrixClientPeg"; import classNames from "classnames"; +import LiveRecordingWaveform from "../voice_messages/LiveRecordingWaveform"; +import {replaceableComponent} from "../../../utils/replaceableComponent"; +import LiveRecordingClock from "../voice_messages/LiveRecordingClock"; interface IProps { room: Room; @@ -31,6 +34,10 @@ interface IState { recorder?: VoiceRecorder; } +/** + * Container tile for rendering the voice message recorder in the composer. + */ +@replaceableComponent("views.rooms.VoiceRecordComposerTile") export default class VoiceRecordComposerTile extends React.PureComponent { public constructor(props) { super(props); @@ -57,13 +64,18 @@ export default class VoiceRecordComposerTile extends React.PureComponent { - // console.log('@@ UPDATE', freq); - // }); this.setState({recorder}); }; + private renderWaveformArea() { + if (!this.state.recorder) return null; + + return
+ + +
; + } + public render() { const classes = classNames({ 'mx_MessageComposer_button': !this.state.recorder, @@ -77,12 +89,13 @@ export default class VoiceRecordComposerTile extends React.PureComponent + {this.renderWaveformArea()} - ); + ); } } diff --git a/src/components/views/voice_messages/Clock.tsx b/src/components/views/voice_messages/Clock.tsx new file mode 100644 index 0000000000..6c256957e9 --- /dev/null +++ b/src/components/views/voice_messages/Clock.tsx @@ -0,0 +1,42 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import React from "react"; +import {replaceableComponent} from "../../../utils/replaceableComponent"; + +interface IProps { + seconds: number; +} + +interface IState { +} + +/** + * Simply converts seconds into minutes and seconds. Note that hours will not be + * displayed, making it possible to see "82:29". + */ +@replaceableComponent("views.voice_messages.Clock") +export default class Clock extends React.PureComponent { + public constructor(props) { + super(props); + } + + public render() { + const minutes = Math.floor(this.props.seconds / 60).toFixed(0).padStart(2, '0'); + const seconds = Math.round(this.props.seconds % 60).toFixed(0).padStart(2, '0'); // hide millis + return {minutes}:{seconds}; + } +} diff --git a/src/components/views/voice_messages/LiveRecordingClock.tsx b/src/components/views/voice_messages/LiveRecordingClock.tsx new file mode 100644 index 0000000000..00316d196a --- /dev/null +++ b/src/components/views/voice_messages/LiveRecordingClock.tsx @@ -0,0 +1,55 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import React from "react"; +import {IRecordingUpdate, VoiceRecorder} from "../../../voice/VoiceRecorder"; +import {replaceableComponent} from "../../../utils/replaceableComponent"; +import Clock from "./Clock"; + +interface IProps { + recorder: VoiceRecorder; +} + +interface IState { + seconds: number; +} + +/** + * A clock for a live recording. + */ +@replaceableComponent("views.voice_messages.LiveRecordingClock") +export default class LiveRecordingClock extends React.Component { + public constructor(props) { + super(props); + + this.state = {seconds: 0}; + this.props.recorder.liveData.onUpdate(this.onRecordingUpdate); + } + + shouldComponentUpdate(nextProps: Readonly, nextState: Readonly, nextContext: any): boolean { + const currentFloor = Math.floor(this.state.seconds); + const nextFloor = Math.floor(nextState.seconds); + return currentFloor !== nextFloor; + } + + private onRecordingUpdate = (update: IRecordingUpdate) => { + this.setState({seconds: update.timeSeconds}); + }; + + public render() { + return ; + } +} diff --git a/src/components/views/voice_messages/LiveRecordingWaveform.tsx b/src/components/views/voice_messages/LiveRecordingWaveform.tsx new file mode 100644 index 0000000000..e7cab4a5cb --- /dev/null +++ b/src/components/views/voice_messages/LiveRecordingWaveform.tsx @@ -0,0 +1,62 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import React from "react"; +import {IRecordingUpdate, VoiceRecorder} from "../../../voice/VoiceRecorder"; +import {replaceableComponent} from "../../../utils/replaceableComponent"; +import {arrayFastResample, arraySeed} from "../../../utils/arrays"; +import {percentageOf} from "../../../utils/numbers"; +import Waveform from "./Waveform"; + +interface IProps { + recorder: VoiceRecorder; +} + +interface IState { + heights: number[]; +} + +const DOWNSAMPLE_TARGET = 35; // number of bars we want + +/** + * A waveform which shows the waveform of a live recording + */ +@replaceableComponent("views.voice_messages.LiveRecordingWaveform") +export default class LiveRecordingWaveform extends React.PureComponent { + public constructor(props) { + super(props); + + this.state = {heights: arraySeed(0, DOWNSAMPLE_TARGET)}; + this.props.recorder.liveData.onUpdate(this.onRecordingUpdate); + } + + private onRecordingUpdate = (update: IRecordingUpdate) => { + // The waveform and the downsample target are pretty close, so we should be fine to + // do this, despite the docs on arrayFastResample. + const bars = arrayFastResample(Array.from(update.waveform), DOWNSAMPLE_TARGET); + this.setState({ + // The incoming data is between zero and one, but typically even screaming into a + // microphone won't send you over 0.6, so we artificially adjust the gain for the + // waveform. This results in a slightly more cinematic/animated waveform for the + // user. + heights: bars.map(b => percentageOf(b, 0, 0.50)), + }); + }; + + public render() { + return ; + } +} diff --git a/src/components/views/voice_messages/Waveform.tsx b/src/components/views/voice_messages/Waveform.tsx new file mode 100644 index 0000000000..5fa68dcadc --- /dev/null +++ b/src/components/views/voice_messages/Waveform.tsx @@ -0,0 +1,45 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import React from "react"; +import {replaceableComponent} from "../../../utils/replaceableComponent"; + +interface IProps { + relHeights: number[]; // relative heights (0-1) +} + +interface IState { +} + +/** + * A simple waveform component. This renders bars (centered vertically) for each + * height provided in the component properties. Updating the properties will update + * the rendered waveform. + */ +@replaceableComponent("views.voice_messages.Waveform") +export default class Waveform extends React.PureComponent { + public constructor(props) { + super(props); + } + + public render() { + return
+ {this.props.relHeights.map((h, i) => { + return ; + })} +
; + } +} diff --git a/src/utils/arrays.ts b/src/utils/arrays.ts index fa5515878f..52308937f7 100644 --- a/src/utils/arrays.ts +++ b/src/utils/arrays.ts @@ -14,6 +14,41 @@ See the License for the specific language governing permissions and limitations under the License. */ +/** + * Quickly resample an array to have less data points. This isn't a perfect representation, + * though this does work best if given a large array to downsample to a much smaller array. + * @param {number[]} input The input array to downsample. + * @param {number} points The number of samples to end up with. + * @returns {number[]} The downsampled array. + */ +export function arrayFastResample(input: number[], points: number): number[] { + // Heavily inpired by matrix-media-repo (used with permission) + // https://github.com/turt2live/matrix-media-repo/blob/abe72c87d2e29/util/util_audio/fastsample.go#L10 + const everyNth = Math.round(input.length / points); + const samples: number[] = []; + for (let i = 0; i < input.length; i += everyNth) { + samples.push(input[i]); + } + while (samples.length < points) { + samples.push(input[input.length - 1]); + } + return samples; +} + +/** + * Creates an array of the given length, seeded with the given value. + * @param {T} val The value to seed the array with. + * @param {number} length The length of the array to create. + * @returns {T[]} The array. + */ +export function arraySeed(val: T, length: number): T[] { + const a: T[] = []; + for (let i = 0; i < length; i++) { + a.push(val); + } + return a; +} + /** * Clones an array as fast as possible, retaining references of the array's values. * @param a The array to clone. Must be defined. diff --git a/src/voice/VoiceRecorder.ts b/src/voice/VoiceRecorder.ts index 6bf4189d8e..077990ac17 100644 --- a/src/voice/VoiceRecorder.ts +++ b/src/voice/VoiceRecorder.ts @@ -19,18 +19,15 @@ import encoderPath from 'opus-recorder/dist/encoderWorker.min.js'; import {MatrixClient} from "matrix-js-sdk/src/client"; import CallMediaHandler from "../CallMediaHandler"; import {SimpleObservable} from "matrix-widget-api"; +import {clamp} from "../utils/numbers"; const CHANNELS = 1; // stereo isn't important const SAMPLE_RATE = 48000; // 48khz is what WebRTC uses. 12khz is where we lose quality. const BITRATE = 24000; // 24kbps is pretty high quality for our use case in opus. -const FREQ_SAMPLE_RATE = 4; // Target rate of frequency data (samples / sec). We don't need this super often. -export interface IFrequencyPackage { - dbBars: Float32Array; - dbMin: number; - dbMax: number; - - // TODO: @@ TravisR: Generalize this for a timing package? +export interface IRecordingUpdate { + waveform: number[]; // floating points between 0 (low) and 1 (high). + timeSeconds: number; // float } export class VoiceRecorder { @@ -38,12 +35,12 @@ export class VoiceRecorder { private recorderContext: AudioContext; private recorderSource: MediaStreamAudioSourceNode; private recorderStream: MediaStream; - private recorderFreqNode: AnalyserNode; + private recorderFFT: AnalyserNode; + private recorderProcessor: ScriptProcessorNode; private buffer = new Uint8Array(0); private mxc: string; private recording = false; - private observable: SimpleObservable; - private freqTimerId: number; + private observable: SimpleObservable; public constructor(private client: MatrixClient) { } @@ -57,11 +54,31 @@ export class VoiceRecorder { }, }); this.recorderContext = new AudioContext({ - latencyHint: "interactive", + // latencyHint: "interactive", // we don't want a latency hint (this causes data smoothing) }); this.recorderSource = this.recorderContext.createMediaStreamSource(this.recorderStream); - this.recorderFreqNode = this.recorderContext.createAnalyser(); - this.recorderSource.connect(this.recorderFreqNode); + this.recorderFFT = this.recorderContext.createAnalyser(); + + // Bring the FFT time domain down a bit. The default is 2048, and this must be a power + // of two. We use 64 points because we happen to know down the line we need less than + // that, but 32 would be too few. Large numbers are not helpful here and do not add + // precision: they introduce higher precision outputs of the FFT (frequency data), but + // it makes the time domain less than helpful. + this.recorderFFT.fftSize = 64; + + // We use an audio processor to get accurate timing information. + // The size of the audio buffer largely decides how quickly we push timing/waveform data + // out of this class. Smaller buffers mean we update more frequently as we can't hold as + // many bytes. Larger buffers mean slower updates. For scale, 1024 gives us about 30Hz of + // updates and 2048 gives us about 20Hz. We use 1024 to get as close to perceived realtime + // as possible. Must be a power of 2. + this.recorderProcessor = this.recorderContext.createScriptProcessor(1024, CHANNELS, CHANNELS); + + // Connect our inputs and outputs + this.recorderSource.connect(this.recorderFFT); + this.recorderSource.connect(this.recorderProcessor); + this.recorderProcessor.connect(this.recorderContext.destination); + this.recorder = new Recorder({ encoderPath, // magic from webpack encoderSampleRate: SAMPLE_RATE, @@ -87,7 +104,7 @@ export class VoiceRecorder { }; } - public get frequencyData(): SimpleObservable { + public get liveData(): SimpleObservable { if (!this.recording) throw new Error("No observable when not recording"); return this.observable; } @@ -107,6 +124,34 @@ export class VoiceRecorder { return this.mxc; } + private tryUpdateLiveData = (ev: AudioProcessingEvent) => { + if (!this.recording) return; + + // The time domain is the input to the FFT, which means we use an array of the same + // size. The time domain is also known as the audio waveform. We're ignoring the + // output of the FFT here (frequency data) because we're not interested in it. + const data = new Float32Array(this.recorderFFT.fftSize); + this.recorderFFT.getFloatTimeDomainData(data); + + // We can't just `Array.from()` the array because we're dealing with 32bit floats + // and the built-in function won't consider that when converting between numbers. + // However, the runtime will convert the float32 to a float64 during the math operations + // which is why the loop works below. Note that a `.map()` call also doesn't work + // and will instead return a Float32Array still. + const translatedData: number[] = []; + for (let i = 0; i < data.length; i++) { + // We're clamping the values so we can do that math operation mentioned above, + // and to ensure that we produce consistent data (it's possible for the array + // to exceed the specified range with some audio input devices). + translatedData.push(clamp(data[i], 0, 1)); + } + + this.observable.update({ + waveform: translatedData, + timeSeconds: ev.playbackTime, + }); + }; + public async start(): Promise { if (this.mxc || this.hasRecording) { throw new Error("Recording already prepared"); @@ -117,18 +162,9 @@ export class VoiceRecorder { if (this.observable) { this.observable.close(); } - this.observable = new SimpleObservable(); + this.observable = new SimpleObservable(); await this.makeRecorder(); - this.freqTimerId = setInterval(() => { - if (!this.recording) return; - const data = new Float32Array(this.recorderFreqNode.frequencyBinCount); - this.recorderFreqNode.getFloatFrequencyData(data); - this.observable.update({ - dbBars: data, - dbMin: this.recorderFreqNode.minDecibels, - dbMax: this.recorderFreqNode.maxDecibels, - }); - }, 1000 / FREQ_SAMPLE_RATE) as any as number; // XXX: Linter doesn't understand timer environment + this.recorderProcessor.addEventListener("audioprocess", this.tryUpdateLiveData); await this.recorder.start(); this.recording = true; } @@ -150,8 +186,8 @@ export class VoiceRecorder { this.recorderStream.getTracks().forEach(t => t.stop()); // Finally do our post-processing and clean up - clearInterval(this.freqTimerId); this.recording = false; + this.recorderProcessor.removeEventListener("audioprocess", this.tryUpdateLiveData); await this.recorder.close(); return this.buffer;