diff --git a/src/voice/VoiceRecorder.ts b/src/voice/VoiceRecorder.ts index 50497438ca..319a6c3a37 100644 --- a/src/voice/VoiceRecorder.ts +++ b/src/voice/VoiceRecorder.ts @@ -16,6 +16,7 @@ limitations under the License. import * as Recorder from 'opus-recorder'; import encoderPath from 'opus-recorder/dist/encoderWorker.min.js'; +import mxVoiceWorkletPath from './mxVoiceWorklet'; import {MatrixClient} from "matrix-js-sdk/src/client"; import CallMediaHandler from "../CallMediaHandler"; import {SimpleObservable} from "matrix-widget-api"; @@ -36,7 +37,7 @@ export class VoiceRecorder { private recorderSource: MediaStreamAudioSourceNode; private recorderStream: MediaStream; private recorderFFT: AnalyserNode; - private recorderProcessor: ScriptProcessorNode; + private recorderWorklet: AudioWorkletNode; private buffer = new Uint8Array(0); private mxc: string; private recording = false; @@ -70,18 +71,20 @@ export class VoiceRecorder { // it makes the time domain less than helpful. this.recorderFFT.fftSize = 64; - // We use an audio processor to get accurate timing information. - // The size of the audio buffer largely decides how quickly we push timing/waveform data - // out of this class. Smaller buffers mean we update more frequently as we can't hold as - // many bytes. Larger buffers mean slower updates. For scale, 1024 gives us about 30Hz of - // updates and 2048 gives us about 20Hz. We use 1024 to get as close to perceived realtime - // as possible. Must be a power of 2. - this.recorderProcessor = this.recorderContext.createScriptProcessor(1024, CHANNELS, CHANNELS); + await this.recorderContext.audioWorklet.addModule(mxVoiceWorkletPath); + this.recorderWorklet = new AudioWorkletNode(this.recorderContext, "mx-voice-worklet"); // Connect our inputs and outputs this.recorderSource.connect(this.recorderFFT); - this.recorderSource.connect(this.recorderProcessor); - this.recorderProcessor.connect(this.recorderContext.destination); + this.recorderSource.connect(this.recorderWorklet); + this.recorderWorklet.connect(this.recorderContext.destination); + + // Dev note: we can't use `addEventListener` for some reason. It just doesn't work. + this.recorderWorklet.port.onmessage = (ev) => { + if (ev.data['ev'] === 'proc') { + this.tryUpdateLiveData(ev.data['timeMs']); + } + }; this.recorder = new Recorder({ encoderPath, // magic from webpack @@ -128,7 +131,7 @@ export class VoiceRecorder { return this.mxc; } - private tryUpdateLiveData = (ev: AudioProcessingEvent) => { + private tryUpdateLiveData = (timeMillis: number) => { if (!this.recording) return; // The time domain is the input to the FFT, which means we use an array of the same @@ -150,7 +153,7 @@ export class VoiceRecorder { this.observable.update({ waveform: translatedData, - timeSeconds: ev.playbackTime, + timeSeconds: timeMillis / 1000, }); }; @@ -166,7 +169,6 @@ export class VoiceRecorder { } this.observable = new SimpleObservable(); await this.makeRecorder(); - this.recorderProcessor.addEventListener("audioprocess", this.tryUpdateLiveData); await this.recorder.start(); this.recording = true; } @@ -178,6 +180,7 @@ export class VoiceRecorder { // Disconnect the source early to start shutting down resources this.recorderSource.disconnect(); + this.recorderWorklet.disconnect(); await this.recorder.stop(); // close the context after the recorder so the recorder doesn't try to @@ -189,7 +192,6 @@ export class VoiceRecorder { // Finally do our post-processing and clean up this.recording = false; - this.recorderProcessor.removeEventListener("audioprocess", this.tryUpdateLiveData); await this.recorder.close(); return this.buffer; diff --git a/src/voice/mxVoiceWorklet.js b/src/voice/mxVoiceWorklet.js new file mode 100644 index 0000000000..a74f5c17c9 --- /dev/null +++ b/src/voice/mxVoiceWorklet.js @@ -0,0 +1,35 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +class MxVoiceWorklet extends AudioWorkletProcessor { + constructor() { + super(); + + this._timeStart = 0; + } + + process(inputs, outputs, parameters) { + const now = (new Date()).getTime(); + if (this._timeStart === 0) { + this._timeStart = now; + } + + this.port.postMessage({ev: 'proc', timeMs: now - this._timeStart}); + return true; + } +} + +registerProcessor('mx-voice-worklet', MxVoiceWorklet);