From 7d9562137ef2768c34214c122ba99b77cf5a28f4 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 19 Apr 2021 21:54:08 -0600 Subject: [PATCH] Replace deprecated processor with a worklet --- src/@types/global.d.ts | 27 ++++++++++++++++++++++++ src/voice/RecorderWorklet.ts | 37 +++++++++++++++++++++++++++++++++ src/voice/VoiceRecording.ts | 40 ++++++++++++++++++++++-------------- src/voice/consts.ts | 29 ++++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 15 deletions(-) create mode 100644 src/voice/RecorderWorklet.ts create mode 100644 src/voice/consts.ts diff --git a/src/@types/global.d.ts b/src/@types/global.d.ts index ee0963e537..78dad28566 100644 --- a/src/@types/global.d.ts +++ b/src/@types/global.d.ts @@ -129,4 +129,31 @@ declare global { // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error/columnNumber columnNumber?: number; } + + // https://github.com/microsoft/TypeScript/issues/28308#issuecomment-650802278 + interface AudioWorkletProcessor { + readonly port: MessagePort; + process( + inputs: Float32Array[][], + outputs: Float32Array[][], + parameters: Record + ): boolean; + + } + + // https://github.com/microsoft/TypeScript/issues/28308#issuecomment-650802278 + const AudioWorkletProcessor: { + prototype: AudioWorkletProcessor; + new (options?: AudioWorkletNodeOptions): AudioWorkletProcessor; + }; + + // https://github.com/microsoft/TypeScript/issues/28308#issuecomment-650802278 + function registerProcessor( + name: string, + processorCtor: (new ( + options?: AudioWorkletNodeOptions + ) => AudioWorkletProcessor) & { + parameterDescriptors?: AudioParamDescriptor[]; + } + ); } diff --git a/src/voice/RecorderWorklet.ts b/src/voice/RecorderWorklet.ts new file mode 100644 index 0000000000..11f24fce4c --- /dev/null +++ b/src/voice/RecorderWorklet.ts @@ -0,0 +1,37 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {ITimingPayload, PayloadEvent, WORKLET_NAME} from "./consts"; + +// from AudioWorkletGlobalScope: https://developer.mozilla.org/en-US/docs/Web/API/AudioWorkletGlobalScope +declare const currentTime: number; +declare const currentFrame: number; +declare const sampleRate: number; + +class MxVoiceWorklet extends AudioWorkletProcessor { + constructor() { + super(); + } + + process(inputs, outputs, parameters) { + this.port.postMessage({ev: PayloadEvent.Timekeep, timeSeconds: currentTime}); + return true; + } +} + +registerProcessor(WORKLET_NAME, MxVoiceWorklet); + +export default null; // to appease module loaders (we never use the export) diff --git a/src/voice/VoiceRecording.ts b/src/voice/VoiceRecording.ts index fc52a38fa9..8e506c235c 100644 --- a/src/voice/VoiceRecording.ts +++ b/src/voice/VoiceRecording.ts @@ -23,6 +23,7 @@ import {clamp} from "../utils/numbers"; import EventEmitter from "events"; import {IDestroyable} from "../utils/IDestroyable"; import {Singleflight} from "../utils/Singleflight"; +import {PayloadEvent, WORKLET_NAME} from "./consts"; const CHANNELS = 1; // stereo isn't important const SAMPLE_RATE = 48000; // 48khz is what WebRTC uses. 12khz is where we lose quality. @@ -49,7 +50,7 @@ export class VoiceRecording extends EventEmitter implements IDestroyable { private recorderSource: MediaStreamAudioSourceNode; private recorderStream: MediaStream; private recorderFFT: AnalyserNode; - private recorderProcessor: ScriptProcessorNode; + private recorderWorklet: AudioWorkletNode; private buffer = new Uint8Array(0); private mxc: string; private recording = false; @@ -93,18 +94,28 @@ export class VoiceRecording extends EventEmitter implements IDestroyable { // it makes the time domain less than helpful. this.recorderFFT.fftSize = 64; - // We use an audio processor to get accurate timing information. - // The size of the audio buffer largely decides how quickly we push timing/waveform data - // out of this class. Smaller buffers mean we update more frequently as we can't hold as - // many bytes. Larger buffers mean slower updates. For scale, 1024 gives us about 30Hz of - // updates and 2048 gives us about 20Hz. We use 1024 to get as close to perceived realtime - // as possible. Must be a power of 2. - this.recorderProcessor = this.recorderContext.createScriptProcessor(1024, CHANNELS, CHANNELS); + // Set up our worklet. We use this for timing information and waveform analysis: the + // web audio API prefers this be done async to avoid holding the main thread with math. + const mxRecorderWorkletPath = document.body.dataset.vectorRecorderWorkletScript; + if (!mxRecorderWorkletPath) { + throw new Error("Unable to create recorder: no worklet script registered"); + } + await this.recorderContext.audioWorklet.addModule(mxRecorderWorkletPath); + this.recorderWorklet = new AudioWorkletNode(this.recorderContext, WORKLET_NAME); // Connect our inputs and outputs this.recorderSource.connect(this.recorderFFT); - this.recorderSource.connect(this.recorderProcessor); - this.recorderProcessor.connect(this.recorderContext.destination); + this.recorderSource.connect(this.recorderWorklet); + this.recorderWorklet.connect(this.recorderContext.destination); + + // Dev note: we can't use `addEventListener` for some reason. It just doesn't work. + this.recorderWorklet.port.onmessage = (ev) => { + switch(ev.data['ev']) { + case PayloadEvent.Timekeep: + this.processAudioUpdate(ev.data['timeSeconds']); + break; + } + }; this.recorder = new Recorder({ encoderPath, // magic from webpack @@ -151,7 +162,7 @@ export class VoiceRecording extends EventEmitter implements IDestroyable { return this.mxc; } - private processAudioUpdate = (ev: AudioProcessingEvent) => { + private processAudioUpdate = (timeSeconds: number) => { if (!this.recording) return; // The time domain is the input to the FFT, which means we use an array of the same @@ -175,12 +186,12 @@ export class VoiceRecording extends EventEmitter implements IDestroyable { this.observable.update({ waveform: translatedData, - timeSeconds: ev.playbackTime, + timeSeconds: timeSeconds, }); // Now that we've updated the data/waveform, let's do a time check. We don't want to // go horribly over the limit. We also emit a warning state if needed. - const secondsLeft = TARGET_MAX_LENGTH - ev.playbackTime; + const secondsLeft = TARGET_MAX_LENGTH - timeSeconds; if (secondsLeft <= 0) { // noinspection JSIgnoredPromiseFromCall - we aren't concerned with it overlapping this.stop(); @@ -204,7 +215,6 @@ export class VoiceRecording extends EventEmitter implements IDestroyable { } this.observable = new SimpleObservable(); await this.makeRecorder(); - this.recorderProcessor.addEventListener("audioprocess", this.processAudioUpdate); await this.recorder.start(); this.recording = true; this.emit(RecordingState.Started); @@ -218,6 +228,7 @@ export class VoiceRecording extends EventEmitter implements IDestroyable { // Disconnect the source early to start shutting down resources this.recorderSource.disconnect(); + this.recorderWorklet.disconnect(); await this.recorder.stop(); // close the context after the recorder so the recorder doesn't try to @@ -229,7 +240,6 @@ export class VoiceRecording extends EventEmitter implements IDestroyable { // Finally do our post-processing and clean up this.recording = false; - this.recorderProcessor.removeEventListener("audioprocess", this.processAudioUpdate); await this.recorder.close(); this.emit(RecordingState.Ended); diff --git a/src/voice/consts.ts b/src/voice/consts.ts new file mode 100644 index 0000000000..dbd3b574f4 --- /dev/null +++ b/src/voice/consts.ts @@ -0,0 +1,29 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export const WORKLET_NAME = "mx-voice-worklet"; + +export enum PayloadEvent { + Timekeep = "timekeep", +} + +export interface IPayload { + ev: PayloadEvent; +} + +export interface ITimingPayload extends IPayload { + timeSeconds: number; +}