Add ability to change audio and video devices during a call (#7173)

res/css/_components.scss

@@ -80,6 +80,7 @@
 @import "./views/avatars/_WidgetAvatar.scss";
 @import "./views/beta/_BetaCard.scss";
 @import "./views/context_menus/_CallContextMenu.scss";
+@import "./views/context_menus/_DeviceContextMenu.scss";
 @import "./views/context_menus/_IconizedContextMenu.scss";
 @import "./views/context_menus/_MessageContextMenu.scss";
 @import "./views/dialogs/_AddExistingToSpaceDialog.scss";

res/css/views/context_menus/_DeviceContextMenu.scss

@@ -0,0 +1,27 @@
+/*
+Copyright 2021 Šimon Brandner <simon.bra.ag@gmail.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+.mx_DeviceContextMenu {
+    max-width: 252px;
+
+    .mx_DeviceContextMenu_device_icon {
+        display: none;
+    }
+
+    .mx_IconizedContextMenu_label {
+        padding-left: 0 !important;
+    }
+}

res/css/views/context_menus/_IconizedContextMenu.scss

@@ -25,6 +25,11 @@ limitations under the License.
             padding-right: 20px;
         }
 
+        .mx_IconizedContextMenu_optionList_label {
+            font-size: $font-15px;
+            font-weight: $font-semi-bold;
+        }
+
         // the notFirst class is for cases where the optionList might be under a header of sorts.
         &:nth-child(n + 2), .mx_IconizedContextMenu_optionList_notFirst {
             // This is a bit of a hack when we could just use a simple border-top property,

res/css/views/voip/CallView/_CallViewButtons.scss

@@ -46,6 +46,10 @@ limitations under the License.
         justify-content: center;
         align-items: center;
 
+        position: relative;
+
+        box-shadow: 0px 4px 4px 0px #00000026; // Same on both themes
+
         &::before {
             content: '';
             display: inline-block;
@@ -60,6 +64,25 @@ limitations under the License.
             width: 24px;
         }
 
+        &.mx_CallViewButtons_dropdownButton {
+            width: 16px;
+            height: 16px;
+
+            position: absolute;
+            right: 0;
+            bottom: 0;
+
+            &::before {
+                width: 14px;
+                height: 14px;
+                mask-image: url('$(res)/img/element-icons/message/chevron-up.svg');
+            }
+
+            &.mx_CallViewButtons_dropdownButton_collapsed::before {
+                transform: rotate(180deg);
+            }
+        }
+
         // State buttons
         &.mx_CallViewButtons_button_on {
             background-color: $call-view-button-on-background;

src/MediaDeviceHandler.ts

@@ -72,12 +72,12 @@ export default class MediaDeviceHandler extends EventEmitter {
     /**
      * Retrieves devices from the SettingsStore and tells the js-sdk to use them
      */
-    public static loadDevices(): void {
+    public static async loadDevices(): Promise<void> {
         const audioDeviceId = SettingsStore.getValue("webrtc_audioinput");
         const videoDeviceId = SettingsStore.getValue("webrtc_videoinput");
 
-        MatrixClientPeg.get().getMediaHandler().setAudioInput(audioDeviceId);
+        await MatrixClientPeg.get().getMediaHandler().setAudioInput(audioDeviceId);
-        MatrixClientPeg.get().getMediaHandler().setVideoInput(videoDeviceId);
+        await MatrixClientPeg.get().getMediaHandler().setVideoInput(videoDeviceId);
     }
 
     public setAudioOutput(deviceId: string): void {
@@ -90,9 +90,9 @@ export default class MediaDeviceHandler extends EventEmitter {
      * need to be ended and started again for this change to take effect
      * @param {string} deviceId
      */
-    public setAudioInput(deviceId: string): void {
+    public async setAudioInput(deviceId: string): Promise<void> {
         SettingsStore.setValue("webrtc_audioinput", null, SettingLevel.DEVICE, deviceId);
-        MatrixClientPeg.get().getMediaHandler().setAudioInput(deviceId);
+        return MatrixClientPeg.get().getMediaHandler().setAudioInput(deviceId);
     }
 
     /**
@@ -100,16 +100,16 @@ export default class MediaDeviceHandler extends EventEmitter {
      * need to be ended and started again for this change to take effect
      * @param {string} deviceId
      */
-    public setVideoInput(deviceId: string): void {
+    public async setVideoInput(deviceId: string): Promise<void> {
         SettingsStore.setValue("webrtc_videoinput", null, SettingLevel.DEVICE, deviceId);
-        MatrixClientPeg.get().getMediaHandler().setVideoInput(deviceId);
+        return MatrixClientPeg.get().getMediaHandler().setVideoInput(deviceId);
     }
 
-    public setDevice(deviceId: string, kind: MediaDeviceKindEnum): void {
+    public async setDevice(deviceId: string, kind: MediaDeviceKindEnum): Promise<void> {
         switch (kind) {
             case MediaDeviceKindEnum.AudioOutput: this.setAudioOutput(deviceId); break;
-            case MediaDeviceKindEnum.AudioInput: this.setAudioInput(deviceId); break;
+            case MediaDeviceKindEnum.AudioInput: await this.setAudioInput(deviceId); break;
-            case MediaDeviceKindEnum.VideoInput: this.setVideoInput(deviceId); break;
+            case MediaDeviceKindEnum.VideoInput: await this.setVideoInput(deviceId); break;
         }
     }
 
@@ -124,4 +124,17 @@ export default class MediaDeviceHandler extends EventEmitter {
     public static getVideoInput(): string {
         return SettingsStore.getValueAt(SettingLevel.DEVICE, "webrtc_videoinput");
     }
+
+    /**
+     * Returns the current set deviceId for a device kind
+     * @param {MediaDeviceKindEnum} kind of the device that will be returned
+     * @returns {string} the deviceId
+     */
+    public static getDevice(kind: MediaDeviceKindEnum): string {
+        switch (kind) {
+            case MediaDeviceKindEnum.AudioOutput: return this.getAudioOutput();
+            case MediaDeviceKindEnum.AudioInput: return this.getAudioInput();
+            case MediaDeviceKindEnum.VideoInput: return this.getVideoInput();
+        }
+    }
 }

src/components/views/context_menus/DeviceContextMenu.tsx

@@ -0,0 +1,89 @@
+/*
+Copyright 2021 Šimon Brandner <simon.bra.ag@gmail.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React, { useEffect, useState } from "react";
+
+import MediaDeviceHandler, { MediaDeviceKindEnum } from "../../../MediaDeviceHandler";
+import IconizedContextMenu, { IconizedContextMenuOptionList, IconizedContextMenuRadio } from "./IconizedContextMenu";
+import { IProps as IContextMenuProps } from "../../structures/ContextMenu";
+import { _t, _td } from "../../../languageHandler";
+
+const SECTION_NAMES: Record<MediaDeviceKindEnum, string> = {
+    [MediaDeviceKindEnum.AudioInput]: _td("Input devices"),
+    [MediaDeviceKindEnum.AudioOutput]: _td("Output devices"),
+    [MediaDeviceKindEnum.VideoInput]: _td("Cameras"),
+};
+
+interface IDeviceContextMenuDeviceProps {
+    label: string;
+    selected: boolean;
+    onClick: () => void;
+}
+
+const DeviceContextMenuDevice: React.FC<IDeviceContextMenuDeviceProps> = ({ label, selected, onClick }) => {
+    return <IconizedContextMenuRadio
+        iconClassName="mx_DeviceContextMenu_device_icon"
+        label={label}
+        active={selected}
+        onClick={onClick}
+    />;
+};
+
+interface IDeviceContextMenuSectionProps {
+    deviceKind: MediaDeviceKindEnum;
+}
+
+const DeviceContextMenuSection: React.FC<IDeviceContextMenuSectionProps> = ({ deviceKind }) => {
+    const [devices, setDevices] = useState<MediaDeviceInfo[]>([]);
+    const [selectedDevice, setSelectedDevice] = useState(MediaDeviceHandler.getDevice(deviceKind));
+
+    useEffect(() => {
+        const getDevices = async () => {
+            return setDevices((await MediaDeviceHandler.getDevices())[deviceKind]);
+        };
+        getDevices();
+    }, [deviceKind]);
+
+    const onDeviceClick = (deviceId: string): void => {
+        MediaDeviceHandler.instance.setDevice(deviceId, deviceKind);
+        setSelectedDevice(deviceId);
+    };
+
+    return <IconizedContextMenuOptionList label={_t(SECTION_NAMES[deviceKind])}>
+        { devices.map(({ label, deviceId }) => {
+            return <DeviceContextMenuDevice
+                key={deviceId}
+                label={label}
+                selected={selectedDevice === deviceId}
+                onClick={() => onDeviceClick(deviceId)}
+            />;
+        }) }
+    </IconizedContextMenuOptionList>;
+};
+
+interface IProps extends IContextMenuProps {
+    deviceKinds: MediaDeviceKind[];
+}
+
+const DeviceContextMenu: React.FC<IProps> = ({ deviceKinds, ...props }) => {
+    return <IconizedContextMenu compact className="mx_DeviceContextMenu" {...props}>
+        { deviceKinds.map((kind) => {
+            return <DeviceContextMenuSection key={kind} deviceKind={kind as MediaDeviceKindEnum} />;
+        }) }
+    </IconizedContextMenu>;
+};
+
+export default DeviceContextMenu;

src/components/views/context_menus/IconizedContextMenu.tsx

@@ -33,6 +33,7 @@ interface IProps extends IContextMenuProps {
 interface IOptionListProps {
     first?: boolean;
     red?: boolean;
+    label?: string;
     className?: string;
 }
 
@@ -126,13 +127,20 @@ export const IconizedContextMenuOption: React.FC<IOptionProps> = ({
     </MenuItem>;
 };
 
-export const IconizedContextMenuOptionList: React.FC<IOptionListProps> = ({ first, red, className, children }) => {
+export const IconizedContextMenuOptionList: React.FC<IOptionListProps> = ({
+    first,
+    red,
+    className,
+    label,
+    children,
+}) => {
     const classes = classNames("mx_IconizedContextMenu_optionList", className, {
         mx_IconizedContextMenu_optionList_notFirst: !first,
         mx_IconizedContextMenu_optionList_red: red,
     });
 
     return <div className={classes}>
+        { label && <div><span className="mx_IconizedContextMenu_optionList_label">{ label }</span></div> }
         { children }
     </div>;
 };

src/components/views/elements/AccessibleTooltipButton.tsx

@@ -28,6 +28,7 @@ interface IProps extends React.ComponentProps<typeof AccessibleButton> {
     forceHide?: boolean;
     yOffset?: number;
     alignment?: Alignment;
+    onHover?: (hovering: boolean) => void;
     onHideTooltip?(ev: SyntheticEvent): void;
 }
 
@@ -52,6 +53,7 @@ export default class AccessibleTooltipButton extends React.PureComponent<IProps,
     }
 
     private showTooltip = () => {
+        if (this.props.onHover) this.props.onHover(true);
         if (this.props.forceHide) return;
         this.setState({
             hover: true,
@@ -59,6 +61,7 @@ export default class AccessibleTooltipButton extends React.PureComponent<IProps,
     };
 
     private hideTooltip = (ev: SyntheticEvent) => {
+        if (this.props.onHover) this.props.onHover(false);
         this.setState({
             hover: false,
         });

src/components/views/voip/CallView/CallViewButtons.tsx

@@ -16,7 +16,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React, { createRef } from "react";
+import React, { createRef, useState } from "react";
 import classNames from "classnames";
 import { MatrixCall } from "matrix-js-sdk/src/webrtc/call";
 
@@ -26,10 +26,14 @@ import DialpadContextMenu from "../../context_menus/DialpadContextMenu";
 import { Alignment } from "../../elements/Tooltip";
 import {
     alwaysAboveLeftOf,
+    alwaysAboveRightOf,
     ChevronFace,
     ContextMenuTooltipButton,
+    useContextMenu,
 } from '../../../structures/ContextMenu';
 import { _t } from "../../../../languageHandler";
+import DeviceContextMenu from "../../context_menus/DeviceContextMenu";
+import { MediaDeviceKindEnum } from "../../../../MediaDeviceHandler";
 
 // Height of the header duplicated from CSS because we need to subtract it from our max
 // height to get the max height of the video
@@ -39,15 +43,22 @@ const TOOLTIP_Y_OFFSET = -24;
 
 const CONTROLS_HIDE_DELAY = 2000;
 
-interface IButtonProps {
+interface IButtonProps extends Omit<React.ComponentProps<typeof AccessibleTooltipButton>, "title"> {
     state: boolean;
     className: string;
-    onLabel: string;
+    onLabel?: string;
-    offLabel: string;
+    offLabel?: string;
-    onClick: () => void;
+    onClick: (event: React.MouseEvent) => void;
 }
 
-const CallViewToggleButton: React.FC<IButtonProps> = ({ state: isOn, className, onLabel, offLabel, onClick }) => {
+const CallViewToggleButton: React.FC<IButtonProps> = ({
+    children,
+    state: isOn,
+    className,
+    onLabel,
+    offLabel,
+    ...props
+}) => {
     const classes = classNames("mx_CallViewButtons_button", className, {
         mx_CallViewButtons_button_on: isOn,
         mx_CallViewButtons_button_off: !isOn,
@@ -56,11 +67,48 @@ const CallViewToggleButton: React.FC<IButtonProps> = ({ state: isOn, className,
     return (
         <AccessibleTooltipButton
             className={classes}
-            onClick={onClick}
             title={isOn ? onLabel : offLabel}
             alignment={Alignment.Top}
             yOffset={TOOLTIP_Y_OFFSET}
-        />
+            {...props}
+        >
+            { children }
+        </AccessibleTooltipButton>
+    );
+};
+
+interface IDropdownButtonProps extends IButtonProps {
+    deviceKinds: MediaDeviceKindEnum[];
+}
+
+const CallViewDropdownButton: React.FC<IDropdownButtonProps> = ({ state, deviceKinds, ...props }) => {
+    const [menuDisplayed, buttonRef, openMenu, closeMenu] = useContextMenu();
+    const [hoveringDropdown, setHoveringDropdown] = useState(false);
+
+    const classes = classNames("mx_CallViewButtons_button", "mx_CallViewButtons_dropdownButton", {
+        mx_CallViewButtons_dropdownButton_collapsed: !menuDisplayed,
+    });
+
+    const onClick = (event: React.MouseEvent): void => {
+        event.stopPropagation();
+        openMenu();
+    };
+
+    return (
+        <CallViewToggleButton inputRef={buttonRef} forceHide={menuDisplayed || hoveringDropdown} state={state} {...props}>
+            <CallViewToggleButton
+                className={classes}
+                onClick={onClick}
+                onHover={(hovering) => setHoveringDropdown(hovering)}
+                state={state}
+            />
+            { menuDisplayed && <DeviceContextMenu
+                {...alwaysAboveRightOf(buttonRef.current?.getBoundingClientRect())}
+
+                onFinished={closeMenu}
+                deviceKinds={deviceKinds}
+            /> }
+        </CallViewToggleButton>
     );
 };
 
@@ -221,19 +269,21 @@ export default class CallViewButtons extends React.Component<IProps, IState> {
                     alignment={Alignment.Top}
                     yOffset={TOOLTIP_Y_OFFSET}
                 /> }
-                <CallViewToggleButton
+                <CallViewDropdownButton
                     state={!this.props.buttonsState.micMuted}
                     className="mx_CallViewButtons_button_mic"
                     onLabel={_t("Mute the microphone")}
                     offLabel={_t("Unmute the microphone")}
                     onClick={this.props.handlers.onMicMuteClick}
+                    deviceKinds={[MediaDeviceKindEnum.AudioInput, MediaDeviceKindEnum.AudioOutput]}
                 />
-                { this.props.buttonsVisibility.vidMute && <CallViewToggleButton
+                { this.props.buttonsVisibility.vidMute && <CallViewDropdownButton
                     state={!this.props.buttonsState.vidMuted}
                     className="mx_CallViewButtons_button_vid"
                     onLabel={_t("Stop the camera")}
                     offLabel={_t("Start the camera")}
                     onClick={this.props.handlers.onVidMuteClick}
+                    deviceKinds={[MediaDeviceKindEnum.VideoInput]}
                 /> }
                 { this.props.buttonsVisibility.screensharing && <CallViewToggleButton
                     state={this.props.buttonsState.screensharing}

src/components/views/voip/VideoFeed.tsx

@@ -161,6 +161,7 @@ export default class VideoFeed extends React.PureComponent<IProps, IState> {
             audioMuted: this.props.feed.isAudioMuted(),
             videoMuted: this.props.feed.isVideoMuted(),
         });
+        this.playMedia();
     };
 
     private onMuteStateChanged = () => {

src/i18n/strings/en_EN.json

@@ -2910,6 +2910,9 @@
     "There was an error finding this widget.": "There was an error finding this widget.",
     "Resume": "Resume",
     "Hold": "Hold",
+    "Input devices": "Input devices",
+    "Output devices": "Output devices",
+    "Cameras": "Cameras",
     "Resend %(unsentCount)s reaction(s)": "Resend %(unsentCount)s reaction(s)",
     "Open in OpenStreetMap": "Open in OpenStreetMap",
     "Forward": "Forward",

src/settings/Settings.tsx

@@ -662,15 +662,15 @@ export const SETTINGS: {[setting: string]: ISetting} = {
     },
     "webrtc_audiooutput": {
         supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS,
-        default: null,
+        default: "default",
     },
     "webrtc_audioinput": {
         supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS,
-        default: null,
+        default: "default",
     },
     "webrtc_videoinput": {
         supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS,
-        default: null,
+        default: "default",
     },
     "language": {
         supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS_WITH_CONFIG,