OpenCloud
/
element-web
mirror of https://github.com/vector-im/element-web
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #5833 from SimonBrandner/feed 

Support for multiple streams (not MSC3077)
pull/21833/head
J. Ryan Stinnett 2021-05-06 11:20:09 +01:00 committed by GitHub
parent 68210b1415 9aaf321e4e
commit 683aec1066
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 471 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
res/css/views/voip/_CallView.scss
View File

@ -65,14 +65,17 @@ limitations under the License.
}
}
.mx_CallView_voice {
.mx_CallView_content {
position: relative;
display: flex;
flex-direction: column;
border-radius: 8px;
}
.mx_CallView_voice {
align-items: center;
justify-content: center;
flex-direction: column;
background-color: $inverted-bg-color;
border-radius: 8px;
}
.mx_CallView_voice_avatarsContainer {
@ -109,9 +112,7 @@ limitations under the License.
.mx_CallView_video {
width: 100%;
height: 100%;
position: relative;
z-index: 30;
border-radius: 8px;
overflow: hidden;
}

24
res/css/views/voip/_VideoFeed.scss
View File

@ -14,21 +14,37 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
.mx_VideoFeed_voice {
// We don't want to collide with the call controls that have 52px of height
padding-bottom: 52px;
background-color: $inverted-bg-color;
}
.mx_VideoFeed_remote {
width: 100%;
height: 100%;
background-color: #000;
z-index: 50;
display: flex;
justify-content: center;
align-items: center;
&.mx_VideoFeed_video {
background-color: #000;
}
}
.mx_VideoFeed_local {
width: 25%;
height: 25%;
max-width: 25%;
max-height: 25%;
position: absolute;
right: 10px;
top: 10px;
z-index: 100;
border-radius: 4px;
&.mx_VideoFeed_video {
background-color: transparent;
}
}
.mx_VideoFeed_mirror {

10
src/@types/global.d.ts vendored
View File

@ -118,6 +118,16 @@ declare global {
interface HTMLAudioElement {
type?: string;
// sinkId & setSinkId are experimental and typescript doesn't know about them
sinkId: string;
setSinkId(outputId: string);
}
interface HTMLVideoElement {
type?: string;
// sinkId & setSinkId are experimental and typescript doesn't know about them
sinkId: string;
setSinkId(outputId: string);
}
interface Element {

58
src/CallHandler.tsx
View File

@ -85,6 +85,7 @@ import { Action } from './dispatcher/actions';
import VoipUserMapper from './VoipUserMapper';
import { addManagedHybridWidget, isManagedHybridWidgetEnabled } from './widgets/ManagedHybrid';
import { randomUppercaseString, randomLowercaseString } from "matrix-js-sdk/src/randomstring";
import EventEmitter from 'events';
import SdkConfig from './SdkConfig';
import { ensureDMExists, findDMForUser } from './createRoom';
@ -138,22 +139,12 @@ export enum PlaceCallType {
ScreenSharing = 'screensharing',
}
function getRemoteAudioElement(): HTMLAudioElement {
// this needs to be somewhere at the top of the DOM which
// always exists to avoid audio interruptions.
// Might as well just use DOM.
const remoteAudioElement = document.getElementById("remoteAudio") as HTMLAudioElement;
if (!remoteAudioElement) {
console.error(
"Failed to find remoteAudio element - cannot play audio!" +
"You need to add an <audio/> to the DOM.",
);
return null;
}
return remoteAudioElement;
export enum CallHandlerEvent {
CallsChanged = "calls_changed",
CallChangeRoom = "call_change_room",
}
export default class CallHandler {
export default class CallHandler extends EventEmitter {
private calls = new Map<string, MatrixCall>(); // roomId -> call
// Calls started as an attended transfer, ie. with the intention of transferring another
// call with a different party to this one.
@ -514,6 +505,7 @@ export default class CallHandler {
}
this.calls.set(mappedRoomId, newCall);
this.emit(CallHandlerEvent.CallsChanged, this.calls);
this.setCallListeners(newCall);
this.setCallState(newCall, newCall.state);
});
@ -546,10 +538,7 @@ export default class CallHandler {
this.removeCallForRoom(mappedRoomId);
mappedRoomId = newMappedRoomId;
this.calls.set(mappedRoomId, call);
dis.dispatch({
action: Action.CallChangeRoom,
call,
});
this.emit(CallHandlerEvent.CallChangeRoom, call);
}
}
});
@ -598,11 +587,6 @@ export default class CallHandler {
}
}
private setCallAudioElement(call: MatrixCall) {
const audioElement = getRemoteAudioElement();
if (audioElement) call.setRemoteAudioElement(audioElement);
}
private setCallState(call: MatrixCall, status: CallState) {
const mappedRoomId = CallHandler.sharedInstance().roomIdForCall(call);
@ -619,6 +603,7 @@ export default class CallHandler {
private removeCallForRoom(roomId: string) {
this.calls.delete(roomId);
this.emit(CallHandlerEvent.CallsChanged, this.calls);
}
private showICEFallbackPrompt() {
@ -679,11 +664,7 @@ export default class CallHandler {
}, null, true);
}
private async placeCall(
roomId: string, type: PlaceCallType,
localElement: HTMLVideoElement, remoteElement: HTMLVideoElement,
transferee: MatrixCall,
) {
private async placeCall(roomId: string, type: PlaceCallType, transferee: MatrixCall) {
Analytics.trackEvent('voip', 'placeCall', 'type', type);
CountlyAnalytics.instance.trackStartCall(roomId, type === PlaceCallType.Video, false);
@ -695,22 +676,19 @@ export default class CallHandler {
const call = MatrixClientPeg.get().createCall(mappedRoomId);
this.calls.set(roomId, call);
this.emit(CallHandlerEvent.CallsChanged, this.calls);
if (transferee) {
this.transferees[call.callId] = transferee;
}
this.setCallListeners(call);
this.setCallAudioElement(call);
this.setActiveCallRoomId(roomId);
if (type === PlaceCallType.Voice) {
call.placeVoiceCall();
} else if (type === 'video') {
call.placeVideoCall(
remoteElement,
localElement,
);
call.placeVideoCall();
} else if (type === PlaceCallType.ScreenSharing) {
const screenCapErrorString = PlatformPeg.get().screenCaptureErrorString();
if (screenCapErrorString) {
@ -724,13 +702,12 @@ export default class CallHandler {
}
call.placeScreenSharingCall(
remoteElement,
localElement,
async (): Promise<DesktopCapturerSource> => {
const {finished} = Modal.createDialog(DesktopCapturerSourcePicker);
const [source] = await finished;
return source;
});
},
);
} else {
console.error("Unknown conf call type: " + type);
}
@ -787,17 +764,12 @@ export default class CallHandler {
} else if (members.length === 2) {
console.info(`Place ${payload.type} call in ${payload.room_id}`);
this.placeCall(
payload.room_id, payload.type, payload.local_element, payload.remote_element,
payload.transferee,
);
this.placeCall(payload.room_id, payload.type, payload.transferee);
} else { // > 2
dis.dispatch({
action: "place_conference_call",
room_id: payload.room_id,
type: payload.type,
remote_element: payload.remote_element,
local_element: payload.local_element,
});
}
}
@ -833,6 +805,7 @@ export default class CallHandler {
Analytics.trackEvent('voip', 'receiveCall', 'type', call.type);
this.calls.set(mappedRoomId, call)
this.emit(CallHandlerEvent.CallsChanged, this.calls);
this.setCallListeners(call);
// get ready to send encrypted events in the room, so if the user does answer
@ -875,7 +848,6 @@ export default class CallHandler {
const call = this.calls.get(payload.room_id);
call.answer();
this.setCallAudioElement(call);
this.setActiveCallRoomId(payload.room_id);
CountlyAnalytics.instance.trackJoinCall(payload.room_id, call.type === CallType.Video, false);
dis.dispatch({

5
src/CallMediaHandler.js
View File

@ -16,7 +16,7 @@
import SettingsStore from "./settings/SettingsStore";
import {SettingLevel} from "./settings/SettingLevel";
import {setMatrixCallAudioInput, setMatrixCallAudioOutput, setMatrixCallVideoInput} from "matrix-js-sdk/src/matrix";
import {setMatrixCallAudioInput, setMatrixCallVideoInput} from "matrix-js-sdk/src/matrix";
export default {
hasAnyLabeledDevices: async function() {
@ -50,18 +50,15 @@ export default {
},
loadDevices: function() {
const audioOutDeviceId = SettingsStore.getValue("webrtc_audiooutput");
const audioDeviceId = SettingsStore.getValue("webrtc_audioinput");
const videoDeviceId = SettingsStore.getValue("webrtc_videoinput");
setMatrixCallAudioOutput(audioOutDeviceId);
setMatrixCallAudioInput(audioDeviceId);
setMatrixCallVideoInput(videoDeviceId);
},
setAudioOutput: function(deviceId) {
SettingsStore.setValue("webrtc_audiooutput", null, SettingLevel.DEVICE, deviceId);
setMatrixCallAudioOutput(deviceId);
},
setAudioInput: function(deviceId) {

20
src/components/structures/LoggedInView.tsx
View File

@ -59,6 +59,9 @@ import { getKeyBindingsManager, NavigationAction, RoomAction } from '../../KeyBi
import { IOpts } from "../../createRoom";
import SpacePanel from "../views/spaces/SpacePanel";
import {replaceableComponent} from "../../utils/replaceableComponent";
import CallHandler, { CallHandlerEvent } from '../../CallHandler';
import { MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
import AudioFeedArrayForCall from '../views/voip/AudioFeedArrayForCall';
// We need to fetch each pinned message individually (if we don't already have it)
// so each pinned message may trigger a request. Limit the number per room for sanity.
@ -119,6 +122,7 @@ interface IState {
usageLimitEventContent?: IUsageLimit;
usageLimitEventTs?: number;
useCompactLayout: boolean;
activeCalls: Array<MatrixCall>;
}
/**
@ -160,6 +164,7 @@ class LoggedInView extends React.Component<IProps, IState> {
// use compact timeline view
useCompactLayout: SettingsStore.getValue('useCompactLayout'),
usageLimitDismissed: false,
activeCalls: [],
};
// stash the MatrixClient in case we log out before we are unmounted
@ -175,6 +180,7 @@ class LoggedInView extends React.Component<IProps, IState> {
componentDidMount() {
document.addEventListener('keydown', this._onNativeKeyDown, false);
CallHandler.sharedInstance().addListener(CallHandlerEvent.CallsChanged, this.onCallsChanged);
this._updateServerNoticeEvents();
@ -199,6 +205,7 @@ class LoggedInView extends React.Component<IProps, IState> {
componentWillUnmount() {
document.removeEventListener('keydown', this._onNativeKeyDown, false);
CallHandler.sharedInstance().removeListener(CallHandlerEvent.CallsChanged, this.onCallsChanged);
this._matrixClient.removeListener("accountData", this.onAccountData);
this._matrixClient.removeListener("sync", this.onSync);
this._matrixClient.removeListener("RoomState.events", this.onRoomStateEvents);
@ -206,6 +213,12 @@ class LoggedInView extends React.Component<IProps, IState> {
this.resizer.detach();
}
private onCallsChanged = () => {
this.setState({
activeCalls: CallHandler.sharedInstance().getAllActiveCalls(),
});
};
// Child components assume that the client peg will not be null, so give them some
// sort of assurance here by only allowing a re-render if the client is truthy.
//
@ -661,6 +674,12 @@ class LoggedInView extends React.Component<IProps, IState> {
bodyClasses += ' mx_MatrixChat_useCompactLayout';
}
const audioFeedArraysForCalls = this.state.activeCalls.map((call) => {
return (
<AudioFeedArrayForCall call={call} key={call.callId} />
);
});
return (
<MatrixClientContext.Provider value={this._matrixClient}>
<div
@ -685,6 +704,7 @@ class LoggedInView extends React.Component<IProps, IState> {
<CallContainer />
<NonUrgentToastContainer />
<HostSignupContainer />
{audioFeedArraysForCalls}
</MatrixClientContext.Provider>
);
}

97
src/components/views/voip/AudioFeed.tsx Normal file
View File

@ -0,0 +1,97 @@
/*
Copyright 2021 Šimon Brandner <simon.bra.ag@gmail.com>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React, {createRef} from 'react';
import { CallFeed, CallFeedEvent } from 'matrix-js-sdk/src/webrtc/callFeed';
import { logger } from 'matrix-js-sdk/src/logger';
import CallMediaHandler from "../../../CallMediaHandler";
interface IProps {
feed: CallFeed,
}
export default class AudioFeed extends React.Component<IProps> {
private element = createRef<HTMLAudioElement>();
componentDidMount() {
this.props.feed.addListener(CallFeedEvent.NewStream, this.onNewStream);
this.playMedia();
}
componentWillUnmount() {
this.props.feed.removeListener(CallFeedEvent.NewStream, this.onNewStream);
this.stopMedia();
}
private playMedia() {
const element = this.element.current;
const audioOutput = CallMediaHandler.getAudioOutput();
if (audioOutput) {
try {
// This seems quite unreliable in Chrome, although I haven't yet managed to make a jsfiddle where
// it fails.
// It seems reliable if you set the sink ID after setting the srcObject and then set the sink ID
// back to the default after the call is over - Dave
element.setSinkId(audioOutput);
} catch (e) {
console.error("Couldn't set requested audio output device: using default", e);
logger.warn("Couldn't set requested audio output device: using default", e);
}
}
element.muted = false;
element.srcObject = this.props.feed.stream;
element.autoplay = true;
try {
// A note on calling methods on media elements:
// We used to have queues per media element to serialise all calls on those elements.
// The reason given for this was that load() and play() were racing. However, we now
// never call load() explicitly so this seems unnecessary. However, serialising every
// operation was causing bugs where video would not resume because some play command
// had got stuck and all media operations were queued up behind it. If necessary, we
// should serialise the ones that need to be serialised but then be able to interrupt
// them with another load() which will cancel the pending one, but since we don't call
// load() explicitly, it shouldn't be a problem. - Dave
element.play()
} catch (e) {
logger.info("Failed to play media element with feed", this.props.feed, e);
}
}
private stopMedia() {
const element = this.element.current;
element.pause();
element.src = null;
// As per comment in componentDidMount, setting the sink ID back to the
// default once the call is over makes setSinkId work reliably. - Dave
// Since we are not using the same element anymore, the above doesn't
// seem to be necessary - Šimon
}
private onNewStream = () => {
this.playMedia();
};
render() {
return (
<audio ref={this.element} />
);
}
}

60
src/components/views/voip/AudioFeedArrayForCall.tsx Normal file
View File

@ -0,0 +1,60 @@
/*
Copyright 2021 Šimon Brandner <simon.bra.ag@gmail.com>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React from "react";
import AudioFeed from "./AudioFeed"
import { CallEvent, MatrixCall } from "matrix-js-sdk/src/webrtc/call";
import { CallFeed } from "matrix-js-sdk/src/webrtc/callFeed";
interface IProps {
call: MatrixCall;
}
interface IState {
feeds: Array<CallFeed>;
}
export default class AudioFeedArrayForCall extends React.Component<IProps, IState> {
constructor(props: IProps) {
super(props);
this.state = {
feeds: [],
};
}
componentDidMount() {
this.props.call.addListener(CallEvent.FeedsChanged, this.onFeedsChanged);
}
componentWillUnmount() {
this.props.call.removeListener(CallEvent.FeedsChanged, this.onFeedsChanged);
}
onFeedsChanged = () => {
this.setState({
feeds: this.props.call.getRemoteFeeds(),
});
}
render() {
return this.state.feeds.map((feed, i) => {
return (
<AudioFeed feed={feed} key={i} />
);
});
}
}

26
src/components/views/voip/CallPreview.tsx
View File

@ -19,7 +19,7 @@ import React from 'react';
import CallView from "./CallView";
import RoomViewStore from '../../../stores/RoomViewStore';
import CallHandler from '../../../CallHandler';
import CallHandler, { CallHandlerEvent } from '../../../CallHandler';
import dis from '../../../dispatcher/dispatcher';
import { ActionPayload } from '../../../dispatcher/payloads';
import PersistentApp from "../elements/PersistentApp";
@ -27,7 +27,6 @@ import SettingsStore from "../../../settings/SettingsStore";
import { CallEvent, CallState, MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
import { MatrixClientPeg } from '../../../MatrixClientPeg';
import {replaceableComponent} from "../../../utils/replaceableComponent";
import { Action } from '../../../dispatcher/actions';
const SHOW_CALL_IN_STATES = [
CallState.Connected,
@ -110,12 +109,14 @@ export default class CallPreview extends React.Component<IProps, IState> {
}
public componentDidMount() {
CallHandler.sharedInstance().addListener(CallHandlerEvent.CallChangeRoom, this.updateCalls);
this.roomStoreToken = RoomViewStore.addListener(this.onRoomViewStoreUpdate);
this.dispatcherRef = dis.register(this.onAction);
MatrixClientPeg.get().on(CallEvent.RemoteHoldUnhold, this.onCallRemoteHold);
}
public componentWillUnmount() {
CallHandler.sharedInstance().removeListener(CallHandlerEvent.CallChangeRoom, this.updateCalls);
MatrixClientPeg.get().removeListener(CallEvent.RemoteHoldUnhold, this.onCallRemoteHold);
if (this.roomStoreToken) {
this.roomStoreToken.remove();
@ -143,21 +144,24 @@ export default class CallPreview extends React.Component<IProps, IState> {
switch (payload.action) {
// listen for call state changes to prod the render method, which
// may hide the global CallView if the call it is tracking is dead
case Action.CallChangeRoom:
case 'call_state': {
const [primaryCall, secondaryCalls] = getPrimarySecondaryCalls(
CallHandler.sharedInstance().getAllActiveCallsNotInRoom(this.state.roomId),
);
this.setState({
primaryCall: primaryCall,
secondaryCall: secondaryCalls[0],
});
this.updateCalls();
break;
}
}
};
private updateCalls = () => {
const [primaryCall, secondaryCalls] = getPrimarySecondaryCalls(
CallHandler.sharedInstance().getAllActiveCallsNotInRoom(this.state.roomId),
);
this.setState({
primaryCall: primaryCall,
secondaryCall: secondaryCalls[0],
});
};
private onCallRemoteHold = () => {
const [primaryCall, secondaryCalls] = getPrimarySecondaryCalls(
CallHandler.sharedInstance().getAllActiveCallsNotInRoom(this.state.roomId),

144
src/components/views/voip/CallView.tsx
View File

@ -20,10 +20,9 @@ import dis from '../../../dispatcher/dispatcher';
import CallHandler from '../../../CallHandler';
import {MatrixClientPeg} from '../../../MatrixClientPeg';
import { _t, _td } from '../../../languageHandler';
import VideoFeed, { VideoFeedType } from "./VideoFeed";
import VideoFeed from './VideoFeed';
import RoomAvatar from "../avatars/RoomAvatar";
import { CallState, CallType, MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
import { CallEvent } from 'matrix-js-sdk/src/webrtc/call';
import { CallState, CallType, MatrixCall, CallEvent } from 'matrix-js-sdk/src/webrtc/call';
import classNames from 'classnames';
import AccessibleButton from '../elements/AccessibleButton';
import {isOnlyCtrlOrCmdKeyEvent, Key} from '../../../Keyboard';
@ -31,6 +30,7 @@ import {alwaysAboveLeftOf, alwaysAboveRightOf, ChevronFace, ContextMenuButton} f
import CallContextMenu from '../context_menus/CallContextMenu';
import { avatarUrlForMember } from '../../../Avatar';
import DialpadContextMenu from '../context_menus/DialpadContextMenu';
import { CallFeed } from 'matrix-js-sdk/src/webrtc/callFeed';
import {replaceableComponent} from "../../../utils/replaceableComponent";
interface IProps {
@ -40,11 +40,11 @@ interface IProps {
// Another ongoing call to display information about
secondaryCall?: MatrixCall,
// a callback which is called when the content in the callview changes
// a callback which is called when the content in the CallView changes
// in a way that is likely to cause a resize.
onResize?: any;
// Whether this call view is for picture-in-pictue mode
// Whether this call view is for picture-in-picture mode
// otherwise, it's the larger call view when viewing the room the call is in.
// This is sort of a proxy for a number of things but we currently have no
// need to control those things separately, so this is simpler.
@ -60,6 +60,7 @@ interface IState {
controlsVisible: boolean,
showMoreMenu: boolean,
showDialpad: boolean,
feeds: CallFeed[],
}
function getFullScreenElement() {
@ -115,6 +116,7 @@ export default class CallView extends React.Component<IProps, IState> {
controlsVisible: true,
showMoreMenu: false,
showDialpad: false,
feeds: this.props.call.getFeeds(),
}
this.updateCallListeners(null, this.props.call);
@ -172,11 +174,13 @@ export default class CallView extends React.Component<IProps, IState> {
oldCall.removeListener(CallEvent.State, this.onCallState);
oldCall.removeListener(CallEvent.LocalHoldUnhold, this.onCallLocalHoldUnhold);
oldCall.removeListener(CallEvent.RemoteHoldUnhold, this.onCallRemoteHoldUnhold);
oldCall.removeListener(CallEvent.FeedsChanged, this.onFeedsChanged);
}
if (newCall) {
newCall.on(CallEvent.State, this.onCallState);
newCall.on(CallEvent.LocalHoldUnhold, this.onCallLocalHoldUnhold);
newCall.on(CallEvent.RemoteHoldUnhold, this.onCallRemoteHoldUnhold);
newCall.on(CallEvent.FeedsChanged, this.onFeedsChanged);
}
}
@ -186,6 +190,10 @@ export default class CallView extends React.Component<IProps, IState> {
});
};
private onFeedsChanged = (newFeeds: Array<CallFeed>) => {
this.setState({feeds: newFeeds});
};
private onCallLocalHoldUnhold = () => {
this.setState({
isLocalOnHold: this.props.call.isLocalOnHold(),
@ -304,7 +312,7 @@ export default class CallView extends React.Component<IProps, IState> {
}
// we register global shortcuts here, they *must not conflict* with local shortcuts elsewhere or both will fire
// Note that this assumes we always have a callview on screen at any given time
// Note that this assumes we always have a CallView on screen at any given time
// CallHandler would probably be a better place for this
private onNativeKeyDown = ev => {
let handled = false;
@ -474,6 +482,8 @@ export default class CallView extends React.Component<IProps, IState> {
{contextMenuButton}
</div>;
const avatarSize = this.props.pipMode ? 76 : 160;
// The 'content' for the call, ie. the videos for a video call and profile picture
// for voice calls (fills the bg)
let contentView: React.ReactNode;
@ -524,41 +534,85 @@ export default class CallView extends React.Component<IProps, IState> {
</div>;
}
if (this.props.call.type === CallType.Video) {
let localVideoFeed = null;
let onHoldBackground = null;
const backgroundStyle: CSSProperties = {};
const containerClasses = classNames({
mx_CallView_video: true,
mx_CallView_video_hold: isOnHold,
});
if (isOnHold) {
// This is a bit messy. I can't see a reason to have two onHold/transfer screens
if (isOnHold || transfereeCall) {
if (this.props.call.type === CallType.Video) {
const containerClasses = classNames({
mx_CallView_content: true,
mx_CallView_video: true,
mx_CallView_video_hold: isOnHold,
});
let onHoldBackground = null;
const backgroundStyle: CSSProperties = {};
const backgroundAvatarUrl = avatarUrlForMember(
// is it worth getting the size of the div to pass here?
// is it worth getting the size of the div to pass here?
this.props.call.getOpponentMember(), 1024, 1024, 'crop',
);
backgroundStyle.backgroundImage = 'url(' + backgroundAvatarUrl + ')';
onHoldBackground = <div className="mx_CallView_video_holdBackground" style={backgroundStyle} />;
}
if (!this.state.vidMuted) {
localVideoFeed = <VideoFeed type={VideoFeedType.Local} call={this.props.call} />;
}
contentView = <div className={containerClasses} ref={this.contentRef} onMouseMove={this.onMouseMove}>
{onHoldBackground}
<VideoFeed type={VideoFeedType.Remote} call={this.props.call} onResize={this.props.onResize} />
{localVideoFeed}
{holdTransferContent}
{callControls}
</div>;
} else {
const avatarSize = this.props.pipMode ? 76 : 160;
contentView = (
<div className={containerClasses} ref={this.contentRef} onMouseMove={this.onMouseMove}>
{onHoldBackground}
{holdTransferContent}
{callControls}
</div>
);
} else {
const classes = classNames({
mx_CallView_content: true,
mx_CallView_voice: true,
mx_CallView_voice_hold: isOnHold,
});
contentView =(
<div className={classes} onMouseMove={this.onMouseMove}>
<div className="mx_CallView_voice_avatarsContainer">
<div
className="mx_CallView_voice_avatarContainer"
style={{width: avatarSize, height: avatarSize}}
>
<RoomAvatar
room={callRoom}
height={avatarSize}
width={avatarSize}
/>
</div>
</div>
{holdTransferContent}
{callControls}
</div>
);
}
} else if (this.props.call.noIncomingFeeds()) {
// Here we're reusing the css classes from voice on hold, because
// I am lazy. If this gets merged, the CallView might be subject
// to change anyway - I might take an axe to this file in order to
// try to get other things working
const classes = classNames({
mx_CallView_content: true,
mx_CallView_voice: true,
mx_CallView_voice_hold: isOnHold,
});
const feeds = this.props.call.getLocalFeeds().map((feed, i) => {
// Here we check to hide local audio feeds to achieve the same UI/UX
// as before. But once again this might be subject to change
if (feed.isVideoMuted()) return;
return (
<VideoFeed
key={i}
feed={feed}
call={this.props.call}
pipMode={this.props.pipMode}
onResize={this.props.onResize}
/>
);
});
// Saying "Connecting" here isn't really true, but the best thing
// I can come up with, but this might be subject to change as well
contentView = <div className={classes} onMouseMove={this.onMouseMove}>
{feeds}
<div className="mx_CallView_voice_avatarsContainer">
<div className="mx_CallView_voice_avatarContainer" style={{width: avatarSize, height: avatarSize}}>
<RoomAvatar
@ -568,7 +622,35 @@ export default class CallView extends React.Component<IProps, IState> {
/>
</div>
</div>
{holdTransferContent}
<div className="mx_CallView_holdTransferContent">{_t("Connecting")}</div>
{callControls}
</div>;
} else {
const containerClasses = classNames({
mx_CallView_content: true,
mx_CallView_video: true,
});
// TODO: Later the CallView should probably be reworked to support
// any number of feeds but now we can always expect there to be two
// feeds. This is because the js-sdk ignores any new incoming streams
const feeds = this.state.feeds.map((feed, i) => {
// Here we check to hide local audio feeds to achieve the same UI/UX
// as before. But once again this might be subject to change
if (feed.isVideoMuted() && feed.isLocal()) return;
return (
<VideoFeed
key={i}
feed={feed}
call={this.props.call}
pipMode={this.props.pipMode}
onResize={this.props.onResize}
/>
);
});
contentView = <div className={containerClasses} ref={this.contentRef} onMouseMove={this.onMouseMove}>
{feeds}
{callControls}
</div>;
}

18
src/components/views/voip/CallViewForRoom.tsx
View File

@ -16,13 +16,12 @@ limitations under the License.
import { CallState, MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
import React from 'react';
import CallHandler from '../../../CallHandler';
import CallHandler, { CallHandlerEvent } from '../../../CallHandler';
import CallView from './CallView';
import dis from '../../../dispatcher/dispatcher';
import {Resizable} from "re-resizable";
import ResizeNotifier from "../../../utils/ResizeNotifier";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import { Action } from '../../../dispatcher/actions';
interface IProps {
// What room we should display the call for
@ -55,25 +54,30 @@ export default class CallViewForRoom extends React.Component<IProps, IState> {
public componentDidMount() {
this.dispatcherRef = dis.register(this.onAction);
CallHandler.sharedInstance().addListener(CallHandlerEvent.CallChangeRoom, this.updateCall);
}
public componentWillUnmount() {
dis.unregister(this.dispatcherRef);
CallHandler.sharedInstance().removeListener(CallHandlerEvent.CallChangeRoom, this.updateCall);
}
private onAction = (payload) => {
switch (payload.action) {
case Action.CallChangeRoom:
case 'call_state': {
const newCall = this.getCall();
if (newCall !== this.state.call) {
this.setState({call: newCall});
}
this.updateCall();
break;
}
}
};
private updateCall = () => {
const newCall = this.getCall();
if (newCall !== this.state.call) {
this.setState({call: newCall});
}
};
private getCall(): MatrixCall {
const call = CallHandler.sharedInstance().getCallForRoom(this.props.roomId);

125
src/components/views/voip/VideoFeed.tsx
View File

@ -18,52 +18,102 @@ import classnames from 'classnames';
import { MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
import React, {createRef} from 'react';
import SettingsStore from "../../../settings/SettingsStore";
import { CallFeed, CallFeedEvent } from 'matrix-js-sdk/src/webrtc/callFeed';
import { logger } from 'matrix-js-sdk/src/logger';
import MemberAvatar from "../avatars/MemberAvatar"
import {replaceableComponent} from "../../../utils/replaceableComponent";
export enum VideoFeedType {
Local,
Remote,
}
interface IProps {
call: MatrixCall,
type: VideoFeedType,
feed: CallFeed,
// Whether this call view is for picture-in-picture mode
// otherwise, it's the larger call view when viewing the room the call is in.
// This is sort of a proxy for a number of things but we currently have no
// need to control those things separately, so this is simpler.
pipMode?: boolean;
// a callback which is called when the video element is resized
// due to a change in video metadata
onResize?: (e: Event) => void,
}
@replaceableComponent("views.voip.VideoFeed")
export default class VideoFeed extends React.Component<IProps> {
private vid = createRef<HTMLVideoElement>();
interface IState {
audioMuted: boolean;
videoMuted: boolean;
}
componentDidMount() {
this.vid.current.addEventListener('resize', this.onResize);
this.setVideoElement();
@replaceableComponent("views.voip.VideoFeed")
export default class VideoFeed extends React.Component<IProps, IState> {
private element = createRef<HTMLVideoElement>();
constructor(props: IProps) {
super(props);
this.state = {
audioMuted: this.props.feed.isAudioMuted(),
videoMuted: this.props.feed.isVideoMuted(),
};
}
componentDidUpdate(prevProps) {
if (this.props.call !== prevProps.call) {
this.setVideoElement();
}
componentDidMount() {
this.props.feed.addListener(CallFeedEvent.NewStream, this.onNewStream);
this.playMedia();
}
componentWillUnmount() {
this.vid.current.removeEventListener('resize', this.onResize);
this.props.feed.removeListener(CallFeedEvent.NewStream, this.onNewStream);
this.element.current?.removeEventListener('resize', this.onResize);
this.stopMedia();
}
private setVideoElement() {
if (this.props.type === VideoFeedType.Local) {
this.props.call.setLocalVideoElement(this.vid.current);
} else {
this.props.call.setRemoteVideoElement(this.vid.current);
private playMedia() {
const element = this.element.current;
if (!element) return;
// We play audio in AudioFeed, not here
element.muted = true;
element.srcObject = this.props.feed.stream;
element.autoplay = true;
try {
// A note on calling methods on media elements:
// We used to have queues per media element to serialise all calls on those elements.
// The reason given for this was that load() and play() were racing. However, we now
// never call load() explicitly so this seems unnecessary. However, serialising every
// operation was causing bugs where video would not resume because some play command
// had got stuck and all media operations were queued up behind it. If necessary, we
// should serialise the ones that need to be serialised but then be able to interrupt
// them with another load() which will cancel the pending one, but since we don't call
// load() explicitly, it shouldn't be a problem. - Dave
element.play()
} catch (e) {
logger.info("Failed to play media element with feed", this.props.feed, e);
}
}
onResize = (e) => {
if (this.props.onResize) {
private stopMedia() {
const element = this.element.current;
if (!element) return;
element.pause();
element.src = null;
// As per comment in componentDidMount, setting the sink ID back to the
// default once the call is over makes setSinkId work reliably. - Dave
// Since we are not using the same element anymore, the above doesn't
// seem to be necessary - Šimon
}
private onNewStream = () => {
this.setState({
audioMuted: this.props.feed.isAudioMuted(),
videoMuted: this.props.feed.isVideoMuted(),
});
this.playMedia();
};
private onResize = (e) => {
if (this.props.onResize && !this.props.feed.isLocal()) {
this.props.onResize(e);
}
};
@ -71,14 +121,33 @@ export default class VideoFeed extends React.Component<IProps> {
render() {
const videoClasses = {
mx_VideoFeed: true,
mx_VideoFeed_local: this.props.type === VideoFeedType.Local,
mx_VideoFeed_remote: this.props.type === VideoFeedType.Remote,
mx_VideoFeed_local: this.props.feed.isLocal(),
mx_VideoFeed_remote: !this.props.feed.isLocal(),
mx_VideoFeed_voice: this.state.videoMuted,
mx_VideoFeed_video: !this.state.videoMuted,
mx_VideoFeed_mirror: (
this.props.type === VideoFeedType.Local &&
this.props.feed.isLocal() &&
SettingsStore.getValue('VideoView.flipVideoHorizontally')
),
};
return <video className={classnames(videoClasses)} ref={this.vid} />;
if (this.state.videoMuted) {
const member = this.props.feed.getMember();
const avatarSize = this.props.pipMode ? 76 : 160;
return (
<div className={classnames(videoClasses)} >
<MemberAvatar
member={member}
height={avatarSize}
width={avatarSize}
/>
</div>
);
} else {
return (
<video className={classnames(videoClasses)} ref={this.element} />
);
}
}
}

3
src/dispatcher/actions.ts
View File

@ -114,9 +114,6 @@ export enum Action {
*/
VirtualRoomSupportUpdated = "virtual_room_support_updated",
// Probably would be better to have a VoIP states in a store and have the store emit changes
CallChangeRoom = "call_change_room",
/**
* Fired when an upload has started. Should be used with UploadStartedPayload.
*/

1
src/i18n/strings/en_EN.json
View File

@ -885,6 +885,7 @@
"You held the call <a>Switch</a>": "You held the call <a>Switch</a>",
"You held the call <a>Resume</a>": "You held the call <a>Resume</a>",
"%(peerName)s held the call": "%(peerName)s held the call",
"Connecting": "Connecting",
"Video Call": "Video Call",
"Voice Call": "Voice Call",
"Fill Screen": "Fill Screen",

12
test/CallHandler-test.ts
View File

@ -16,7 +16,7 @@ limitations under the License.
import './skinned-sdk';
import CallHandler, { PlaceCallType } from '../src/CallHandler';
import CallHandler, { PlaceCallType, CallHandlerEvent } from '../src/CallHandler';
import { stubClient, mkStubRoom } from './test-utils';
import { MatrixClientPeg } from '../src/MatrixClientPeg';
import dis from '../src/dispatcher/dispatcher';
@ -172,11 +172,9 @@ describe('CallHandler', () => {
let callRoomChangeEventCount = 0;
const roomChangePromise = new Promise<void>(resolve => {
dispatchHandle = dis.register(payload => {
if (payload.action === Action.CallChangeRoom) {
++callRoomChangeEventCount;
resolve();
}
callHandler.addListener(CallHandlerEvent.CallChangeRoom, () => {
++callRoomChangeEventCount;
resolve();
});
});
@ -201,7 +199,7 @@ describe('CallHandler', () => {
fakeCall.emit(CallEvent.AssertedIdentityChanged);
await roomChangePromise;
dis.unregister(dispatchHandle);
callHandler.removeAllListeners();
// If everything's gone well, we should have seen only one room change
// event and the call should now be in user 3's room.