OpenCloud
/
element-web
mirror of https://github.com/vector-im/element-web
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #5308 from matrix-org/dbkr/call_state_machine 

Rewrite call state machine
pull/21833/head
David Baker 2020-10-13 10:39:15 +01:00 committed by GitHub
parent d007b06d7c ed9584bbac
commit 40ba342aa2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 173 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

197
src/CallHandler.tsx
View File

@ -77,13 +77,28 @@ import ErrorDialog from "./components/views/dialogs/ErrorDialog";
import WidgetStore from "./stores/WidgetStore";
import { WidgetMessagingStore } from "./stores/widgets/WidgetMessagingStore";
import { ElementWidgetActions } from "./stores/widgets/ElementWidgetActions";
import { MatrixCall, CallErrorCode, CallState, CallEvent, CallParty } from "matrix-js-sdk/lib/webrtc/call";
// until we ts-ify the js-sdk voip code
type Call = any;
enum AudioID {
Ring = 'ringAudio',
Ringback = 'ringbackAudio',
CallEnd = 'callendAudio',
Busy = 'busyAudio',
}
// Unlike 'CallType' in js-sdk, this one includes screen sharing
// (because a screen sharing call is only a screen sharing call to the caller,
// to the callee it's just a video call, at least as far as the current impl
// is concerned).
export enum PlaceCallType {
Voice = 'voice',
Video = 'video',
ScreenSharing = 'screensharing',
}
export default class CallHandler {
private calls = new Map<string, Call>();
private audioPromises = new Map<string, Promise<void>>();
private calls = new Map<string, MatrixCall>();
private audioPromises = new Map<AudioID, Promise<void>>();
static sharedInstance() {
if (!window.mxCallHandler) {
@ -108,20 +123,20 @@ export default class CallHandler {
}
}
getCallForRoom(roomId: string): Call {
getCallForRoom(roomId: string): MatrixCall {
return this.calls.get(roomId) || null;
}
getAnyActiveCall() {
for (const call of this.calls.values()) {
if (call.state !== "ended") {
if (call.state !== CallState.Ended) {
return call;
}
}
return null;
}
play(audioId: string) {
play(audioId: AudioID) {
// TODO: Attach an invisible element for this instead
// which listens?
const audio = document.getElementById(audioId) as HTMLMediaElement;
@ -150,7 +165,7 @@ export default class CallHandler {
}
}
pause(audioId: string) {
pause(audioId: AudioID) {
// TODO: Attach an invisible element for this instead
// which listens?
const audio = document.getElementById(audioId) as HTMLMediaElement;
@ -164,8 +179,8 @@ export default class CallHandler {
}
}
private setCallListeners(call: Call) {
call.on("error", (err) => {
private setCallListeners(call: MatrixCall) {
call.on(CallEvent.Error, (err) => {
console.error("Call error:", err);
if (
MatrixClientPeg.get().getTurnServers().length === 0 &&
@ -180,74 +195,60 @@ export default class CallHandler {
description: err.message,
});
});
call.on("hangup", () => {
call.on(CallEvent.Hangup, () => {
this.removeCallForRoom(call.roomId);
});
// map web rtc states to dummy UI state
// ringing|ringback|connected|ended|busy|stop_ringback|stop_ringing
call.on("state", (newState, oldState) => {
if (newState === "ringing") {
this.setCallState(call, call.roomId, "ringing");
this.pause("ringbackAudio");
} else if (newState === "invite_sent") {
this.setCallState(call, call.roomId, "ringback");
this.play("ringbackAudio");
} else if (newState === "ended" && oldState === "connected") {
this.removeCallForRoom(call.roomId);
this.pause("ringbackAudio");
this.play("callendAudio");
} else if (newState === "ended" && oldState === "invite_sent" &&
(call.hangupParty === "remote" ||
(call.hangupParty === "local" && call.hangupReason === "invite_timeout")
call.on(CallEvent.State, (newState: CallState, oldState: CallState) => {
this.setCallState(call, newState);
switch (oldState) {
case CallState.Ringing:
this.pause(AudioID.Ring);
break;
case CallState.InviteSent:
this.pause(AudioID.Ringback);
break;
}
switch (newState) {
case CallState.Ringing:
this.play(AudioID.Ring);
break;
case CallState.InviteSent:
this.play(AudioID.Ringback);
break;
case CallState.Ended:
this.removeCallForRoom(call.roomId);
if (oldState === CallState.InviteSent && (
call.hangupParty === CallParty.Remote ||
(call.hangupParty === CallParty.Local && call.hangupReason === CallErrorCode.InviteTimeout)
)) {
this.setCallState(call, call.roomId, "busy");
this.pause("ringbackAudio");
this.play("busyAudio");
Modal.createTrackedDialog('Call Handler', 'Call Timeout', ErrorDialog, {
title: _t('Call Timeout'),
description: _t('The remote side failed to pick up') + '.',
});
} else if (oldState === "invite_sent") {
this.setCallState(call, call.roomId, "stop_ringback");
this.pause("ringbackAudio");
} else if (oldState === "ringing") {
this.setCallState(call, call.roomId, "stop_ringing");
this.pause("ringbackAudio");
} else if (newState === "connected") {
this.setCallState(call, call.roomId, "connected");
this.pause("ringbackAudio");
this.play(AudioID.Busy);
Modal.createTrackedDialog('Call Handler', 'Call Timeout', ErrorDialog, {
title: _t('Call Timeout'),
description: _t('The remote side failed to pick up') + '.',
});
} else {
this.play(AudioID.CallEnd);
}
}
});
}
private setCallState(call: Call, roomId: string, status: string) {
private setCallState(call: MatrixCall, status: CallState) {
console.log(
`Call state in ${roomId} changed to ${status} (${call ? call.call_state : "-"})`,
`Call state in ${call.roomId} changed to ${status}`,
);
if (call) {
this.calls.set(roomId, call);
} else {
this.calls.delete(roomId);
}
if (status === "ringing") {
this.play("ringAudio");
} else if (call && call.call_state === "ringing") {
this.pause("ringAudio");
}
if (call) {
call.call_state = status;
}
dis.dispatch({
action: 'call_state',
room_id: roomId,
room_id: call.roomId,
state: status,
});
}
private removeCallForRoom(roomId: string) {
this.setCallState(null, roomId, null);
this.calls.delete(roomId);
}
private showICEFallbackPrompt() {
@ -279,36 +280,39 @@ export default class CallHandler {
}, null, true);
}
private onAction = (payload: ActionPayload) => {
const placeCall = (newCall) => {
this.setCallListeners(newCall);
if (payload.type === 'voice') {
newCall.placeVoiceCall();
} else if (payload.type === 'video') {
newCall.placeVideoCall(
payload.remote_element,
payload.local_element,
);
} else if (payload.type === 'screensharing') {
const screenCapErrorString = PlatformPeg.get().screenCaptureErrorString();
if (screenCapErrorString) {
this.removeCallForRoom(newCall.roomId);
console.log("Can't capture screen: " + screenCapErrorString);
Modal.createTrackedDialog('Call Handler', 'Unable to capture screen', ErrorDialog, {
title: _t('Unable to capture screen'),
description: screenCapErrorString,
});
return;
}
newCall.placeScreenSharingCall(
payload.remote_element,
payload.local_element,
);
} else {
console.error("Unknown conf call type: %s", payload.type);
}
}
private placeCall(
roomId: string, type: PlaceCallType,
localElement: HTMLVideoElement, remoteElement: HTMLVideoElement,
) {
const call = Matrix.createNewMatrixCall(MatrixClientPeg.get(), roomId);
this.calls.set(roomId, call);
this.setCallListeners(call);
if (type === PlaceCallType.Voice) {
call.placeVoiceCall();
} else if (type === 'video') {
call.placeVideoCall(
remoteElement,
localElement,
);
} else if (type === PlaceCallType.ScreenSharing) {
const screenCapErrorString = PlatformPeg.get().screenCaptureErrorString();
if (screenCapErrorString) {
this.removeCallForRoom(roomId);
console.log("Can't capture screen: " + screenCapErrorString);
Modal.createTrackedDialog('Call Handler', 'Unable to capture screen', ErrorDialog, {
title: _t('Unable to capture screen'),
description: screenCapErrorString,
});
return;
}
call.placeScreenSharingCall(remoteElement, localElement);
} else {
console.error("Unknown conf call type: %s", type);
}
}
private onAction = (payload: ActionPayload) => {
switch (payload.action) {
case 'place_call':
{
@ -343,8 +347,8 @@ export default class CallHandler {
return;
} else if (members.length === 2) {
console.info("Place %s call in %s", payload.type, payload.room_id);
const call = Matrix.createNewMatrixCall(MatrixClientPeg.get(), payload.room_id);
placeCall(call);
this.placeCall(payload.room_id, payload.type, payload.local_element, payload.remote_element);
} else { // > 2
dis.dispatch({
action: "place_conference_call",
@ -383,24 +387,23 @@ export default class CallHandler {
return;
}
const call = payload.call;
const call = payload.call as MatrixCall;
this.calls.set(call.roomId, call)
this.setCallListeners(call);
this.setCallState(call, call.roomId, "ringing");
}
break;
case 'hangup':
if (!this.calls.get(payload.room_id)) {
return; // no call to hangup
}
this.calls.get(payload.room_id).hangup();
this.calls.get(payload.room_id).hangup(CallErrorCode.UserHangup, false)
this.removeCallForRoom(payload.room_id);
break;
case 'answer':
if (!this.calls.get(payload.room_id)) {
if (!this.calls.has(payload.room_id)) {
return; // no call to answer
}
this.calls.get(payload.room_id).answer();
this.setCallState(this.calls.get(payload.room_id), payload.room_id, "connected");
dis.dispatch({
action: "view_room",
room_id: payload.room_id,

46
src/components/structures/RoomStatusBar.js
View File

@ -1,7 +1,5 @@
/*
Copyright 2015, 2016 OpenMarket Ltd
Copyright 2017, 2018 New Vector Ltd
Copyright 2019 The Matrix.org Foundation C.I.C.
Copyright 2015-2020 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -26,6 +24,7 @@ import Resend from '../../Resend';
import dis from '../../dispatcher/dispatcher';
import {messageForResourceLimitError, messageForSendError} from '../../utils/ErrorUtils';
import {Action} from "../../dispatcher/actions";
import { CallState, CallType } from 'matrix-js-sdk/lib/webrtc/call';
const STATUS_BAR_HIDDEN = 0;
const STATUS_BAR_EXPANDED = 1;
@ -46,10 +45,12 @@ export default class RoomStatusBar extends React.Component {
// Used to suggest to the user to invite someone
sentMessageAndIsAlone: PropTypes.bool,
// true if there is an active call in this room (means we show
// the 'Active Call' text in the status bar if there is nothing
// more interesting)
hasActiveCall: PropTypes.bool,
// The active call in the room, if any (means we show the call bar
// along with the status of the call)
callState: PropTypes.string,
// The type of the call in progress, or null if no call is in progress
callType: PropTypes.string,
// true if the room is being peeked at. This affects components that shouldn't
// logically be shown when peeking, such as a prompt to invite people to a room.
@ -121,6 +122,10 @@ export default class RoomStatusBar extends React.Component {
});
};
_showCallBar() {
return this.props.callState !== CallState.Ended && this.props.callState !== CallState.Ringing;
}
_onResendAllClick = () => {
Resend.resendUnsentEvents(this.props.room);
dis.fire(Action.FocusComposer);
@ -153,7 +158,7 @@ export default class RoomStatusBar extends React.Component {
// indicate other sizes.
_getSize() {
if (this._shouldShowConnectionError() ||
this.props.hasActiveCall ||
this._showCallBar() ||
this.props.sentMessageAndIsAlone
) {
return STATUS_BAR_EXPANDED;
@ -165,7 +170,7 @@ export default class RoomStatusBar extends React.Component {
// return suitable content for the image on the left of the status bar.
_getIndicator() {
if (this.props.hasActiveCall) {
if (this._showCallBar()) {
const TintableSvg = sdk.getComponent("elements.TintableSvg");
return (
<TintableSvg src={require("../../../res/img/element-icons/room/in-call.svg")} width="23" height="20" />
@ -269,6 +274,25 @@ export default class RoomStatusBar extends React.Component {
</div>;
}
_getCallStatusText() {
switch (this.props.callState) {
case CallState.CreateOffer:
case CallState.InviteSent:
return _t('Calling...');
case CallState.Connecting:
case CallState.CreateAnswer:
return _t('Call connecting...');
case CallState.Connected:
return _t('Active call');
case CallState.WaitLocalMedia:
if (this.props.callType === CallType.Video) {
return _t('Starting camera...');
} else {
return _t('Starting microphone...');
}
}
}
// return suitable content for the main (text) part of the status bar.
_getContent() {
if (this._shouldShowConnectionError()) {
@ -291,10 +315,10 @@ export default class RoomStatusBar extends React.Component {
return this._getUnsentMessageContent();
}
if (this.props.hasActiveCall) {
if (this._showCallBar()) {
return (
<div className="mx_RoomStatusBar_callBar">
<b>{ _t('Active call') }</b>
<b>{ this._getCallStatusText() }</b>
</div>
);
}

43
src/components/structures/RoomView.tsx
View File

@ -71,6 +71,7 @@ import RoomHeader from "../views/rooms/RoomHeader";
import TintableSvg from "../views/elements/TintableSvg";
import {XOR} from "../../@types/common";
import { IThreepidInvite } from "../../stores/ThreepidInviteStore";
import { CallState, CallType, MatrixCall } from "matrix-js-sdk/lib/webrtc/call";
const DEBUG = false;
let debuglog = function(msg: string) {};
@ -141,7 +142,7 @@ export interface IState {
}>;
searchHighlights?: string[];
searchInProgress?: boolean;
callState?: string;
callState?: CallState;
guestsCanJoin: boolean;
canPeek: boolean;
showApps: boolean;
@ -479,7 +480,7 @@ export default class RoomView extends React.Component<IProps, IState> {
componentDidMount() {
const call = this.getCallForRoom();
const callState = call ? call.call_state : "ended";
const callState = call ? call.state : null;
this.setState({
callState: callState,
});
@ -712,14 +713,9 @@ export default class RoomView extends React.Component<IProps, IState> {
}
const call = this.getCallForRoom();
let callState = "ended";
if (call) {
callState = call.call_state;
}
this.setState({
callState: callState,
callState: call ? call.state : null,
});
break;
}
@ -1605,7 +1601,7 @@ export default class RoomView extends React.Component<IProps, IState> {
/**
* get any current call for this room
*/
private getCallForRoom() {
private getCallForRoom(): MatrixCall {
if (!this.state.room) {
return null;
}
@ -1742,10 +1738,13 @@ export default class RoomView extends React.Component<IProps, IState> {
// We have successfully loaded this room, and are not previewing.
// Display the "normal" room view.
const call = this.getCallForRoom();
let inCall = false;
if (call && (this.state.callState !== 'ended' && this.state.callState !== 'ringing')) {
inCall = true;
let activeCall = null;
{
// New block because this variable doesn't need to hang around for the rest of the function
const call = this.getCallForRoom();
if (call && (this.state.callState !== 'ended' && this.state.callState !== 'ringing')) {
activeCall = call;
}
}
const scrollheaderClasses = classNames({
@ -1764,7 +1763,8 @@ export default class RoomView extends React.Component<IProps, IState> {
statusBar = <RoomStatusBar
room={this.state.room}
sentMessageAndIsAlone={this.state.isAlone}
hasActiveCall={inCall}
callState={this.state.callState}
callType={activeCall ? activeCall.type : null}
isPeeking={myMembership !== "join"}
onInviteClick={this.onInviteButtonClick}
onStopWarningClick={this.onStopAloneWarningClick}
@ -1890,10 +1890,10 @@ export default class RoomView extends React.Component<IProps, IState> {
};
}
if (inCall) {
if (activeCall) {
let zoomButton; let videoMuteButton;
if (call.type === "video") {
if (activeCall.type === CallType.Video) {
zoomButton = (
<div className="mx_RoomView_voipButton" onClick={this.onFullscreenClick} title={_t("Fill screen")}>
<TintableSvg
@ -1908,10 +1908,11 @@ export default class RoomView extends React.Component<IProps, IState> {
videoMuteButton =
<div className="mx_RoomView_voipButton" onClick={this.onMuteVideoClick}>
<TintableSvg
src={call.isLocalVideoMuted() ?
src={activeCall.isLocalVideoMuted() ?
require("../../../res/img/element-icons/call/video-muted.svg") :
require("../../../res/img/element-icons/call/video-call.svg")}
alt={call.isLocalVideoMuted() ? _t("Click to unmute video") : _t("Click to mute video")}
alt={activeCall.isLocalVideoMuted() ? _t("Click to unmute video") :
_t("Click to mute video")}
width=""
height="27"
/>
@ -1920,10 +1921,10 @@ export default class RoomView extends React.Component<IProps, IState> {
const voiceMuteButton =
<div className="mx_RoomView_voipButton" onClick={this.onMuteAudioClick}>
<TintableSvg
src={call.isMicrophoneMuted() ?
src={activeCall.isMicrophoneMuted() ?
require("../../../res/img/element-icons/call/voice-muted.svg") :
require("../../../res/img/element-icons/call/voice-unmuted.svg")}
alt={call.isMicrophoneMuted() ? _t("Click to unmute audio") : _t("Click to mute audio")}
alt={activeCall.isMicrophoneMuted() ? _t("Click to unmute audio") : _t("Click to mute audio")}
width="21"
height="26"
/>
@ -2041,7 +2042,7 @@ export default class RoomView extends React.Component<IProps, IState> {
});
const mainClasses = classNames("mx_RoomView", {
mx_RoomView_inCall: inCall,
mx_RoomView_inCall: Boolean(activeCall),
});
return (

5
src/components/views/rooms/MessageComposer.js
View File

@ -37,6 +37,7 @@ import WidgetStore from "../../../stores/WidgetStore";
import WidgetUtils from "../../../utils/WidgetUtils";
import {UPDATE_EVENT} from "../../../stores/AsyncStore";
import ActiveWidgetStore from "../../../stores/ActiveWidgetStore";
import { PlaceCallType } from "../../../CallHandler";
function ComposerAvatar(props) {
const MemberStatusMessageAvatar = sdk.getComponent('avatars.MemberStatusMessageAvatar');
@ -53,7 +54,7 @@ function CallButton(props) {
const onVoiceCallClick = (ev) => {
dis.dispatch({
action: 'place_call',
type: "voice",
type: PlaceCallType.Voice,
room_id: props.roomId,
});
};
@ -73,7 +74,7 @@ function VideoCallButton(props) {
const onCallClick = (ev) => {
dis.dispatch({
action: 'place_call',
type: ev.shiftKey ? "screensharing" : "video",
type: ev.shiftKey ? PlaceCallType.ScreenSharing : PlaceCallType.Video,
room_id: props.roomId,
});
};

7
src/components/views/voip/CallPreview.tsx
View File

@ -24,13 +24,14 @@ import dis from '../../../dispatcher/dispatcher';
import { ActionPayload } from '../../../dispatcher/payloads';
import PersistentApp from "../elements/PersistentApp";
import SettingsStore from "../../../settings/SettingsStore";
import { CallState, MatrixCall } from 'matrix-js-sdk/lib/webrtc/call';
interface IProps {
}
interface IState {
roomId: string;
activeCall: any;
activeCall: MatrixCall;
}
export default class CallPreview extends React.Component<IProps, IState> {
@ -84,7 +85,7 @@ export default class CallPreview extends React.Component<IProps, IState> {
if (call) {
dis.dispatch({
action: 'view_room',
room_id: call.groupRoomId || call.roomId,
room_id: call.roomId,
});
}
};
@ -93,7 +94,7 @@ export default class CallPreview extends React.Component<IProps, IState> {
const callForRoom = CallHandler.sharedInstance().getCallForRoom(this.state.roomId);
const showCall = (
this.state.activeCall &&
this.state.activeCall.call_state === 'connected' &&
this.state.activeCall.state === CallState.Connected &&
!callForRoom
);

5
src/components/views/voip/CallView.tsx
View File

@ -25,6 +25,7 @@ import AccessibleButton from '../elements/AccessibleButton';
import VideoView from "./VideoView";
import RoomAvatar from "../avatars/RoomAvatar";
import PulsedAvatar from '../avatars/PulsedAvatar';
import { CallState, MatrixCall } from 'matrix-js-sdk/lib/webrtc/call';
interface IProps {
// js-sdk room object. If set, we will only show calls for the given
@ -87,7 +88,7 @@ export default class CallView extends React.Component<IProps, IState> {
};
private showCall() {
let call;
let call: MatrixCall;
if (this.props.room) {
const roomId = this.props.room.roomId;
@ -120,7 +121,7 @@ export default class CallView extends React.Component<IProps, IState> {
call.setRemoteAudioElement(this.getVideoView().getRemoteAudioElement());
}
}
if (call && call.type === "video" && call.call_state !== "ended" && call.call_state !== "ringing") {
if (call && call.type === "video" && call.state !== CallState.Ended && call.state !== CallState.Ringing) {
this.getVideoView().getLocalVideoElement().style.display = "block";
this.getVideoView().getRemoteVideoElement().style.display = "block";
} else {

3
src/components/views/voip/IncomingCallBox.tsx
View File

@ -25,6 +25,7 @@ import CallHandler from '../../../CallHandler';
import PulsedAvatar from '../avatars/PulsedAvatar';
import RoomAvatar from '../avatars/RoomAvatar';
import FormButton from '../elements/FormButton';
import { CallState } from 'matrix-js-sdk/lib/webrtc/call';
interface IProps {
}
@ -53,7 +54,7 @@ export default class IncomingCallBox extends React.Component<IProps, IState> {
switch (payload.action) {
case 'call_state': {
const call = CallHandler.sharedInstance().getCallForRoom(payload.room_id);
if (call && call.call_state === 'ringing') {
if (call && call.state === CallState.Ringing) {
this.setState({
incomingCall: call,
});

4
src/i18n/strings/en_EN.json
View File

@ -2095,6 +2095,10 @@
"%(count)s of your messages have not been sent.|one": "Your message was not sent.",
"%(count)s <resendText>Resend all</resendText> or <cancelText>cancel all</cancelText> now. You can also select individual messages to resend or cancel.|other": "<resendText>Resend all</resendText> or <cancelText>cancel all</cancelText> now. You can also select individual messages to resend or cancel.",
"%(count)s <resendText>Resend all</resendText> or <cancelText>cancel all</cancelText> now. You can also select individual messages to resend or cancel.|one": "<resendText>Resend message</resendText> or <cancelText>cancel message</cancelText> now.",
"Calling...": "Calling...",
"Call connecting...": "Call connecting...",
"Starting camera...": "Starting camera...",
"Starting microphone...": "Starting microphone...",
"Connectivity to the server has been lost.": "Connectivity to the server has been lost.",
"Sent messages will be stored until your connection has returned.": "Sent messages will be stored until your connection has returned.",
"There's no one else here! Would you like to <inviteText>invite others</inviteText> or <nowarnText>stop warning about the empty room</nowarnText>?": "There's no one else here! Would you like to <inviteText>invite others</inviteText> or <nowarnText>stop warning about the empty room</nowarnText>?",