diff --git a/res/css/_components.scss b/res/css/_components.scss
index 445ed70ff4..414e2dc6a6 100644
--- a/res/css/_components.scss
+++ b/res/css/_components.scss
@@ -53,6 +53,7 @@
 @import "./views/avatars/_MemberStatusMessageAvatar.scss";
 @import "./views/avatars/_PulsedAvatar.scss";
 @import "./views/avatars/_WidgetAvatar.scss";
+@import "./views/context_menus/_CallContextMenu.scss";
 @import "./views/context_menus/_IconizedContextMenu.scss";
 @import "./views/context_menus/_MessageContextMenu.scss";
 @import "./views/context_menus/_StatusMessageContextMenu.scss";
diff --git a/res/css/views/context_menus/_CallContextMenu.scss b/res/css/views/context_menus/_CallContextMenu.scss
new file mode 100644
index 0000000000..55b73b0344
--- /dev/null
+++ b/res/css/views/context_menus/_CallContextMenu.scss
@@ -0,0 +1,23 @@
+/*
+Copyright 2020 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+.mx_CallContextMenu_item {
+    width: 205px;
+    height: 40px;
+    padding-left: 16px;
+    line-height: 40px;
+    vertical-align: center;
+}
diff --git a/res/css/views/voip/_CallView.scss b/res/css/views/voip/_CallView.scss
index 2b87181b1e..d5e58c94c5 100644
--- a/res/css/views/voip/_CallView.scss
+++ b/res/css/views/voip/_CallView.scss
@@ -43,17 +43,99 @@ limitations under the License.
 .mx_CallView_voice {
     position: relative;
     display: flex;
+    flex-direction: column;
     align-items: center;
     justify-content: center;
     background-color: $inverted-bg-color;
 }
 
+.mx_CallView_voice_hold {
+    // This masks the avatar image so when it's blurred, the edge is still crisp
+    .mx_CallView_voice_avatarContainer {
+        border-radius: 2000px;
+        overflow: hidden;
+        position: relative;
+        &::after {
+            position: absolute;
+            content: '';
+            width: 20px;
+            height: 20px;
+            top: 50%;
+            left: 50%;
+            transform: translate(-50%, -50%);
+            background-image: url('$(res)/img/voip/paused.svg');
+            background-position: center;
+            background-size: cover;
+        }
+    }
+    .mx_BaseAvatar {
+        filter: blur(20px);
+        overflow: hidden;
+    }
+}
+
+.mx_CallView_voice_holdText {
+    height: 16px;
+    color: $accent-fg-color;
+    .mx_AccessibleButton_hasKind {
+        padding: 0px;
+    }
+}
+
 .mx_CallView_video {
     width: 100%;
     position: relative;
     z-index: 30;
 }
 
+.mx_CallView_video_hold {
+    overflow: hidden;
+
+    // we keep these around in the DOM: it saved wiring them up again when the call
+    // is resumed and keeps the container the right size
+    .mx_VideoFeed {
+        visibility: hidden;
+    }
+}
+
+.mx_CallView_video_holdBackground {
+    position: absolute;
+    width: 100%;
+    height: 100%;
+    left: 0;
+    right: 0;
+    background-repeat: no-repeat;
+    background-size: cover;
+    background-position: center;
+    filter: blur(20px);
+}
+
+.mx_CallView_video_holdContent {
+    position: absolute;
+    top: 50%;
+    left: 50%;
+    transform: translate(-50%, -50%);
+    font-weight: bold;
+    color: $accent-fg-color;
+    text-align: center;
+
+    &::before {
+        display: block;
+        margin-left: auto;
+        margin-right: auto;
+        content: '';
+        width: 20px;
+        height: 20px;
+        background-image: url('$(res)/img/voip/paused.svg');
+        background-position: center;
+        background-size: cover;
+    }
+    .mx_AccessibleButton_hasKind {
+        display: block;
+        padding: 0px;
+    }
+}
+
 .mx_CallView_header {
     height: 44px;
     display: flex;
@@ -173,6 +255,12 @@ limitations under the License.
     }
 }
 
+// Makes the alignment correct
+.mx_CallView_callControls_nothing {
+    margin-right: auto;
+    cursor: initial;
+}
+
 .mx_CallView_callControls_button_micOn {
     &::before {
         background-image: url('$(res)/img/voip/mic-on.svg');
@@ -203,6 +291,18 @@ limitations under the License.
     }
 }
 
+.mx_CallView_callControls_button_more {
+    margin-left: auto;
+    &::before {
+        background-image: url('$(res)/img/voip/more.svg');
+    }
+}
+
+.mx_CallView_callControls_button_more_hidden {
+    margin-left: auto;
+    cursor: initial;
+}
+
 .mx_CallView_callControls_button_invisible {
     visibility: hidden;
     pointer-events: none;
diff --git a/src/components/structures/ContextMenu.tsx b/src/components/structures/ContextMenu.tsx
index fa0d6682dd..190b231b74 100644
--- a/src/components/structures/ContextMenu.tsx
+++ b/src/components/structures/ContextMenu.tsx
@@ -398,7 +398,7 @@ export const toRightOf = (elementRect: DOMRect, chevronOffset = 12) => {
 };
 
 // Placement method for <ContextMenu /> to position context menu right-aligned and flowing to the left of elementRect
-export const aboveLeftOf = (elementRect: DOMRect, chevronFace = ChevronFace.None) => {
+export const aboveLeftOf = (elementRect: DOMRect, chevronFace = ChevronFace.None, vPadding = 0) => {
     const menuOptions: IPosition & { chevronFace: ChevronFace } = { chevronFace };
 
     const buttonRight = elementRect.right + window.pageXOffset;
@@ -408,9 +408,9 @@ export const aboveLeftOf = (elementRect: DOMRect, chevronFace = ChevronFace.None
     menuOptions.right = window.innerWidth - buttonRight;
     // Align the menu vertically on whichever side of the button has more space available.
     if (buttonBottom < window.innerHeight / 2) {
-        menuOptions.top = buttonBottom;
+        menuOptions.top = buttonBottom + vPadding;
     } else {
-        menuOptions.bottom = window.innerHeight - buttonTop;
+        menuOptions.bottom = (window.innerHeight - buttonTop) + vPadding;
     }
 
     return menuOptions;
diff --git a/src/components/views/context_menus/CallContextMenu.tsx b/src/components/views/context_menus/CallContextMenu.tsx
new file mode 100644
index 0000000000..31e82c19b1
--- /dev/null
+++ b/src/components/views/context_menus/CallContextMenu.tsx
@@ -0,0 +1,51 @@
+/*
+Copyright 2020 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React from 'react';
+import PropTypes from 'prop-types';
+import { _t } from '../../../languageHandler';
+import { ContextMenu, IProps as IContextMenuProps, MenuItem } from '../../structures/ContextMenu';
+import { MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
+
+interface IProps extends IContextMenuProps {
+    call: MatrixCall;
+}
+
+export default class CallContextMenu extends React.Component<IProps> {
+    static propTypes = {
+        // js-sdk User object. Not required because it might not exist.
+        user: PropTypes.object,
+    };
+
+    constructor(props) {
+        super(props);
+    }
+
+    onHoldUnholdClick = () => {
+        this.props.call.setRemoteOnHold(!this.props.call.isRemoteOnHold());
+        this.props.onFinished();
+    }
+
+    render() {
+        const holdUnholdCaption = this.props.call.isRemoteOnHold() ? _t("Resume") : _t("Hold");
+
+        return <ContextMenu {...this.props}>
+            <MenuItem className="mx_CallContextMenu_item" onClick={this.onHoldUnholdClick}>
+                {holdUnholdCaption}
+            </MenuItem>
+        </ContextMenu>;
+    }
+}
diff --git a/src/components/views/voip/CallView.tsx b/src/components/views/voip/CallView.tsx
index db6d2b7ae0..c9f5db77e6 100644
--- a/src/components/views/voip/CallView.tsx
+++ b/src/components/views/voip/CallView.tsx
@@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React, { createRef } from 'react';
+import React, { createRef, CSSProperties } from 'react';
 import Room from 'matrix-js-sdk/src/models/room';
 import dis from '../../../dispatcher/dispatcher';
 import CallHandler from '../../../CallHandler';
@@ -28,6 +28,9 @@ import { CallEvent } from 'matrix-js-sdk/src/webrtc/call';
 import classNames from 'classnames';
 import AccessibleButton from '../elements/AccessibleButton';
 import {isOnlyCtrlOrCmdKeyEvent, Key} from '../../../Keyboard';
+import {aboveLeftOf, ChevronFace, ContextMenuButton} from '../../structures/ContextMenu';
+import CallContextMenu from '../context_menus/CallContextMenu';
+import { avatarUrlForMember } from '../../../Avatar';
 
 interface IProps {
         // js-sdk room object. If set, we will only show calls for the given
@@ -51,10 +54,12 @@ interface IProps {
 interface IState {
     call: MatrixCall;
     isLocalOnHold: boolean,
+    isRemoteOnHold: boolean,
     micMuted: boolean,
     vidMuted: boolean,
     callState: CallState,
     controlsVisible: boolean,
+    showMoreMenu: boolean,
 }
 
 function getFullScreenElement() {
@@ -89,11 +94,14 @@ const CONTROLS_HIDE_DELAY = 1000;
 // Height of the header duplicated from CSS because we need to subtract it from our max
 // height to get the max height of the video
 const HEADER_HEIGHT = 44;
+const CONTEXT_MENU_VPADDING = 8; // How far the context menu sits above the button (px)
 
 export default class CallView extends React.Component<IProps, IState> {
     private dispatcherRef: string;
     private contentRef = createRef<HTMLDivElement>();
     private controlsHideTimer: number = null;
+    private contextMenuButton = createRef<HTMLDivElement>();
+
     constructor(props: IProps) {
         super(props);
 
@@ -101,10 +109,12 @@ export default class CallView extends React.Component<IProps, IState> {
         this.state = {
             call,
             isLocalOnHold: call ? call.isLocalOnHold() : null,
+            isRemoteOnHold: call ? call.isRemoteOnHold() : null,
             micMuted: call ? call.isMicrophoneMuted() : null,
             vidMuted: call ? call.isLocalVideoMuted() : null,
             callState: call ? call.state : null,
             controlsVisible: true,
+            showMoreMenu: false,
         }
 
         this.updateCallListeners(null, call);
@@ -149,11 +159,16 @@ export default class CallView extends React.Component<IProps, IState> {
                     this.setState({
                         call: newCall,
                         isLocalOnHold: newCall ? newCall.isLocalOnHold() : null,
+                        isRemoteOnHold: newCall ? newCall.isRemoteOnHold() : null,
                         micMuted: newCall ? newCall.isMicrophoneMuted() : null,
                         vidMuted: newCall ? newCall.isLocalVideoMuted() : null,
                         callState: newCall ? newCall.state : null,
                         controlsVisible: newControlsVisible,
                     });
+                } else {
+                    this.setState({
+                        callState: newCall ? newCall.state : null,
+                    });
                 }
                 if (!newCall && getFullScreenElement()) {
                     exitFullscreen();
@@ -187,16 +202,30 @@ export default class CallView extends React.Component<IProps, IState> {
     private updateCallListeners(oldCall: MatrixCall, newCall: MatrixCall) {
         if (oldCall === newCall) return;
 
-        if (oldCall) oldCall.removeListener(CallEvent.HoldUnhold, this.onCallHoldUnhold);
-        if (newCall) newCall.on(CallEvent.HoldUnhold, this.onCallHoldUnhold);
+        if (oldCall) {
+            oldCall.removeListener(CallEvent.LocalHoldUnhold, this.onCallLocalHoldUnhold);
+            oldCall.removeListener(CallEvent.RemoteHoldUnhold, this.onCallRemoteHoldUnhold);
+        }
+        if (newCall) {
+            newCall.on(CallEvent.LocalHoldUnhold, this.onCallLocalHoldUnhold);
+            newCall.on(CallEvent.RemoteHoldUnhold, this.onCallRemoteHoldUnhold);
+        }
     }
 
-    private onCallHoldUnhold = () => {
+    private onCallLocalHoldUnhold = () => {
         this.setState({
             isLocalOnHold: this.state.call ? this.state.call.isLocalOnHold() : null,
         });
     };
 
+    private onCallRemoteHoldUnhold = () => {
+        this.setState({
+            isRemoteOnHold: this.state.call ? this.state.call.isRemoteOnHold() : null,
+            // update both here because isLocalOnHold changes when we hold the call too
+            isLocalOnHold: this.state.call ? this.state.call.isLocalOnHold() : null,
+        });
+    };
+
     private onFullscreenClick = () => {
         dis.dispatch({
             action: 'video_fullscreen',
@@ -223,6 +252,8 @@ export default class CallView extends React.Component<IProps, IState> {
     }
 
     private showControls() {
+        if (this.state.showMoreMenu) return;
+
         if (!this.state.controlsVisible) {
             this.setState({
                 controlsVisible: true,
@@ -252,6 +283,25 @@ export default class CallView extends React.Component<IProps, IState> {
         this.setState({vidMuted: newVal});
     }
 
+    private onMoreClick = () => {
+        if (this.controlsHideTimer) {
+            clearTimeout(this.controlsHideTimer);
+            this.controlsHideTimer = null;
+        }
+
+        this.setState({
+            showMoreMenu: true,
+            controlsVisible: true,
+        });
+    }
+
+    private closeContextMenu = () => {
+        this.setState({
+            showMoreMenu: false,
+        });
+        this.controlsHideTimer = window.setTimeout(this.onControlsHideTimer, CONTROLS_HIDE_DELAY);
+    }
+
     // we register global shortcuts here, they *must not conflict* with local shortcuts elsewhere or both will fire
     // Note that this assumes we always have a callview on screen at any given time
     // CallHandler would probably be a better place for this
@@ -292,14 +342,32 @@ export default class CallView extends React.Component<IProps, IState> {
         });
     }
 
+    private onCallResumeClick = () => {
+        this.state.call.setRemoteOnHold(false);
+    }
+
     public render() {
         if (!this.state.call) return null;
 
         const client = MatrixClientPeg.get();
         const callRoom = client.getRoom(this.state.call.roomId);
 
+        let contextMenu;
+
         let callControls;
         if (this.props.room) {
+            if (this.state.showMoreMenu) {
+                contextMenu = <CallContextMenu
+                    {...aboveLeftOf(
+                        this.contextMenuButton.current.getBoundingClientRect(),
+                        ChevronFace.None,
+                        CONTEXT_MENU_VPADDING,
+                    )}
+                    onFinished={this.closeContextMenu}
+                    call={this.state.call}
+                />;
+            }
+
             const micClasses = classNames({
                 mx_CallView_callControls_button: true,
                 mx_CallView_callControls_button_micOn: !this.state.micMuted,
@@ -333,17 +401,29 @@ export default class CallView extends React.Component<IProps, IState> {
                 mx_CallView_callControls_hidden: !this.state.controlsVisible,
             });
 
-            const vidMuteButton = this.state.call.type === CallType.Video ? <div
+            const vidMuteButton = this.state.call.type === CallType.Video ? <AccessibleButton
                 className={vidClasses}
                 onClick={this.onVidMuteClick}
             /> : null;
 
+            // The 'more' button actions are only relevant in a connected call
+            // When not connected, we have to put something there to make the flexbox alignment correct
+            const contextMenuButton = this.state.callState === CallState.Connected ? <ContextMenuButton
+                className="mx_CallView_callControls_button mx_CallView_callControls_button_more"
+                onClick={this.onMoreClick}
+                inputRef={this.contextMenuButton}
+                isExpanded={this.state.showMoreMenu}
+            /> : <div className="mx_CallView_callControls_button mx_CallView_callControls_button_more_hidden" />;
+
+            // in the near future, the dial pad button will go on the left. For now, it's the nothing button
+            // because something needs to have margin-right: auto to make the alignment correct.
             callControls = <div className={callControlsClasses}>
-                <div
+                <div className="mx_CallView_callControls_button mx_CallView_callControls_nothing" />
+                <AccessibleButton
                     className={micClasses}
                     onClick={this.onMicMuteClick}
                 />
-                <div
+                <AccessibleButton
                     className="mx_CallView_callControls_button mx_CallView_callControls_button_hangup"
                     onClick={() => {
                         dis.dispatch({
@@ -355,6 +435,7 @@ export default class CallView extends React.Component<IProps, IState> {
                 {vidMuteButton}
                 <div className={micCacheClasses} />
                 <div className={vidCacheClasses} />
+                {contextMenuButton}
             </div>;
         }
 
@@ -362,24 +443,66 @@ export default class CallView extends React.Component<IProps, IState> {
         // for voice calls (fills the bg)
         let contentView: React.ReactNode;
 
+        const isOnHold = this.state.isLocalOnHold || this.state.isRemoteOnHold;
+        let onHoldText = null;
+        if (this.state.isRemoteOnHold) {
+            onHoldText = _t("You held the call <a>Resume</a>", {}, {
+                a: sub => <AccessibleButton kind="link" onClick={this.onCallResumeClick}>
+                    {sub}
+                </AccessibleButton>,
+            });
+        } else if (this.state.isLocalOnHold) {
+            onHoldText = _t("%(peerName)s held the call", {
+                peerName: this.state.call.getOpponentMember().name,
+            });
+        }
+
         if (this.state.call.type === CallType.Video) {
+            let onHoldContent = null;
+            let onHoldBackground = null;
+            const backgroundStyle: CSSProperties = {};
+            const containerClasses = classNames({
+                mx_CallView_video: true,
+                mx_CallView_video_hold: isOnHold,
+            });
+            if (isOnHold) {
+                onHoldContent = <div className="mx_CallView_video_holdContent">
+                    {onHoldText}
+                </div>;
+                const backgroundAvatarUrl = avatarUrlForMember(
+                    // is it worth getting the size of the div to pass here?
+                    this.state.call.getOpponentMember(), 1024, 1024, 'crop',
+                );
+                backgroundStyle.backgroundImage = 'url(' + backgroundAvatarUrl + ')';
+                onHoldBackground = <div className="mx_CallView_video_holdBackground" style={backgroundStyle} />;
+            }
+
             // if we're fullscreen, we don't want to set a maxHeight on the video element.
             const maxVideoHeight = getFullScreenElement() ? null : this.props.maxVideoHeight - HEADER_HEIGHT;
-            contentView = <div className="mx_CallView_video" ref={this.contentRef} onMouseMove={this.onMouseMove}>
+            contentView = <div className={containerClasses} ref={this.contentRef} onMouseMove={this.onMouseMove}>
+                {onHoldBackground}
                 <VideoFeed type={VideoFeedType.Remote} call={this.state.call} onResize={this.props.onResize}
                     maxHeight={maxVideoHeight}
                 />
                 <VideoFeed type={VideoFeedType.Local} call={this.state.call} />
+                {onHoldContent}
                 {callControls}
             </div>;
         } else {
             const avatarSize = this.props.room ? 200 : 75;
-            contentView = <div className="mx_CallView_voice" onMouseMove={this.onMouseMove}>
-                <RoomAvatar
-                    room={callRoom}
-                    height={avatarSize}
-                    width={avatarSize}
-                />
+            const classes = classNames({
+                mx_CallView_voice: true,
+                mx_CallView_voice_hold: isOnHold,
+            });
+            contentView = <div className={classes} onMouseMove={this.onMouseMove}>
+                <div className="mx_CallView_voice_avatarContainer" style={{width: avatarSize, height: avatarSize}}>
+                    <RoomAvatar
+                        room={callRoom}
+                        height={avatarSize}
+                        width={avatarSize}
+                    />
+                </div>
+                <div className="mx_CallView_voice_holdText">{onHoldText}</div>
                 {callControls}
             </div>;
         }
@@ -431,6 +554,7 @@ export default class CallView extends React.Component<IProps, IState> {
         return <div className={"mx_CallView " + myClassName}>
             {header}
             {contentView}
+            {contextMenu}
         </div>;
     }
 }
diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json
index 0d50128f32..de1c20be1b 100644
--- a/src/i18n/strings/en_EN.json
+++ b/src/i18n/strings/en_EN.json
@@ -836,6 +836,8 @@
     "When rooms are upgraded": "When rooms are upgraded",
     "My Ban List": "My Ban List",
     "This is your list of users/servers you have blocked - don't leave the room!": "This is your list of users/servers you have blocked - don't leave the room!",
+    "You held the call <a>Resume</a>": "You held the call <a>Resume</a>",
+    "%(peerName)s held the call": "%(peerName)s held the call",
     "Video Call": "Video Call",
     "Voice Call": "Voice Call",
     "Fill Screen": "Fill Screen",
@@ -2231,6 +2233,8 @@
     "<b>Warning</b>: You should only set up key backup from a trusted computer.": "<b>Warning</b>: You should only set up key backup from a trusted computer.",
     "Access your secure message history and set up secure messaging by entering your recovery key.": "Access your secure message history and set up secure messaging by entering your recovery key.",
     "If you've forgotten your recovery key you can <button>set up new recovery options</button>": "If you've forgotten your recovery key you can <button>set up new recovery options</button>",
+    "Resume": "Resume",
+    "Hold": "Hold",
     "Reject invitation": "Reject invitation",
     "Are you sure you want to reject the invitation?": "Are you sure you want to reject the invitation?",
     "Unable to reject invite": "Unable to reject invite",