Merge remote-tracking branch 'upstream/develop' into feature/improved-composer

pull/21833/head
Šimon Brandner 2021-08-06 08:02:28 +02:00
commit 3677d0c5f2
No known key found for this signature in database
GPG Key ID: CC823428E9B582FB
15 changed files with 180 additions and 76 deletions

View File

@ -85,7 +85,7 @@ limitations under the License.
.mx_InteractiveAuthEntryComponents_termsPolicy {
display: flex;
flex-direction: row;
justify-content: start;
justify-content: flex-start;
align-items: center;
}

View File

@ -35,7 +35,6 @@ limitations under the License.
.mx_desktopCapturerSourcePicker_source_thumbnail {
margin: 4px;
padding: 4px;
width: 312px;
border-width: 2px;
border-radius: 8px;
border-style: solid;
@ -53,6 +52,5 @@ limitations under the License.
white-space: nowrap;
text-overflow: ellipsis;
overflow: hidden;
width: 312px;
}
}

View File

@ -271,7 +271,7 @@ limitations under the License.
display: flex;
align-items: center;
justify-content: start;
justify-content: flex-start;
padding: 5px 0;
.mx_EventTile_avatar {

View File

@ -46,6 +46,21 @@ limitations under the License.
mask-image: url('$(res)/img/element-icons/trashcan.svg');
}
.mx_VoiceRecordComposerTile_uploadingState {
margin-right: 10px;
color: $secondary-fg-color;
}
.mx_VoiceRecordComposerTile_failedState {
margin-right: 21px;
.mx_VoiceRecordComposerTile_uploadState_badge {
display: inline-block;
margin-right: 4px;
vertical-align: middle;
}
}
.mx_MessageComposer_row .mx_VoiceMessagePrimaryContainer {
// Note: remaining class properties are in the PlayerContainer CSS.
@ -68,7 +83,7 @@ limitations under the License.
height: 10px;
position: absolute;
left: 12px; // 12px from the left edge for container padding
top: 18px; // vertically center (middle align with clock)
top: 16px; // vertically center (middle align with clock)
border-radius: 10px;
}
}

View File

@ -75,8 +75,6 @@ limitations under the License.
height: 100%;
&.mx_VideoFeed_voice {
// We don't want to collide with the call controls that have 52px of height
margin-bottom: 52px;
background-color: $inverted-bg-color;
display: flex;
justify-content: center;

View File

@ -40,8 +40,6 @@ limitations under the License.
display: flex;
align-items: center;
justify-content: center;
aspect-ratio: 16 / 9;
}
.mx_VideoFeed_video {

View File

@ -20,6 +20,7 @@ limitations under the License.
&.mx_VideoFeed_voice {
background-color: $inverted-bg-color;
aspect-ratio: 16 / 9;
}
.mx_VideoFeed_video {

View File

@ -209,6 +209,14 @@ async function loadImageElement(imageFile: File) {
return { width, height, img };
}
// Minimum size for image files before we generate a thumbnail for them.
const IMAGE_SIZE_THRESHOLD_THUMBNAIL = 1 << 15; // 32KB
// Minimum size improvement for image thumbnails, if both are not met then don't bother uploading thumbnail.
const IMAGE_THUMBNAIL_MIN_REDUCTION_SIZE = 1 << 16; // 1MB
const IMAGE_THUMBNAIL_MIN_REDUCTION_PERCENT = 0.1; // 10%
// We don't apply these thresholds to video thumbnails as a poster image is always useful
// and videos tend to be much larger.
/**
* Read the metadata for an image file and create and upload a thumbnail of the image.
*
@ -217,23 +225,33 @@ async function loadImageElement(imageFile: File) {
* @param {File} imageFile The image to read and thumbnail.
* @return {Promise} A promise that resolves with the attachment info.
*/
function infoForImageFile(matrixClient, roomId, imageFile) {
async function infoForImageFile(matrixClient: MatrixClient, roomId: string, imageFile: File) {
let thumbnailType = "image/png";
if (imageFile.type === "image/jpeg") {
thumbnailType = "image/jpeg";
}
let imageInfo;
return loadImageElement(imageFile).then((r) => {
return createThumbnail(r.img, r.width, r.height, thumbnailType);
}).then((result) => {
imageInfo = result.info;
return uploadFile(matrixClient, roomId, result.thumbnail);
}).then((result) => {
imageInfo.thumbnail_url = result.url;
imageInfo.thumbnail_file = result.file;
const imageElement = await loadImageElement(imageFile);
const result = await createThumbnail(imageElement.img, imageElement.width, imageElement.height, thumbnailType);
const imageInfo = result.info;
// we do all sizing checks here because we still rely on thumbnail generation for making a blurhash from.
const sizeDifference = imageFile.size - imageInfo.thumbnail_info.size;
if (
imageFile.size <= IMAGE_SIZE_THRESHOLD_THUMBNAIL || // image is small enough already
(sizeDifference <= IMAGE_THUMBNAIL_MIN_REDUCTION_SIZE && // thumbnail is not sufficiently smaller than original
sizeDifference <= (imageFile.size * IMAGE_THUMBNAIL_MIN_REDUCTION_PERCENT))
) {
delete imageInfo["thumbnail_info"];
return imageInfo;
});
}
const uploadResult = await uploadFile(matrixClient, roomId, result.thumbnail);
imageInfo["thumbnail_url"] = uploadResult.url;
imageInfo["thumbnail_file"] = uploadResult.file;
return imageInfo;
}
/**

View File

@ -38,17 +38,9 @@ function makePlaybackWaveform(input: number[]): number[] {
// First, convert negative amplitudes to positive so we don't detect zero as "noisy".
const noiseWaveform = input.map(v => Math.abs(v));
// Next, we'll resample the waveform using a smoothing approach so we can keep the same rough shape.
// We also rescale the waveform to be 0-1 for the remaining function logic.
const resampled = arrayRescale(arraySmoothingResample(noiseWaveform, PLAYBACK_WAVEFORM_SAMPLES), 0, 1);
// Then, we'll do a high and low pass filter to isolate actual speaking volumes within the rescaled
// waveform. Most speech happens below the 0.5 mark.
const filtered = resampled.map(v => clamp(v, 0.1, 0.5));
// Finally, we'll rescale the filtered waveform (0.1-0.5 becomes 0-1 again) so the user sees something
// sensible. This is what we return to keep our contract of "values between zero and one".
return arrayRescale(filtered, 0, 1);
// Then, we'll resample the waveform using a smoothing approach so we can keep the same rough shape.
// We also rescale the waveform to be 0-1 so we end up with a clamped waveform to rely upon.
return arrayRescale(arraySmoothingResample(noiseWaveform, PLAYBACK_WAVEFORM_SAMPLES), 0, 1);
}
export class Playback extends EventEmitter implements IDestroyable {

View File

@ -17,8 +17,7 @@ limitations under the License.
import React from "react";
import { IRecordingUpdate, RECORDING_PLAYBACK_SAMPLES, VoiceRecording } from "../../../audio/VoiceRecording";
import { replaceableComponent } from "../../../utils/replaceableComponent";
import { arrayFastResample } from "../../../utils/arrays";
import { percentageOf } from "../../../utils/numbers";
import { arrayFastResample, arraySeed } from "../../../utils/arrays";
import Waveform from "./Waveform";
import { MarkedExecution } from "../../../utils/MarkedExecution";
@ -48,18 +47,14 @@ export default class LiveRecordingWaveform extends React.PureComponent<IProps, I
constructor(props) {
super(props);
this.state = {
waveform: [],
waveform: arraySeed(0, RECORDING_PLAYBACK_SAMPLES),
};
}
componentDidMount() {
this.props.recorder.liveData.onUpdate((update: IRecordingUpdate) => {
const bars = arrayFastResample(Array.from(update.waveform), RECORDING_PLAYBACK_SAMPLES);
// The incoming data is between zero and one, but typically even screaming into a
// microphone won't send you over 0.6, so we artificially adjust the gain for the
// waveform. This results in a slightly more cinematic/animated waveform for the
// user.
this.waveform = bars.map(b => percentageOf(b, 0, 0.50));
// The incoming data is between zero and one, so we don't need to clamp/rescale it.
this.waveform = arrayFastResample(Array.from(update.waveform), RECORDING_PLAYBACK_SAMPLES);
this.scheduledUpdate.mark();
});
}

View File

@ -17,10 +17,7 @@ limitations under the License.
import AccessibleTooltipButton from "../elements/AccessibleTooltipButton";
import { _t } from "../../../languageHandler";
import React, { ReactNode } from "react";
import {
RecordingState,
VoiceRecording,
} from "../../../audio/VoiceRecording";
import { IUpload, RecordingState, VoiceRecording } from "../../../audio/VoiceRecording";
import { Room } from "matrix-js-sdk/src/models/room";
import { MatrixClientPeg } from "../../../MatrixClientPeg";
import classNames from "classnames";
@ -34,6 +31,11 @@ import { MsgType } from "matrix-js-sdk/src/@types/event";
import Modal from "../../../Modal";
import ErrorDialog from "../dialogs/ErrorDialog";
import MediaDeviceHandler, { MediaDeviceKindEnum } from "../../../MediaDeviceHandler";
import NotificationBadge from "./NotificationBadge";
import { StaticNotificationState } from "../../../stores/notifications/StaticNotificationState";
import { NotificationColor } from "../../../stores/notifications/NotificationColor";
import InlineSpinner from "../elements/InlineSpinner";
import { PlaybackManager } from "../../../audio/PlaybackManager";
interface IProps {
room: Room;
@ -42,6 +44,7 @@ interface IProps {
interface IState {
recorder?: VoiceRecording;
recordingPhase?: RecordingState;
didUploadFail?: boolean;
}
/**
@ -69,9 +72,19 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
await this.state.recorder.stop();
let upload: IUpload;
try {
const upload = await this.state.recorder.upload(this.props.room.roomId);
upload = await this.state.recorder.upload(this.props.room.roomId);
} catch (e) {
console.error("Error uploading voice message:", e);
// Flag error and move on. The recording phase will be reset by the upload function.
this.setState({ didUploadFail: true });
return; // don't dispose the recording: the user has a chance to re-upload
}
try {
// noinspection ES6MissingAwait - we don't care if it fails, it'll get queued.
MatrixClientPeg.get().sendMessage(this.props.room.roomId, {
"body": "Voice message",
@ -104,12 +117,11 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
"org.matrix.msc3245.voice": {}, // No content, this is a rendering hint
});
} catch (e) {
console.error("Error sending/uploading voice message:", e);
Modal.createTrackedDialog('Upload failed', '', ErrorDialog, {
title: _t('Upload Failed'),
description: _t("The voice message failed to upload."),
});
return; // don't dispose the recording so the user can retry, maybe
console.error("Error sending voice message:", e);
// Voice message should be in the timeline at this point, so let other things take care
// of error handling. We also shouldn't need the recording anymore, so fall through to
// disposal.
}
await this.disposeRecording();
}
@ -118,7 +130,7 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
await VoiceRecordingStore.instance.disposeRecording();
// Reset back to no recording, which means no phase (ie: restart component entirely)
this.setState({ recorder: null, recordingPhase: null });
this.setState({ recorder: null, recordingPhase: null, didUploadFail: false });
}
private onCancel = async () => {
@ -166,6 +178,9 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
}
try {
// stop any noises which might be happening
await PlaybackManager.instance.playOnly(null);
const recorder = VoiceRecordingStore.instance.startRecording();
await recorder.start();
@ -209,9 +224,9 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
'mx_VoiceRecordComposerTile_stop': this.state.recorder?.isRecording,
});
let tooltip = _t("Record a voice message");
let tooltip = _t("Send voice message");
if (!!this.state.recorder) {
tooltip = _t("Stop the recording");
tooltip = _t("Stop recording");
}
let stopOrRecordBtn = <AccessibleTooltipButton
@ -229,12 +244,30 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
if (this.state.recorder && this.state.recordingPhase !== RecordingState.Uploading) {
deleteButton = <AccessibleTooltipButton
className='mx_VoiceRecordComposerTile_delete'
title={_t("Delete recording")}
title={_t("Delete")}
onClick={this.onCancel}
/>;
}
let uploadIndicator;
if (this.state.recordingPhase === RecordingState.Uploading) {
uploadIndicator = <span className='mx_VoiceRecordComposerTile_uploadingState'>
<InlineSpinner w={16} h={16} />
</span>;
} else if (this.state.didUploadFail && this.state.recordingPhase === RecordingState.Ended) {
uploadIndicator = <span className='mx_VoiceRecordComposerTile_failedState'>
<span className='mx_VoiceRecordComposerTile_uploadState_badge'>
{ /* Need to stick the badge in a span to ensure it doesn't create a block component */ }
<NotificationBadge
notification={StaticNotificationState.forSymbol("!", NotificationColor.Red)}
/>
</span>
<span className='text-warning'>{ _t("Failed to send") }</span>
</span>;
}
return (<>
{ uploadIndicator }
{ deleteButton }
{ this.renderWaveformArea() }
{ recordingInfo }

View File

@ -23,11 +23,16 @@ import { MatrixClientPeg } from '../../../MatrixClientPeg';
import { _t, _td } from '../../../languageHandler';
import VideoFeed from './VideoFeed';
import RoomAvatar from "../avatars/RoomAvatar";
import { CallState, CallType, MatrixCall, CallEvent } from 'matrix-js-sdk/src/webrtc/call';
import { CallEvent, CallState, CallType, MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
import classNames from 'classnames';
import AccessibleButton from '../elements/AccessibleButton';
import { isOnlyCtrlOrCmdKeyEvent, Key } from '../../../Keyboard';
import { alwaysAboveLeftOf, alwaysAboveRightOf, ChevronFace, ContextMenuButton } from '../../structures/ContextMenu';
import {
alwaysAboveLeftOf,
alwaysAboveRightOf,
ChevronFace,
ContextMenuTooltipButton,
} from '../../structures/ContextMenu';
import CallContextMenu from '../context_menus/CallContextMenu';
import { avatarUrlForMember } from '../../../Avatar';
import DialpadContextMenu from '../context_menus/DialpadContextMenu';
@ -37,6 +42,8 @@ import DesktopCapturerSourcePicker from "../elements/DesktopCapturerSourcePicker
import Modal from '../../../Modal';
import { SDPStreamMetadataPurpose } from 'matrix-js-sdk/src/webrtc/callEventTypes';
import CallViewSidebar from './CallViewSidebar';
import AccessibleTooltipButton from "../elements/AccessibleTooltipButton";
import { Alignment } from "../elements/Tooltip";
interface IProps {
// The call for us to display
@ -75,6 +82,8 @@ interface IState {
sidebarShown: boolean;
}
const tooltipYOffset = -24;
function getFullScreenElement() {
return (
document.fullscreenElement ||
@ -115,7 +124,6 @@ export default class CallView extends React.Component<IProps, IState> {
private controlsHideTimer: number = null;
private dialpadButton = createRef<HTMLDivElement>();
private contextMenuButton = createRef<HTMLDivElement>();
private contextMenu = createRef<HTMLDivElement>();
constructor(props: IProps) {
super(props);
@ -479,9 +487,12 @@ export default class CallView extends React.Component<IProps, IState> {
let vidMuteButton;
if (this.props.call.type === CallType.Video) {
vidMuteButton = (
<AccessibleButton
<AccessibleTooltipButton
className={vidClasses}
onClick={this.onVidMuteClick}
title={this.state.vidMuted ? _t("Start the camera") : _t("Stop the camera")}
alignment={Alignment.Top}
yOffset={tooltipYOffset}
/>
);
}
@ -496,9 +507,15 @@ export default class CallView extends React.Component<IProps, IState> {
this.props.call.state === CallState.Connected
) {
screensharingButton = (
<AccessibleButton
<AccessibleTooltipButton
className={screensharingClasses}
onClick={this.onScreenshareClick}
title={this.state.screensharing
? _t("Stop sharing your screen")
: _t("Start sharing your screen")
}
alignment={Alignment.Top}
yOffset={tooltipYOffset}
/>
);
}
@ -518,6 +535,7 @@ export default class CallView extends React.Component<IProps, IState> {
<AccessibleButton
className={sidebarButtonClasses}
onClick={this.onToggleSidebar}
aria-label={this.state.sidebarShown ? _t("Hide sidebar") : _t("Show sidebar")}
/>
);
}
@ -526,22 +544,28 @@ export default class CallView extends React.Component<IProps, IState> {
let contextMenuButton;
if (this.state.callState === CallState.Connected) {
contextMenuButton = (
<ContextMenuButton
<ContextMenuTooltipButton
className="mx_CallView_callControls_button mx_CallView_callControls_button_more"
onClick={this.onMoreClick}
inputRef={this.contextMenuButton}
isExpanded={this.state.showMoreMenu}
title={_t("More")}
alignment={Alignment.Top}
yOffset={tooltipYOffset}
/>
);
}
let dialpadButton;
if (this.state.callState === CallState.Connected && this.props.call.opponentSupportsDTMF()) {
dialpadButton = (
<ContextMenuButton
<ContextMenuTooltipButton
className="mx_CallView_callControls_button mx_CallView_callControls_dialpad"
inputRef={this.dialpadButton}
onClick={this.onDialpadClick}
isExpanded={this.state.showDialpad}
title={_t("Dialpad")}
alignment={Alignment.Top}
yOffset={tooltipYOffset}
/>
);
}
@ -554,7 +578,11 @@ export default class CallView extends React.Component<IProps, IState> {
ChevronFace.None,
CONTEXT_MENU_VPADDING,
)}
mountAsChild={true}
// We mount the context menus as a as a child typically in order to include the
// context menus when fullscreening the call content.
// However, this does not work as well when the call is embedded in a
// picture-in-picture frame. Thus, only mount as child when we are *not* in PiP.
mountAsChild={!this.props.pipMode}
onFinished={this.closeDialpad}
call={this.props.call}
/>;
@ -568,7 +596,7 @@ export default class CallView extends React.Component<IProps, IState> {
ChevronFace.None,
CONTEXT_MENU_VPADDING,
)}
mountAsChild={true}
mountAsChild={!this.props.pipMode}
onFinished={this.closeContextMenu}
call={this.props.call}
/>;
@ -583,9 +611,12 @@ export default class CallView extends React.Component<IProps, IState> {
{ dialPad }
{ contextMenu }
{ dialpadButton }
<AccessibleButton
<AccessibleTooltipButton
className={micClasses}
onClick={this.onMicMuteClick}
title={this.state.micMuted ? _t("Unmute the microphone") : _t("Mute the microphone")}
alignment={Alignment.Top}
yOffset={tooltipYOffset}
/>
{ vidMuteButton }
<div className={micCacheClasses} />
@ -593,9 +624,12 @@ export default class CallView extends React.Component<IProps, IState> {
{ screensharingButton }
{ sidebarButton }
{ contextMenuButton }
<AccessibleButton
<AccessibleTooltipButton
className="mx_CallView_callControls_button mx_CallView_callControls_button_hangup"
onClick={this.onHangupClick}
title={_t("Hangup")}
alignment={Alignment.Top}
yOffset={tooltipYOffset}
/>
</div>
);
@ -820,7 +854,7 @@ export default class CallView extends React.Component<IProps, IState> {
let fullScreenButton;
if (!this.props.pipMode) {
fullScreenButton = (
<div
<AccessibleTooltipButton
className="mx_CallView_header_button mx_CallView_header_button_fullscreen"
onClick={this.onFullscreenClick}
title={_t("Fill Screen")}
@ -830,7 +864,7 @@ export default class CallView extends React.Component<IProps, IState> {
let expandButton;
if (this.props.pipMode) {
expandButton = <div
expandButton = <AccessibleTooltipButton
className="mx_CallView_header_button mx_CallView_header_button_expand"
onClick={this.onExpandClick}
title={_t("Return to call")}

View File

@ -35,6 +35,16 @@ export const EMOTICON_TO_EMOJI = new Map<string, IEmoji>();
export const getEmojiFromUnicode = unicode => UNICODE_TO_EMOJI.get(stripVariation(unicode));
const isRegionalIndicator = (x: string): boolean => {
// First verify that the string is a single character. We use Array.from
// to make sure we count by characters, not UTF-8 code units.
return Array.from(x).length === 1 &&
// Next verify that the character is within the code point range for
// regional indicators.
// http://unicode.org/charts/PDF/Unicode-6.0/U60-1F100.pdf
x >= '\u{1f1e6}' && x <= '\u{1f1ff}';
};
const EMOJIBASE_GROUP_ID_TO_CATEGORY = [
"people", // smileys
"people", // actually people
@ -72,7 +82,11 @@ export const EMOJI: IEmoji[] = EMOJIBASE.map((emojiData: Omit<IEmoji, "shortcode
shortcodes: typeof shortcodeData === "string" ? [shortcodeData] : shortcodeData,
};
const categoryId = EMOJIBASE_GROUP_ID_TO_CATEGORY[emoji.group];
// We manually include regional indicators in the symbols group, since
// Emojibase intentionally leaves them uncategorized
const categoryId = EMOJIBASE_GROUP_ID_TO_CATEGORY[emoji.group] ??
(isRegionalIndicator(emoji.unicode) ? "symbols" : null);
if (DATA_BY_CATEGORY.hasOwnProperty(categoryId)) {
DATA_BY_CATEGORY[categoryId].push(emoji);
}

View File

@ -904,6 +904,17 @@
"sends snowfall": "sends snowfall",
"Sends the given message with a space themed effect": "Sends the given message with a space themed effect",
"sends space invaders": "sends space invaders",
"Start the camera": "Start the camera",
"Stop the camera": "Stop the camera",
"Stop sharing your screen": "Stop sharing your screen",
"Start sharing your screen": "Start sharing your screen",
"Hide sidebar": "Hide sidebar",
"Show sidebar": "Show sidebar",
"More": "More",
"Dialpad": "Dialpad",
"Unmute the microphone": "Unmute the microphone",
"Mute the microphone": "Mute the microphone",
"Hangup": "Hangup",
"unknown person": "unknown person",
"Consulting with %(transferTarget)s. <a>Transfer to %(transferee)s</a>": "Consulting with %(transferTarget)s. <a>Transfer to %(transferee)s</a>",
"You held the call <a>Switch</a>": "You held the call <a>Switch</a>",
@ -1697,14 +1708,12 @@
"Invited by %(sender)s": "Invited by %(sender)s",
"Jump to first unread message.": "Jump to first unread message.",
"Mark all as read": "Mark all as read",
"The voice message failed to upload.": "The voice message failed to upload.",
"Unable to access your microphone": "Unable to access your microphone",
"We were unable to access your microphone. Please check your browser settings and try again.": "We were unable to access your microphone. Please check your browser settings and try again.",
"No microphone found": "No microphone found",
"We didn't find a microphone on your device. Please check your settings and try again.": "We didn't find a microphone on your device. Please check your settings and try again.",
"Record a voice message": "Record a voice message",
"Stop the recording": "Stop the recording",
"Delete recording": "Delete recording",
"Send voice message": "Send voice message",
"Stop recording": "Stop recording",
"Error updating main address": "Error updating main address",
"There was an error updating the room's main address. It may not be allowed by the server or a temporary failure occurred.": "There was an error updating the room's main address. It may not be allowed by the server or a temporary failure occurred.",
"There was an error updating the room's alternative addresses. It may not be allowed by the server or a temporary failure occurred.": "There was an error updating the room's alternative addresses. It may not be allowed by the server or a temporary failure occurred.",

View File

@ -43,9 +43,8 @@ function getManagedIframe(): { iframe: HTMLIFrameElement, onLoadPromise: Promise
// Dev note: the reassignment warnings are entirely incorrect here.
// @ts-ignore
// noinspection JSConstantReassignment
managedIframe.style = { display: "none" };
managedIframe.style.display = "none";
// @ts-ignore
// noinspection JSConstantReassignment
managedIframe.sandbox = "allow-scripts allow-downloads allow-downloads-without-user-activation";