From 53db386731422c88d4e24e299e9e56a150dad3cf Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 10 Aug 2020 22:06:30 -0600 Subject: [PATCH] Add support for blurhash (MSC2448) MSC: https://github.com/matrix-org/matrix-doc/pull/2448 While the image loads, we can show a blurred version of it (calculated at upload time) so we don't have a blank space in the timeline. --- package.json | 1 + src/ContentMessages.tsx | 10 +++- .../views/elements/BlurhashPlaceholder.tsx | 56 +++++++++++++++++++ src/components/views/messages/MImageBody.js | 18 +++--- yarn.lock | 5 ++ 5 files changed, 81 insertions(+), 9 deletions(-) create mode 100644 src/components/views/elements/BlurhashPlaceholder.tsx diff --git a/package.json b/package.json index 548b33f353..989672d414 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "@babel/runtime": "^7.10.5", "await-lock": "^2.0.1", "blueimp-canvas-to-blob": "^3.27.0", + "blurhash": "^1.1.3", "browser-encrypt-attachment": "^0.3.0", "browser-request": "^0.3.3", "classnames": "^2.2.6", diff --git a/src/ContentMessages.tsx b/src/ContentMessages.tsx index 6f55a75d0c..7e57b34ff7 100644 --- a/src/ContentMessages.tsx +++ b/src/ContentMessages.tsx @@ -334,6 +334,7 @@ function uploadFile(matrixClient: MatrixClient, roomId: string, file: File | Blo if (file.type) { encryptInfo.mimetype = file.type; } + // TODO: Blurhash for encrypted media? return {"file": encryptInfo}; }); (prom as IAbortablePromise).abort = () => { @@ -344,11 +345,15 @@ function uploadFile(matrixClient: MatrixClient, roomId: string, file: File | Blo } else { const basePromise = matrixClient.uploadContent(file, { progressHandler: progressHandler, + onlyContentUri: false, }); - const promise1 = basePromise.then(function(url) { + const promise1 = basePromise.then(function(body) { if (canceled) throw new UploadCanceledError(); // If the attachment isn't encrypted then include the URL directly. - return {"url": url}; + return { + "url": body.content_uri, + "blurhash": body["xyz.amorgan.blurhash"], // TODO: Use `body.blurhash` when MSC2448 lands + }; }); promise1.abort = () => { canceled = true; @@ -550,6 +555,7 @@ export default class ContentMessages { return upload.promise.then(function(result) { content.file = result.file; content.url = result.url; + content.info['xyz.amorgan.blurhash'] = result.blurhash; // TODO: Use `blurhash` when MSC2448 lands }); }).then(() => { // Await previous message being sent into the room diff --git a/src/components/views/elements/BlurhashPlaceholder.tsx b/src/components/views/elements/BlurhashPlaceholder.tsx new file mode 100644 index 0000000000..bed45bfe5a --- /dev/null +++ b/src/components/views/elements/BlurhashPlaceholder.tsx @@ -0,0 +1,56 @@ +/* + Copyright 2020 The Matrix.org Foundation C.I.C. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +import React from 'react'; +import {decode} from "blurhash"; + +interface IProps { + blurhash: string; + width: number; + height: number; +} + +export default class BlurhashPlaceholder extends React.PureComponent { + private canvas: React.RefObject = React.createRef(); + + public componentDidMount() { + this.draw(); + } + + public componentDidUpdate() { + this.draw(); + } + + private draw() { + if (!this.canvas.current) return; + + try { + const {width, height} = this.props; + + const pixels = decode(this.props.blurhash, Math.ceil(width), Math.ceil(height)); + const ctx = this.canvas.current.getContext("2d"); + const imgData = ctx.createImageData(width, height); + imgData.data.set(pixels); + ctx.putImageData(imgData, 0, 0); + } catch (e) { + console.error("Error rendering blurhash: ", e); + } + } + + public render() { + return ; + } +} diff --git a/src/components/views/messages/MImageBody.js b/src/components/views/messages/MImageBody.js index c92ae475bf..e0ac24c641 100644 --- a/src/components/views/messages/MImageBody.js +++ b/src/components/views/messages/MImageBody.js @@ -27,6 +27,7 @@ import { _t } from '../../../languageHandler'; import SettingsStore from "../../../settings/SettingsStore"; import MatrixClientContext from "../../../contexts/MatrixClientContext"; import InlineSpinner from '../elements/InlineSpinner'; +import BlurhashPlaceholder from "../elements/BlurhashPlaceholder"; export default class MImageBody extends React.Component { static propTypes = { @@ -53,6 +54,8 @@ export default class MImageBody extends React.Component { this.onClick = this.onClick.bind(this); this._isGif = this._isGif.bind(this); + const imageInfo = this.props.mxEvent.getContent().info; + this.state = { decryptedUrl: null, decryptedThumbnailUrl: null, @@ -63,6 +66,7 @@ export default class MImageBody extends React.Component { loadedImageDimensions: null, hover: false, showImage: SettingsStore.getValue("showImages"), + blurhash: imageInfo ? imageInfo['xyz.amorgan.blurhash'] : null, // TODO: Use `blurhash` when MSC2448 lands. }; this._image = createRef(); @@ -329,7 +333,8 @@ export default class MImageBody extends React.Component { infoWidth = content.info.w; infoHeight = content.info.h; } else { - // Whilst the image loads, display nothing. + // Whilst the image loads, display nothing. We also don't display a blurhash image + // because we don't really know what size of image we'll end up with. // // Once loaded, use the loaded image dimensions stored in `loadedImageDimensions`. // @@ -368,8 +373,7 @@ export default class MImageBody extends React.Component { if (content.file !== undefined && this.state.decryptedUrl === null) { placeholder = ; } else if (!this.state.imgLoaded) { - // Deliberately, getSpinner is left unimplemented here, MStickerBody overides - placeholder = this.getPlaceholder(); + placeholder = this.getPlaceholder(maxWidth, maxHeight); } let showPlaceholder = Boolean(placeholder); @@ -391,7 +395,7 @@ export default class MImageBody extends React.Component { if (!this.state.showImage) { img = ; - showPlaceholder = false; // because we're hiding the image, so don't show the sticker icon. + showPlaceholder = false; // because we're hiding the image, so don't show the placeholder. } if (this._isGif() && !SettingsStore.getValue("autoplayGifsAndVideos") && !this.state.hover) { @@ -433,9 +437,9 @@ export default class MImageBody extends React.Component { } // Overidden by MStickerBody - getPlaceholder() { - // MImageBody doesn't show a placeholder whilst the image loads, (but it could do) - return null; + getPlaceholder(width, height) { + if (!this.state.blurhash) return null; + return ; } // Overidden by MStickerBody diff --git a/yarn.lock b/yarn.lock index 98fe42ef13..f1cace67a1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2493,6 +2493,11 @@ blueimp-canvas-to-blob@^3.27.0: resolved "https://registry.yarnpkg.com/blueimp-canvas-to-blob/-/blueimp-canvas-to-blob-3.27.0.tgz#a2bd5c43587b95dedf0f6998603452d1bfcc9b9e" integrity sha512-AcIj+hCw6WquxzJuzC6KzgYmqxLFeTWe88KuY2BEIsW1zbEOfoinDAGlhyvFNGt+U3JElkVSK7anA1FaSdmmfA== +blurhash@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/blurhash/-/blurhash-1.1.3.tgz#dc325af7da836d07a0861d830bdd63694382483e" + integrity sha512-yUhPJvXexbqbyijCIE/T2NCXcj9iNPhWmOKbPTuR/cm7Q5snXYIfnVnz6m7MWOXxODMz/Cr3UcVkRdHiuDVRDw== + bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.4.0: version "4.11.9" resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.11.9.tgz#26d556829458f9d1e81fc48952493d0ba3507828"