Merge pull request #2439 from matrix-org/matthew/retina

Fix for retina thumbnails being massive

package.json

@@ -83,6 +83,7 @@
     "matrix-js-sdk": "1.0.4",
     "optimist": "^0.6.1",
     "pako": "^1.0.5",
+    "png-chunks-extract": "^1.0.0",
     "prop-types": "^15.5.8",
     "qrcode-react": "^0.1.16",
     "qs": "^6.6.0",

src/ContentMessages.js

@@ -25,8 +25,8 @@ import sdk from './index';
 import { _t } from './languageHandler';
 import Modal from './Modal';
 import RoomViewStore from './stores/RoomViewStore';
-
 import encrypt from "browser-encrypt-attachment";
+import extractPngChunks from "png-chunks-extract";
 
 // Polyfill for Canvas.toBlob API using Canvas.toDataURL
 import "blueimp-canvas-to-blob";
@@ -34,6 +34,10 @@ import "blueimp-canvas-to-blob";
 const MAX_WIDTH = 800;
 const MAX_HEIGHT = 600;
 
+// scraped out of a macOS hidpi (5660ppm) screenshot png
+//                  5669 px (x-axis)      , 5669 px (y-axis)      , per metre
+const PHYS_HIDPI = [0x00, 0x00, 0x16, 0x25, 0x00, 0x00, 0x16, 0x25, 0x01];
+
 export class UploadCanceledError extends Error {}
 
 /**
@@ -97,24 +101,48 @@ function createThumbnail(element, inputWidth, inputHeight, mimeType) {
  * @param {File} imageFile The file to load in an image element.
  * @return {Promise} A promise that resolves with the html image element.
  */
-function loadImageElement(imageFile) {
+async function loadImageElement(imageFile) {
-    const deferred = Promise.defer();
-
     // Load the file into an html element
     const img = document.createElement("img");
     const objectUrl = URL.createObjectURL(imageFile);
+    const imgPromise = new Promise((resolve, reject) => {
+        img.onload = function() {
+            URL.revokeObjectURL(objectUrl);
+            resolve(img);
+        };
+        img.onerror = function(e) {
+            reject(e);
+        };
+    });
     img.src = objectUrl;
 
-    // Once ready, create a thumbnail
+    // check for hi-dpi PNGs and fudge display resolution as needed.
-    img.onload = function() {
+    // this is mainly needed for macOS screencaps
-        URL.revokeObjectURL(objectUrl);
+    let parsePromise;
-        deferred.resolve(img);
+    if (imageFile.type === "image/png") {
-    };
+        // in practice macOS happens to order the chunks so they fall in
-    img.onerror = function(e) {
+        // the first 0x1000 bytes (thanks to a massive ICC header).
-        deferred.reject(e);
+        // Thus we could slice the file down to only sniff the first 0x1000
-    };
+        // bytes (but this makes extractPngChunks choke on the corrupt file)
+        const headers = imageFile; //.slice(0, 0x1000);
+        parsePromise = readFileAsArrayBuffer(headers).then(arrayBuffer => {
+            const buffer = new Uint8Array(arrayBuffer);
+            const chunks = extractPngChunks(buffer);
+            for (const chunk of chunks) {
+                if (chunk.name === 'pHYs') {
+                    if (chunk.data.byteLength !== PHYS_HIDPI.length) return;
+                    const hidpi = chunk.data.every((val, i) => val === PHYS_HIDPI[i]);
+                    return hidpi;
+                }
+            }
+            return false;
+        });
+    }
 
-    return deferred.promise;
+    const [hidpi] = await Promise.all([parsePromise, imgPromise]);
+    const width = hidpi ? (img.width >> 1) : img.width;
+    const height = hidpi ? (img.height >> 1) : img.height;
+    return {width, height, img};
 }
 
 /**
@@ -132,8 +160,8 @@ function infoForImageFile(matrixClient, roomId, imageFile) {
     }
 
     let imageInfo;
-    return loadImageElement(imageFile).then(function(img) {
+    return loadImageElement(imageFile).then(function(r) {
-        return createThumbnail(img, img.width, img.height, thumbnailType);
+        return createThumbnail(r.img, r.width, r.height, thumbnailType);
     }).then(function(result) {
         imageInfo = result.info;
         return uploadFile(matrixClient, roomId, result.thumbnail);

src/components/views/messages/MImageBody.js

@@ -150,7 +150,7 @@ export default class MImageBody extends React.Component {
 
         if (this.refs.image) {
             const { naturalWidth, naturalHeight } = this.refs.image;
-
+            // this is only used as a fallback in case content.info.w/h is missing
             loadedImageDimensions = { naturalWidth, naturalHeight };
         }
 
@@ -167,6 +167,14 @@ export default class MImageBody extends React.Component {
     }
 
     _getThumbUrl() {
+        // FIXME: the dharma skin lets images grow as wide as you like, rather than capped to 800x600.
+        // So either we need to support custom timeline widths here, or reimpose the cap, otherwise the
+        // thumbnail resolution will be unnecessarily reduced.
+        // custom timeline widths seems preferable.
+        const pixelRatio = window.devicePixelRatio;
+        const thumbWidth = 800 * pixelRatio;
+        const thumbHeight = 600 * pixelRatio;
+
         const content = this.props.mxEvent.getContent();
         if (content.file !== undefined) {
             // Don't use the thumbnail for clients wishing to autoplay gifs.
@@ -175,14 +183,61 @@ export default class MImageBody extends React.Component {
             }
             return this.state.decryptedUrl;
         } else if (content.info && content.info.mimetype === "image/svg+xml" && content.info.thumbnail_url) {
-            // special case to return client-generated thumbnails for SVGs, if any,
+            // special case to return clientside sender-generated thumbnails for SVGs, if any,
             // given we deliberately don't thumbnail them serverside to prevent
             // billion lol attacks and similar
             return this.context.matrixClient.mxcUrlToHttp(
-                content.info.thumbnail_url, 800, 600,
+                content.info.thumbnail_url,
+                thumbWidth,
+                thumbHeight,
             );
         } else {
-            return this.context.matrixClient.mxcUrlToHttp(content.url, 800, 600);
+            // we try to download the correct resolution
+            // for hi-res images (like retina screenshots).
+            // synapse only supports 800x600 thumbnails for now though,
+            // so we'll need to download the original image for this to work
+            // well for now. First, let's try a few cases that let us avoid
+            // downloading the original:
+            if (pixelRatio === 1.0 ||
+                    (!content.info || !content.info.w ||
+                     !content.info.h || !content.info.size)) {
+                // always thumbnail. it may look a bit worse, but it'll save bandwidth.
+                // which is probably desirable on a lo-dpi device anyway.
+                return this.context.matrixClient.mxcUrlToHttp(content.url, thumbWidth, thumbHeight);
+            } else {
+                // we should only request thumbnails if the image is bigger than 800x600
+                // (or 1600x1200 on retina) otherwise the image in the timeline will just
+                // end up resampled and de-retina'd for no good reason.
+                // Ideally the server would pregen 1600x1200 thumbnails in order to provide retina
+                // thumbnails, but we don't do this currently in synapse for fear of disk space.
+                // As a compromise, let's switch to non-retina thumbnails only if the original
+                // image is both physically too large and going to be massive to load in the
+                // timeline (e.g. >1MB).
+
+                const isLargerThanThumbnail = (
+                    content.info.w > thumbWidth ||
+                    content.info.h > thumbHeight
+                );
+                const isLargeFileSize = content.info.size > 1*1024*1024;
+
+                if (isLargeFileSize && isLargerThanThumbnail) {
+                    // image is too large physically and bytewise to clutter our timeline so
+                    // we ask for a thumbnail, despite knowing that it will be max 800x600
+                    // despite us being retina (as synapse doesn't do 1600x1200 thumbs yet).
+                    return this.context.matrixClient.mxcUrlToHttp(
+                        content.url,
+                        thumbWidth,
+                        thumbHeight,
+                    );
+                } else {
+                    // download the original image otherwise, so we can scale it client side
+                    // to take pixelRatio into account.
+                    // ( no width/height means we want the original image)
+                    return this.context.matrixClient.mxcUrlToHttp(
+                        content.url,
+                    );
+                }
+            }
         }
     }
 

yarn.lock

@@ -2240,6 +2240,11 @@ counterpart@^0.18.0:
     pluralizers "^0.1.7"
     sprintf-js "^1.0.3"
 
+crc-32@^0.3.0:
+  version "0.3.0"
+  resolved "https://registry.yarnpkg.com/crc-32/-/crc-32-0.3.0.tgz#6a3d3687f5baec41f7e9b99fe1953a2e5d19775e"
+  integrity sha1-aj02h/W67EH36bmf4ZU6Ll0Zd14=
+
 create-ecdh@^4.0.0:
   version "4.0.3"
   resolved "https://registry.yarnpkg.com/create-ecdh/-/create-ecdh-4.0.3.tgz#c9111b6f33045c4697f144787f9254cdc77c45ff"
@@ -5863,6 +5868,13 @@ pluralizers@^0.1.7:
   resolved "https://registry.yarnpkg.com/pluralizers/-/pluralizers-0.1.7.tgz#8d38dd0a1b660e739b10ab2eab10b684c9d50142"
   integrity sha512-mw6AejUiCaMQ6uPN9ObjJDTnR5AnBSmnHHy3uVTbxrSFSxO5scfwpTs8Dxyb6T2v7GSulhvOq+pm9y+hXUvtOA==
 
+png-chunks-extract@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/png-chunks-extract/-/png-chunks-extract-1.0.0.tgz#fad4a905e66652197351c65e35b92c64311e472d"
+  integrity sha1-+tSpBeZmUhlzUcZeNbksZDEeRy0=
+  dependencies:
+    crc-32 "^0.3.0"
+
 posix-character-classes@^0.1.0:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab"