new: [feed] Support brotli compression

app/Lib/Tools/HttpSocketExtended.php

@@ -0,0 +1,111 @@
+<?php
+App::uses('HttpSocketResponse', 'Network/Http');
+App::uses('HttpSocket', 'Network/Http');
+
+class HttpClientJsonException extends Exception
+{
+    /** @var HttpSocketResponse */
+    private $response;
+
+    public function __construct($message, HttpSocketResponseExtended $response, Throwable $previous = null)
+    {
+        $this->response = $response;
+        parent::__construct($message, 0, $previous);
+    }
+
+    /**
+     * @return HttpSocketResponse
+     */
+    public function getResponse()
+    {
+        return $this->response;
+    }
+}
+
+class HttpSocketResponseExtended extends HttpSocketResponse
+{
+    /**
+     * @param string $message
+     * @throws SocketException
+     */
+    public function parseResponse($message)
+    {
+        parent::parseResponse($message);
+
+        $contentEncoding = $this->getHeader('Content-Encoding');
+        if ($contentEncoding === 'gzip' && function_exists('gzdecode')) {
+            $this->body = gzdecode($this->body);
+            if ($this->body === false) {
+                throw new SocketException("Response should be gzip encoded, but gzip decoding failed.");
+            }
+        } else if ($contentEncoding === 'br' && function_exists('brotli_uncompress')) {
+            $this->body = brotli_uncompress($this->body);
+            if ($this->body === false) {
+                throw new SocketException("Response should be brotli encoded, but brotli decoding failed.");
+            }
+        } else if ($contentEncoding) {
+            throw new SocketException("Remote server returns unsupported content encoding '$contentEncoding'");
+        }
+    }
+
+    /**
+     * Decodes JSON string and throws exception if string is not valid JSON.
+     *
+     * @return array
+     * @throws HttpClientJsonException
+     */
+    public function json()
+    {
+        try {
+            if (defined('JSON_THROW_ON_ERROR')) {
+                // JSON_THROW_ON_ERROR is supported since PHP 7.3
+                $decoded = json_decode($this->body, true, 512, JSON_THROW_ON_ERROR);
+            } else {
+                $decoded = json_decode($this->body, true);
+                if ($decoded === null) {
+                    throw new UnexpectedValueException('Could not parse JSON: ' . json_last_error_msg(), json_last_error());
+                }
+            }
+            return $decoded;
+        } catch (Exception $e) {
+            throw new HttpClientJsonException('Could not parse response as JSON.', $this, $e);
+        }
+    }
+}
+
+/**
+ * Supports response compression and also decodes response as JSON
+ */
+class HttpSocketExtended extends HttpSocket
+{
+    public $responseClass = 'HttpSocketResponseExtended';
+
+    public function __construct($config = array())
+    {
+        parent::__construct($config);
+        if (isset($config['compress']) && $config['compress']) {
+            $acceptEncoding = $this->acceptedEncodings();
+            if (!empty($acceptEncoding)) {
+                $this->config['request']['header']['Accept-Encoding'] = implode(', ', $this->acceptedEncodings());
+            }
+        }
+    }
+
+    /**
+     * Returns accepted content encodings (compression algorithms)
+     * @return string[]
+     */
+    private function acceptedEncodings()
+    {
+        $supportedEncoding = [];
+        // Enable brotli compressed responses if PHP has 'brotli_uncompress' method
+        if (function_exists('brotli_uncompress')) {
+            $supportedEncoding[] = 'br';
+        }
+        // Enable gzipped responses if PHP has 'gzdecode' method
+        if (function_exists('gzdecode')) {
+            $supportedEncoding[] = 'gzip';
+        }
+        return $supportedEncoding;
+    }
+}

app/Lib/Tools/SyncTool.php

@@ -2,7 +2,14 @@
 
 class SyncTool
 {
-    // take a server as parameter and return a HttpSocket object using the ssl options defined in the server settings
+    /**
+     * Take a server as parameter and return a HttpSocket object using the ssl options defined in the server settings
+     * @param array|null $server
+     * @param false $timeout
+     * @param string $model
+     * @return HttpSocketExtended
+     * @throws Exception
+     */
     public function setupHttpSocket($server = null, $timeout = false, $model = 'Server')
     {
         $params = array();
@@ -33,12 +40,12 @@ class SyncTool
 
     public function setupHttpSocketFeed($feed = null)
     {
-        return $this->setupHttpSocket();
+        return $this->createHttpSocket(['compress' => true]);
     }
 
     /**
      * @param array $params
-     * @return HttpSocket
+     * @return HttpSocketExtended
      * @throws Exception
      */
     public function createHttpSocket($params = array())
@@ -52,8 +59,8 @@ class SyncTool
             $params['ssl_cafile'] = $caPath;
         }
 
-        App::uses('HttpSocket', 'Network/Http');
-        $HttpSocket = new HttpSocket($params);
+        App::uses('HttpSocketExtended', 'Tools');
+        $HttpSocket = new HttpSocketExtended($params);
         $proxy = Configure::read('Proxy');
         if (empty($params['skip_proxy']) && isset($proxy['host']) && !empty($proxy['host'])) {
             $HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']);

app/Model/Feed.php

@@ -605,11 +605,6 @@ class Feed extends AppModel
             )
         );
 
-        // Enable gzipped responses if PHP has 'gzdecode' method
-        if (function_exists('gzdecode')) {
-            $result['header']['Accept-Encoding'] = 'gzip';
-        }
-
         $commit = $this->checkMIPSCommit();
         if ($commit) {
             $result['header']['commit'] = $commit;
@@ -1780,16 +1775,6 @@ class Feed extends AppModel
 
         $data = $response->body;
 
-        $contentEncoding = $response->getHeader('Content-Encoding');
-        if ($contentEncoding === 'gzip') {
-            $data = gzdecode($data);
-            if ($data === false) {
-                throw new Exception("Fetching the '$uri' failed, response should be gzip encoded, but gzip decoding failed.");
-            }
-        } else if ($contentEncoding) {
-            throw new Exception("Fetching the '$uri' failed, because remote server returns unsupported content encoding '$contentEncoding'");
-        }
-
         $contentType = $response->getHeader('Content-Type');
         if ($contentType === 'application/zip') {
             $zipFile = new File($this->tempFileName());

app/composer.json

@@ -23,6 +23,7 @@
         "ext-redis": "For working background jobs and feed and warninglist caches",
         "ext-zip": "Enabling processing feeds that are ZIP compressed",
         "ext-zlib": "Allow gzip compression of HTTP responses",
+        "ext-brotli": "Allow brotli compression of HTTP responses",
         "ext-intl": "For handling IDN domain names",
         "ext-ssdeep": "For ssdeep hashes correlation",
         "ext-bcmath": "For faster validating IBAN numbers",