From 4dec19dedfe3e5c48c44947133fe3e78ba41dc03 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Tue, 28 Apr 2020 15:31:27 +0200
Subject: [PATCH] new: [feed] Support for compressed feeds

---
 app/Model/AppModel.php |  17 +++++++
 app/Model/Feed.php     | 110 ++++++++++++++++++++++++++++++++++-------
 2 files changed, 108 insertions(+), 19 deletions(-)

diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php
index f25cc18bd..a62223a1a 100644
--- a/app/Model/AppModel.php
+++ b/app/Model/AppModel.php
@@ -2847,4 +2847,21 @@ class AppModel extends Model
         $message .= "\nStack Trace:\n" . $exception->getTraceAsString();
         return $this->log($message, $type);
     }
+
+    /**
+     * Generates random file name in tmp dir.
+     * @return string
+     */
+    protected function tempFileName()
+    {
+        return $this->tempDir() . DS . $this->generateRandomFileName();
+    }
+
+    /**
+     * @return string
+     */
+    protected function tempDir()
+    {
+        return Configure::read('MISP.tmpdir') ?: sys_get_temp_dir();
+    }
 }
diff --git a/app/Model/Feed.php b/app/Model/Feed.php
index ceefb22fa..b96a016cd 100644
--- a/app/Model/Feed.php
+++ b/app/Model/Feed.php
@@ -247,10 +247,7 @@ class Feed extends AppModel
             $data = $this->feedGetUri($feed, $feedUrl, $HttpSocket, true);
 
             if (!$isLocal) {
-                $redis = $this->setupRedis();
-                if ($redis === false) {
-                    throw new Exception('Could not reach Redis.');
-                }
+                $redis = $this->setupRedisWithException();
                 $redis->del('misp:feed_cache:' . $feed['Feed']['id']);
                 file_put_contents($feedCache, $data);
             }
@@ -502,12 +499,17 @@ class Feed extends AppModel
 
         $result = array(
             'header' => array(
-                    'Accept' => array('application/json', 'text/plain'),
-                    'Content-Type' => 'application/json',
-                    'MISP-version' => $version,
-                    'MISP-uuid' => Configure::read('MISP.uuid')
+                'Accept' => array('application/json', 'text/plain'),
+                'MISP-version' => $version,
+                'MISP-uuid' => Configure::read('MISP.uuid'),
             )
         );
+
+        // Enable gzipped responses if PHP has 'gzdecode' method
+        if (function_exists('gzdecode')) {
+            $result['header']['Accept-Encoding'] = 'gzip';
+        }
+
         if ($commit) {
             $result['header']['commit'] = $commit;
         }
@@ -1590,24 +1592,51 @@ class Feed extends AppModel
                 if ($data === false) {
                     throw new Exception("Could not read local file '$uri'.");
                 }
+                return $data;
             } else {
                 throw new Exception("Local file '$uri' doesn't exists.");
             }
+        }
+
+        $request = $this->__createFeedRequest($feed['Feed']['headers']);
+
+        if ($followRedirect) {
+            $response = $this->getFollowRedirect($HttpSocket, $uri, $request);
         } else {
-            $request = $this->__createFeedRequest($feed['Feed']['headers']);
+            $response = $HttpSocket->get($uri, array(), $request);
+        }
 
-            if ($followRedirect) {
-                $response = $this->getFollowRedirect($HttpSocket, $uri, $request);
-            } else {
-                $response = $HttpSocket->get($uri, array(), $request);
-            }
+        if ($response === false) {
+            throw new Exception("Could not reach '$uri'.");
+        } else if ($response->code != 200) { // intentionally !=
+            throw new Exception("Fetching the '$uri' failed with HTTP error {$response->code}: {$response->reasonPhrase}");
+        }
 
-            if ($response === false) {
-                throw new Exception("Could not reach '$uri'.");
-            } else if ($response->code != 200) { // intentionally !=
-                throw new Exception("Fetching the '$uri' failed with HTTP error {$response->code}: {$response->reasonPhrase}");
+        $data = $response->body;
+
+        $contentEncoding = $response->getHeader('Content-Encoding');
+        if ($contentEncoding === 'gzip') {
+            $data = gzdecode($data);
+            if ($data === false) {
+                throw new Exception("Fetching the '$uri' failed, response should be gzip encoded, but gzip decoding failed.");
+            }
+        } else if ($contentEncoding) {
+            throw new Exception("Fetching the '$uri' failed, because remote server returns unsupported content encoding '$contentEncoding'");
+        }
+
+        $contentType = $response->getHeader('Content-Type');
+        if ($contentType === 'application/zip') {
+            $zipFile = new File($this->tempFileName());
+            $zipFile->write($data);
+            $zipFile->close();
+
+            try {
+                $data = $this->unzipFirstFile($zipFile);
+            } catch (Exception $e) {
+                throw new Exception("Fetching the '$uri' failed: {$e->getMessage()}");
+            } finally {
+                $zipFile->delete();
             }
-            $data = $response->body;
         }
 
         return $data;
@@ -1720,4 +1749,47 @@ class Feed extends AppModel
         $this->save($feed);
         return $count;
     }
+
+    /**
+     * @param File $zipFile
+     * @return string Uncompressed data
+     * @throws Exception
+     */
+    private function unzipFirstFile(File $zipFile)
+    {
+        if (!class_exists('ZipArchive')) {
+            throw new Exception("ZIP archive decompressing is not supported.");
+        }
+
+        $zip = new ZipArchive();
+        $result = $zip->open($zipFile->pwd());
+        if ($result !== true) {
+            throw new Exception("Remote server returns ZIP file, that cannot be open (error $result)");
+        }
+
+        if ($zip->numFiles !== 1) {
+            throw new Exception("Remote server returns ZIP file, that contains multiple files.");
+        }
+
+        $filename = $zip->getNameIndex(0);
+        if ($filename === false) {
+            throw new Exception("Remote server returns ZIP file, but there is a problem with reading filename.");
+        }
+
+        $zip->close();
+
+        $destinationFile = $this->tempFileName();
+        $result = copy("zip://{$zipFile->pwd()}#$filename", $destinationFile);
+        if ($result === false) {
+            throw new Exception("Remote server returns ZIP file, that contains '$filename' file, that cannot be extracted.");
+        }
+
+        $unzipped = new File($destinationFile);
+        $data = $unzipped->read();
+        if ($data === false) {
+            throw new Exception("Couldn't read extracted file content.");
+        }
+        $unzipped->delete();
+        return $data;
+    }
 }