mirror of https://github.com/MISP/MISP
new: [feed] Support for compressed feeds
parent
cf9fcc45ce
commit
4dec19dedf
|
@ -2847,4 +2847,21 @@ class AppModel extends Model
|
|||
$message .= "\nStack Trace:\n" . $exception->getTraceAsString();
|
||||
return $this->log($message, $type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates random file name in tmp dir.
|
||||
* @return string
|
||||
*/
|
||||
protected function tempFileName()
|
||||
{
|
||||
return $this->tempDir() . DS . $this->generateRandomFileName();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return string
|
||||
*/
|
||||
protected function tempDir()
|
||||
{
|
||||
return Configure::read('MISP.tmpdir') ?: sys_get_temp_dir();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -247,10 +247,7 @@ class Feed extends AppModel
|
|||
$data = $this->feedGetUri($feed, $feedUrl, $HttpSocket, true);
|
||||
|
||||
if (!$isLocal) {
|
||||
$redis = $this->setupRedis();
|
||||
if ($redis === false) {
|
||||
throw new Exception('Could not reach Redis.');
|
||||
}
|
||||
$redis = $this->setupRedisWithException();
|
||||
$redis->del('misp:feed_cache:' . $feed['Feed']['id']);
|
||||
file_put_contents($feedCache, $data);
|
||||
}
|
||||
|
@ -502,12 +499,17 @@ class Feed extends AppModel
|
|||
|
||||
$result = array(
|
||||
'header' => array(
|
||||
'Accept' => array('application/json', 'text/plain'),
|
||||
'Content-Type' => 'application/json',
|
||||
'MISP-version' => $version,
|
||||
'MISP-uuid' => Configure::read('MISP.uuid')
|
||||
'Accept' => array('application/json', 'text/plain'),
|
||||
'MISP-version' => $version,
|
||||
'MISP-uuid' => Configure::read('MISP.uuid'),
|
||||
)
|
||||
);
|
||||
|
||||
// Enable gzipped responses if PHP has 'gzdecode' method
|
||||
if (function_exists('gzdecode')) {
|
||||
$result['header']['Accept-Encoding'] = 'gzip';
|
||||
}
|
||||
|
||||
if ($commit) {
|
||||
$result['header']['commit'] = $commit;
|
||||
}
|
||||
|
@ -1590,24 +1592,51 @@ class Feed extends AppModel
|
|||
if ($data === false) {
|
||||
throw new Exception("Could not read local file '$uri'.");
|
||||
}
|
||||
return $data;
|
||||
} else {
|
||||
throw new Exception("Local file '$uri' doesn't exists.");
|
||||
}
|
||||
}
|
||||
|
||||
$request = $this->__createFeedRequest($feed['Feed']['headers']);
|
||||
|
||||
if ($followRedirect) {
|
||||
$response = $this->getFollowRedirect($HttpSocket, $uri, $request);
|
||||
} else {
|
||||
$request = $this->__createFeedRequest($feed['Feed']['headers']);
|
||||
$response = $HttpSocket->get($uri, array(), $request);
|
||||
}
|
||||
|
||||
if ($followRedirect) {
|
||||
$response = $this->getFollowRedirect($HttpSocket, $uri, $request);
|
||||
} else {
|
||||
$response = $HttpSocket->get($uri, array(), $request);
|
||||
}
|
||||
if ($response === false) {
|
||||
throw new Exception("Could not reach '$uri'.");
|
||||
} else if ($response->code != 200) { // intentionally !=
|
||||
throw new Exception("Fetching the '$uri' failed with HTTP error {$response->code}: {$response->reasonPhrase}");
|
||||
}
|
||||
|
||||
if ($response === false) {
|
||||
throw new Exception("Could not reach '$uri'.");
|
||||
} else if ($response->code != 200) { // intentionally !=
|
||||
throw new Exception("Fetching the '$uri' failed with HTTP error {$response->code}: {$response->reasonPhrase}");
|
||||
$data = $response->body;
|
||||
|
||||
$contentEncoding = $response->getHeader('Content-Encoding');
|
||||
if ($contentEncoding === 'gzip') {
|
||||
$data = gzdecode($data);
|
||||
if ($data === false) {
|
||||
throw new Exception("Fetching the '$uri' failed, response should be gzip encoded, but gzip decoding failed.");
|
||||
}
|
||||
} else if ($contentEncoding) {
|
||||
throw new Exception("Fetching the '$uri' failed, because remote server returns unsupported content encoding '$contentEncoding'");
|
||||
}
|
||||
|
||||
$contentType = $response->getHeader('Content-Type');
|
||||
if ($contentType === 'application/zip') {
|
||||
$zipFile = new File($this->tempFileName());
|
||||
$zipFile->write($data);
|
||||
$zipFile->close();
|
||||
|
||||
try {
|
||||
$data = $this->unzipFirstFile($zipFile);
|
||||
} catch (Exception $e) {
|
||||
throw new Exception("Fetching the '$uri' failed: {$e->getMessage()}");
|
||||
} finally {
|
||||
$zipFile->delete();
|
||||
}
|
||||
$data = $response->body;
|
||||
}
|
||||
|
||||
return $data;
|
||||
|
@ -1720,4 +1749,47 @@ class Feed extends AppModel
|
|||
$this->save($feed);
|
||||
return $count;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param File $zipFile
|
||||
* @return string Uncompressed data
|
||||
* @throws Exception
|
||||
*/
|
||||
private function unzipFirstFile(File $zipFile)
|
||||
{
|
||||
if (!class_exists('ZipArchive')) {
|
||||
throw new Exception("ZIP archive decompressing is not supported.");
|
||||
}
|
||||
|
||||
$zip = new ZipArchive();
|
||||
$result = $zip->open($zipFile->pwd());
|
||||
if ($result !== true) {
|
||||
throw new Exception("Remote server returns ZIP file, that cannot be open (error $result)");
|
||||
}
|
||||
|
||||
if ($zip->numFiles !== 1) {
|
||||
throw new Exception("Remote server returns ZIP file, that contains multiple files.");
|
||||
}
|
||||
|
||||
$filename = $zip->getNameIndex(0);
|
||||
if ($filename === false) {
|
||||
throw new Exception("Remote server returns ZIP file, but there is a problem with reading filename.");
|
||||
}
|
||||
|
||||
$zip->close();
|
||||
|
||||
$destinationFile = $this->tempFileName();
|
||||
$result = copy("zip://{$zipFile->pwd()}#$filename", $destinationFile);
|
||||
if ($result === false) {
|
||||
throw new Exception("Remote server returns ZIP file, that contains '$filename' file, that cannot be extracted.");
|
||||
}
|
||||
|
||||
$unzipped = new File($destinationFile);
|
||||
$data = $unzipped->read();
|
||||
if ($data === false) {
|
||||
throw new Exception("Couldn't read extracted file content.");
|
||||
}
|
||||
$unzipped->delete();
|
||||
return $data;
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue