mirror of https://github.com/MISP/MISP
new: [feed] Support brotli compression
parent
ab6a9ea502
commit
8c686304a0
|
@ -0,0 +1,111 @@
|
|||
<?php
|
||||
App::uses('HttpSocketResponse', 'Network/Http');
|
||||
App::uses('HttpSocket', 'Network/Http');
|
||||
|
||||
class HttpClientJsonException extends Exception
|
||||
{
|
||||
/** @var HttpSocketResponse */
|
||||
private $response;
|
||||
|
||||
public function __construct($message, HttpSocketResponseExtended $response, Throwable $previous = null)
|
||||
{
|
||||
$this->response = $response;
|
||||
parent::__construct($message, 0, $previous);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return HttpSocketResponse
|
||||
*/
|
||||
public function getResponse()
|
||||
{
|
||||
return $this->response;
|
||||
}
|
||||
}
|
||||
|
||||
class HttpSocketResponseExtended extends HttpSocketResponse
|
||||
{
|
||||
/**
|
||||
* @param string $message
|
||||
* @throws SocketException
|
||||
*/
|
||||
public function parseResponse($message)
|
||||
{
|
||||
parent::parseResponse($message);
|
||||
|
||||
$contentEncoding = $this->getHeader('Content-Encoding');
|
||||
if ($contentEncoding === 'gzip' && function_exists('gzdecode')) {
|
||||
$this->body = gzdecode($this->body);
|
||||
if ($this->body === false) {
|
||||
throw new SocketException("Response should be gzip encoded, but gzip decoding failed.");
|
||||
}
|
||||
} else if ($contentEncoding === 'br' && function_exists('brotli_uncompress')) {
|
||||
$this->body = brotli_uncompress($this->body);
|
||||
if ($this->body === false) {
|
||||
throw new SocketException("Response should be brotli encoded, but brotli decoding failed.");
|
||||
}
|
||||
} else if ($contentEncoding) {
|
||||
throw new SocketException("Remote server returns unsupported content encoding '$contentEncoding'");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decodes JSON string and throws exception if string is not valid JSON.
|
||||
*
|
||||
* @return array
|
||||
* @throws HttpClientJsonException
|
||||
*/
|
||||
public function json()
|
||||
{
|
||||
try {
|
||||
if (defined('JSON_THROW_ON_ERROR')) {
|
||||
// JSON_THROW_ON_ERROR is supported since PHP 7.3
|
||||
$decoded = json_decode($this->body, true, 512, JSON_THROW_ON_ERROR);
|
||||
} else {
|
||||
$decoded = json_decode($this->body, true);
|
||||
if ($decoded === null) {
|
||||
throw new UnexpectedValueException('Could not parse JSON: ' . json_last_error_msg(), json_last_error());
|
||||
}
|
||||
}
|
||||
return $decoded;
|
||||
} catch (Exception $e) {
|
||||
throw new HttpClientJsonException('Could not parse response as JSON.', $this, $e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Supports response compression and also decodes response as JSON
|
||||
*/
|
||||
class HttpSocketExtended extends HttpSocket
|
||||
{
|
||||
public $responseClass = 'HttpSocketResponseExtended';
|
||||
|
||||
public function __construct($config = array())
|
||||
{
|
||||
parent::__construct($config);
|
||||
if (isset($config['compress']) && $config['compress']) {
|
||||
$acceptEncoding = $this->acceptedEncodings();
|
||||
if (!empty($acceptEncoding)) {
|
||||
$this->config['request']['header']['Accept-Encoding'] = implode(', ', $this->acceptedEncodings());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns accepted content encodings (compression algorithms)
|
||||
* @return string[]
|
||||
*/
|
||||
private function acceptedEncodings()
|
||||
{
|
||||
$supportedEncoding = [];
|
||||
// Enable brotli compressed responses if PHP has 'brotli_uncompress' method
|
||||
if (function_exists('brotli_uncompress')) {
|
||||
$supportedEncoding[] = 'br';
|
||||
}
|
||||
// Enable gzipped responses if PHP has 'gzdecode' method
|
||||
if (function_exists('gzdecode')) {
|
||||
$supportedEncoding[] = 'gzip';
|
||||
}
|
||||
return $supportedEncoding;
|
||||
}
|
||||
}
|
|
@ -2,7 +2,14 @@
|
|||
|
||||
class SyncTool
|
||||
{
|
||||
// take a server as parameter and return a HttpSocket object using the ssl options defined in the server settings
|
||||
/**
|
||||
* Take a server as parameter and return a HttpSocket object using the ssl options defined in the server settings
|
||||
* @param array|null $server
|
||||
* @param false $timeout
|
||||
* @param string $model
|
||||
* @return HttpSocketExtended
|
||||
* @throws Exception
|
||||
*/
|
||||
public function setupHttpSocket($server = null, $timeout = false, $model = 'Server')
|
||||
{
|
||||
$params = array();
|
||||
|
@ -33,12 +40,12 @@ class SyncTool
|
|||
|
||||
public function setupHttpSocketFeed($feed = null)
|
||||
{
|
||||
return $this->setupHttpSocket();
|
||||
return $this->createHttpSocket(['compress' => true]);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $params
|
||||
* @return HttpSocket
|
||||
* @return HttpSocketExtended
|
||||
* @throws Exception
|
||||
*/
|
||||
public function createHttpSocket($params = array())
|
||||
|
@ -52,8 +59,8 @@ class SyncTool
|
|||
$params['ssl_cafile'] = $caPath;
|
||||
}
|
||||
|
||||
App::uses('HttpSocket', 'Network/Http');
|
||||
$HttpSocket = new HttpSocket($params);
|
||||
App::uses('HttpSocketExtended', 'Tools');
|
||||
$HttpSocket = new HttpSocketExtended($params);
|
||||
$proxy = Configure::read('Proxy');
|
||||
if (empty($params['skip_proxy']) && isset($proxy['host']) && !empty($proxy['host'])) {
|
||||
$HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']);
|
||||
|
|
|
@ -605,11 +605,6 @@ class Feed extends AppModel
|
|||
)
|
||||
);
|
||||
|
||||
// Enable gzipped responses if PHP has 'gzdecode' method
|
||||
if (function_exists('gzdecode')) {
|
||||
$result['header']['Accept-Encoding'] = 'gzip';
|
||||
}
|
||||
|
||||
$commit = $this->checkMIPSCommit();
|
||||
if ($commit) {
|
||||
$result['header']['commit'] = $commit;
|
||||
|
@ -1780,16 +1775,6 @@ class Feed extends AppModel
|
|||
|
||||
$data = $response->body;
|
||||
|
||||
$contentEncoding = $response->getHeader('Content-Encoding');
|
||||
if ($contentEncoding === 'gzip') {
|
||||
$data = gzdecode($data);
|
||||
if ($data === false) {
|
||||
throw new Exception("Fetching the '$uri' failed, response should be gzip encoded, but gzip decoding failed.");
|
||||
}
|
||||
} else if ($contentEncoding) {
|
||||
throw new Exception("Fetching the '$uri' failed, because remote server returns unsupported content encoding '$contentEncoding'");
|
||||
}
|
||||
|
||||
$contentType = $response->getHeader('Content-Type');
|
||||
if ($contentType === 'application/zip') {
|
||||
$zipFile = new File($this->tempFileName());
|
||||
|
|
|
@ -23,6 +23,7 @@
|
|||
"ext-redis": "For working background jobs and feed and warninglist caches",
|
||||
"ext-zip": "Enabling processing feeds that are ZIP compressed",
|
||||
"ext-zlib": "Allow gzip compression of HTTP responses",
|
||||
"ext-brotli": "Allow brotli compression of HTTP responses",
|
||||
"ext-intl": "For handling IDN domain names",
|
||||
"ext-ssdeep": "For ssdeep hashes correlation",
|
||||
"ext-bcmath": "For faster validating IBAN numbers",
|
||||
|
|
Loading…
Reference in New Issue