MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #8717 from JakubOnderka/experimental-curl-client 

new: [sync] Experimental curl client
pull/9491/head
Jakub Onderka 2024-01-12 12:18:54 +01:00 committed by GitHub
parent f859fe37a5 305b3be44d
commit 9616e07e95
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 459 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
app/Console/Command/AdminShell.php
View File

@ -109,6 +109,17 @@ class AdminShell extends AppShell
$parser->addSubcommand('configLint', [
'help' => __('Check if settings has correct value.'),
]);
$parser->addSubcommand('scanAttachment', [
'help' => __('Scan attachments with AV.'),
'parser' => [
'arguments' => [
'type' => ['help' => __('all, Attribute or ShadowAttribute'), 'required' => true],
'attributeId' => ['help' => __('ID to scan.')],
'jobId' => ['help' => __('Job ID')],
],
],
]);
return $parser;
}
@ -839,8 +850,8 @@ class AdminShell extends AppShell
public function scanAttachment()
{
$input = $this->args[0];
$attributeId = isset($this->args[1]) ? $this->args[1] : null;
$jobId = isset($this->args[2]) ? $this->args[2] : null;
$attributeId = $this->args[1] ?? null;
$jobId = $this->args[2] ?? null;
$this->loadModel('AttachmentScan');
$result = $this->AttachmentScan->scan($input, $attributeId, $jobId);

376
app/Lib/Tools/CurlClient.php Normal file
View File

@ -0,0 +1,376 @@
<?php
App::uses('HttpSocketExtended', 'Tools');
class CurlClient extends HttpSocketExtended
{
/** @var resource */
private $ch;
/** @var int */
private $timeout = 30;
/** @var string|null */
private $caFile;
/** @var string|null */
private $localCert;
/** @var int */
private $cryptoMethod;
/** @var bool */
private $allowSelfSigned;
/** @var bool */
private $verifyPeer;
/** @var bool */
private $compress = true;
/** @var array */
private $proxy = [];
/** @var array */
private $defaultOptions;
/**
* @param array $params
* @noinspection PhpMissingParentConstructorInspection
*/
public function __construct(array $params)
{
if (isset($params['timeout'])) {
$this->timeout = $params['timeout'];
}
if (isset($params['ssl_cafile'])) {
$this->caFile = $params['ssl_cafile'];
}
if (isset($params['ssl_local_cert'])) {
$this->localCert = $params['ssl_local_cert'];
}
if (isset($params['compress'])) {
$this->compress = $params['compress'];
}
if (isset($params['ssl_crypto_method'])) {
$this->cryptoMethod = $this->convertCryptoMethod($params['ssl_crypto_method']);
}
if (isset($params['ssl_allow_self_signed'])) {
$this->allowSelfSigned = $params['ssl_allow_self_signed'];
}
if (isset($params['ssl_verify_peer'])) {
$this->verifyPeer = $params['ssl_verify_peer'];
}
$this->defaultOptions = $this->generateDefaultOptions();
}
/**
* @param string $uri
* @param array $query
* @param array $request
* @return HttpSocketResponseExtended
*/
public function head($uri = null, $query = [], $request = [])
{
return $this->internalRequest('HEAD', $uri, $query, $request);
}
/**
* @param string $uri
* @param array $query
* @param array $request
* @return HttpSocketResponseExtended
*/
public function get($uri = null, $query = [], $request = [])
{
return $this->internalRequest('GET', $uri, $query, $request);
}
/**
* @param string $uri
* @param array $data
* @param array $request
* @return HttpSocketResponseExtended
*/
public function post($uri = null, $data = [], $request = [])
{
return $this->internalRequest('POST', $uri, $data, $request);
}
/**
* @param string $uri
* @param array$data
* @param $request
* @return HttpSocketResponseExtended
*/
public function put($uri = null, $data = [], $request = [])
{
return $this->internalRequest('PUT', $uri, $data, $request);
}
/**
* @param string $uri
* @param array $data
* @param array $request
* @return HttpSocketResponseExtended
*/
public function patch($uri = null, $data = [], $request = [])
{
return $this->internalRequest('PATCH', $uri, $data, $request);
}
/**
* @param string $uri
* @param array $data
* @param array $request
* @return HttpSocketResponseExtended
*/
public function delete($uri = null, $data = array(), $request = array())
{
return $this->internalRequest('DELETE', $uri, $data, $request);
}
public function url($url = null, $uriTemplate = null)
{
throw new Exception('Not implemented');
}
public function request($request = array())
{
throw new Exception('Not implemented');
}
public function setContentResource($resource)
{
throw new Exception('Not implemented');
}
public function getMetaData()
{
return null; // not supported by curl extension
}
/**
* @param string $host
* @param int $port
* @param string $method
* @param string $user
* @param string $pass
* @return void
*/
public function configProxy($host, $port = 3128, $method = null, $user = null, $pass = null)
{
if (empty($host)) {
$this->proxy = [];
return;
}
if (is_array($host)) {
$this->proxy = $host + ['host' => null];
return;
}
$this->proxy = compact('host', 'port', 'method', 'user', 'pass');
}
/**
* @param string $method
* @param string $url
* @param array|string $query
* @param array $request
* @return HttpSocketResponseExtended
*/
private function internalRequest($method, $url, $query, $request)
{
if (empty($url)) {
throw new InvalidArgumentException("No URL provided.");
}
if (!$this->ch) {
// Share handle between requests to allow keep connection alive between requests
$this->ch = curl_init();
if (!$this->ch) {
throw new \RuntimeException("Could not initialize cURL");
}
} else {
// Reset options, so we can do another request
curl_reset($this->ch);
}
if (($method === 'GET' || $method === 'HEAD') && !empty($query)) {
$url .= '?' . http_build_query($query, '', '&', PHP_QUERY_RFC3986);
}
$options = $this->defaultOptions;
$options[CURLOPT_URL] = $url;
$options[CURLOPT_CUSTOMREQUEST] = $method;
if (($method === 'POST' || $method === 'DELETE' || $method === 'PUT' || $method === 'PATCH') && !empty($query)) {
$options[CURLOPT_POSTFIELDS] = $query;
}
if (!empty($request['header'])) {
$headers = [];
foreach ($request['header'] as $key => $value) {
if (is_array($value)) {
$value = implode(', ', $value);
}
$headers[] = "$key: $value";
}
$options[CURLOPT_HTTPHEADER] = $headers;
}
// Parse response headers
$responseHeaders = [];
$options[CURLOPT_HEADERFUNCTION] = function ($curl, $header) use (&$responseHeaders){
$len = strlen($header);
$header = explode(':', $header, 2);
if (count($header) < 2) { // ignore invalid headers
return $len;
}
$key = strtolower(trim($header[0]));
$value = trim($header[1]);
if (isset($responseHeaders[$key])) {
$responseHeaders[$key] = array_merge((array)$responseHeaders[$key], [$value]);
} else {
$responseHeaders[$key] = $value;
}
return $len;
};
if (!curl_setopt_array($this->ch, $options)) {
throw new \RuntimeException('cURL error: Could not set options');
}
// Download the given URL, and return output
$output = curl_exec($this->ch);
if ($output === false) {
$errorMessage = curl_error($this->ch);
if (!empty($errorMessage)) {
$errorMessage = ": $errorMessage";
}
throw new SocketException('cURL error ' . curl_strerror(curl_errno($this->ch)) . $errorMessage);
}
$code = curl_getinfo($this->ch, CURLINFO_HTTP_CODE);
return $this->constructResponse($output, $responseHeaders, $code);
}
public function disconnect()
{
if ($this->ch) {
curl_close($this->ch);
$this->ch = null;
}
}
/**
* @param string $body
* @param array $headers
* @param int $code
* @return HttpSocketResponseExtended
*/
private function constructResponse($body, array $headers, $code)
{
if (isset($responseHeaders['content-encoding']) && $responseHeaders['content-encoding'] === 'zstd') {
if (!function_exists('zstd_uncompress')) {
throw new SocketException('Response is zstd encoded, but PHP do not support zstd decoding.');
}
$body = zstd_uncompress($body);
if ($body === false) {
throw new SocketException('Could not decode zstd encoded response.');
}
}
$response = new HttpSocketResponseExtended();
$response->code = $code;
$response->body = $body;
$response->headers = $headers;
return $response;
}
/**
* @param int $cryptoMethod
* @return int
*/
private function convertCryptoMethod($cryptoMethod)
{
switch ($cryptoMethod) {
case STREAM_CRYPTO_METHOD_TLSv1_0_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_1_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
return CURL_SSLVERSION_TLSv1;
case STREAM_CRYPTO_METHOD_TLSv1_1_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
return CURL_SSLVERSION_TLSv1_1;
case STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
return CURL_SSLVERSION_TLSv1_2;
case STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
return CURL_SSLVERSION_TLSv1_3;
default:
throw new InvalidArgumentException("Unsupported crypto method value $cryptoMethod");
}
}
/**
* @return array
*/
private function generateDefaultOptions()
{
$options = [
CURLOPT_FOLLOWLOCATION => true, // Allows to follow redirect
CURLOPT_MAXREDIRS => 10,
CURLOPT_RETURNTRANSFER => true, // Should cURL return or print out the data? (true = return, false = print)
CURLOPT_HEADER => false, // Include header in result?
CURLOPT_TIMEOUT => $this->timeout, // Timeout in seconds
CURLOPT_PROTOCOLS => CURLPROTO_HTTPS | CURLPROTO_HTTP, // be sure that only HTTP and HTTPS protocols are enabled,
];
if ($this->caFile) {
$options[CURLOPT_CAINFO] = $this->caFile;
}
if ($this->localCert) {
$options[CURLOPT_SSLCERT] = $this->localCert;
}
if ($this->cryptoMethod) {
$options[CURLOPT_SSLVERSION] = $this->cryptoMethod;
}
if ($this->compress) {
$options[CURLOPT_ACCEPT_ENCODING] = $this->supportedEncodings();
}
if ($this->allowSelfSigned) {
$options[CURLOPT_SSL_VERIFYPEER] = $this->verifyPeer;
$options[CURLOPT_SSL_VERIFYHOST] = 0;
}
if (!empty($this->proxy)) {
$options[CURLOPT_PROXY] = "{$this->proxy['host']}:{$this->proxy['port']}";
if (!empty($this->proxy['method']) && isset($this->proxy['user'], $this->proxy['pass'])) {
$options[CURLOPT_PROXYUSERPWD] = "{$this->proxy['user']}:{$this->proxy['pass']}";
}
}
return $options;
}
/**
* @return string
*/
private function supportedEncodings()
{
$encodings = [];
// zstd is not supported by curl itself, but add support if PHP zstd extension is installed
if (function_exists('zstd_uncompress')) {
$encodings[] = 'zstd';
}
// brotli and gzip is supported by curl itself if it is compiled with these features
$info = curl_version();
if (defined('CURL_VERSION_BROTLI') && $info['features'] & CURL_VERSION_BROTLI) {
$encodings[] = 'br';
}
if ($info['features'] & CURL_VERSION_LIBZ) {
$encodings[] = 'gzip, deflate';
}
return implode(', ', $encodings);
}
}

11
app/Lib/Tools/SyncTool.php
View File

@ -1,5 +1,4 @@
<?php
class SyncTool
{
@ -84,8 +83,14 @@ class SyncTool
$params['ssl_crypto_method'] = $version;
}
App::uses('HttpSocketExtended', 'Tools');
$HttpSocket = new HttpSocketExtended($params);
if (function_exists('curl_init')) {
App::uses('CurlClient', 'Tools');
$HttpSocket = new CurlClient($params);
} else {
App::uses('HttpSocketExtended', 'Tools');
$HttpSocket = new HttpSocketExtended($params);
}
$proxy = Configure::read('Proxy');
if (empty($params['skip_proxy']) && isset($proxy['host']) && !empty($proxy['host'])) {
$HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']);

48
app/Model/Module.php
View File

@ -50,6 +50,8 @@ class Module extends AppModel
)
);
private $httpSocket = [];
public function validateIPField($value)
{
if (!filter_var($value, FILTER_VALIDATE_IP) === false) {
@ -309,16 +311,9 @@ class Module extends AppModel
if (!$serverUrl) {
throw new Exception("Module type $moduleFamily is not enabled.");
}
App::uses('HttpSocketExtended', 'Tools');
$httpSocketSetting = ['timeout' => $timeout];
$sslSettings = array('ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_verify_peer', 'ssl_cafile');
foreach ($sslSettings as $sslSetting) {
$value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
if ($value && $value !== '') {
$httpSocketSetting[$sslSetting] = $value;
}
}
$httpSocket = new HttpSocketExtended($httpSocketSetting);
$httpSocket = $this->initHttpSocket($moduleFamily, $timeout);
$request = [];
if ($moduleFamily === 'Cortex') {
if (!empty(Configure::read('Plugin.' . $moduleFamily . '_authkey'))) {
@ -422,4 +417,37 @@ class Module extends AppModel
return false;
}
/**
* @param string $moduleFamily
* @param int $timeout
* @return HttpSocketExtended|CurlClient
*/
private function initHttpSocket($moduleFamily, $timeout)
{
$unique = "$moduleFamily:$timeout";
if (isset($this->httpSocket[$unique])) {
return $this->httpSocket[$unique];
}
$httpSocketSetting = ['timeout' => $timeout];
$sslSettings = ['ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_cafile'];
foreach ($sslSettings as $sslSetting) {
$value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
if ($value && $value !== '') {
$httpSocketSetting[$sslSetting] = $value;
}
}
if (function_exists('curl_init')) {
App::uses('CurlClient', 'Tools');
$httpSocket = new CurlClient($httpSocketSetting);
} else {
App::uses('HttpSocketExtended', 'Tools');
$httpSocket = new HttpSocketExtended($httpSocketSetting);
}
return $this->httpSocket[$unique] = $httpSocket;
}
}

34
app/Model/Server.php
View File

@ -472,7 +472,21 @@ class Server extends AppModel
return false;
}
private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, Event $eventModel, $server, $user, $jobId, $force = false, $headers = false, $body = false)
/**
* @param array $event
* @param int|string $eventId
* @param array $successes
* @param array $fails
* @param Event $eventModel
* @param array $server
* @param array $user
* @param int $jobId
* @param bool $force
* @param HttpSocketResponseExtended $response
* @return false|void
* @throws Exception
*/
private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, Event $eventModel, $server, $user, $jobId, $force = false, $response)
{
// check if the event already exist (using the uuid)
$existingEvent = $eventModel->find('first', [
@ -485,7 +499,7 @@ class Server extends AppModel
if (!$existingEvent) {
// add data for newly imported events
if (isset($event['Event']['protected']) && $event['Event']['protected']) {
if (!$eventModel->CryptographicKey->validateProtectedEvent($body, $user, $headers['x-pgp-signature'], $event)) {
if (!$eventModel->CryptographicKey->validateProtectedEvent($response->body, $user, $response->getHeader('x-pgp-signature'), $event)) {
$fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.');
return false;
}
@ -505,7 +519,7 @@ class Server extends AppModel
$fails[$eventId] = __('Blocked an edit to an event that was created locally. This can happen if a synchronised event that was created on this instance was modified by an administrator on the remote side.');
} else {
if ($existingEvent['Event']['protected']) {
if (!$eventModel->CryptographicKey->validateProtectedEvent($body, $user, $headers['x-pgp-signature'], $existingEvent)) {
if (!$eventModel->CryptographicKey->validateProtectedEvent($response->body, $user, $response->getHeader('x-pgp-signature'), $existingEvent)) {
$fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.');
}
}
@ -549,10 +563,8 @@ class Server extends AppModel
$params['excludeLocalTags'] = 1;
}
try {
$event = $serverSync->fetchEvent($eventId, $params);
$headers = $event->headers;
$body = $event->body;
$event = $event->json();
$response = $serverSync->fetchEvent($eventId, $params);
$event = $response->json();
} catch (Exception $e) {
$this->logException("Failed downloading the event $eventId from remote server {$serverSync->serverId()}", $e);
$fails[$eventId] = __('failed downloading the event');
@ -568,7 +580,7 @@ class Server extends AppModel
}
return false;
}
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $headers, $body);
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $response);
return true;
}
@ -4793,11 +4805,11 @@ class Server extends AppModel
$results = [
__('User') => $user['User']['email'],
__('Role name') => isset($user['Role']['name']) ? $user['Role']['name'] : __('Unknown, outdated instance'),
__('Role name') => $user['Role']['name'] ?? __('Unknown, outdated instance'),
__('Sync flag') => isset($user['Role']['perm_sync']) ? ($user['Role']['perm_sync'] ? __('Yes') : __('No')) : __('Unknown, outdated instance'),
];
if (isset($response->headers['X-Auth-Key-Expiration'])) {
$date = new DateTime($response->headers['X-Auth-Key-Expiration']);
if ($response->getHeader('X-Auth-Key-Expiration')) {
$date = new DateTime($response->getHeader('X-Auth-Key-Expiration'));
$results[__('Auth key expiration')] = $date->format('Y-m-d H:i:s');
}
return $results;

1
app/composer.json
View File

@ -36,6 +36,7 @@
"ext-rdkafka": "Required for publishing events to Kafka broker",
"ext-apcu": "To cache data in memory instead of file system",
"ext-simdjson": "To decode JSON structures faster",
"ext-curl": "For faster remote requests",
"elasticsearch/elasticsearch": "For logging to elasticsearch",
"aws/aws-sdk-php": "To upload samples to S3",
"jakub-onderka/openid-connect-php": "For OIDC authentication",