MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

fix: [sync] Better error handling when fetching IDs for push/pull

pull/7433/head
Jakub Onderka 2021-05-24 15:17:01 +02:00
parent 672476001b
commit f65a923f6e
3 changed files with 110 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
app/Lib/Tools/HttpSocketExtended.php
View File

@ -2,9 +2,29 @@
App::uses('HttpSocketResponse', 'Network/Http');
App::uses('HttpSocket', 'Network/Http');
class HttpClientJsonException extends Exception
class HttpSocketHttpException extends Exception
{
/** @var HttpSocketResponse */
/** @var HttpSocketResponseExtended */
private $response;
public function __construct(HttpSocketResponseExtended $response)
{
$this->response = $response;
parent::__construct("Remote server returns HTTP error code {$response->code}", (int)$response->code);
}
/**
* @return HttpSocketResponseExtended
*/
public function getResponse()
{
return $this->response;
}
}
class HttpSocketJsonException extends Exception
{
/** @var HttpSocketResponseExtended */
private $response;
public function __construct($message, HttpSocketResponseExtended $response, Throwable $previous = null)
@ -14,7 +34,7 @@ class HttpClientJsonException extends Exception
}
/**
* @return HttpSocketResponse
* @return HttpSocketResponseExtended
*/
public function getResponse()
{
@ -56,7 +76,7 @@ class HttpSocketResponseExtended extends HttpSocketResponse
* Decodes JSON string and throws exception if string is not valid JSON.
*
* @return array
* @throws HttpClientJsonException
* @throws HttpSocketJsonException
*/
public function json()
{
@ -72,7 +92,7 @@ class HttpSocketResponseExtended extends HttpSocketResponse
}
return $decoded;
} catch (Exception $e) {
throw new HttpClientJsonException('Could not parse response as JSON.', $this, $e);
throw new HttpSocketJsonException('Could not parse response as JSON.', $this, $e);
}
}
}

67
app/Model/GalaxyCluster.php
View File

@ -1964,38 +1964,45 @@ class GalaxyCluster extends AppModel
private function getClusterIdListBasedOnPullTechnique(array $user, $technique, array $server)
{
$this->Server = ClassRegistry::init('Server');
if ("update" === $technique) {
$localClustersToUpdate = $this->getElligibleLocalClustersToUpdate($user);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket=null, $onlyUpdateLocalCluster=true, $elligibleClusters=$localClustersToUpdate);
} elseif ("pull_relevant_clusters" === $technique) {
// Fetch all local custom cluster tags then fetch their corresponding clusters on the remote end
$tagNames = $this->Tag->find('column', array(
'conditions' => array(
'Tag.is_custom_galaxy' => true
),
'fields' => array('Tag.name'),
));
$clusterUUIDs = array();
$re = '/^misp-galaxy:[^:="]+="(?<uuid>[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})"$/m';
foreach ($tagNames as $tagName) {
preg_match($re, $tagName, $matches);
if (isset($matches['uuid'])) {
$clusterUUIDs[$matches['uuid']] = true;
try {
if ("update" === $technique) {
$localClustersToUpdate = $this->getElligibleLocalClustersToUpdate($user);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = true, $elligibleClusters = $localClustersToUpdate);
} elseif ("pull_relevant_clusters" === $technique) {
// Fetch all local custom cluster tags then fetch their corresponding clusters on the remote end
$tagNames = $this->Tag->find('column', array(
'conditions' => array(
'Tag.is_custom_galaxy' => true
),
'fields' => array('Tag.name'),
));
$clusterUUIDs = array();
$re = '/^misp-galaxy:[^:="]+="(?<uuid>[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})"$/m';
foreach ($tagNames as $tagName) {
preg_match($re, $tagName, $matches);
if (isset($matches['uuid'])) {
$clusterUUIDs[$matches['uuid']] = true;
}
}
$localClustersToUpdate = $this->getElligibleLocalClustersToUpdate($user);
$conditions = array('uuid' => array_keys($clusterUUIDs));
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = false, $elligibleClusters = $localClustersToUpdate, $conditions = $conditions);
} elseif (is_numeric($technique)) {
$conditions = array('eventid' => $technique);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = false, $elligibleClusters = array(), $conditions = $conditions);
} else {
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = false);
}
$localClustersToUpdate = $this->getElligibleLocalClustersToUpdate($user);
$conditions = array('uuid' => array_keys($clusterUUIDs));
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket=null, $onlyUpdateLocalCluster=false, $elligibleClusters=$localClustersToUpdate, $conditions=$conditions);
} elseif (is_numeric($technique)) {
$conditions = array('eventid' => $technique);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket=null, $onlyUpdateLocalCluster=false, $elligibleClusters=array(), $conditions=$conditions);
} else {
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket=null, $onlyUpdateLocalCluster=false);
}
if ($clusterIds === 403) {
return array('error' => array(1, null));
} elseif (is_string($clusterIds)) {
return array('error' => array(2, $clusterIds));
} catch (HttpSocketHttpException $e) {
if ($e->getCode() === 403) {
return array('error' => array(1, null));
} else {
$this->logException("Could not get eligible cluster IDs from server {$server['Server']['id']} for pull.", $e);
return array('error' => array(2, $e->getMessage()));
}
} catch (Exception $e) {
$this->logException("Could not get eligible cluster IDs from server {$server['Server']['id']} for pull.", $e);
return array('error' => array(2, $e->getMessage()));
}
return $clusterIds;
}

84
app/Model/Server.php
View File

@ -648,55 +648,52 @@ class Server extends AppModel
/**
* fetchCustomClusterIdsFromServer Fetch custom-published remote clusters' UUIDs and versions
*
* @param array $server
* @param mixed $HttpSocket
* @param array $conditions
* @return mixed The list of clusters or the error
* @param array $server
* @param HttpSocketExtended|null $HttpSocket
* @param array $conditions
* @return array The list of clusters
* @throws JsonException|HttpSocketHttpException|HttpSocketJsonException
*/
public function fetchCustomClusterIdsFromServer(array $server, $HttpSocket=null, array $conditions=array())
private function fetchCustomClusterIdsFromServer(array $server, HttpSocketExtended $HttpSocket=null, array $conditions=array())
{
$url = $server['Server']['url'];
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
$uri = $url . '/galaxy_clusters/restSearch';
$filterRules['published'] = 1;
$filterRules['minimal'] = 1;
$filterRules['custom'] = 1;
$filterRules = [
'published' => 1,
'minimal' => 1,
'custom' => 1,
];
$filterRules = array_merge($filterRules, $conditions);
try {
$response = $HttpSocket->post($uri, json_encode($filterRules), $request);
if ($response->isOk()) {
$clusterArray = json_decode($response->body, true);
if (isset($clusterArray['response'])) {
$clusterArray = $clusterArray['response'];
}
return $clusterArray;
}
if ($response->code == '403') {
return 403;
}
} catch (SocketException $e) {
return $e->getMessage();
$response = $HttpSocket->post($uri, json_encode($filterRules), $request);
if (!$response->isOk()) {
throw new HttpSocketHttpException($response);
}
// error, so return error message, since that is handled and everything is expecting an array
return __('Error: got response code %s', $response->code);
$clusterArray = $response->json();
if (isset($clusterArray['response'])) {
$clusterArray = $clusterArray['response'];
}
return $clusterArray;
}
/**
* getElligibleClusterIdsFromServerForPull Get a list of cluster IDs that are present on the remote server and returns clusters that should be pulled
*
* @param array $server
* @param mixed $HttpSocket
* @param bool $onlyUpdateLocalCluster If set to true, only cluster present locally will be returned
* @param array $elligibleClusters Array of cluster present locally that could potentially be updated. Linked to $onlyUpdateLocalCluster
* @param array $conditions Conditions to be sent to the remote server while fetching accessible clusters IDs
* @param array $server
* @param mixed $HttpSocket
* @param bool $onlyUpdateLocalCluster If set to true, only cluster present locally will be returned
* @param array $elligibleClusters Array of cluster present locally that could potentially be updated. Linked to $onlyUpdateLocalCluster
* @param array $conditions Conditions to be sent to the remote server while fetching accessible clusters IDs
* @return array List of cluster IDs to be pulled
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
* @throws JsonException
*/
public function getElligibleClusterIdsFromServerForPull(array $server, $HttpSocket=null, $onlyUpdateLocalCluster=true, array $elligibleClusters=array(), array $conditions=array())
{
$clusterArray = $this->fetchCustomClusterIdsFromServer($server, $HttpSocket=null, $conditions=$conditions);
$clusterArray = $this->fetchCustomClusterIdsFromServer($server, $HttpSocket, $conditions=$conditions);
if (!empty($clusterArray)) {
foreach ($clusterArray as $cluster) {
if (isset($elligibleClusters[$cluster['GalaxyCluster']['uuid']])) {
@ -717,10 +714,20 @@ class Server extends AppModel
return $clusterArray;
}
// Get an array of cluster_ids that are present on the remote server and returns clusters that should be pushed
public function getElligibleClusterIdsFromServerForPush($server, $HttpSocket=null, $localClusters=array(), $conditions=array())
/**
* Get an array of cluster_ids that are present on the remote server and returns clusters that should be pushed.
* @param array $server
* @param HttpSocket|null $HttpSocket
* @param array $localClusters
* @param array $conditions
* @return array
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
* @throws JsonException
*/
public function getElligibleClusterIdsFromServerForPush(array $server, $HttpSocket=null, $localClusters=array(), $conditions=array())
{
$clusterArray = $this->fetchCustomClusterIdsFromServer($server, $HttpSocket=null, $conditions=$conditions);
$clusterArray = $this->fetchCustomClusterIdsFromServer($server, $HttpSocket, $conditions=$conditions);
$keyedClusterArray = Hash::combine($clusterArray, '{n}.GalaxyCluster.uuid', '{n}.GalaxyCluster.version');
if (!empty($localClusters)) {
foreach ($localClusters as $k => $localCluster) {
@ -1153,8 +1160,13 @@ class Server extends AppModel
}
}
$localClusterUUIDs = Hash::extract($clusters, '{n}.GalaxyCluster.uuid');
$clustersToPush = $this->getElligibleClusterIdsFromServerForPush($server, $HttpSocket=$HttpSocket, $localClusters=$clusters, $conditions=array('uuid' => $localClusterUUIDs));
foreach ($clustersToPush as $k => $cluster) {
try {
$clustersToPush = $this->getElligibleClusterIdsFromServerForPush($server, $HttpSocket = $HttpSocket, $localClusters = $clusters, $conditions = array('uuid' => $localClusterUUIDs));
} catch (Exception $e) {
$this->logException("Could not get eligible cluster IDs from server #{$server['Server']['id']} for push.", $e);
return [];
}
foreach ($clustersToPush as $cluster) {
$result = $this->GalaxyCluster->uploadClusterToServer($cluster, $server, $HttpSocket, $user);
if ($result === 'Success') {
$successes[] = __('GalaxyCluster %s', $cluster['GalaxyCluster']['uuid']);