MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #8347 from JakubOnderka/revert 

Revert "chg: [sync] Use server sync when fetching galaxy clusters"
pull/8348/head
Jakub Onderka 2022-05-06 21:16:56 +02:00 committed by GitHub
parent 8ea8f6ef2c 9c0072496d
commit 1a6589270f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 61 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
app/Lib/Tools/ServerSyncTool.php
View File

@ -9,8 +9,7 @@ class ServerSyncTool
FEATURE_FILTER_SIGHTINGS = 'filter_sightings',
FEATURE_PROPOSALS = 'proposals',
FEATURE_POST_TEST = 'post_test',
FEATURE_PROTECTED_EVENT = 'protected_event',
FEATURE_GALAXY_CLUSTER_SYNC = 'galaxy_cluster_sync';
FEATURE_PROTECTED_EVENT = 'protected_event';
/** @var array */
private $server;
@ -213,27 +212,6 @@ class ServerSyncTool
$this->post('/sightings/bulkSaveSightings/' . $eventUuid, $sightings, $logMessage);
}
/**
* @param array $filterRules
* @return HttpSocketResponseExtended
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
*/
public function galaxyClusterSearch(array $filterRules = [])
{
return $this->post('/galaxy_clusters/restSearch', $filterRules);
}
/**
* @param int|string $clusterId Cluster ID or UUID
* @return HttpSocketResponseExtended
* @throws HttpSocketHttpException
*/
public function fetchGalaxyCluster($clusterId)
{
return $this->get('/galaxy_clusters/view/' . $clusterId);
}
/**
* @return HttpSocketResponseExtended
* @throws HttpSocketHttpException
@ -354,8 +332,6 @@ class ServerSyncTool
case self::FEATURE_PROTECTED_EVENT:
$version = explode('.', $info['version']);
return $version[0] == 2 && $version[1] == 4 && $version[2] > 155;
case self::FEATURE_GALAXY_CLUSTER_SYNC:
return isset($version['perm_galaxy_editor']);
default:
throw new InvalidArgumentException("Invalid flag `$flag` provided");
}

77
app/Model/GalaxyCluster.php
View File

@ -1943,7 +1943,7 @@ class GalaxyCluster extends AppModel
* pullGalaxyClusters
*
* @param array $user
* @param ServerSyncTool $serverSync
* @param array $server
* @param string|int $technique The technique startegy used for pulling
* allowed:
* - int <event id> event containing the clusters to pulled
@ -1952,20 +1952,21 @@ class GalaxyCluster extends AppModel
* - string <pull_relevant_clusters> pull clusters based on tags present locally
* @return int The number of pulled clusters
*/
public function pullGalaxyClusters(array $user, ServerSyncTool $serverSync, $technique = 'full')
public function pullGalaxyClusters(array $user, array $server, $technique = 'full')
{
if (!$serverSync->isSupported(ServerSyncTool::FEATURE_GALAXY_CLUSTER_SYNC)) {
$this->Server = ClassRegistry::init('Server');
$compatible = $this->Server->checkVersionCompatibility($server, $user)['supportEditOfGalaxyCluster'];
if (!$compatible) {
return 0;
}
$clusterIds = $this->getClusterIdListBasedOnPullTechnique($user, $technique, $serverSync);
$clusterIds = $this->getClusterIdListBasedOnPullTechnique($user, $technique, $server);
$successes = array();
$fails = array();
// now process the $clusterIds to pull each of the events sequentially
if (!empty($clusterIds)) {
// download each cluster
foreach ($clusterIds as $clusterId) {
$this->__pullGalaxyCluster($clusterId, $successes, $fails, $serverSync, $user);
foreach ($clusterIds as $k => $clusterId) {
$this->__pullGalaxyCluster($clusterId, $successes, $fails, $server, $user);
}
}
return count($successes);
@ -1976,16 +1977,16 @@ class GalaxyCluster extends AppModel
*
* @param array $user
* @param string|int $technique
* @param ServerSyncTool $serverSync
* @param array $server
* @return array cluster ID list to be pulled
*/
private function getClusterIdListBasedOnPullTechnique(array $user, $technique, ServerSyncTool $serverSync)
private function getClusterIdListBasedOnPullTechnique(array $user, $technique, array $server)
{
$this->Server = ClassRegistry::init('Server');
try {
if ("update" === $technique) {
$localClustersToUpdate = $this->getElligibleLocalClustersToUpdate($user);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($serverSync, $onlyUpdateLocalCluster = true, $elligibleClusters = $localClustersToUpdate);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = true, $elligibleClusters = $localClustersToUpdate);
} elseif ("pull_relevant_clusters" === $technique) {
// Fetch all local custom cluster tags then fetch their corresponding clusters on the remote end
$tagNames = $this->Tag->find('column', array(
@ -2004,55 +2005,57 @@ class GalaxyCluster extends AppModel
}
$localClustersToUpdate = $this->getElligibleLocalClustersToUpdate($user);
$conditions = array('uuid' => array_keys($clusterUUIDs));
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($serverSync, $onlyUpdateLocalCluster = false, $elligibleClusters = $localClustersToUpdate, $conditions = $conditions);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = false, $elligibleClusters = $localClustersToUpdate, $conditions = $conditions);
} elseif (is_numeric($technique)) {
$conditions = array('eventid' => $technique);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($serverSync, $onlyUpdateLocalCluster = false, $elligibleClusters = array(), $conditions = $conditions);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = false, $elligibleClusters = array(), $conditions = $conditions);
} else {
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($serverSync, $onlyUpdateLocalCluster = false);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = false);
}
} catch (HttpSocketHttpException $e) {
if ($e->getCode() === 403) {
return array('error' => array(1, null));
} else {
$this->logException("Could not get eligible cluster IDs from server {$serverSync->serverId()} for pull.", $e);
$this->logException("Could not get eligible cluster IDs from server {$server['Server']['id']} for pull.", $e);
return array('error' => array(2, $e->getMessage()));
}
} catch (Exception $e) {
$this->logException("Could not get eligible cluster IDs from server {$serverSync->serverId()} for pull.", $e);
$this->logException("Could not get eligible cluster IDs from server {$server['Server']['id']} for pull.", $e);
return array('error' => array(2, $e->getMessage()));
}
return $clusterIds;
}
/**
* @param $clusterId
* @param array $successes
* @param array $fails
* @param ServerSyncTool $serverSync
* @param array $user
* @return bool
*/
private function __pullGalaxyCluster($clusterId, array &$successes, array &$fails, ServerSyncTool $serverSync, array $user)
private function __pullGalaxyCluster($clusterId, &$successes, &$fails, $server, $user)
{
try {
$cluster = $serverSync->fetchGalaxyCluster($clusterId)->json();
} catch (Exception $e) {
$this->logException("Failed downloading the galaxy cluster $clusterId from server {$serverSync->serverId()}", $e);
$fails[$clusterId] = __('failed downloading the galaxy cluster');
return false;
}
$cluster = $this->updatePulledClusterBeforeInsert($cluster, $serverSync->server(), $user);
$result = $this->captureCluster($user, $cluster, $fromPull=true, $orgId=$serverSync->server()['Server']['org_id']);
if ($result['success']) {
$successes[] = $clusterId;
$cluster = $this->downloadGalaxyClusterFromServer($clusterId, $server);
if (!empty($cluster)) {
$cluster = $this->updatePulledClusterBeforeInsert($cluster, $server, $user);
$result = $this->captureCluster($user, $cluster, $fromPull=true, $orgId=$server['Server']['org_id']);
if ($result['success']) {
$successes[] = $clusterId;
} else {
$fails[$clusterId] = __('Failed because of errors: ') . json_encode($result['errors']);
}
} else {
$fails[$clusterId] = __('Failed because of errors: ') . json_encode($result['errors']);
$fails[$clusterId] = __('failed downloading the galaxy cluster');
}
return true;
}
public function downloadGalaxyClusterFromServer($clusterId, $server, $HttpSocket=null)
{
$url = $server['Server']['url'];
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
$uri = $url . '/galaxy_clusters/view/' . $clusterId;
$response = $HttpSocket->get($uri, $data = '', $request);
if ($response->isOk()) {
return json_decode($response->body, true);
}
return null;
}
private function updatePulledClusterBeforeInsert($cluster, $server, $user)
{
// The cluster came from a pull, so it should be locked and distribution should be adapted.

30
app/Model/Server.php
View File

@ -551,7 +551,7 @@ class Server extends AppModel
if ($jobId) {
$job->saveProgress($jobId, $technique === 'pull_relevant_clusters' ? __('Pulling relevant galaxy clusters.') : __('Pulling galaxy clusters.'));
}
$pulledClusters = $this->GalaxyCluster->pullGalaxyClusters($user, $serverSync, $technique);
$pulledClusters = $this->GalaxyCluster->pullGalaxyClusters($user, $server, $technique);
if ($technique === 'pull_relevant_clusters') {
if ($jobId) {
$job->saveStatus($jobId, true, 'Pulling complete.');
@ -662,21 +662,30 @@ class Server extends AppModel
/**
* fetchCustomClusterIdsFromServer Fetch custom-published remote clusters' UUIDs and versions
*
* @param ServerSyncTool $serverSync
* @param array $server
* @param HttpSocketExtended|null $HttpSocket
* @param array $conditions
* @return array The list of clusters
* @throws HttpSocketHttpException|HttpSocketJsonException
* @throws JsonException|HttpSocketHttpException|HttpSocketJsonException
*/
private function fetchCustomClusterIdsFromServer(ServerSyncTool $serverSync, array $conditions=array())
private function fetchCustomClusterIdsFromServer(array $server, HttpSocketExtended $HttpSocket=null, array $conditions=array())
{
$url = $server['Server']['url'];
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
$uri = $url . '/galaxy_clusters/restSearch';
$filterRules = [
'published' => 1,
'minimal' => 1,
'custom' => 1,
];
$filterRules = array_merge($filterRules, $conditions);
$response = $HttpSocket->post($uri, json_encode($filterRules), $request);
if (!$response->isOk()) {
throw new HttpSocketHttpException($response);
}
$clusterArray = $serverSync->galaxyClusterSearch($filterRules)->json();
$clusterArray = $response->json();
if (isset($clusterArray['response'])) {
$clusterArray = $clusterArray['response'];
}
@ -686,17 +695,19 @@ class Server extends AppModel
/**
* getElligibleClusterIdsFromServerForPull Get a list of cluster IDs that are present on the remote server and returns clusters that should be pulled
*
* @param ServerSyncTool $serverSync
* @param array $server
* @param mixed $HttpSocket
* @param bool $onlyUpdateLocalCluster If set to true, only cluster present locally will be returned
* @param array $elligibleClusters Array of cluster present locally that could potentially be updated. Linked to $onlyUpdateLocalCluster
* @param array $conditions Conditions to be sent to the remote server while fetching accessible clusters IDs
* @return array List of cluster IDs to be pulled
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
* @throws JsonException
*/
public function getElligibleClusterIdsFromServerForPull(ServerSyncTool $serverSync, $onlyUpdateLocalCluster=true, array $elligibleClusters=array(), array $conditions=array())
public function getElligibleClusterIdsFromServerForPull(array $server, $HttpSocket=null, $onlyUpdateLocalCluster=true, array $elligibleClusters=array(), array $conditions=array())
{
$clusterArray = $this->fetchCustomClusterIdsFromServer($serverSync, $conditions=$conditions);
$clusterArray = $this->fetchCustomClusterIdsFromServer($server, $HttpSocket, $conditions=$conditions);
if (!empty($clusterArray)) {
foreach ($clusterArray as $cluster) {
if (isset($elligibleClusters[$cluster['GalaxyCluster']['uuid']])) {
@ -730,8 +741,7 @@ class Server extends AppModel
*/
public function getElligibleClusterIdsFromServerForPush(array $server, $HttpSocket=null, $localClusters=array(), $conditions=array())
{
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
$clusterArray = $this->fetchCustomClusterIdsFromServer($serverSync, $conditions=$conditions);
$clusterArray = $this->fetchCustomClusterIdsFromServer($server, $HttpSocket, $conditions=$conditions);
$keyedClusterArray = Hash::combine($clusterArray, '{n}.GalaxyCluster.uuid', '{n}.GalaxyCluster.version');
if (!empty($localClusters)) {
foreach ($localClusters as $k => $localCluster) {