MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #8205 from JakubOnderka/pull-optim 

Pull optim
pull/8490/head
Jakub Onderka 2022-07-09 09:01:54 +02:00 committed by GitHub
parent 31b0c3998b d8fd3e937a
commit 26cc86fde2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 272 additions and 359 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
app/Controller/EventsController.php
View File

@ -4115,29 +4115,31 @@ class EventsController extends AppController
public function filterEventIdsForPush()
{
if ($this->request->is('post')) {
$incomingIDs = array();
$incomingEvents = array();
foreach ($this->request->data as $event) {
$incomingIDs[] = $event['Event']['uuid'];
$incomingEvents[$event['Event']['uuid']] = $event['Event']['timestamp'];
}
$events = $this->Event->find('all', array(
'conditions' => array('Event.uuid' => $incomingIDs),
'recursive' => -1,
'fields' => array('Event.uuid', 'Event.timestamp', 'Event.locked'),
));
foreach ($events as $event) {
if ($event['Event']['timestamp'] >= $incomingEvents[$event['Event']['uuid']]) {
unset($incomingEvents[$event['Event']['uuid']]);
continue;
}
if ($event['Event']['locked'] == 0) {
unset($incomingEvents[$event['Event']['uuid']]);
}
}
return $this->RestResponse->viewData(array_keys($incomingEvents), $this->response->type());
if (!$this->request->is('post')) {
throw new MethodNotAllowedException(__('This endpoint requires a POST request.'));
}
$incomingUuids = [];
$incomingEvents = [];
foreach ($this->request->data as $event) {
$incomingUuids[] = $event['Event']['uuid'];
$incomingEvents[$event['Event']['uuid']] = $event['Event']['timestamp'];
}
$events = $this->Event->find('all', [
'conditions' => ['Event.uuid' => $incomingUuids],
'recursive' => -1,
'fields' => ['Event.uuid', 'Event.timestamp', 'Event.locked'],
]);
foreach ($events as $event) {
if ($event['Event']['timestamp'] >= $incomingEvents[$event['Event']['uuid']]) {
unset($incomingEvents[$event['Event']['uuid']]);
continue;
}
if ($event['Event']['locked'] == 0) {
unset($incomingEvents[$event['Event']['uuid']]);
}
}
return $this->RestResponse->viewData(array_keys($incomingEvents), $this->response->type());
}
public function checkuuid($uuid)

58
app/Lib/Tools/ServerSyncTool.php
View File

@ -1,5 +1,6 @@
<?php
App::uses('SyncTool', 'Tools');
App::uses('JsonTool', 'Tools');
class ServerSyncTool
{
@ -8,8 +9,11 @@ class ServerSyncTool
FEATURE_ORG_RULE = 'org_rule',
FEATURE_FILTER_SIGHTINGS = 'filter_sightings',
FEATURE_PROPOSALS = 'proposals',
FEATURE_PROTECTED_EVENT = 'protected_event',
FEATURE_POST_TEST = 'post_test',
FEATURE_PROTECTED_EVENT = 'protected_event';
FEATURE_EDIT_OF_GALAXY_CLUSTER = 'edit_of_galaxy_cluster',
PERM_SYNC = 'perm_sync',
PERM_GALAXY_EDITOR = 'perm_galaxy_editor';
/** @var array */
private $server;
@ -89,6 +93,17 @@ class ServerSyncTool
return $this->get($url);
}
/**
* @param array $events
* @return HttpSocketResponseExtended
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
*/
public function filterEventIdsForPush(array $events)
{
return $this->post('/events/filterEventIdsForPush', $events);
}
/**
* @param array $event
* @return HttpSocketResponseExtended
@ -164,6 +179,39 @@ class ServerSyncTool
return $this->post('/attributes/restSearch.json', $rules);
}
/**
* @param array $rules
* @return HttpSocketResponseExtended
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
*/
public function galaxyClusterSearch(array $rules)
{
return $this->post('/galaxy_clusters/restSearch', $rules);
}
/**
* @param int|string $galaxyClusterId Galaxy Cluster ID or UUID
* @return HttpSocketResponseExtended
* @throws HttpSocketHttpException
*/
public function fetchGalaxyCluster($galaxyClusterId)
{
return $this->get('/galaxy_clusters/view/' . $galaxyClusterId);
}
/**
* @param array $cluster
* @return HttpSocketResponseExtended
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
*/
public function pushGalaxyCluster(array $cluster)
{
$logMessage = "Pushing Galaxy Cluster #{$cluster['GalaxyCluster']['id']} to Server #{$this->serverId()}";
return $this->post('/galaxies/pushCluster', [$cluster], $logMessage);
}
/**
* @param array $params
* @return HttpSocketResponseExtended
@ -332,6 +380,12 @@ class ServerSyncTool
case self::FEATURE_PROTECTED_EVENT:
$version = explode('.', $info['version']);
return $version[0] == 2 && $version[1] == 4 && $version[2] > 155;
case self::FEATURE_EDIT_OF_GALAXY_CLUSTER:
return isset($info['perm_galaxy_editor']);
case self::PERM_SYNC:
return isset($info['perm_sync']) && $info['perm_sync'];
case self::PERM_GALAXY_EDITOR:
return isset($info['perm_galaxy_editor']) && $info['perm_galaxy_editor'];
default:
throw new InvalidArgumentException("Invalid flag `$flag` provided");
}
@ -373,7 +427,7 @@ class ServerSyncTool
private function post($url, $data, $logMessage = null)
{
$protectedMode = !empty($data['Event']['protected']);
$data = json_encode($data, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
$data = JsonTool::encode($data);
if ($logMessage && !empty(Configure::read('Security.sync_audit'))) {
$pushLogEntry = sprintf(

90
app/Model/Event.php
View File

@ -17,6 +17,7 @@ App::uses('ProcessTool', 'Tools');
* @property ThreatLevel $ThreatLevel
* @property Sighting $Sighting
* @property Organisation $Org
* @property Organisation $Orgc
* @property CryptographicKey $CryptographicKey
*/
class Event extends AppModel
@ -301,6 +302,9 @@ class Event extends AppModel
private $assetCache = [];
/** @var array|null */
private $eventBlockRule;
public function beforeDelete($cascade = true)
{
// blocklist the event UUID if the feature is enabled
@ -836,12 +840,13 @@ class Event extends AppModel
/**
* @param array $event
* @param array $server
* @param ServerSyncTool $serverSync
* @return false|string
* @throws HttpSocketJsonException
* @throws JsonException
* @throws Exception
*/
public function uploadEventToServer(array $event, array $server)
public function uploadEventToServer(array $event, array $server, ServerSyncTool $serverSync)
{
$this->Server = ClassRegistry::init('Server');
@ -852,8 +857,6 @@ class Event extends AppModel
return 'The distribution level of this event blocks it from being pushed.';
}
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
$push = $this->Server->checkVersionCompatibility($server, false, $serverSync);
if (empty($push['canPush'])) {
return 'The remote user is not a sync user - the upload of the event has been blocked.';
@ -3437,17 +3440,18 @@ class Event extends AppModel
return $attributes;
}
public function checkEventBlockRules($event)
/**
* @param array $event
* @return bool
*/
private function checkEventBlockRules(array $event)
{
if (!isset($this->AdminSetting)) {
if (!isset($this->eventBlockRule)) {
$this->AdminSetting = ClassRegistry::init('AdminSetting');
$setting = $this->AdminSetting->getSetting('eventBlockRule');
$this->eventBlockRule = $setting ? json_decode($setting, true) : false;
}
$setting = $this->AdminSetting->getSetting('eventBlockRule');
if (empty($setting)) {
return true;
}
$rules = json_decode($setting, true);
if (empty($rules)) {
if (empty($this->eventBlockRule)) {
return true;
}
if (!empty($rules['tags'])) {
@ -3548,7 +3552,9 @@ class Event extends AppModel
public function _add(array &$data, $fromXml, array $user, $org_id = 0, $passAlong = null, $fromPull = false, $jobId = null, &$created_id = 0, &$validationErrors = array())
{
if (Configure::read('MISP.enableEventBlocklisting') !== false && isset($data['Event']['uuid'])) {
$this->EventBlocklist = ClassRegistry::init('EventBlocklist');
if (!isset($this->EventBlocklist)) {
$this->EventBlocklist = ClassRegistry::init('EventBlocklist');
}
if ($this->EventBlocklist->isBlocked($data['Event']['uuid'])) {
return 'Blocked by blocklist';
}
@ -4219,8 +4225,15 @@ class Event extends AppModel
$failedServers = [];
foreach ($servers as $server) {
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
try {
$this->pushSightingsToServer($server, $event, $sightingsUuidsToPush);
try {
if ($serverSync->eventExists($event) === false) {
continue; // skip if event not exists on remote server
}
} catch (Exception $e) {}
$this->pushSightingsToServer($serverSync, $event, $sightingsUuidsToPush);
} catch (Exception $e) {
$this->logException("Uploading sightings to server {$server['Server']['id']} failed.", $e);
$failedServers[] = $server['Server']['url'];
@ -4233,25 +4246,16 @@ class Event extends AppModel
}
/**
* @param array $server
* @param ServerSyncTool $serverSync
* @param array $event
* @param array $sightingsUuidsToPush
* @throws HttpSocketJsonException
* @throws JsonException
* @throws Exception
*/
private function pushSightingsToServer(array $server, array $event, array $sightingsUuidsToPush = [])
private function pushSightingsToServer(ServerSyncTool $serverSync, array $event, array $sightingsUuidsToPush = [])
{
App::uses('ServerSyncTool', 'Tools');
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
try {
if ($serverSync->eventExists($event) === false) {
return; // skip if event not exists on remote server
}
} catch (Exception $e) {}
$fakeSyncUser = [
'org_id' => $server['Server']['remote_org_id'],
'org_id' => $serverSync->server()['Server']['remote_org_id'],
'Role' => [
'perm_site_admin' => 0,
],
@ -4320,8 +4324,6 @@ class Event extends AppModel
}
$uploaded = true;
$failedServers = [];
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
foreach ($servers as $server) {
if (
@ -4331,7 +4333,7 @@ class Event extends AppModel
}
// Skip servers where the event has come from.
if ($passAlong != $server['Server']['id']) {
$HttpSocket = $syncTool->setupHttpSocket($server);
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
$params = [
'eventid' => $id,
'includeAttachments' => true,
@ -4359,17 +4361,17 @@ class Event extends AppModel
'perm_site_admin' => 0
)
);
$this->Server->syncGalaxyClusters($HttpSocket, $server, $fakeSyncUser, $technique=$event['Event']['id'], $event=$event);
$thisUploaded = $this->uploadEventToServer($event, $server);
$this->Server->syncGalaxyClusters($serverSync, $server, $fakeSyncUser, $technique=$event['Event']['id'], $event=$event);
$thisUploaded = $this->uploadEventToServer($event, $server, $serverSync);
if ($thisUploaded === 'Success') {
try {
$this->pushSightingsToServer($server, $event); // push sighting by method that check for duplicates
$this->pushSightingsToServer($serverSync, $event); // push sighting by method that check for duplicates
} catch (Exception $e) {
$this->logException("Uploading sightings to server {$server['Server']['id']} failed.", $e);
}
}
if (isset($this->data['ShadowAttribute'])) {
$this->Server->syncProposals($HttpSocket, $server, null, $id, $this);
$this->Server->syncProposals(null, $server, null, $id, $this);
}
if (!$thisUploaded) {
$uploaded = !$uploaded ? $uploaded : $thisUploaded;
@ -4757,30 +4759,6 @@ class Event extends AppModel
return $xmlArray;
}
public function removeOlder(array &$events, $scope = 'events')
{
$field = $scope === 'sightings' ? 'sighting_timestamp' : 'timestamp';
$localEvents = $this->find('all', [
'recursive' => -1,
'fields' => ['Event.uuid', 'Event.' . $field, 'Event.locked'],
]);
$localEvents = array_column(array_column($localEvents, 'Event'), null, 'uuid');
foreach ($events as $k => $event) {
// remove all events for the sighting sync if the remote is not aware of the new field yet
if (!isset($event[$field])) {
unset($events[$k]);
} else {
$uuid = $event['uuid'];
if (isset($localEvents[$uuid])
&& ($localEvents[$uuid][$field] >= $event[$field]
|| ($scope === 'events' && !$localEvents[$uuid]['locked'])))
{
unset($events[$k]);
}
}
}
}
public function sharingGroupRequired($field)
{
if ($this->data[$this->alias]['distribution'] == 4) {

240
app/Model/GalaxyCluster.php
View File

@ -628,13 +628,11 @@ class GalaxyCluster extends AppModel
$elevatedUser = array(
'Role' => array(
'perm_site_admin' => 1,
'perm_sync' => 1
'perm_sync' => 1,
'perm_audit' => 0,
),
'org_id' => $clusterOrgcId['GalaxyCluster']['orgc_id']
);
$elevatedUser['Role']['perm_site_admin'] = 1;
$elevatedUser['Role']['perm_sync'] = 1;
$elevatedUser['Role']['perm_audit'] = 0;
$cluster = $this->fetchGalaxyClusters($elevatedUser, array('minimal' => true, 'conditions' => array('id' => $clusterId)), $full=false);
if (empty($cluster)) {
return true;
@ -655,14 +653,10 @@ class GalaxyCluster extends AppModel
return true;
}
$uploaded = false;
$failedServers = array();
App::uses('SyncTool', 'Tools');
foreach ($servers as &$server) {
foreach ($servers as $server) {
if ((!isset($server['Server']['internal']) || !$server['Server']['internal']) && $cluster['GalaxyCluster']['distribution'] < 2) {
continue;
}
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
$fakeSyncUser = array(
'id' => 0,
'email' => 'fakeSyncUser@user.test',
@ -678,14 +672,13 @@ class GalaxyCluster extends AppModel
);
$cluster = $this->fetchGalaxyClusters($fakeSyncUser, array('conditions' => array('GalaxyCluster.id' => $clusterId)), $full=true);
if (empty($cluster)) {
return true;
continue;
}
$cluster = $cluster[0];
$result = $this->uploadClusterToServer($cluster, $server, $HttpSocket, $fakeSyncUser);
if ($result == 'Success') {
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
$result = $this->uploadClusterToServer($cluster, $server, $serverSync, $fakeSyncUser);
if ($result === 'Success') {
$uploaded = true;
} else {
$failedServers[] = $server;
}
}
return $uploaded;
@ -1672,79 +1665,29 @@ class GalaxyCluster extends AppModel
/**
* @return string|bool The result of the upload. True if success, a string otherwise
* @throws Exception
*/
public function uploadClusterToServer($cluster, $server, $HttpSocket, $user)
{
$this->Server = ClassRegistry::init('Server');
$this->Log = ClassRegistry::init('Log');
$push = $this->Server->checkVersionCompatibility($server, false);
if (empty($push['canPush']) && empty($push['canPushGalaxyCluster'])) {
return __('The remote user does not have the permission to manipulate galaxies - the upload of the galaxy clusters has been blocked.');
}
$updated = null;
$newLocation = $newTextBody = '';
$result = $this->__executeRestfulGalaxyClusterToServer($cluster, $server, null, $newLocation, $newTextBody, $HttpSocket, $user);
if ($result !== true) {
return $result;
}
if (strlen($newLocation)) { // HTTP/1.1 302 Found and Location: http://<newLocation>
$result = $this->__executeRestfulGalaxyClusterToServer($cluster, $server, $newLocation, $newLocation, $newTextBody, $HttpSocket, $user);
if ($result !== true) {
return $result;
}
}
$uploadFailed = false;
try {
$json = json_decode($newTextBody, true);
} catch (Exception $e) {
$uploadFailed = true;
}
if (!is_array($json) || $uploadFailed) {
$this->Log->createLogEntry($user, 'push', 'GalaxyCluster', $cluster['GalaxyCluster']['id'], 'push', $newTextBody);
}
return 'Success';
}
private function __executeRestfulGalaxyClusterToServer($cluster, $server, $resourceId, &$newLocation, &$newTextBody, $HttpSocket, $user)
{
$result = $this->restfulGalaxyClusterToServer($cluster, $server, $resourceId, $newLocation, $newTextBody, $HttpSocket);
if (is_numeric($result)) {
$error = $this->__resolveErrorCode($result, $cluster, $server, $user);
if ($error) {
return $error . ' Error code: ' . $result;
}
}
return true;
}
/**
* @return string|bool|int The result of the upload.
*/
public function restfulGalaxyClusterToServer($cluster, $server, $urlPath, &$newLocation, &$newTextBody, $HttpSocket = null)
public function uploadClusterToServer(array $cluster, array $server, ServerSyncTool $serverSync, array $user)
{
$cluster = $this->__prepareForPushToServer($cluster, $server);
if (is_numeric($cluster)) {
return $cluster;
}
$url = $server['Server']['url'];
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
$scope = 'galaxies/pushCluster';
$uri = $url . '/' . $scope;
$clusters = array($cluster);
$data = json_encode($clusters);
if (!empty(Configure::read('Security.sync_audit'))) {
$pushLogEntry = sprintf(
"==============================================================\n\n[%s] Pushing Galaxy Cluster #%d to Server #%d:\n\n%s\n\n",
date("Y-m-d H:i:s"),
$cluster['GalaxyCluster']['id'],
$server['Server']['id'],
$data
);
file_put_contents(APP . 'files/scripts/tmp/debug_server_' . $server['Server']['id'] . '.log', $pushLogEntry, FILE_APPEND);
try {
if (!$serverSync->isSupported(ServerSyncTool::PERM_SYNC) || $serverSync->isSupported(ServerSyncTool::PERM_GALAXY_EDITOR)) {
return __('The remote user does not have the permission to manipulate galaxies - the upload of the galaxy clusters has been blocked.');
}
$serverSync->pushGalaxyCluster($cluster)->json();
} catch (Exception $e) {
$title = __('Uploading GalaxyCluster (%s) to Server (%s)', $cluster['GalaxyCluster']['id'], $server['Server']['id']);
$this->loadLog()->createLogEntry($user, 'push', 'GalaxyCluster', $cluster['GalaxyCluster']['id'], $title, $e->getMessage());
$this->logException("Could not push galaxy cluster to remote server {$serverSync->serverId()}", $e);
return $e->getMessage();
}
$response = $HttpSocket->post($uri, $data, $request);
return $this->__handleRestfulGalaxyClusterToServerResponse($response, $newLocation, $newTextBody);
return 'Success';
}
/**
@ -1752,7 +1695,7 @@ class GalaxyCluster extends AppModel
*
* @param array $cluster
* @param array $server
* @return array The cluster ready to be pushed
* @return array|int The cluster ready to be pushed
*/
private function __prepareForPushToServer(array $cluster, array $server)
{
@ -1773,11 +1716,9 @@ class GalaxyCluster extends AppModel
}
$this->Event = ClassRegistry::init('Event');
if ($this->Event->checkDistributionForPush($cluster, $server, 'GalaxyCluster')) {
$cluster = $this->__updateClusterForSync($cluster, $server);
} else {
return 403;
return $this->__updateClusterForSync($cluster, $server);
}
return $cluster;
return 403;
}
/**
@ -1886,90 +1827,39 @@ class GalaxyCluster extends AppModel
return $relation;
}
/**
* @return string|bool|int The result of the upload.
*/
private function __handleRestfulGalaxyClusterToServerResponse($response, &$newLocation, &$newTextBody)
{
switch ($response->code) {
case '200': // 200 (OK) + entity-action-result
if ($response->isOk()) {
$newTextBody = $response->body();
return true;
} else {
try {
$jsonArray = json_decode($response->body, true);
} catch (Exception $e) {
return true;
}
return $jsonArray['name'];
}
// no break
case '302': // Found
$newLocation = $response->headers['Location'];
$newTextBody = $response->body();
return true;
case '404': // Not Found
$newLocation = $response->headers['Location'];
$newTextBody = $response->body();
return 404;
case '405':
return 405;
case '403': // Not authorised
return 403;
}
}
private function __resolveErrorCode($code, &$cluster, &$server, $user)
{
$this->Log = ClassRegistry::init('Log');
$error = false;
switch ($code) {
case 403:
return __('The distribution level of the cluster blocks it from being pushed.');
case 405:
$error = __('The sync user on the remote instance does not have the required privileges to handle this cluster.');
break;
}
if ($error) {
$newTextBody = 'Uploading GalaxyCluster (' . $cluster['GalaxyCluster']['id'] . ') to Server (' . $server['Server']['id'] . ')';
$newTextBody = __('Uploading GalaxyCluster (%s) to Server (%s)', $cluster['GalaxyCluster']['id'], $server['Server']['id']);
$this->Log->createLogEntry($user, 'push', 'GalaxyCluster', $cluster['GalaxyCluster']['id'], 'push', $newTextBody);
}
return $error;
}
/**
* pullGalaxyClusters
*
* @param array $user
* @param array $server
* @param string|int $technique The technique startegy used for pulling
* @param array $user
* @param ServerSyncTool $serverSync
* @param string|int $technique The technique startegy used for pulling
* allowed:
* - int <event id> event containing the clusters to pulled
* - string <full> pull everything
* - string <update> pull updates of cluster present locally
* - string <pull_relevant_clusters> pull clusters based on tags present locally
* @return int The number of pulled clusters
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
*/
public function pullGalaxyClusters(array $user, array $server, $technique = 'full')
public function pullGalaxyClusters(array $user, ServerSyncTool $serverSync, $technique = 'full')
{
$this->Server = ClassRegistry::init('Server');
$compatible = $this->Server->checkVersionCompatibility($server, $user)['supportEditOfGalaxyCluster'];
$compatible = $serverSync->isSupported(ServerSyncTool::FEATURE_EDIT_OF_GALAXY_CLUSTER);
if (!$compatible) {
return 0;
}
$clusterIds = $this->getClusterIdListBasedOnPullTechnique($user, $technique, $server);
$successes = array();
$fails = array();
$clusterIds = $this->getClusterIdListBasedOnPullTechnique($user, $technique, $serverSync);
$successes = 0;
// now process the $clusterIds to pull each of the events sequentially
if (!empty($clusterIds)) {
// download each cluster
foreach ($clusterIds as $k => $clusterId) {
$this->__pullGalaxyCluster($clusterId, $successes, $fails, $server, $user);
foreach ($clusterIds as $clusterId) {
if ($this->__pullGalaxyCluster($clusterId, $serverSync, $user)) {
$successes++;
}
}
}
return count($successes);
return $successes;
}
/**
@ -1977,16 +1867,16 @@ class GalaxyCluster extends AppModel
*
* @param array $user
* @param string|int $technique
* @param array $server
* @param ServerSyncTool $serverSync
* @return array cluster ID list to be pulled
*/
private function getClusterIdListBasedOnPullTechnique(array $user, $technique, array $server)
private function getClusterIdListBasedOnPullTechnique(array $user, $technique, ServerSyncTool $serverSync)
{
$this->Server = ClassRegistry::init('Server');
try {
if ("update" === $technique) {
$localClustersToUpdate = $this->getElligibleLocalClustersToUpdate($user);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = true, $elligibleClusters = $localClustersToUpdate);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($serverSync, $onlyUpdateLocalCluster = true, $elligibleClusters = $localClustersToUpdate);
} elseif ("pull_relevant_clusters" === $technique) {
// Fetch all local custom cluster tags then fetch their corresponding clusters on the remote end
$tagNames = $this->Tag->find('column', array(
@ -2005,55 +1895,39 @@ class GalaxyCluster extends AppModel
}
$localClustersToUpdate = $this->getElligibleLocalClustersToUpdate($user);
$conditions = array('uuid' => array_keys($clusterUUIDs));
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = false, $elligibleClusters = $localClustersToUpdate, $conditions = $conditions);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($serverSync, $onlyUpdateLocalCluster = false, $elligibleClusters = $localClustersToUpdate, $conditions = $conditions);
} elseif (is_numeric($technique)) {
$conditions = array('eventid' => $technique);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = false, $elligibleClusters = array(), $conditions = $conditions);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($serverSync, $onlyUpdateLocalCluster = false, $elligibleClusters = array(), $conditions = $conditions);
} else {
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket = null, $onlyUpdateLocalCluster = false);
$clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($serverSync, $onlyUpdateLocalCluster = false);
}
} catch (HttpSocketHttpException $e) {
if ($e->getCode() === 403) {
return array('error' => array(1, null));
} else {
$this->logException("Could not get eligible cluster IDs from server {$server['Server']['id']} for pull.", $e);
$this->logException("Could not get eligible cluster IDs from server {$serverSync->serverId()} for pull.", $e);
return array('error' => array(2, $e->getMessage()));
}
} catch (Exception $e) {
$this->logException("Could not get eligible cluster IDs from server {$server['Server']['id']} for pull.", $e);
$this->logException("Could not get eligible cluster IDs from server {$serverSync->serverId()} for pull.", $e);
return array('error' => array(2, $e->getMessage()));
}
return $clusterIds;
}
private function __pullGalaxyCluster($clusterId, &$successes, &$fails, $server, $user)
private function __pullGalaxyCluster($clusterId, ServerSyncTool $serverSync, array $user)
{
$cluster = $this->downloadGalaxyClusterFromServer($clusterId, $server);
if (!empty($cluster)) {
$cluster = $this->updatePulledClusterBeforeInsert($cluster, $server, $user);
$result = $this->captureCluster($user, $cluster, $fromPull=true, $orgId=$server['Server']['org_id']);
if ($result['success']) {
$successes[] = $clusterId;
} else {
$fails[$clusterId] = __('Failed because of errors: ') . json_encode($result['errors']);
}
} else {
$fails[$clusterId] = __('failed downloading the galaxy cluster');
try {
$cluster = $serverSync->fetchGalaxyCluster($clusterId)->json();
} catch (Exception $e) {
$this->logException("Could not fetch galaxy cluster $clusterId from server {$serverSync->serverId()}", $e);
return false;
}
return true;
}
public function downloadGalaxyClusterFromServer($clusterId, $server, $HttpSocket=null)
{
$url = $server['Server']['url'];
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
$uri = $url . '/galaxy_clusters/view/' . $clusterId;
$response = $HttpSocket->get($uri, $data = '', $request);
if ($response->isOk()) {
return json_decode($response->body, true);
}
return null;
$cluster = $this->updatePulledClusterBeforeInsert($cluster, $serverSync->server(), $user);
$result = $this->captureCluster($user, $cluster, $fromPull=true, $orgId=$serverSync->server()['Server']['org_id']);
return $result['success'];
}
private function updatePulledClusterBeforeInsert($cluster, $server, $user)

197
app/Model/Server.php
View File

@ -222,25 +222,14 @@ class Server extends AppModel
throw new InvalidArgumentException("Invalid pull technique `$technique`.");
}
private function __checkIfEventIsBlockedBeforePull($event)
{
if (Configure::read('MISP.enableEventBlocklisting') !== false) {
if (!isset($this->EventBlocklist)) {
$this->EventBlocklist = ClassRegistry::init('EventBlocklist');
}
if ($this->EventBlocklist->isBlocked($event['Event']['uuid'])) {
return true;
}
}
return false;
}
/**
* @param array $event
* @param array $server
* @param array $user
* @param array $pullRules
* @param bool $pullRulesEmptiedEvent
*/
private function __updatePulledEventBeforeInsert(array &$event, array $server, array $user, array $pullRules, bool &$pullRulesEmptiedEvent=false)
private function __updatePulledEventBeforeInsert(array &$event, array $server, array $user, array $pullRules, bool &$pullRulesEmptiedEvent = false)
{
// we have an Event array
// The event came from a pull, so it should be locked.
@ -269,14 +258,12 @@ class Server extends AppModel
}
}
$typeFilteringEnabled = !empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) &&
!empty($pullRules['type_attributes']['NOT']);
if (isset($event['Event']['Attribute'])) {
$originalCount = count($event['Event']['Attribute']);
foreach ($event['Event']['Attribute'] as $key => $attribute) {
if (
!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) &&
!empty($pullRules['type_attributes']['NOT']) &&
in_array($attribute['type'], $pullRules['type_attributes']['NOT'])
) {
if ($typeFilteringEnabled && in_array($attribute['type'], $pullRules['type_attributes']['NOT'])) {
unset($event['Event']['Attribute'][$key]);
continue;
}
@ -297,7 +284,7 @@ class Server extends AppModel
}
}
}
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalCount > 0 && count($event['Event']['Attribute']) == 0) {
if ($typeFilteringEnabled && $originalCount > 0 && empty($event['Event']['Attribute'])) {
$pullRulesEmptiedEvent = true;
}
}
@ -323,11 +310,7 @@ class Server extends AppModel
if (isset($object['Attribute'])) {
$originalAttributeCount = count($object['Attribute']);
foreach ($object['Attribute'] as $j => $a) {
if (
!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) &&
!empty($pullRules['type_attributes']['NOT']) &&
in_array($a['type'], $pullRules['type_attributes']['NOT'])
) {
if ($typeFilteringEnabled && in_array($a['type'], $pullRules['type_attributes']['NOT'])) {
unset($event['Event']['Object'][$i]['Attribute'][$j]);
continue;
}
@ -348,13 +331,13 @@ class Server extends AppModel
}
}
}
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalAttributeCount > 0 && empty($event['Event']['Object'][$i]['Attribute'])) {
if ($typeFilteringEnabled && $originalAttributeCount > 0 && empty($event['Event']['Object'][$i]['Attribute'])) {
unset($event['Event']['Object'][$i]); // Object is empty, get rid of it
$pullRulesEmptiedEvent = true;
}
}
}
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalObjectCount > 0 && count($event['Event']['Object']) == 0) {
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalObjectCount > 0 && empty($event['Event']['Object'])) {
$pullRulesEmptiedEvent = true;
}
}
@ -466,7 +449,18 @@ class Server extends AppModel
}
}
private function __pullEvent($eventId, &$successes, &$fails, Event $eventModel, ServerSyncTool $serverSync, $user, $jobId, $force = false)
/**
* @param int|string $eventId Event ID or UUID
* @param array $successes
* @param array $fails
* @param Event $eventModel
* @param ServerSyncTool $serverSync
* @param array $user
* @param int $jobId
* @param bool $force
* @return bool
*/
private function __pullEvent($eventId, array &$successes, array &$fails, Event $eventModel, ServerSyncTool $serverSync, $user, $jobId, $force = false)
{
$params = [
'deleted' => [0, 1],
@ -489,23 +483,16 @@ class Server extends AppModel
return false;
}
if (!empty($event)) {
if ($this->__checkIfEventIsBlockedBeforePull($event)) {
return false;
$pullRulesEmptiedEvent = false;
$this->__updatePulledEventBeforeInsert($event, $serverSync->server(), $user, $serverSync->pullRules(), $pullRulesEmptiedEvent);
if (!$this->__checkIfEventSaveAble($event)) {
if (!$pullRulesEmptiedEvent) { // The event is empty because of the filtering rule. This is not considered a failure
$fails[$eventId] = __('Empty event detected.');
}
$pullRulesEmptiedEvent = false;
$this->__updatePulledEventBeforeInsert($event, $serverSync->server(), $user, $serverSync->pullRules(), $pullRulesEmptiedEvent);
if (!$this->__checkIfEventSaveAble($event)) {
if (!$pullRulesEmptiedEvent) { // The event is empty because of the filtering rule. This is not considered a failure
$fails[$eventId] = __('Empty event detected.');
}
} else {
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $headers, $body);
}
} else {
// error
$fails[$eventId] = __('failed downloading the event');
return false;
}
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $headers, $body);
return true;
}
@ -551,7 +538,7 @@ class Server extends AppModel
if ($jobId) {
$job->saveProgress($jobId, $technique === 'pull_relevant_clusters' ? __('Pulling relevant galaxy clusters.') : __('Pulling galaxy clusters.'));
}
$pulledClusters = $this->GalaxyCluster->pullGalaxyClusters($user, $server, $technique);
$pulledClusters = $this->GalaxyCluster->pullGalaxyClusters($user, $serverSync, $technique);
if ($technique === 'pull_relevant_clusters') {
if ($jobId) {
$job->saveStatus($jobId, true, 'Pulling complete.');
@ -579,8 +566,8 @@ class Server extends AppModel
/** @var Event $eventModel */
$eventModel = ClassRegistry::init('Event');
$successes = array();
$fails = array();
$successes = [];
$fails = [];
// now process the $eventIds to pull each of the events sequentially
if (!empty($eventIds)) {
// download each event
@ -622,7 +609,7 @@ class Server extends AppModel
count($fails)
);
$this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['Server']['id'], 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $email, $change);
return array($successes, $fails, $pulledProposals, $pulledSightings, $pulledClusters);
return [$successes, $fails, $pulledProposals, $pulledSightings, $pulledClusters];
}
public function filterRuleToParameter($filter_rules)
@ -662,30 +649,20 @@ class Server extends AppModel
/**
* fetchCustomClusterIdsFromServer Fetch custom-published remote clusters' UUIDs and versions
*
* @param array $server
* @param HttpSocketExtended|null $HttpSocket
* @param ServerSyncTool $serverSync
* @param array $conditions
* @return array The list of clusters
* @throws JsonException|HttpSocketHttpException|HttpSocketJsonException
*/
private function fetchCustomClusterIdsFromServer(array $server, HttpSocketExtended $HttpSocket=null, array $conditions=array())
private function fetchCustomClusterIdsFromServer(ServerSyncTool $serverSync, array $conditions = [])
{
$url = $server['Server']['url'];
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
$uri = $url . '/galaxy_clusters/restSearch';
$filterRules = [
'published' => 1,
'minimal' => 1,
'custom' => 1,
];
$filterRules = array_merge($filterRules, $conditions);
$response = $HttpSocket->post($uri, json_encode($filterRules), $request);
if (!$response->isOk()) {
throw new HttpSocketHttpException($response);
}
$clusterArray = $response->json();
$clusterArray = $serverSync->galaxyClusterSearch($filterRules)->json();
if (isset($clusterArray['response'])) {
$clusterArray = $clusterArray['response'];
}
@ -705,9 +682,9 @@ class Server extends AppModel
* @throws HttpSocketJsonException
* @throws JsonException
*/
public function getElligibleClusterIdsFromServerForPull(array $server, $HttpSocket=null, $onlyUpdateLocalCluster=true, array $elligibleClusters=array(), array $conditions=array())
public function getElligibleClusterIdsFromServerForPull(ServerSyncTool $serverSyncTool, $onlyUpdateLocalCluster=true, array $elligibleClusters=array(), array $conditions=array())
{
$clusterArray = $this->fetchCustomClusterIdsFromServer($server, $HttpSocket, $conditions=$conditions);
$clusterArray = $this->fetchCustomClusterIdsFromServer($serverSyncTool, $conditions=$conditions);
if (!empty($clusterArray)) {
foreach ($clusterArray as $cluster) {
if (isset($elligibleClusters[$cluster['GalaxyCluster']['uuid']])) {
@ -730,8 +707,7 @@ class Server extends AppModel
/**
* Get an array of cluster_ids that are present on the remote server and returns clusters that should be pushed.
* @param array $server
* @param HttpSocket|null $HttpSocket
* @param ServerSyncTool $serverSync
* @param array $localClusters
* @param array $conditions
* @return array
@ -739,9 +715,9 @@ class Server extends AppModel
* @throws HttpSocketJsonException
* @throws JsonException
*/
public function getElligibleClusterIdsFromServerForPush(array $server, $HttpSocket=null, $localClusters=array(), $conditions=array())
private function getElligibleClusterIdsFromServerForPush(ServerSyncTool $serverSync, array $localClusters=array(), array $conditions=array())
{
$clusterArray = $this->fetchCustomClusterIdsFromServer($server, $HttpSocket, $conditions=$conditions);
$clusterArray = $this->fetchCustomClusterIdsFromServer($serverSync, $conditions=$conditions);
$keyedClusterArray = Hash::combine($clusterArray, '{n}.GalaxyCluster.uuid', '{n}.GalaxyCluster.version');
if (!empty($localClusters)) {
foreach ($localClusters as $k => $localCluster) {
@ -784,13 +760,35 @@ class Server extends AppModel
return $eventIndex;
}
/**
* @param array $events
* @return void
*/
private function removeOlderEvents(array &$events)
{
$conditions = (count($events) > 10000) ? [] : ['Event.uuid' => array_column($events, 'uuid')];
$this->Event = ClassRegistry::init('Event');
$localEvents = $this->Event->find('all', [
'recursive' => -1,
'conditions' => $conditions,
'fields' => ['Event.uuid', 'Event.timestamp', 'Event.locked'],
]);
$localEvents = array_column(array_column($localEvents, 'Event'), null, 'uuid');
foreach ($events as $k => $event) {
$uuid = $event['uuid'];
if (isset($localEvents[$uuid]) && ($localEvents[$uuid]['timestamp'] >= $event['timestamp'] || !$localEvents[$uuid]['locked'])) {
unset($events[$k]);
}
}
}
/**
* Get an array of event UUIDs that are present on the remote server.
*
* @param ServerSyncTool $serverSync
* @param bool $all
* @param bool $ignoreFilterRules
* @param bool $force
* @param bool $ignoreFilterRules Ignore defined server pull rules
* @param bool $force If true, returns all events regardless their update timestamp
* @return array Array of event UUIDs.
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
@ -820,8 +818,7 @@ class Server extends AppModel
}
}
if (!$force) {
$this->Event = ClassRegistry::init('Event');
$this->Event->removeOlder($eventArray);
$this->removeOlderEvents($eventArray);
}
return array_column($eventArray, 'uuid');
}
@ -940,7 +937,7 @@ class Server extends AppModel
// sync custom galaxy clusters if user is capable
if ($push['canEditGalaxyCluster'] && $server['Server']['push_galaxy_clusters'] && "full" == $technique) {
$clustersSuccesses = $this->syncGalaxyClusters($HttpSocket, $this->data, $user, $technique='full');
$clustersSuccesses = $this->syncGalaxyClusters($serverSync, $this->data, $user, $technique='full');
} else {
$clustersSuccesses = array();
}
@ -989,7 +986,7 @@ class Server extends AppModel
'fields' => array('Event.id', 'Event.timestamp', 'Event.sighting_timestamp', 'Event.uuid', 'Event.orgc_id'), // array of field names
);
$eventIds = $this->Event->find('all', $findParams);
$eventUUIDsFiltered = $this->getEventIdsForPush($server, $HttpSocket, $eventIds);
$eventUUIDsFiltered = $this->getEventIdsForPush($server, $serverSync, $eventIds);
if (!empty($eventUUIDsFiltered)) {
$eventCount = count($eventUUIDsFiltered);
// now process the $eventIds to push each of the events sequentially
@ -1018,11 +1015,11 @@ class Server extends AppModel
$event = $event[0];
$event['Event']['locked'] = 1;
if ($push['canEditGalaxyCluster'] && $server['Server']['push_galaxy_clusters'] && "full" != $technique) {
$clustersSuccesses = $this->syncGalaxyClusters($HttpSocket, $this->data, $user, $technique=$event['Event']['id'], $event=$event);
$clustersSuccesses = $this->syncGalaxyClusters($serverSync, $this->data, $user, $technique=$event['Event']['id'], $event=$event);
} else {
$clustersSuccesses = array();
}
$result = $this->Event->uploadEventToServer($event, $server, $HttpSocket);
$result = $this->Event->uploadEventToServer($event, $server, $serverSync);
if ('Success' === $result) {
$successes[] = $event['Event']['id'];
} else {
@ -1049,7 +1046,7 @@ class Server extends AppModel
if ($push['canPush'] || $push['canSight']) {
$this->Sighting = ClassRegistry::init('Sighting');
$sightingSuccesses =$this->Sighting->pushSightings($user, $serverSync);
$sightingSuccesses = $this->Sighting->pushSightings($user, $serverSync);
} else {
$sightingSuccesses = array();
}
@ -1083,23 +1080,35 @@ class Server extends AppModel
return true;
}
public function getEventIdsForPush(array $server, HttpSocket $HttpSocket, array $eventIds)
/**
* @param array $server
* @param ServerSyncTool $serverSync
* @param array $events
* @return array|false
*/
private function getEventIdsForPush(array $server, ServerSyncTool $serverSync, array $events)
{
foreach ($eventIds as $k => $event) {
if (empty($this->eventFilterPushableServers($event, array($server)))) {
unset($eventIds[$k]);
$request = [];
foreach ($events as $event) {
if (empty($this->eventFilterPushableServers($event, [$server]))) {
continue;
}
unset($eventIds[$k]['Event']['id']);
$request[] = ['Event' => [
'uuid' => $event['Event']['uuid'],
'timestamp' => $event['Event']['timestamp'],
]];
}
$request = $this->setupSyncRequest($server);
$data = json_encode($eventIds);
$uri = $server['Server']['url'] . '/events/filterEventIdsForPush';
$response = $HttpSocket->post($uri, $data, $request);
if ($response->code == '200') {
return $this->jsonDecode($response->body());
if (empty($request)) {
return [];
}
try {
return $serverSync->filterEventIdsForPush($request)->json();
} catch (Exception $e) {
$this->logException("Could not filter events for push when pushing to server {$serverSync->serverId()}", $e);
return false;
}
return false;
}
/**
@ -1112,7 +1121,7 @@ class Server extends AppModel
* @param array|bool $event
* @return array List of successfully pushed clusters
*/
public function syncGalaxyClusters($HttpSocket, array $server, array $user, $technique='full', $event=false)
public function syncGalaxyClusters(ServerSyncTool $serverSync, array $server, array $user, $technique='full', $event=false)
{
$successes = array();
if (!$server['Server']['push_galaxy_clusters']) {
@ -1120,7 +1129,6 @@ class Server extends AppModel
}
$this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
$this->Event = ClassRegistry::init('Event');
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$clusters = array();
if ($technique == 'full') {
$clusters = $this->GalaxyCluster->getElligibleClustersToPush($user, $conditions=array(), $full=true);
@ -1135,13 +1143,13 @@ class Server extends AppModel
}
$localClusterUUIDs = Hash::extract($clusters, '{n}.GalaxyCluster.uuid');
try {
$clustersToPush = $this->getElligibleClusterIdsFromServerForPush($server, $HttpSocket = $HttpSocket, $localClusters = $clusters, $conditions = array('uuid' => $localClusterUUIDs));
$clustersToPush = $this->getElligibleClusterIdsFromServerForPush($serverSync, $localClusters = $clusters, $conditions = array('uuid' => $localClusterUUIDs));
} catch (Exception $e) {
$this->logException("Could not get eligible cluster IDs from server #{$server['Server']['id']} for push.", $e);
return [];
}
foreach ($clustersToPush as $cluster) {
$result = $this->GalaxyCluster->uploadClusterToServer($cluster, $server, $HttpSocket, $user);
$result = $this->GalaxyCluster->uploadClusterToServer($cluster, $server, $serverSync, $user);
if ($result === 'Success') {
$successes[] = __('GalaxyCluster %s', $cluster['GalaxyCluster']['uuid']);
}
@ -1153,10 +1161,10 @@ class Server extends AppModel
{
$saModel = ClassRegistry::init('ShadowAttribute');
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
if ($sa_id == null) {
if ($event_id == null) {
// event_id is null when we are doing a push
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
try {
$ids = $this->getEventIdsFromServer($serverSync, true, true);
} catch (Exception $e) {
@ -2503,7 +2511,6 @@ class Server extends AppModel
$canPush = isset($remoteVersion['perm_sync']) ? $remoteVersion['perm_sync'] : false;
$canSight = isset($remoteVersion['perm_sighting']) ? $remoteVersion['perm_sighting'] : false;
$supportEditOfGalaxyCluster = isset($remoteVersion['perm_galaxy_editor']);
$canEditGalaxyCluster = isset($remoteVersion['perm_galaxy_editor']) ? $remoteVersion['perm_galaxy_editor'] : false;
$remoteVersionString = $remoteVersion['version'];
$remoteVersion = explode('.', $remoteVersion['version']);
@ -2555,7 +2562,6 @@ class Server extends AppModel
'canPush' => $canPush,
'canSight' => $canSight,
'canEditGalaxyCluster' => $canEditGalaxyCluster,
'supportEditOfGalaxyCluster' => $supportEditOfGalaxyCluster,
'version' => $remoteVersion,
'protectedMode' => $protectedMode,
];
@ -3770,8 +3776,7 @@ class Server extends AppModel
public function extensionDiagnostics()
{
try {
$file = new File(APP . DS . 'composer.json');
$composer = $this->jsonDecode($file->read());
$composer = FileAccessTool::readJsonFromFile(APP . DS . 'composer.json');
$extensions = [];
foreach ($composer['require'] as $require => $foo) {
if (substr($require, 0, 4) === 'ext-') {