MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #6001 from JakubOnderka/get-events-refactoring 

chg: [internal] Refactor Server::getEventIdsFromServer
pull/6241/head
Jakub Onderka 2020-08-10 16:36:37 +02:00 committed by GitHub
parent 1233a88783 31ea1d6eb3
commit 345dce51d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 193 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

233
app/Model/Server.php
View File

@ -2391,40 +2391,46 @@ class Server extends AppModel
return true; return true;
} }
private function __getEventIdListBasedOnPullTechnique($technique, $server, $force = false) /**
* @param int|string $technique 'full', 'update', remote event ID or remote event UUID
* @param array $server
* @param bool $force
* @return array
*/
private function __getEventIdListBasedOnPullTechnique($technique, array $server, $force = false)
{ {
try {
if ("full" === $technique) { if ("full" === $technique) {
// get a list of the event_ids on the server // get a list of the event_ids on the server
$eventIds = $this->getEventIdsFromServer($server, false, null, false, false, 'events', $force); $eventIds = $this->getEventIdsFromServer($server, false, null, false, 'events', $force);
if ($eventIds === 403) {
return array('error' => array(1, null));
} elseif (is_string($eventIds)) {
return array('error' => array(2, $eventIds));
}
// reverse array of events, to first get the old ones, and then the new ones // reverse array of events, to first get the old ones, and then the new ones
if (!empty($eventIds)) { return array_reverse($eventIds);
$eventIds = array_reverse($eventIds);
}
} elseif ("update" === $technique) { } elseif ("update" === $technique) {
$eventIds = $this->getEventIdsFromServer($server, false, null, true, true, 'events', $force); $eventIds = $this->getEventIdsFromServer($server, false, null, true, 'events', $force);
if ($eventIds === 403) {
return array('error' => array(1, null));
} elseif (is_string($eventIds)) {
return array('error' => array(2, $eventIds));
}
$eventModel = ClassRegistry::init('Event'); $eventModel = ClassRegistry::init('Event');
$local_event_ids = $eventModel->find('list', array( $local_event_ids = $eventModel->find('list', array(
'fields' => array('uuid'), 'fields' => array('uuid'),
'recursive' => -1, 'recursive' => -1,
)); ));
$eventIds = array_intersect($eventIds, $local_event_ids); return array_intersect($eventIds, $local_event_ids);
} elseif (is_numeric($technique)) { } elseif (is_numeric($technique)) {
$eventIds[] = intval($technique); return array(intval($technique));
} elseif (Validation::uuid($technique)) {
return array($technique);
} else { } else {
return array('error' => array(4, null)); return array('error' => array(4, null));
} }
return $eventIds; } catch (HttpException $e) {
$this->logException("Could not fetch event IDs from server {$server['Server']['name']}", $e);
if ($e->getCode() === 403) {
return array('error' => array(1, null));
} else {
return array('error' => array(2, $e->getMessage()));
}
} catch (Exception $e) {
$this->logException("Could not fetch event IDs from server {$server['Server']['name']}", $e);
return array('error' => array(2, $e->getMessage()));
}
} }
private function __checkIfEventIsBlockedBeforePull($event) private function __checkIfEventIsBlockedBeforePull($event)
@ -2737,17 +2743,19 @@ class Server extends AppModel
return $final; return $final;
} }
private function __orgRuleDowngrade($HttpSocket, $request, $server, $filter_rules) /**
* @param HttpSocket $HttpSocket
* @param array $request
* @param array $server
* @param array $filter_rules
* @return array
* @throws JsonException
*/
private function __orgRuleDowngrade(HttpSocket $HttpSocket, array $request, array $server, array $filter_rules)
{ {
$uri = $server['Server']['url'] . '/servers/getVersion'; $uri = $server['Server']['url'] . '/servers/getVersion';
try {
$version_response = $HttpSocket->get($uri, false, $request); $version_response = $HttpSocket->get($uri, false, $request);
$body = $version_response->body; $version = $this->jsonDecode($version_response->body)['version'];
$version_response = json_decode($body, true);
$version = $version_response['version'];
} catch (Exception $e) {
return $e->getMessage();
}
$version = explode('.', $version); $version = explode('.', $version);
if ($version[0] <= 2 && $version[1] <= 4 && $version[0] <= 123) { if ($version[0] <= 2 && $version[1] <= 4 && $version[0] <= 123) {
$filter_rules['org'] = implode('|', $filter_rules['org']); $filter_rules['org'] = implode('|', $filter_rules['org']);
@ -2755,115 +2763,112 @@ class Server extends AppModel
return $filter_rules; return $filter_rules;
} }
// Get an array of event_ids that are present on the remote server /**
public function getEventIdsFromServer($server, $all = false, $HttpSocket=null, $force_uuid=false, $ignoreFilterRules = false, $scope = 'events', $force = false) * Get an array of event UUIDs that are present on the remote server.
*
* @param array $server
* @param bool $all
* @param HttpSocket|null $HttpSocket
* @param bool $ignoreFilterRules
* @param string $scope 'events' or 'sightings'
* @param bool $force
* @return array Array of event UUIDs.
* @throws JsonException
* @throws InvalidArgumentException
*/
public function getEventIdsFromServer(array $server, $all = false, HttpSocket $HttpSocket = null, $ignoreFilterRules = false, $scope = 'events', $force = false)
{ {
$url = $server['Server']['url']; if (!in_array($scope, array('events', 'sightings'))) {
if ($ignoreFilterRules) { throw new InvalidArgumentException("Scope mus be 'events' or 'sightings', '$scope' given.");
$filter_rules = array();
} else {
$filter_rules = $this->filterRuleToParameter($server['Server']['pull_rules']);
} }
if ($ignoreFilterRules) {
$filterRules = array();
} else {
$filterRules = $this->filterRuleToParameter($server['Server']['pull_rules']);
}
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket); $HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server); $request = $this->setupSyncRequest($server);
if (!empty($filter_rules['org'])) { if (!empty($filterRules['org'])) {
$filter_rules = $this->__orgRuleDowngrade($HttpSocket, $request, $server, $filter_rules); $filterRules = $this->__orgRuleDowngrade($HttpSocket, $request, $server, $filterRules);
} }
$uri = $url . '/events/index'; $filterRules['minimal'] = 1;
$filter_rules['minimal'] = 1; $filterRules['published'] = 1;
$filter_rules['published'] = 1;
try { $uri = $server['Server']['url'] . '/events/index';
$response = $HttpSocket->post($uri, json_encode($filter_rules), $request); $response = $HttpSocket->post($uri, json_encode($filterRules), $request);
if ($response->isOk()) { if ($response === false) {
$eventArray = json_decode($response->body, true); throw new Exception("Could not reach '$uri'.");
}
if (!$response->isOk()) {
throw new HttpException("Fetching the '$uri' failed with HTTP error {$response->code}: {$response->reasonPhrase}", intval($response->code));
}
$eventArray = $this->jsonDecode($response->body);
// correct $eventArray if just one event // correct $eventArray if just one event
if (is_array($eventArray) && isset($eventArray['id'])) { if (isset($eventArray['id'])) {
$tmp = $eventArray; $eventArray = array($eventArray);
unset($eventArray);
$eventArray[0] = $tmp;
unset($tmp);
} }
$eventIds = array();
if ($all) { if ($all) {
if (!empty($eventArray)) {
if ($scope === 'sightings') { if ($scope === 'sightings') {
foreach ($eventArray as $event) { $this->Event = ClassRegistry::init('Event');
$localEvent = $this->Event->find('first', array( $localEvents = $this->Event->find('list', array(
'recursive' => -1, 'recursive' => -1,
'fields' => array('Event.uuid', 'Event.sighting_timestamp'), 'fields' => array('Event.uuid', 'Event.sighting_timestamp'),
'conditions' => array('Event.uuid' => $event['uuid']) 'conditions' => array('Event.uuid' => array_column($eventArray, 'uuid'))
)); ));
if (!empty($localEvent) && $localEvent['Event']['sighting_timestamp'] > $event['sighting_timestamp']) {
$eventIds[] = $event['uuid']; $eventUuids = array();
}
}
} else {
foreach ($eventArray as $event) { foreach ($eventArray as $event) {
$eventIds[] = $event['uuid']; if (!isset($localEvents[$event['uuid']]) && $localEvents[$event['uuid']] > $event['sighting_timestamp']) {
} $eventUuids[] = $event['uuid'];
} }
} }
} else { } else {
// multiple events, iterate over the array $eventUuids = array_column($eventArray, 'uuid');
$this->Event = ClassRegistry::init('Event'); }
$blacklisting = array(); } else {
if (Configure::read('MISP.enableEventBlacklisting') !== false) { if (Configure::read('MISP.enableEventBlacklisting') !== false) {
$this->EventBlacklist = ClassRegistry::init('EventBlacklist'); $this->EventBlacklist = ClassRegistry::init('EventBlacklist');
$blacklisting['EventBlacklist'] = array( $blacklistHits = $this->EventBlacklist->find('list', array(
'index_field' => 'uuid', 'recursive' => -1,
'blacklist_field' => 'event_uuid' 'conditions' => array('EventBlacklist.event_uuid' => array_column($eventArray, 'uuid')),
); 'fields' => array('EventBlacklist.event_uuid', 'EventBlacklist.event_uuid'),
));
foreach ($eventArray as $k => $event) {
if (isset($blacklistHits[$event['uuid']])) {
unset($eventArray[$k]);
} }
}
}
if (Configure::read('MISP.enableOrgBlacklisting') !== false) { if (Configure::read('MISP.enableOrgBlacklisting') !== false) {
$this->OrgBlacklist = ClassRegistry::init('OrgBlacklist'); $this->OrgBlacklist = ClassRegistry::init('OrgBlacklist');
$blacklisting['OrgBlacklist'] = array( $blacklistHits = $this->OrgBlacklist->find('list', array(
'index_field' => 'orgc_uuid', 'recursive' => -1,
'blacklist_field' => 'org_uuid' 'conditions' => array('OrgBlacklist.org_uuid' => array_unique(array_column($eventArray, 'orgc_uuid'))),
); 'fields' => array('OrgBlacklist.org_uuid', 'OrgBlacklist.org_uuid'),
));
foreach ($eventArray as $k => $event) {
if (isset($blacklistHits[$event['orgc_uuid']])) {
unset($eventArray[$k]);
} }
}
}
foreach ($eventArray as $k => $event) { foreach ($eventArray as $k => $event) {
if (1 != $event['published']) { if (1 != $event['published']) {
unset($eventArray[$k]); // do not keep non-published events unset($eventArray[$k]); // do not keep non-published events
continue;
}
foreach ($blacklisting as $type => $blacklist) {
if (!empty($eventArray[$k][$blacklist['index_field']])) {
$blacklist_hit = $this->{$type}->find('first', array(
'conditions' => array($blacklist['blacklist_field'] => $eventArray[$k][$blacklist['index_field']]),
'recursive' => -1,
'fields' => array($type . '.id')
));
if (!empty($blacklist_hit)) {
unset($eventArray[$k]);
continue 2;
}
}
} }
} }
if (!$force) { if (!$force) {
$this->Event = ClassRegistry::init('Event');
$this->Event->removeOlder($eventArray, $scope); $this->Event->removeOlder($eventArray, $scope);
} }
if (!empty($eventArray)) { $eventUuids = array_column($eventArray, 'uuid');
foreach ($eventArray as $event) {
if ($force_uuid) {
$eventIds[] = $event['uuid'];
} else {
$eventIds[] = $event['uuid'];
} }
} return $eventUuids;
}
}
return $eventIds;
}
if ($response->code == '403') {
return 403;
}
} catch (SocketException $e) {
return $e->getMessage();
}
// error, so return error message, since that is handled and everything is expecting an array
return "Error: got response code " . $response->code;
} }
public function push($id = null, $technique=false, $jobId = false, $HttpSocket, $user) public function push($id = null, $technique=false, $jobId = false, $HttpSocket, $user)
@ -3073,9 +3078,13 @@ class Server extends AppModel
} }
$this->Sighting = ClassRegistry::init('Sighting'); $this->Sighting = ClassRegistry::init('Sighting');
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket); $HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$eventIds = $this->getEventIdsFromServer($server, true, $HttpSocket, false, true, 'sightings'); try {
$eventIds = $this->getEventIdsFromServer($server, true, $HttpSocket, true, 'sightings');
} catch (Exception $e) {
$this->logException("Could not fetch event IDs from server {$server['Server']['name']}", $e);
return $successes;
}
// now process the $eventIds to push each of the events sequentially // now process the $eventIds to push each of the events sequentially
if (!empty($eventIds)) {
// check each event and push sightings when needed // check each event and push sightings when needed
foreach ($eventIds as $k => $eventId) { foreach ($eventIds as $k => $eventId) {
$event = $eventModel->fetchEvent($user, $options = array('event_uuid' => $eventId, 'metadata' => true)); $event = $eventModel->fetchEvent($user, $options = array('event_uuid' => $eventId, 'metadata' => true));
@ -3088,7 +3097,6 @@ class Server extends AppModel
} }
} }
} }
}
return $successes; return $successes;
} }
@ -3099,9 +3107,10 @@ class Server extends AppModel
if ($sa_id == null) { if ($sa_id == null) {
if ($event_id == null) { if ($event_id == null) {
// event_id is null when we are doing a push // event_id is null when we are doing a push
$ids = $this->getEventIdsFromServer($server, true, $HttpSocket, false, true); try {
// error return strings or ints or throw exceptions $ids = $this->getEventIdsFromServer($server, true, $HttpSocket, true);
if (!is_array($ids)) { } catch (Exception $e) {
$this->logException("Could not fetch event IDs from server {$server['Server']['name']}", $e);
return false; return false;
} }
$conditions = array('uuid' => $ids); $conditions = array('uuid' => $ids);

12
app/Model/Sighting.php
View File

@ -825,19 +825,22 @@ class Sighting extends AppModel
{ {
$HttpSocket = $this->setupHttpSocket($server); $HttpSocket = $this->setupHttpSocket($server);
$this->Server = ClassRegistry::init('Server'); $this->Server = ClassRegistry::init('Server');
$eventIds = $this->Server->getEventIdsFromServer($server, false, $HttpSocket, false, false, 'sightings'); try {
$eventIds = $this->Server->getEventIdsFromServer($server, false, $HttpSocket, false, 'sightings');
} catch (Exception $e) {
$this->logException("Could not fetch event IDs from server {$server['Server']['name']}", $e);
return 0;
}
$saved = 0; $saved = 0;
// now process the $eventIds to pull each of the events sequentially // now process the $eventIds to pull each of the events sequentially
if (!empty($eventIds)) {
// download each event and save sightings // download each event and save sightings
foreach ($eventIds as $k => $eventId) { foreach ($eventIds as $k => $eventId) {
try { try {
$event = $this->Event->downloadEventFromServer($eventId, $server); $event = $this->Event->downloadEventFromServer($eventId, $server);
} catch (Exception $e) { } catch (Exception $e) {
$this->logException('Failed downloading the event ' . $eventId, $e); $this->logException("Failed downloading the event $eventId from {$server['Server']['name']}.", $e);
continue; continue;
} }
$sightings = array(); $sightings = array();
if (!empty($event) && !empty($event['Event']['Attribute'])) { if (!empty($event) && !empty($event['Event']['Attribute'])) {
foreach ($event['Event']['Attribute'] as $attribute) { foreach ($event['Event']['Attribute'] as $attribute) {
@ -853,7 +856,6 @@ class Sighting extends AppModel
} }
} }
} }
}
return $saved; return $saved;
} }