MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

fix: [sync] Fixed major performance blocker 

- fix based on the insights of @RichieB2B, the hero we need, not the one we deserve
- added orgc_uuid to the minimal event index
- added handlers for it on the pull side
- when pulling from old instances the new functionality is skipped, resulting in the behaviour we had pre-patch
- both sides of the sync are encouraged to update, especially if the slow pulls are causing issues
pull/5116/head
iglocska 2019-09-06 10:59:48 +02:00
parent d783e0d039
commit 41f5c88c74
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
2 changed files with 33 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/Controller/EventsController.php
View File

@ -730,6 +730,7 @@ class EventsController extends AppController
unset($rules['contain']);
$rules['recursive'] = -1;
$rules['fields'] = array('id', 'timestamp', 'published', 'uuid');
$rules['contain'] = array('Orgc.uuid');
}
$paginationRules = array('page', 'limit', 'sort', 'direction', 'order');
foreach ($paginationRules as $paginationRule) {
@ -835,6 +836,7 @@ class EventsController extends AppController
return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, array('X-Result-Count' => $absolute_total));
} else {
foreach ($events as $key => $event) {
$event['Event']['orgc_uuid'] = $event['Orgc']['uuid'];
$events[$key] = $event['Event'];
}
return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, array('X-Result-Count' => $absolute_total));

46
app/Model/Server.php
View File

@ -2528,6 +2528,7 @@ class Server extends AppModel
$request = $this->setupSyncRequest($server);
$uri = $url . '/events/index';
$filter_rules['minimal'] = 1;
$filter_rules['published'] = 1;
try {
$response = $HttpSocket->post($uri, json_encode($filter_rules), $request);
if ($response->isOk()) {
@ -2549,9 +2550,38 @@ class Server extends AppModel
} else {
// multiple events, iterate over the array
$this->Event = ClassRegistry::init('Event');
$blacklisting = array();
if (Configure::read('MISP.enableEventBlacklisting') !== false) {
$this->EventBlacklist = ClassRegistry::init('EventBlacklist');
$blacklisting['EventBlacklist'] = array(
'index_field' => 'uuid',
'blacklist_field' => 'event_uuid'
);
}
if (Configure::read('MISP.enableOrgBlacklisting') !== false) {
$this->OrgBlacklist = ClassRegistry::init('OrgBlacklist');
$blacklisting['OrgBlacklist'] = array(
'index_field' => 'orgc_uuid',
'blacklist_field' => 'org_uuid'
);
}
foreach ($eventArray as $k => $event) {
if (1 != $event['published']) {
unset($eventArray[$k]); // do not keep non-published events
continue;
}
foreach ($blacklisting as $type => $blacklist) {
if (!empty($eventArray[$k][$blacklist['index_field']])) {
$blacklist_hit = $this->{$type}->find('first', array(
'conditions' => array($blacklist['blacklist_field'] => $eventArray[$k][$blacklist['index_field']]),
'recursive' => -1,
'fields' => array($type . '.id')
));
if (!empty($blacklist_hit)) {
unset($eventArray[$k]);
continue 2;
}
}
}
}
$this->Event->removeOlder($eventArray);
@ -2565,20 +2595,6 @@ class Server extends AppModel
}
}
}
if (!empty($eventIds) && Configure::read('MISP.enableEventBlacklisting') !== false) {
$this->EventBlacklist = ClassRegistry::init('EventBlacklist');
foreach ($eventIds as $k => $eventUuid) {
$blacklistEntry = $this->EventBlacklist->find('first', array(
'conditions' => array('event_uuid' => $eventUuid),
'recursive' => -1,
'fields' => array('EventBlacklist.id')
));
if (!empty($blacklistEntry)) {
unset($eventIds[$k]);
}
}
}
$eventIds = array_values($eventIds);
return $eventIds;
}
if ($response->code == '403') {
@ -4193,7 +4209,7 @@ class Server extends AppModel
}
return $result;
}
}
public function writeableDirsDiagnostics(&$diagnostic_errors)