MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

new: [API] events/restsearch rework - chunked export for performance gains

pull/3729/head
iglocska 2018-09-29 23:43:52 +02:00
parent bef609607a
commit f18f8b579a
3 changed files with 88 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
app/Controller/EventsController.php
View File

@ -3002,6 +3002,43 @@ class EventsController extends AppController
return $this->response;
}
/*
* Receive a list of eventids in the id=>count format
* Chunk them by the attribute count to fit the memory limits
*
*/
private function __clusterEventIds($exportTool, $eventIds) {
$memory_in_mb = $this->Event->Attribute->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
$memory_scaling_factor = isset($exportTool->memory_scaling_factor) ? $exportTool->memory_scaling_factor : 100;
$limit = $memory_in_mb * $memory_scaling_factor;
$eventIdList = array();
$continue = true;
$i = 0;
$current_chunk_size = 0;
while (!empty($eventIds)) {
foreach ($eventIds as $id => $count) {
if ($current_chunk_size == 0 && $count > $limit) {
$eventIdList[$i][] = $id;
$current_chunk_size = $count;
unset($eventIds[$id]);
$i++;
break;
} else {
if (($current_chunk_size + $count) > $limit) {
$i++;
$current_chunk_size = 0;
break;
} else {
$current_chunk_size += $count;
$eventIdList[$i][] = $id;
unset($eventIds[$id]);
}
}
}
}
return $eventIdList;
}
// Use the REST interface to search for attributes or events. Usage:
// MISP-base-url/events/restSearch/[api-key]/[value]/[type]/[category]/[orgc]
// value, type, category, orgc are optional
@ -3058,7 +3095,24 @@ class EventsController extends AppController
$filters['published'] = 1;
}
}
if (isset($filters['ignore'])) {
$filters['to_ids'] = array(0, 1);
$filters['published'] = array(0, 1);
}
if (isset($filters['searchall'])) {
$filters['tags'] = $filters['searchall'];
$filters['eventinfo'] = $filters['searchall'];
$filters['value'] = $filters['searchall'];
$filters['comment'] = $filters['searchall'];
}
if (!empty($filters['quickfilter']) && !empty($filters['value'])) {
$filters['tags'] = $filters['value'];
$filters['eventinfo'] = $filters['value'];
$filters['comment'] = $filters['value'];
}
$filters['include_attribute_count'] = 1;
$eventid = $this->Event->filterEventIds($user, $filters);
$eventids_chunked = $this->__clusterEventIds($exportTool, $eventid);
if (!empty($exportTool->additional_params)) {
$filters = array_merge($filters, $exportTool->additional_params);
}
@ -3077,14 +3131,15 @@ class EventsController extends AppController
$filters['published'] = 1;
}
}
$final = $exportTool->header($exportToolParams);
$tmpfile = tmpfile();
fwrite($tmpfile, $exportTool->header($exportToolParams));
$eventCount = count($eventid);
$i = 0;
if (!empty($filters['withAttachments'])) {
$filters['includeAttachments'] = 1;
}
foreach ($eventid as $k => $currentEventId) {
$filters['eventid'] = $currentEventId;
foreach ($eventids_chunked as $chunk_index => $chunk) {
$filters['eventid'] = $chunk;
if (!empty($filters['tags']['NOT'])) {
$filters['blockedAttributeTags'] = $filters['tags']['NOT'];
}
@ -3093,20 +3148,25 @@ class EventsController extends AppController
$filters,
true
);
if (!empty($result)) {
$this->loadModel('Whitelist');
$result = $this->Whitelist->removeWhitelistedFromArray($result, false);
$temp = $exportTool->handler($result[0], $exportToolParams);
if ($temp !== '') {
if ($k !== 0) {
$final .= $exportTool->separator($exportToolParams);
if (!empty($result)) {
foreach ($result as $event) {
$this->loadModel('Whitelist');
$result = $this->Whitelist->removeWhitelistedFromArray($result, false);
$temp = $exportTool->handler($event, $exportToolParams);
if ($temp !== '') {
if ($i !== 0) {
$temp = $exportTool->separator($exportToolParams) . $temp;
}
fwrite($tmpfile, $temp);
$i++;
}
$final .= $temp;
}
$i++;
}
}
$final .= $exportTool->footer($exportToolParams);
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
$final = fread($tmpfile, fstat($tmpfile)['size']);
fclose($tmpfile);
$responseType = $validFormats[$returnFormat][0];
return $this->RestResponse->viewData($final, $responseType, false, true);
}

11
app/Model/Attribute.php
View File

@ -3662,17 +3662,6 @@ class Attribute extends AppModel
$params['to_ids'] = array(0, 1);
$params['published'] = array(0, 1);
}
if (isset($params['searchall'])) {
$params['tags'] = $params['searchall'];
$params['eventinfo'] = $params['searchall'];
$params['value'] = $params['searchall'];
$params['comment'] = $params['searchall'];
}
if (!empty($params['quickfilter']) && !empty($params['value'])) {
$params['tags'] = $params['value'];
$params['eventinfo'] = $params['value'];
$params['comment'] = $params['value'];
}
$simple_params = array(
'Attribute' => array(
'value' => array('function' => 'set_filter_value'),

34
app/Model/Event.php
View File

@ -1297,21 +1297,6 @@ class Event extends AppModel
public function filterEventIds($user, &$params = array())
{
$conditions = $this->createEventConditions($user);
if (isset($params['ignore'])) {
$params['to_ids'] = array(0, 1);
$params['published'] = array(0, 1);
}
if (isset($params['searchall'])) {
$params['tags'] = $params['searchall'];
$params['eventinfo'] = $params['searchall'];
$params['value'] = $params['searchall'];
$params['comment'] = $params['searchall'];
}
if (!empty($params['quickfilter']) && !empty($params['value'])) {
$params['tags'] = $params['value'];
$params['eventinfo'] = $params['value'];
$params['comment'] = $params['value'];
}
$simple_params = array(
'Event' => array(
'eventid' => array('function' => 'set_filter_eventid', 'pop' => true),
@ -1370,11 +1355,22 @@ class Event extends AppModel
}
}
}
$results = array_values($this->find('list', array(
'conditions' => $conditions,
$fields = array('Event.id');
if (!empty($params['include_attribute_count'])) {
$fields[] = 'Event.attribute_count';
}
$find_params = array(
'conditions' => $conditions,
'recursive' => -1,
'fields' => array('Event.id')
)));
'fields' => $fields
);
if (isset($params['limit'])) {
$find_params['limit'] = $params['limit'];
if (isset($params['page'])) {
$find_params['page'] = $params['page'];
}
}
$results = $this->find('list', $find_params);
return $results;
}