mirror of https://github.com/MISP/MISP
new: [API] events/restSearch reworked, added CSV export
iglocska
parent
cea4f857bb
commit
f40f802862
|
|
@ -2734,35 +2734,6 @@ class EventsController extends AppController
|
|||
}
|
||||
}
|
||||
$final = array();
|
||||
$requested_attributes = array('uuid', 'event_id', 'category', 'type',
|
||||
'value', 'comment', 'to_ids', 'timestamp', 'object_relation');
|
||||
$requested_obj_attributes = array('uuid', 'name', 'meta-category');
|
||||
if ($includeContext) {
|
||||
$requested_attributes[] = 'attribute_tag';
|
||||
}
|
||||
if (isset($this->params['url']['attributes'])) {
|
||||
if (!isset($this->params['url']['obj_attributes'])) {
|
||||
$requested_obj_attributes = array();
|
||||
}
|
||||
$requested_attributes = explode(',', $this->params['url']['attributes']);
|
||||
}
|
||||
if (isset($this->params['url']['obj_attributes'])) {
|
||||
$requested_obj_attributes = explode(',', $this->params['url']['obj_attributes']);
|
||||
}
|
||||
if (isset($data['request']['attributes'])) {
|
||||
if (!isset($data['request']['obj_attributes'])) {
|
||||
$requested_obj_attributes = array();
|
||||
}
|
||||
$requested_attributes = $data['request']['attributes'];
|
||||
}
|
||||
if (isset($data['request']['obj_attributes'])) {
|
||||
$requested_obj_attributes = $data['request']['obj_attributes'];
|
||||
}
|
||||
$possibleParams = array(
|
||||
'ignore', 'list', 'category', 'type', 'includeContext',
|
||||
'enforceWarninglist', 'value', 'timestamp', 'tags',
|
||||
'last', 'from', 'to'
|
||||
);
|
||||
if (isset($params['eventid']) && $params['eventid'] == 'all') {
|
||||
unset($params['eventid']);
|
||||
}
|
||||
|
|
@ -3006,43 +2977,6 @@ class EventsController extends AppController
|
|||
return $this->response;
|
||||
}
|
||||
|
||||
/*
|
||||
* Receive a list of eventids in the id=>count format
|
||||
* Chunk them by the attribute count to fit the memory limits
|
||||
*
|
||||
*/
|
||||
private function __clusterEventIds($exportTool, $eventIds) {
|
||||
$memory_in_mb = $this->Event->Attribute->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
|
||||
$memory_scaling_factor = isset($exportTool->memory_scaling_factor) ? $exportTool->memory_scaling_factor : 100;
|
||||
$limit = $memory_in_mb * $memory_scaling_factor;
|
||||
$eventIdList = array();
|
||||
$continue = true;
|
||||
$i = 0;
|
||||
$current_chunk_size = 0;
|
||||
while (!empty($eventIds)) {
|
||||
foreach ($eventIds as $id => $count) {
|
||||
if ($current_chunk_size == 0 && $count > $limit) {
|
||||
$eventIdList[$i][] = $id;
|
||||
$current_chunk_size = $count;
|
||||
unset($eventIds[$id]);
|
||||
$i++;
|
||||
break;
|
||||
} else {
|
||||
if (($current_chunk_size + $count) > $limit) {
|
||||
$i++;
|
||||
$current_chunk_size = 0;
|
||||
break;
|
||||
} else {
|
||||
$current_chunk_size += $count;
|
||||
$eventIdList[$i][] = $id;
|
||||
unset($eventIds[$id]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return $eventIdList;
|
||||
}
|
||||
|
||||
// Use the REST interface to search for attributes or events. Usage:
|
||||
// MISP-base-url/events/restSearch/[api-key]/[value]/[type]/[category]/[orgc]
|
||||
// value, type, category, orgc are optional
|
||||
|
|
@ -3066,7 +3000,8 @@ class EventsController extends AppController
|
|||
'suricata' => array('txt', 'NidsSuricataExport', 'rules'),
|
||||
'snort' => array('txt', 'NidsSnortExport', 'rules'),
|
||||
'rpz' => array('rpz', 'RPZExport', 'rpz'),
|
||||
'text' => array('text', 'TextExport', 'txt')
|
||||
'text' => array('text', 'TextExport', 'txt'),
|
||||
'csv' => array('csv', 'CsvExport', 'csv')
|
||||
);
|
||||
$exception = false;
|
||||
$filters = $this->_harvestParameters($filterData, $exception);
|
||||
|
|
@ -3085,92 +3020,7 @@ class EventsController extends AppController
|
|||
if ($returnFormat === 'download') {
|
||||
$returnFormat = 'json';
|
||||
}
|
||||
if (!isset($validFormats[$returnFormat][1])) {
|
||||
throw new NotFoundException('Invalid output format.');
|
||||
}
|
||||
App::uses($validFormats[$returnFormat][1], 'Export');
|
||||
$exportTool = new $validFormats[$returnFormat][1]();
|
||||
|
||||
if (empty($exportTool->non_restrictive_export)) {
|
||||
if (!isset($filters['to_ids'])) {
|
||||
$filters['to_ids'] = 1;
|
||||
}
|
||||
if (!isset($filters['published'])) {
|
||||
$filters['published'] = 1;
|
||||
}
|
||||
}
|
||||
if (isset($filters['ignore'])) {
|
||||
$filters['to_ids'] = array(0, 1);
|
||||
$filters['published'] = array(0, 1);
|
||||
}
|
||||
if (isset($filters['searchall'])) {
|
||||
$filters['tags'] = $filters['searchall'];
|
||||
$filters['eventinfo'] = $filters['searchall'];
|
||||
$filters['value'] = $filters['searchall'];
|
||||
$filters['comment'] = $filters['searchall'];
|
||||
}
|
||||
if (!empty($filters['quickfilter']) && !empty($filters['value'])) {
|
||||
$filters['tags'] = $filters['value'];
|
||||
$filters['eventinfo'] = $filters['value'];
|
||||
$filters['comment'] = $filters['value'];
|
||||
}
|
||||
$filters['include_attribute_count'] = 1;
|
||||
$eventid = $this->Event->filterEventIds($user, $filters);
|
||||
$eventids_chunked = $this->__clusterEventIds($exportTool, $eventid);
|
||||
if (!empty($exportTool->additional_params)) {
|
||||
$filters = array_merge($filters, $exportTool->additional_params);
|
||||
}
|
||||
$exportToolParams = array(
|
||||
'user' => $this->Auth->user(),
|
||||
'params' => array(),
|
||||
'returnFormat' => $returnFormat,
|
||||
'scope' => 'Event',
|
||||
'filters' => $filters
|
||||
);
|
||||
if (empty($exportTool->non_restrictive_export)) {
|
||||
if (!isset($filters['to_ids'])) {
|
||||
$filters['to_ids'] = 1;
|
||||
}
|
||||
if (!isset($filters['published'])) {
|
||||
$filters['published'] = 1;
|
||||
}
|
||||
}
|
||||
$tmpfile = tmpfile();
|
||||
fwrite($tmpfile, $exportTool->header($exportToolParams));
|
||||
$eventCount = count($eventid);
|
||||
$i = 0;
|
||||
if (!empty($filters['withAttachments'])) {
|
||||
$filters['includeAttachments'] = 1;
|
||||
}
|
||||
foreach ($eventids_chunked as $chunk_index => $chunk) {
|
||||
$filters['eventid'] = $chunk;
|
||||
if (!empty($filters['tags']['NOT'])) {
|
||||
$filters['blockedAttributeTags'] = $filters['tags']['NOT'];
|
||||
}
|
||||
$result = $this->Event->fetchEvent(
|
||||
$this->Auth->user(),
|
||||
$filters,
|
||||
true
|
||||
);
|
||||
if (!empty($result)) {
|
||||
foreach ($result as $event) {
|
||||
$this->loadModel('Whitelist');
|
||||
$result = $this->Whitelist->removeWhitelistedFromArray($result, false);
|
||||
$temp = $exportTool->handler($event, $exportToolParams);
|
||||
if ($temp !== '') {
|
||||
if ($i !== 0) {
|
||||
$temp = $exportTool->separator($exportToolParams) . $temp;
|
||||
}
|
||||
fwrite($tmpfile, $temp);
|
||||
$i++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fwrite($tmpfile, $exportTool->footer($exportToolParams));
|
||||
fseek($tmpfile, 0);
|
||||
$final = fread($tmpfile, fstat($tmpfile)['size']);
|
||||
fclose($tmpfile);
|
||||
$final = $this->Event->restSearch($user, $validFormats, $returnFormat, $filters);
|
||||
$responseType = $validFormats[$returnFormat][0];
|
||||
return $this->RestResponse->viewData($final, $responseType, false, true);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue