new: [API] Rework of the restSearch APIs

- peformance tuning
  - removed some redundant looping
  - internal memory profiling for attributes/restSearch
  - saving the intermediary results to file instead of keeping it all in memory to reduce the memory footprint
- added the searchall parameter
- fixed the ignore parameter
- added the event_timestamp parameter
- added manual pagination to the attribute level restsearch (limit, page)
pull/3725/head
iglocska 2018-09-28 09:21:29 +02:00
parent cfe86512a2
commit 0216c9ea33
4 changed files with 227 additions and 81 deletions

View File

@ -2140,6 +2140,12 @@ class AttributesController extends AppController
if (isset($filters['include_event_uuid'])) {
$params['includeEventUuid'] = $filters['include_event_uuid'];
}
if (isset($filters['limit'])) {
$params['limit'] = $filters['limit'];
}
if (isset($filters['page'])) {
$params['page'] = $filters['page'];
}
if (!empty($filtes['deleted'])) {
$params['deleted'] = 1;
if ($params['deleted'] === 'only') {
@ -2161,35 +2167,53 @@ class AttributesController extends AppController
if (!empty($exportTool->additional_params)) {
$params = array_merge($params, $exportTool->additional_params);
}
$final = $exportTool->header($exportToolParams);
$continue = false;
$tmpfile = tmpfile();
fwrite($tmpfile, $exportTool->header($exportToolParams));
$loop = false;
if (empty($params['limit'])) {
$params['limit'] = 20000;
$continue = true;
$memory_in_mb = $this->Attribute->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
$memory_scaling_factor = isset($exportTool->memory_scaling_factor) ? $exportTool->memory_scaling_factor : 100;
$params['limit'] = $memory_in_mb * $memory_scaling_factor;
$loop = true;
$params['page'] = 1;
}
$this->loadModel('Whitelist');
$this->__iteratedFetch($params, $loop, $tmpfile, $exportTool, $exportToolParams);
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
$final = fread($tmpfile, fstat($tmpfile)['size']);
$responseType = $validFormats[$returnFormat][0];
return $this->RestResponse->viewData($final, $responseType, false, true);
}
private function __iteratedFetch(&$params, &$loop, &$tmpfile, $exportTool, $exportToolParams) {
$continue = true;
while ($continue) {
$this->loadModel('Whitelist');
$results = $this->Attribute->fetchAttributes($this->Auth->user(), $params, $continue);
$params['page'] += 1;
$results = $this->Whitelist->removeWhitelistedFromArray($results, true);
$results = array_values($results);
$i = 0;
$temp = '';
foreach ($results as $attribute) {
$temp = $exportTool->handler($attribute, $exportToolParams);
$temp .= $exportTool->handler($attribute, $exportToolParams);
if ($temp !== '') {
$final .= $temp;
if ($i != count($results) -1) {
$final .= $exportTool->separator($exportToolParams);
$temp .= $exportTool->separator($exportToolParams);
}
}
$i++;
}
if (!$loop) {
$continue = false;
}
if ($continue) {
$temp .= $exportTool->separator($exportToolParams);
}
fwrite($tmpfile, $temp);
}
$final .= $exportTool->footer($exportToolParams);
$responseType = $validFormats[$returnFormat][0];
return $this->RestResponse->viewData($final, $responseType, false, true);
}
return true;
}
// returns an XML with attributes that belong to an event. The type of attributes to be returned can be restricted by type using the 3rd parameter.
// Similar to the restSearch, this parameter can be chained with '&&' and negations are accepted too. For example filename&&!filename|md5 would return all filenames that don't have an md5

View File

@ -1673,7 +1673,7 @@ class AppModel extends Model
}
// take filters in the {"OR" => [foo], "NOT" => [bar]} format along with conditions and set the conditions
public function generic_add_filter($conditions, &$filter, $keys)
public function generic_add_filter($conditions, &$filter, $keys, $searchall = false)
{
$operator_composition = array(
'NOT' => 'AND',
@ -1711,7 +1711,11 @@ class AppModel extends Model
}
}
}
$conditions['AND'][] = array($operator_composition[$operator] => $temp);
if ($searchall && $operator === 'OR') {
$conditions['AND']['OR'][] = array($operator_composition[$operator] => $temp);
} else {
$conditions['AND'][] = array($operator_composition[$operator] => $temp);
}
if ($operator !== 'NOT') {
unset($filter[$operator]);
}
@ -1754,4 +1758,28 @@ class AppModel extends Model
}
return $filter;
}
public function convert_to_memory_limit_to_mb($val) {
$val = trim($val);
if ($val == -1) {
// default to 8GB if no limit is set
return 8 * 1024;
}
$unit = $val[strlen($val)-1];
if (is_numeric($unit)) {
$unit = 'b';
} else {
$val = intval($val);
}
$unit = strtolower($unit);
switch($unit) {
case 'g':
$val *= 1024;
case 'm':
$val *= 1024;
case 'k':
$val *= 1024;
}
return $val / (1024 * 1024);
}
}

View File

@ -603,15 +603,19 @@ class Attribute extends AppModel
public function afterSave($created, $options = array())
{
$passedEvent = false;
if (isset($options['parentEvent'])) {
$passedEvent = $options['parentEvent'];
}
parent::afterSave($created, $options);
// update correlation...
if (isset($this->data['Attribute']['deleted']) && $this->data['Attribute']['deleted']) {
$this->__beforeSaveCorrelation($this->data['Attribute']);
if (isset($this->data['Attribute']['event_id'])) {
$this->__alterAttributeCount($this->data['Attribute']['event_id'], false);
$this->__alterAttributeCount($this->data['Attribute']['event_id'], false, $passedEvent);
}
} else {
$this->__afterSaveCorrelation($this->data['Attribute']);
$this->__afterSaveCorrelation($this->data['Attribute'], false, $passedEvent);
}
$result = true;
// if the 'data' field is set on the $this->data then save the data to the correct file
@ -2075,7 +2079,7 @@ class Attribute extends AppModel
$tag = ClassRegistry::init('Tag');
$params['tags'] = $this->dissectArgs($params['tags']);
$tagArray = $tag->fetchTagIds($params['tags'][0], $params['tags'][1]);
if (!empty($params['tags'][0]) && empty($tagArray[0])) {
if (!empty($params['tags'][0]) && empty($tagArray[0]) && empty($params['lax_tags'])) {
$tagArray[0] = array(-1);
}
$temp = array();
@ -2093,8 +2097,7 @@ class Attribute extends AppModel
$temp,
$this->subQueryGenerator($tag->EventTag, $subquery_options, $lookup_field)
);
$subquery_options = array(
$subquery_options = array(
'conditions' => array(
'tag_id' => $tagArray[0]
),
@ -2107,7 +2110,11 @@ class Attribute extends AppModel
$temp,
$this->subQueryGenerator($tag->AttributeTag, $subquery_options, $lookup_field)
);
$conditions['AND'][] = array('OR' => $temp);
if (!empty($params['searchall'])) {
$conditions['AND']['OR'][] = array('OR' => $temp);
} else {
$conditions['AND'][] = array('OR' => $temp);
}
}
$temp = array();
if (!empty($tagArray[1])) {
@ -2772,12 +2779,12 @@ class Attribute extends AppModel
'Event' => array(
'fields' => array('id', 'info', 'org_id', 'orgc_id', 'uuid'),
),
'AttributeTag' => array('Tag' => array()),
'Object' => array(
'fields' => array('id', 'distribution', 'sharing_group_id')
)
)
);
$params['contain']['AttributeTag'] = array('Tag' => array('conditions' => array()));
if (empty($options['includeAllTags'])) {
$params['contain']['AttributeTag']['Tag']['conditions']['exportable'] = 1;
}
@ -2854,9 +2861,9 @@ class Attribute extends AppModel
return $results;
}
if ($options['enforceWarninglist']) {
if ($options['enforceWarninglist'] && !isset($this->warninglists)) {
$this->Warninglist = ClassRegistry::init('Warninglist');
$warninglists = $this->Warninglist->fetchForEventView();
$this->warninglists = $this->Warninglist->fetchForEventView();
}
if (empty($params['limit'])) {
$loopLimit = 50000;
@ -2894,38 +2901,16 @@ class Attribute extends AppModel
$proposals_block_attributes = Configure::read('MISP.proposals_block_attributes');
foreach ($results as $key => $attribute) {
if (!empty($options['includeEventTags'])) {
if (!isset($eventTags[$results[$key]['Event']['id']])) {
$tagConditions = array('EventTag.event_id' => $attribute['Event']['id']);
if (empty($options['includeAllTags'])) {
$tagConditions['Tag.exportable'] = 1;
}
$temp = $this->Event->EventTag->find('all', array(
'recursive' => -1,
'contain' => array('Tag'),
'conditions' => $tagConditions
));
foreach ($temp as $tag) {
$tag['EventTag']['Tag'] = $tag['Tag'];
unset($tag['Tag']);
$eventTags[$results[$key]['Event']['id']][] = $tag;
}
}
foreach ($eventTags[$results[$key]['Event']['id']] as $eventTag) {
$results[$key]['EventTag'][] = $eventTag['EventTag'];
}
$results = $this->__attachEventTagsToAttributes($eventTags, $results, $key, $options);
}
if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttributes($warninglists, $attribute['Attribute'])) {
if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttributes($this->warninglists, $attribute['Attribute'])) {
continue;
}
if (!empty($options['includeAttributeUuid']) || !empty($options['includeEventUuid'])) {
$results[$key]['Attribute']['event_uuid'] = $results[$key]['Event']['uuid'];
}
if ($proposals_block_attributes) {
if (!empty($attribute['ShadowAttribute'])) {
continue;
} else {
unset($results[$key]['ShadowAttribute']);
}
$results = $this->__blockAttributeViaProposal($results, $k);
}
if ($options['withAttachments']) {
if ($this->typeIsAttachment($attribute['Attribute']['type'])) {
@ -2942,6 +2927,47 @@ class Attribute extends AppModel
return $attributes;
}
private function __attachEventTagsToAttributes($eventTags, &$results, $key, $options) {
if (!isset($eventTags[$results[$key]['Event']['id']])) {
$tagConditions = array('EventTag.event_id' => $results[$key]['Event']['id']);
if (empty($options['includeAllTags'])) {
$tagConditions['Tag.exportable'] = 1;
}
$temp = $this->Event->EventTag->find('all', array(
'recursive' => -1,
'contain' => array('Tag'),
'conditions' => $tagConditions
));
foreach ($temp as $tag) {
$tag['EventTag']['Tag'] = $tag['Tag'];
unset($tag['Tag']);
$eventTags[$results[$key]['Event']['id']][] = $tag;
}
}
foreach ($eventTags[$results[$key]['Event']['id']] as $eventTag) {
$results[$key]['EventTag'][] = $eventTag['EventTag'];
}
return $results;
}
private function __blockAttributeViaProposal(&$attributes, $k) {
if (!empty($attributes[$k]['ShadowAttribute'])) {
foreach ($attributes[$k]['ShadowAttribute'] as $sa) {
if ($sa['value'] === $attributes[$k]['Attribute']['value'] &&
$sa['type'] === $attributes[$k]['Attribute']['type'] &&
$sa['category'] === $attributes[$k]['Attribute']['category'] &&
$sa['to_ids'] == 0 &&
$attribute['to_ids'] == 1
) {
continue;
}
}
} else {
unset($results[$key]['ShadowAttribute']);
}
return $results;
}
// Method gets and converts the contents of a file passed along as a base64 encoded string with the original filename into a zip archive
// The zip archive is then passed back as a base64 encoded string along with the md5 hash and a flag whether the transaction was successful
// The archive is password protected using the "infected" password
@ -3395,7 +3421,7 @@ class Attribute extends AppModel
// gets an attribute, saves it
// handles encryption, attaching to event/object, logging of issues, tag capturing
public function captureAttribute($attribute, $eventId, $user, $objectId = false, $log = false)
public function captureAttribute($attribute, $eventId, $user, $objectId = false, $log = false, $parentEvent = false)
{
if ($log == false) {
$log = ClassRegistry::init('Log');
@ -3414,7 +3440,13 @@ class Attribute extends AppModel
$attribute['distribution'] = 5;
}
}
if (!$this->save($attribute, array('fieldList' => $fieldList))) {
$params = array(
'fieldList' => $fieldList
);
if (!empty($parentEvent)) {
$params['parentEvent'] = $parentEvent;
}
if (!$this->save($attribute, $params)) {
$attribute_short = (isset($attribute['category']) ? $attribute['category'] : 'N/A') . '/' . (isset($attribute['type']) ? $attribute['type'] : 'N/A') . ' ' . (isset($attribute['value']) ? $attribute['value'] : 'N/A');
$log->create();
$log->save(array(
@ -3626,6 +3658,16 @@ class Attribute extends AppModel
$conditions = $this->buildConditions($user);
$attribute_conditions = array();
$object_conditions = array();
if (isset($params['ignore'])) {
$params['to_ids'] = array(0, 1);
$params['published'] = array(0, 1);
}
if (isset($params['searchall'])) {
$params['tags'] = $params['searchall'];
$params['eventinfo'] = $params['searchall'];
$params['value'] = $params['searchall'];
$params['comment'] = $params['searchall'];
}
$simple_params = array(
'Attribute' => array(
'value' => array('function' => 'set_filter_value'),
@ -3635,16 +3677,18 @@ class Attribute extends AppModel
'uuid' => array('function' => 'set_filter_uuid'),
'deleted' => array('function' => 'set_filter_deleted'),
'timestamp' => array('function' => 'set_filter_timestamp'),
'to_ids' => array('function' => 'set_filter_to_ids')
'to_ids' => array('function' => 'set_filter_to_ids'),
'comment' => array('function' => 'set_filter_comment')
),
'Event' => array(
'eventid' => array('function' => 'set_filter_eventid'),
'eventinfo' => array('function' => 'set_filter_eventinfo'),
'ignore' => array('function' => 'set_filter_ignore'),
'tags' => array('function' => 'set_filter_tags'),
'tag' => array('function' => 'set_filter_tags'),
'from' => array('function' => 'set_filter_timestamp'),
'to' => array('function' => 'set_filter_timestamp'),
'last' => array('function' => 'set_filter_timestamp'),
'last' => array('function' => 'set_filter_timestamp', 'pop' => true),
'timestamp' => array('function' => 'set_filter_timestamp', 'pop' => true),
'event_timestamp' => array('function' => 'set_filter_timestamp', 'pop' => true),
'publish_timestamp' => array('function' => 'set_filter_timestamp'),
'org' => array('function' => 'set_filter_org'),
'uuid' => array('function' => 'set_filter_uuid'),
@ -3661,7 +3705,8 @@ class Attribute extends AppModel
$options = array(
'filter' => $param,
'scope' => $scope,
'pop' => !empty($simple_param_scoped[$param]['pop'])
'pop' => !empty($simple_param_scoped[$param]['pop']),
'context' => 'Attribute'
);
$conditions = $this->Event->{$simple_param_scoped[$param]['function']}($params, $conditions, $options);
}

View File

@ -1297,30 +1297,45 @@ class Event extends AppModel
public function filterEventIds($user, &$params = array())
{
$conditions = $this->createEventConditions($user);
if (isset($params['ignore'])) {
$params['to_ids'] = array(0, 1);
$params['published'] = array(0, 1);
}
if (isset($params['searchall'])) {
$params['tags'] = $params['searchall'];
$params['eventinfo'] = $params['searchall'];
$params['value'] = $params['searchall'];
$params['comment'] = $params['searchall'];
}
$simple_params = array(
'Event' => array(
'eventid' => array('function' => 'set_filter_eventid', 'pop' => true),
'eventinfo' => array('function' => 'set_filter_eventinfo'),
'ignore' => array('function' => 'set_filter_ignore'),
'tags' => array('function' => 'set_filter_tags'),
'tag' => array('function' => 'set_filter_tags'),
'from' => array('function' => 'set_filter_timestamp', 'pop' => true),
'to' => array('function' => 'set_filter_timestamp', 'pop' => true),
'last' => array('function' => 'set_filter_timestamp', 'pop' => true),
'timestamp' => array('function' => 'set_filter_timestamp', 'pop' => true),
'event_timestamp' => array('function' => 'set_filter_timestamp', 'pop' => true),
'publish_timestamp' => array('function' => 'set_filter_timestamp', 'pop' => true),
'org' => array('function' => 'set_filter_org', 'pop' => true),
'uuid' => array('function' => 'set_filter_uuid', 'pop' => true),
'published' => array('function' => 'set_filter_published', 'pop' => true)
),
'Object' => array(
'object_name' => array('function' => 'set_filter_object_name')
'object_name' => array('function' => 'set_filter_object_name'),
'deleted' => array('function' => 'set_filter_deleted')
),
'Attribute' => array(
'value' => array('function' => 'set_filter_value', 'pop' => true),
'category' => array('function' => 'set_filter_simple_attribute'),
'type' => array('function' => 'set_filter_simple_attribute'),
'tags' => array('function' => 'set_filter_tags', 'pop' => true),
'uuid' => array('function' => 'set_filter_uuid')
'uuid' => array('function' => 'set_filter_uuid'),
'deleted' => array('function' => 'set_filter_deleted'),
'to_ids' => array('function' => 'set_filter_to_ids'),
'comment' => array('function' => 'set_filter_comment')
)
);
foreach ($params as $param => $paramData) {
@ -1329,7 +1344,8 @@ class Event extends AppModel
$options = array(
'filter' => $param,
'scope' => $scope,
'pop' => !empty($simple_param_scoped[$param]['pop'])
'pop' => !empty($simple_param_scoped[$param]['pop']),
'context' => 'Event'
);
if ($scope === 'Event') {
$conditions = $this->{$simple_param_scoped[$param]['function']}($params, $conditions, $options);
@ -1470,6 +1486,7 @@ class Event extends AppModel
'last',
'to_ids',
'includeAllTags',
'withAttachments',
'includeAttachments',
'event_uuid',
'distribution',
@ -2016,6 +2033,16 @@ class Event extends AppModel
return $conditions;
}
public function set_filter_eventinfo(&$params, $conditions, $options)
{
if (!empty($params['eventinfo'])) {
$params['eventinfo'] = $this->convert_filters($params['eventinfo']);
$searchall = empty($params['searchall']) ? false : $params['searchall'];
$conditions = $this->generic_add_filter($conditions, $params['eventinfo'], 'Event.info', $searchall);
}
return $conditions;
}
public function set_filter_uuid(&$params, $conditions, $options)
{
if (!empty($params['uuid'])) {
@ -2100,11 +2127,22 @@ class Event extends AppModel
{
if (!empty($params['value'])) {
$params[$options['filter']] = $this->convert_filters($params[$options['filter']]);
$conditions = $this->generic_add_filter($conditions, $params[$options['filter']], array('Attribute.value1', 'Attribute.value2'));
$searchall = empty($params['searchall']) ? false : $params['searchall'];
$conditions = $this->generic_add_filter($conditions, $params[$options['filter']], array('Attribute.value1', 'Attribute.value2'), $searchall);
}
return $conditions;
}
public function set_filter_comment(&$params, $conditions, $options)
{
if (!empty($params['comment'])) {
$params['comment'] = $this->convert_filters($params['comment']);
$searchall = empty($params['searchall']) ? false : $params['searchall'];
$conditions = $this->generic_add_filter($conditions, $params['comment'], 'Attribute.comment', $searchall);
}
return $conditions;
}
public function set_filter_timestamp(&$params, $conditions, $options)
{
if ($options['filter'] == 'from') {
@ -2126,7 +2164,10 @@ class Event extends AppModel
),
'last' => array(
'Event.publish_timestamp'
)
),
'event_timestamp' => array(
'Event.timestamp'
)
);
foreach ($filters[$options['filter']] as $f) {
$conditions = $this->Attribute->setTimestampConditions($params[$options['filter']], $conditions, $f);
@ -2987,6 +3028,10 @@ class Event extends AppModel
$this->EventTag->save($et);
}
}
$parentEvent = $this->find('first', array(
'conditions' => array('Event.id' => $this->id),
'recursive' => -1
));
if (isset($data['Event']['Attribute']) && !empty($data['Event']['Attribute'])) {
foreach ($data['Event']['Attribute'] as $k => $attribute) {
$block = false;
@ -3005,7 +3050,7 @@ class Event extends AppModel
}
}
if (!$block) {
$data['Event']['Attribute'][$k] = $this->Attribute->captureAttribute($attribute, $this->id, $user, 0, $this->Log);
$data['Event']['Attribute'][$k] = $this->Attribute->captureAttribute($attribute, $this->id, $user, 0, $this->Log, $parentEvent);
}
}
$data['Event']['Attribute'] = array_values($data['Event']['Attribute']);
@ -4232,35 +4277,39 @@ class Event extends AppModel
unset($event['Object']);
unset($event['ShadowAttribute']);
$referencedObjectFields = array('meta-category', 'name', 'uuid', 'id');
$objectReferenceCount = 0;
$referencedByArray = array();
foreach ($event['objects'] as $object) {
if (!in_array($object['objectType'], array('attribute', 'object'))) {
continue;
}
if (!empty($object['ObjectReference'])) {
if (!empty($object['ObjectReference'])) {
foreach ($object['ObjectReference'] as $reference) {
if (isset($reference['referenced_uuid'])) {
foreach ($event['objects'] as $k => $v) {
if ($v['uuid'] == $reference['referenced_uuid']) {
$temp = array();
foreach ($referencedObjectFields as $field) {
if (isset($object[$field])) {
$temp[$field] = $object[$field];
}
}
$temp['relationship_type'] = $reference['relationship_type'];
$event['objects'][$k]['referenced_by'][$object['objectType']][] = $temp;
}
}
}
}
}
}
App::uses('CustomPaginationTool', 'Tools');
if (isset($reference['referenced_uuid'])) {
$referencedByArray[$reference['referenced_uuid']][$object['objectType']][] = array(
'meta-category' => $object['meta-category'],
'name' => $object['name'],
'uuid' => $object['uuid'],
'id' => $object['id'],
'object_type' => $object['objectType']
);
}
}
}
}
App::uses('CustomPaginationTool', 'Tools');
$customPagination = new CustomPaginationTool();
if ($all) {
$passedArgs['page'] = 0;
}
$params = $customPagination->applyRulesOnArray($event['objects'], $passedArgs, 'events', 'category');
foreach ($event['objects'] as $k => $object) {
if (isset($referencedByArray[$object['uuid']])) {
foreach ($referencedByArray[$object['uuid']] as $objectType => $references) {
$event['objects'][$k]['referenced_by'][$objectType] = $references;
}
}
}
$params['total_elements'] = count($event['objects']);
$event['Event']['warnings'] = $eventWarnings;
return $params;