new: [refactor] CSV api refactor

- performance gains
- first step in unifying all APIs
- moved the CSV data lookup into fetchattributes
- internal pagination is now more clever with a watchdog flag that can prevent unneeded executions by whatever calls fetchattributes
pull/3551/head
iglocska 2018-08-05 19:10:52 +02:00
parent 34ba484b06
commit 76ede22308
3 changed files with 237 additions and 195 deletions

View File

@ -2697,18 +2697,9 @@ class EventsController extends AppController
}
}
$exportType = $eventid;
if ($from) {
$from = $this->Event->dateFieldCheck($from);
}
if ($to) {
$to = $this->Event->dateFieldCheck($to);
}
if ($tags) {
$tags = str_replace(';', ':', $tags);
}
if ($last) {
$last = $this->Event->resolveTimeDelta($last);
}
$list = array();
if ($key != 'download') {
// check if the key is valid -> search for users based on key
@ -2750,30 +2741,10 @@ class EventsController extends AppController
$list[] = $attribute['Attribute']['id'];
}
$events = array($eventid);
} elseif ($eventid === false) {
$events = $this->Event->fetchEventIds($this->Auth->user(), $from, $to, $last, true);
if (empty($events)) {
$events = array(0 => -1);
}
} else {
$events = array($eventid);
} else if ($eventid !== 'all') {
$events = $eventid;
}
$final = array();
$this->loadModel('Whitelist');
if ($tags) {
$args = $this->Event->Attribute->dissectArgs($tags);
$tagArray = $this->Event->EventTag->Tag->fetchEventTagIds($args[0], $args[1]);
if (!empty($tagArray[0])) {
$events = array_intersect($events, $tagArray[0]);
}
if (!empty($tagArray[1])) {
foreach ($events as $k => $eventid) {
if (in_array($eventid, $tagArray[1])) {
unset($events[$k]);
}
}
}
}
$requested_attributes = array('uuid', 'event_id', 'category', 'type',
'value', 'comment', 'to_ids', 'timestamp', 'object_relation');
$requested_obj_attributes = array('uuid', 'name', 'meta-category');
@ -2798,46 +2769,52 @@ class EventsController extends AppController
if (isset($data['request']['obj_attributes'])) {
$requested_obj_attributes = $data['request']['obj_attributes'];
}
if (isset($events)) {
$events = array_chunk($events, 100);
foreach ($events as $k => $eventid) {
$attributes = $this->Event->csv($user, $eventid, $ignore, $list, false, $category, $type, $includeContext, false, false, false, $enforceWarninglist, $value, $timestamp);
$attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true);
foreach ($attributes as $attribute) {
$line1 = '';
$line2 = '';
foreach ($requested_attributes as $requested_attribute) {
$line1 .= $attribute['Attribute'][$requested_attribute] . ',';
}
$line1 = rtrim($line1, ",");
foreach ($requested_obj_attributes as $requested_obj_attribute) {
$line2 .= $attribute['Object'][$requested_obj_attribute] . ',';
}
$line2 = rtrim($line2, ",");
$line = $line1 . ',' . $line2;
$line = rtrim($line, ",");
if ($includeContext) {
foreach ($this->Event->csv_event_context_fields_to_fetch as $header => $field) {
if ($field['object']) {
$line .= ',' . $attribute['Event'][$field['object']][$field['var']];
} else {
$line .= ',' . str_replace(array("\n","\t","\r"), " ", $attribute['Event'][$field['var']]);
}
$possibleParams = array(
'ignore', 'list', 'category', 'type', 'includeContext',
'enforceWarninglist', 'value', 'timestamp', 'tags',
'last', 'from', 'to'
);
$params = array();
if (!empty($events)) {
$params = array(
'eventid' => $events
);
}
foreach ($possibleParams as $possibleParam) {
$params[$possibleParam] = ${$possibleParam};
}
$params['limit'] = 1000;
$params['page'] = 1;
$i = 0;
$continue = true;
while ($continue) {
$attributes = $this->Event->csv($user, $params, false, $continue);
$params['page'] += 1;
foreach ($attributes as $attribute) {
$line1 = '';
$line2 = '';
foreach ($requested_attributes as $requested_attribute) {
$line1 .= $attribute['Attribute'][$requested_attribute] . ',';
}
$line1 = rtrim($line1, ",");
foreach ($requested_obj_attributes as $requested_obj_attribute) {
$line2 .= $attribute['Object'][$requested_obj_attribute] . ',';
}
$line2 = rtrim($line2, ",");
$line = $line1 . ',' . $line2;
$line = rtrim($line, ",");
if ($includeContext) {
foreach ($this->Event->csv_event_context_fields_to_fetch as $header => $field) {
if ($field['object']) {
$line .= ',' . $attribute['Event'][$field['object']][$field['var']];
} else {
$line .= ',' . str_replace(array("\n","\t","\r"), " ", $attribute['Event'][$field['var']]);
}
}
$final[] = $line;
}
$final[] = $line;
}
}
$this->response->type('csv'); // set the content type
if (!$exportType) {
$filename = "misp.all_attributes.csv";
} elseif ($exportType === 'search') {
$filename = "misp.search_result.csv";
} else {
$filename = "misp.event_" . $exportType . ".csv";
}
$this->layout = 'text/default';
if (!empty($requested_obj_attributes)) {
array_walk($requested_obj_attributes, function (&$value, $key) {
$value = 'object-'.$value;
@ -2857,6 +2834,15 @@ class EventsController extends AppController
$final = array_merge(array($headers), $final);
$final = implode(PHP_EOL, $final);
$final .= PHP_EOL;
$this->response->type('csv'); // set the content type
if (!$exportType) {
$filename = "misp.all_attributes.csv";
} elseif ($exportType === 'search') {
$filename = "misp.search_result.csv";
} else {
$filename = "misp.event_" . $exportType . ".csv";
}
$this->layout = 'text/default';
return $this->RestResponse->viewData($final, 'csv', false, true, $filename);
}

View File

@ -2033,6 +2033,65 @@ class Attribute extends AppModel
return $rules;
}
public function set_filter_tags($params, $conditions, $scope = 'all') {
if (empty($params['tags'])) {
return $conditions;
}
$tag = ClassRegistry::init('Tag');
$args = $this->dissectArgs($params['tags']);
$tagArray = $tag->fetchTagIds($args[0], $args[1]);
$temp = array();
if (!empty($tagArray[0])) {
if ($scope == 'all' || $scope == 'Event') {
$options = array(
'conditions' => array(
'tag_id' => $tagArray[0]
),
'fields' => array(
'event_id'
)
);
$temp = array_merge($temp, $this->subQueryGenerator($tag->EventTag, $options, 'Attribute.event_id'));
}
$options = array(
'conditions' => array(
'tag_id' => $tagArray[0]
),
'fields' => array(
'attribute_id'
)
);
$temp = array_merge($temp, $this->subQueryGenerator($tag->AttributeTag, $options, 'Attribute.id'));
$conditions['AND'][] = array('OR' => $temp);
}
$temp = array();
if (!empty($tagArray[1])) {
if ($scope == 'all' || $scope == 'Event') {
$options = array(
'conditions' => array(
'tag_id' => $tagArray[1]
),
'fields' => array(
'event_id'
)
);
$conditions['AND'][] = array_merge($temp, $this->subQueryGenerator($tag->EventTag, $options, 'Attribute.event_id', 1));
}
if ($scope == 'all' || $scope == 'Attribute') {
$options = array(
'conditions' => array(
'tag_id' => $tagArray[1]
),
'fields' => array(
'attribute_id'
)
);
$conditions['AND'][] = array_merge($temp, $this->subQueryGenerator($tag->AttributeTag, $options, 'Attribute.id', 1));
}
}
return $conditions;
}
public function text($user, $type, $tags = false, $eventId = false, $allowNonIDS = false, $from = false, $to = false, $last = false, $enforceWarninglist = false, $allowNotPublished = false)
{
//permissions are taken care of in fetchAttributes()
@ -2059,56 +2118,7 @@ class Attribute extends AppModel
if ($eventId !== false) {
$conditions['AND'][] = array('Event.id' => $eventId);
} elseif ($tags !== false) {
// If we sent any tags along, load the associated tag names for each attribute
$tag = ClassRegistry::init('Tag');
$args = $this->dissectArgs($tags);
$tagArray = $tag->fetchTagIds($args[0], $args[1]);
$temp = array();
if (!empty($tagArray[0])) {
$options = array(
'conditions' => array(
'tag_id' => $tagArray[0]
),
'fields' => array(
'event_id'
)
);
$temp = array_merge($temp, $this->subQueryGenerator($tag->EventTag, $options, 'Attribute.event_id'));
$options = array(
'conditions' => array(
'tag_id' => $tagArray[0]
),
'fields' => array(
'attribute_id'
)
);
$temp = array_merge($temp, $this->subQueryGenerator($tag->AttributeTag, $options, 'Attribute.id'));
$temp2 = array('OR' => $temp);
$conditions['AND'][] = $temp2;
}
$temp = array();
if (!empty($tagArray[1])) {
$options = array(
'conditions' => array(
'tag_id' => $tagArray[1]
),
'fields' => array(
'event_id'
)
);
$temp = array_merge($temp, $this->subQueryGenerator($tag->EventTag, $options, 'Attribute.event_id', 1));
$options = array(
'conditions' => array(
'tag_id' => $tagArray[1]
),
'fields' => array(
'attribute_id'
)
);
$temp = array_merge($temp, $this->subQueryGenerator($tag->AttributeTag, $options, 'Attribute.id', 1));
$temp2 = array('AND' => $temp);
$conditions['AND'][] = $temp2;
}
$conditions = $this->set_filter_tags(array('tags' => $tags), $conditions);
}
$attributes = $this->fetchAttributes($user, array(
'conditions' => $conditions,
@ -2690,7 +2700,7 @@ class Attribute extends AppModel
// conditions
// order
// group
public function fetchAttributes($user, $options = array())
public function fetchAttributes($user, $options = array(), &$continue = true)
{
$params = array(
'conditions' => $this->buildConditions($user),
@ -2787,7 +2797,6 @@ class Attribute extends AppModel
$this->Warninglist = ClassRegistry::init('Warninglist');
$warninglists = $this->Warninglist->fetchForEventView();
}
if (empty($params['limit'])) {
$loopLimit = 100000;
$loop = true;
@ -2798,7 +2807,6 @@ class Attribute extends AppModel
$pagesToFetch = 1;
}
$attributes = array();
$continue = true;
while ($continue) {
if ($loop) {
$params['page'] = $params['page'] + 1;
@ -2812,7 +2820,7 @@ class Attribute extends AppModel
$results = $this->find('all', $params);
// return false if we're paginating
if (isset($options['limit']) && empty($results)) {
return false;
return array();
}
$results = array_values($results);
$proposals_block_attributes = Configure::read('MISP.proposals_block_attributes');

View File

@ -1242,8 +1242,8 @@ class Event extends AppModel
public function downloadProposalsFromServer($uuidList, $server, $HttpSocket = null)
{
$url = $server['Server']['url'];
$HttpSocket = $this->__setupHttpSocket($server, $HttpSocket);
$request = $this->__setupPushRequest($server);
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
$uri = $url . '/shadow_attributes/getProposalsByUuidList';
$response = $HttpSocket->post($uri, json_encode($uuidList), $request);
if ($response->isOk()) {
@ -1882,90 +1882,133 @@ class Event extends AppModel
$field = '"' . $field . '"';
}
public function csv($user, $eventid=false, $ignore=false, $attributeIDList = array(), $tags = false, $category = false, $type = false, $includeContext = false, $from = false, $to = false, $last = false, $enforceWarninglist = false, $value = false, $timestamp = false)
public function set_filter_eventid($params, $conditions, $filter) {
if (!empty($params['eventid']) && $params['eventid'] !== 'all') {
$conditions['AND'][] = array('Event.id' => $params['eventid']);
}
return $conditions;
}
public function set_filter_ignore($params, $conditions, $filter) {
if (empty($params['ignore'])) {
$conditions['AND']['Event.published'] = 1;
$conditions['AND']['Attribute.to_ids'] = 1;
}
return $conditions;
}
public function set_filter_tags($params, $conditions, $filter) {
if (!empty($params['tags'])) {
$conditions = $this->Attribute->set_filter_tags($params, $conditions, 'Event');
}
return $conditions;
}
public function set_filter_simple_attribute($params, $conditions, $filter) {
if (!empty($params[$filter])) {
$conditions['AND']['Attribute.' . $filter] = $params[$filter];
}
return $conditions;
}
public function set_filter_attribute_id($params, $conditions, $filter) {
if (!empty($params[$filter])) {
$conditions['AND']['Attribute.id'] = $params[$filter];
}
return $conditions;
}
public function set_filter_value($params, $conditions, $filter)
{
$this->recursive = -1;
$conditions = array();
// If we are not in the search result csv download function then we need to check what can be downloaded. CSV downloads are already filtered by the search function.
if ($eventid !== 'search') {
if ($from) {
$conditions['AND']['Event.date >='] = $from;
}
if ($to) {
$conditions['AND']['Event.date <='] = $to;
}
if ($last) {
$conditions['AND']['Event.publish_timestamp >='] = $last;
}
if ($timestamp) {
$conditions['AND']['Attribute.timestamp >='] = $timestamp;
$conditions['AND']['Event.timestamp >='] = $timestamp;
}
// This is for both single event downloads and for full downloads. Org has to be the same as the user's or distribution not org only - if the user is no siteadmin
if ($ignore == false) {
$conditions['AND']['Event.published'] = 1;
}
if (!empty($params['value'])) {
$temp = array(
'OR' => array(
'Attribute.value1' => $value,
'Attribute.value2' => $value
)
);
$conditions['AND'][] = $temp;
}
return $conditions;
}
// If we sent any tags along, load the associated tag names for each attribute
if ($tags) {
$tag = ClassRegistry::init('Tag');
$args = $this->Attribute->dissectArgs($tags);
$tagArray = $tag->fetchEventTagIds($args[0], $args[1]);
$temp = array();
foreach ($tagArray[0] as $accepted) {
$temp['OR'][] = array('Event.id' => $accepted);
}
if (!empty($temp)) {
$conditions['AND'][] = $temp;
}
$temp = array();
foreach ($tagArray[1] as $rejected) {
$temp['AND'][] = array('Event.id !=' => $rejected);
}
if (!empty($temp)) {
$conditions['AND'][] = $temp;
}
}
// if we're downloading a single event, set it as a condition
if ($eventid) {
$conditions['AND'][] = array('Event.id' => $eventid);
}
//restricting to non-private or same org if the user is not a site-admin.
if (!$ignore) {
$conditions['AND']['Attribute.to_ids'] = 1;
}
if ($type) {
$conditions['AND']['Attribute.type'] = $type;
}
if ($category) {
$conditions['AND']['Attribute.category'] = $category;
}
if ($value) {
$temp = array(
'OR' => array(
'Attribute.value1' => $value,
'Attribute.value2' => $value
)
);
$conditions['AND'][] = $temp;
public function set_filter_timestamp($params, $conditions, $filter)
{
if ($filter == 'from') {
$conditions['AND']['Event.date >='] = $from;
} else if ($filter == 'to') {
$conditions['AND']['Event.date <='] = $from;
} else {
$filters = array(
'timestamp' => array(
'Event.timestamp',
'Attribute.timestamp'
),
'publish_timestamp' => array(
'Event.publish_timestamp'
),
'last' => array(
'Event.publish_timestamp'
)
);
foreach ($filters[$filter] as $f) {
$conditions = $this->Attribute->setTimestampConditions($params[$filter], $conditions, $f);
}
}
return $conditions;
}
if ($eventid === 'search') {
foreach ($attributeIDList as $aID) {
public function csv($user, $params, $search = false, &$continue = true)
{
$conditions = array();
$simple_params = array(
'eventid' => array('function' => 'set_filter_eventid'),
'ignore' => array('function' => 'set_filter_ignore'),
'tags' => array('function' => 'set_filter_tags'),
'category' => array('function' => 'set_filter_simple_attribute'),
'type' => array('function' => 'set_filter_simple_attribute'),
'from' => array('function' => 'set_filter_timestamp'),
'to' => array('function' => 'set_filter_timestamp'),
'last' => array('function' => 'set_filter_timestamp'),
'value' => array('function' => 'set_filter_value'),
'timestamp' => array('function' => 'set_filter_timestamp'),
'attributeIDList' => array('functon' => 'set_filter_attribute_id')
);
foreach ($params as $param => $paramData) {
if (isset($simple_params[$param]) && $params[$param] !== false) {
$conditions = $this->{$simple_params[$param]['function']}($params, $conditions, $param);
}
}
//$attributeIDList = array(), $includeContext = false, $enforceWarninglist = false
$this->recursive = -1;
if (!empty($params['eventid']) && $params['eventid'] === 'search') {
foreach ($params['attributeIDList'] as $aID) {
$conditions['AND']['OR'][] = array('Attribute.id' => $aID);
}
}
$params = array(
$csv_params = array(
'conditions' => $conditions, //array of conditions
'fields' => array('Attribute.event_id', 'Attribute.distribution', 'Attribute.category', 'Attribute.type', 'Attribute.value', 'Attribute.comment', 'Attribute.uuid', 'Attribute.to_ids', 'Attribute.timestamp', 'Attribute.id', 'Attribute.object_relation'),
'order' => array('Attribute.uuid ASC'),
'enforceWarninglist' => $enforceWarninglist,
'flatten' => true
);
// copy over the parameters that have to deal with pagination or additional functionality to be executed
$control_params = array(
'limit', 'page', 'enforceWarninglist'
);
foreach ($control_params as $control_param) {
if (!empty($params[$control_param])) {
$csv_params[$control_param] = $params[$control_param];
}
}
$csv_params = $this->__appendIncludesCSV($csv_params, !empty($params['includeContext']));
$attributes = $this->Attribute->fetchAttributes($user, $csv_params, $continue);
$attributes = $this->__sanitiseCSVAttributes($attributes, !empty($params['includeContext']), !empty($params['ignore']));
return $attributes;
}
private function __appendIncludesCSV($params, $includeContext) {
if ($includeContext) {
$params['contain'] = array(
'Event' => array(
@ -1985,9 +2028,14 @@ class Event extends AppModel
);
}
$params['contain']['Object'] = array('fields' => array('id', 'uuid', 'name', 'meta-category'));
$attributes = $this->Attribute->fetchAttributes($user, $params);
if (empty($attributes)) {
return array();
return $params;
}
private function __sanitiseCSVAttributes($attributes, $includeContext, $ignore)
{
if (!empty($ignore)) {
$this->Log = ClassRegistry::init('Log');
$attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true);
}
foreach ($attributes as &$attribute) {
$this->__escapeCSVField($attribute['Attribute']['value']);