diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index af6773409..a3ff1ee00 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -2697,18 +2697,9 @@ class EventsController extends AppController } } $exportType = $eventid; - if ($from) { - $from = $this->Event->dateFieldCheck($from); - } - if ($to) { - $to = $this->Event->dateFieldCheck($to); - } if ($tags) { $tags = str_replace(';', ':', $tags); } - if ($last) { - $last = $this->Event->resolveTimeDelta($last); - } $list = array(); if ($key != 'download') { // check if the key is valid -> search for users based on key @@ -2750,30 +2741,10 @@ class EventsController extends AppController $list[] = $attribute['Attribute']['id']; } $events = array($eventid); - } elseif ($eventid === false) { - $events = $this->Event->fetchEventIds($this->Auth->user(), $from, $to, $last, true); - if (empty($events)) { - $events = array(0 => -1); - } - } else { - $events = array($eventid); + } else if ($eventid !== 'all') { + $events = $eventid; } $final = array(); - $this->loadModel('Whitelist'); - if ($tags) { - $args = $this->Event->Attribute->dissectArgs($tags); - $tagArray = $this->Event->EventTag->Tag->fetchEventTagIds($args[0], $args[1]); - if (!empty($tagArray[0])) { - $events = array_intersect($events, $tagArray[0]); - } - if (!empty($tagArray[1])) { - foreach ($events as $k => $eventid) { - if (in_array($eventid, $tagArray[1])) { - unset($events[$k]); - } - } - } - } $requested_attributes = array('uuid', 'event_id', 'category', 'type', 'value', 'comment', 'to_ids', 'timestamp', 'object_relation'); $requested_obj_attributes = array('uuid', 'name', 'meta-category'); @@ -2798,46 +2769,52 @@ class EventsController extends AppController if (isset($data['request']['obj_attributes'])) { $requested_obj_attributes = $data['request']['obj_attributes']; } - if (isset($events)) { - $events = array_chunk($events, 100); - foreach ($events as $k => $eventid) { - $attributes = $this->Event->csv($user, $eventid, $ignore, $list, false, $category, $type, $includeContext, false, false, false, $enforceWarninglist, $value, $timestamp); - $attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true); - foreach ($attributes as $attribute) { - $line1 = ''; - $line2 = ''; - foreach ($requested_attributes as $requested_attribute) { - $line1 .= $attribute['Attribute'][$requested_attribute] . ','; - } - $line1 = rtrim($line1, ","); - foreach ($requested_obj_attributes as $requested_obj_attribute) { - $line2 .= $attribute['Object'][$requested_obj_attribute] . ','; - } - $line2 = rtrim($line2, ","); - $line = $line1 . ',' . $line2; - $line = rtrim($line, ","); - if ($includeContext) { - foreach ($this->Event->csv_event_context_fields_to_fetch as $header => $field) { - if ($field['object']) { - $line .= ',' . $attribute['Event'][$field['object']][$field['var']]; - } else { - $line .= ',' . str_replace(array("\n","\t","\r"), " ", $attribute['Event'][$field['var']]); - } + $possibleParams = array( + 'ignore', 'list', 'category', 'type', 'includeContext', + 'enforceWarninglist', 'value', 'timestamp', 'tags', + 'last', 'from', 'to' + ); + $params = array(); + if (!empty($events)) { + $params = array( + 'eventid' => $events + ); + } + foreach ($possibleParams as $possibleParam) { + $params[$possibleParam] = ${$possibleParam}; + } + $params['limit'] = 1000; + $params['page'] = 1; + $i = 0; + $continue = true; + while ($continue) { + $attributes = $this->Event->csv($user, $params, false, $continue); + $params['page'] += 1; + foreach ($attributes as $attribute) { + $line1 = ''; + $line2 = ''; + foreach ($requested_attributes as $requested_attribute) { + $line1 .= $attribute['Attribute'][$requested_attribute] . ','; + } + $line1 = rtrim($line1, ","); + foreach ($requested_obj_attributes as $requested_obj_attribute) { + $line2 .= $attribute['Object'][$requested_obj_attribute] . ','; + } + $line2 = rtrim($line2, ","); + $line = $line1 . ',' . $line2; + $line = rtrim($line, ","); + if ($includeContext) { + foreach ($this->Event->csv_event_context_fields_to_fetch as $header => $field) { + if ($field['object']) { + $line .= ',' . $attribute['Event'][$field['object']][$field['var']]; + } else { + $line .= ',' . str_replace(array("\n","\t","\r"), " ", $attribute['Event'][$field['var']]); } } - $final[] = $line; } + $final[] = $line; } } - $this->response->type('csv'); // set the content type - if (!$exportType) { - $filename = "misp.all_attributes.csv"; - } elseif ($exportType === 'search') { - $filename = "misp.search_result.csv"; - } else { - $filename = "misp.event_" . $exportType . ".csv"; - } - $this->layout = 'text/default'; if (!empty($requested_obj_attributes)) { array_walk($requested_obj_attributes, function (&$value, $key) { $value = 'object-'.$value; @@ -2857,6 +2834,15 @@ class EventsController extends AppController $final = array_merge(array($headers), $final); $final = implode(PHP_EOL, $final); $final .= PHP_EOL; + $this->response->type('csv'); // set the content type + if (!$exportType) { + $filename = "misp.all_attributes.csv"; + } elseif ($exportType === 'search') { + $filename = "misp.search_result.csv"; + } else { + $filename = "misp.event_" . $exportType . ".csv"; + } + $this->layout = 'text/default'; return $this->RestResponse->viewData($final, 'csv', false, true, $filename); } diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index b6da831fd..1f84d47d5 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -2033,6 +2033,65 @@ class Attribute extends AppModel return $rules; } + public function set_filter_tags($params, $conditions, $scope = 'all') { + if (empty($params['tags'])) { + return $conditions; + } + $tag = ClassRegistry::init('Tag'); + $args = $this->dissectArgs($params['tags']); + $tagArray = $tag->fetchTagIds($args[0], $args[1]); + $temp = array(); + if (!empty($tagArray[0])) { + if ($scope == 'all' || $scope == 'Event') { + $options = array( + 'conditions' => array( + 'tag_id' => $tagArray[0] + ), + 'fields' => array( + 'event_id' + ) + ); + $temp = array_merge($temp, $this->subQueryGenerator($tag->EventTag, $options, 'Attribute.event_id')); + } + $options = array( + 'conditions' => array( + 'tag_id' => $tagArray[0] + ), + 'fields' => array( + 'attribute_id' + ) + ); + $temp = array_merge($temp, $this->subQueryGenerator($tag->AttributeTag, $options, 'Attribute.id')); + $conditions['AND'][] = array('OR' => $temp); + } + $temp = array(); + if (!empty($tagArray[1])) { + if ($scope == 'all' || $scope == 'Event') { + $options = array( + 'conditions' => array( + 'tag_id' => $tagArray[1] + ), + 'fields' => array( + 'event_id' + ) + ); + $conditions['AND'][] = array_merge($temp, $this->subQueryGenerator($tag->EventTag, $options, 'Attribute.event_id', 1)); + } + if ($scope == 'all' || $scope == 'Attribute') { + $options = array( + 'conditions' => array( + 'tag_id' => $tagArray[1] + ), + 'fields' => array( + 'attribute_id' + ) + ); + $conditions['AND'][] = array_merge($temp, $this->subQueryGenerator($tag->AttributeTag, $options, 'Attribute.id', 1)); + } + } + return $conditions; + } + public function text($user, $type, $tags = false, $eventId = false, $allowNonIDS = false, $from = false, $to = false, $last = false, $enforceWarninglist = false, $allowNotPublished = false) { //permissions are taken care of in fetchAttributes() @@ -2059,56 +2118,7 @@ class Attribute extends AppModel if ($eventId !== false) { $conditions['AND'][] = array('Event.id' => $eventId); } elseif ($tags !== false) { - // If we sent any tags along, load the associated tag names for each attribute - $tag = ClassRegistry::init('Tag'); - $args = $this->dissectArgs($tags); - $tagArray = $tag->fetchTagIds($args[0], $args[1]); - $temp = array(); - if (!empty($tagArray[0])) { - $options = array( - 'conditions' => array( - 'tag_id' => $tagArray[0] - ), - 'fields' => array( - 'event_id' - ) - ); - $temp = array_merge($temp, $this->subQueryGenerator($tag->EventTag, $options, 'Attribute.event_id')); - $options = array( - 'conditions' => array( - 'tag_id' => $tagArray[0] - ), - 'fields' => array( - 'attribute_id' - ) - ); - $temp = array_merge($temp, $this->subQueryGenerator($tag->AttributeTag, $options, 'Attribute.id')); - $temp2 = array('OR' => $temp); - $conditions['AND'][] = $temp2; - } - $temp = array(); - if (!empty($tagArray[1])) { - $options = array( - 'conditions' => array( - 'tag_id' => $tagArray[1] - ), - 'fields' => array( - 'event_id' - ) - ); - $temp = array_merge($temp, $this->subQueryGenerator($tag->EventTag, $options, 'Attribute.event_id', 1)); - $options = array( - 'conditions' => array( - 'tag_id' => $tagArray[1] - ), - 'fields' => array( - 'attribute_id' - ) - ); - $temp = array_merge($temp, $this->subQueryGenerator($tag->AttributeTag, $options, 'Attribute.id', 1)); - $temp2 = array('AND' => $temp); - $conditions['AND'][] = $temp2; - } + $conditions = $this->set_filter_tags(array('tags' => $tags), $conditions); } $attributes = $this->fetchAttributes($user, array( 'conditions' => $conditions, @@ -2690,7 +2700,7 @@ class Attribute extends AppModel // conditions // order // group - public function fetchAttributes($user, $options = array()) + public function fetchAttributes($user, $options = array(), &$continue = true) { $params = array( 'conditions' => $this->buildConditions($user), @@ -2787,7 +2797,6 @@ class Attribute extends AppModel $this->Warninglist = ClassRegistry::init('Warninglist'); $warninglists = $this->Warninglist->fetchForEventView(); } - if (empty($params['limit'])) { $loopLimit = 100000; $loop = true; @@ -2798,7 +2807,6 @@ class Attribute extends AppModel $pagesToFetch = 1; } $attributes = array(); - $continue = true; while ($continue) { if ($loop) { $params['page'] = $params['page'] + 1; @@ -2812,7 +2820,7 @@ class Attribute extends AppModel $results = $this->find('all', $params); // return false if we're paginating if (isset($options['limit']) && empty($results)) { - return false; + return array(); } $results = array_values($results); $proposals_block_attributes = Configure::read('MISP.proposals_block_attributes'); diff --git a/app/Model/Event.php b/app/Model/Event.php index 2ea48587d..bef6148a7 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -1242,8 +1242,8 @@ class Event extends AppModel public function downloadProposalsFromServer($uuidList, $server, $HttpSocket = null) { $url = $server['Server']['url']; - $HttpSocket = $this->__setupHttpSocket($server, $HttpSocket); - $request = $this->__setupPushRequest($server); + $HttpSocket = $this->setupHttpSocket($server, $HttpSocket); + $request = $this->setupSyncRequest($server); $uri = $url . '/shadow_attributes/getProposalsByUuidList'; $response = $HttpSocket->post($uri, json_encode($uuidList), $request); if ($response->isOk()) { @@ -1882,90 +1882,133 @@ class Event extends AppModel $field = '"' . $field . '"'; } - public function csv($user, $eventid=false, $ignore=false, $attributeIDList = array(), $tags = false, $category = false, $type = false, $includeContext = false, $from = false, $to = false, $last = false, $enforceWarninglist = false, $value = false, $timestamp = false) + public function set_filter_eventid($params, $conditions, $filter) { + if (!empty($params['eventid']) && $params['eventid'] !== 'all') { + $conditions['AND'][] = array('Event.id' => $params['eventid']); + } + return $conditions; + } + + public function set_filter_ignore($params, $conditions, $filter) { + if (empty($params['ignore'])) { + $conditions['AND']['Event.published'] = 1; + $conditions['AND']['Attribute.to_ids'] = 1; + } + return $conditions; + } + + public function set_filter_tags($params, $conditions, $filter) { + if (!empty($params['tags'])) { + $conditions = $this->Attribute->set_filter_tags($params, $conditions, 'Event'); + } + return $conditions; + } + + public function set_filter_simple_attribute($params, $conditions, $filter) { + if (!empty($params[$filter])) { + $conditions['AND']['Attribute.' . $filter] = $params[$filter]; + } + return $conditions; + } + + public function set_filter_attribute_id($params, $conditions, $filter) { + if (!empty($params[$filter])) { + $conditions['AND']['Attribute.id'] = $params[$filter]; + } + return $conditions; + } + + public function set_filter_value($params, $conditions, $filter) { - $this->recursive = -1; - $conditions = array(); - // If we are not in the search result csv download function then we need to check what can be downloaded. CSV downloads are already filtered by the search function. - if ($eventid !== 'search') { - if ($from) { - $conditions['AND']['Event.date >='] = $from; - } - if ($to) { - $conditions['AND']['Event.date <='] = $to; - } - if ($last) { - $conditions['AND']['Event.publish_timestamp >='] = $last; - } - if ($timestamp) { - $conditions['AND']['Attribute.timestamp >='] = $timestamp; - $conditions['AND']['Event.timestamp >='] = $timestamp; - } - // This is for both single event downloads and for full downloads. Org has to be the same as the user's or distribution not org only - if the user is no siteadmin - if ($ignore == false) { - $conditions['AND']['Event.published'] = 1; - } + if (!empty($params['value'])) { + $temp = array( + 'OR' => array( + 'Attribute.value1' => $value, + 'Attribute.value2' => $value + ) + ); + $conditions['AND'][] = $temp; + } + return $conditions; + } - // If we sent any tags along, load the associated tag names for each attribute - if ($tags) { - $tag = ClassRegistry::init('Tag'); - $args = $this->Attribute->dissectArgs($tags); - $tagArray = $tag->fetchEventTagIds($args[0], $args[1]); - $temp = array(); - foreach ($tagArray[0] as $accepted) { - $temp['OR'][] = array('Event.id' => $accepted); - } - if (!empty($temp)) { - $conditions['AND'][] = $temp; - } - $temp = array(); - foreach ($tagArray[1] as $rejected) { - $temp['AND'][] = array('Event.id !=' => $rejected); - } - if (!empty($temp)) { - $conditions['AND'][] = $temp; - } - } - // if we're downloading a single event, set it as a condition - if ($eventid) { - $conditions['AND'][] = array('Event.id' => $eventid); - } - - //restricting to non-private or same org if the user is not a site-admin. - if (!$ignore) { - $conditions['AND']['Attribute.to_ids'] = 1; - } - if ($type) { - $conditions['AND']['Attribute.type'] = $type; - } - if ($category) { - $conditions['AND']['Attribute.category'] = $category; - } - if ($value) { - $temp = array( - 'OR' => array( - 'Attribute.value1' => $value, - 'Attribute.value2' => $value - ) - ); - - $conditions['AND'][] = $temp; + public function set_filter_timestamp($params, $conditions, $filter) + { + if ($filter == 'from') { + $conditions['AND']['Event.date >='] = $from; + } else if ($filter == 'to') { + $conditions['AND']['Event.date <='] = $from; + } else { + $filters = array( + 'timestamp' => array( + 'Event.timestamp', + 'Attribute.timestamp' + ), + 'publish_timestamp' => array( + 'Event.publish_timestamp' + ), + 'last' => array( + 'Event.publish_timestamp' + ) + ); + foreach ($filters[$filter] as $f) { + $conditions = $this->Attribute->setTimestampConditions($params[$filter], $conditions, $f); } } + return $conditions; + } - if ($eventid === 'search') { - foreach ($attributeIDList as $aID) { + public function csv($user, $params, $search = false, &$continue = true) + { + $conditions = array(); + $simple_params = array( + 'eventid' => array('function' => 'set_filter_eventid'), + 'ignore' => array('function' => 'set_filter_ignore'), + 'tags' => array('function' => 'set_filter_tags'), + 'category' => array('function' => 'set_filter_simple_attribute'), + 'type' => array('function' => 'set_filter_simple_attribute'), + 'from' => array('function' => 'set_filter_timestamp'), + 'to' => array('function' => 'set_filter_timestamp'), + 'last' => array('function' => 'set_filter_timestamp'), + 'value' => array('function' => 'set_filter_value'), + 'timestamp' => array('function' => 'set_filter_timestamp'), + 'attributeIDList' => array('functon' => 'set_filter_attribute_id') + ); + foreach ($params as $param => $paramData) { + if (isset($simple_params[$param]) && $params[$param] !== false) { + $conditions = $this->{$simple_params[$param]['function']}($params, $conditions, $param); + } + } + //$attributeIDList = array(), $includeContext = false, $enforceWarninglist = false + $this->recursive = -1; + if (!empty($params['eventid']) && $params['eventid'] === 'search') { + foreach ($params['attributeIDList'] as $aID) { $conditions['AND']['OR'][] = array('Attribute.id' => $aID); } } - $params = array( + $csv_params = array( 'conditions' => $conditions, //array of conditions 'fields' => array('Attribute.event_id', 'Attribute.distribution', 'Attribute.category', 'Attribute.type', 'Attribute.value', 'Attribute.comment', 'Attribute.uuid', 'Attribute.to_ids', 'Attribute.timestamp', 'Attribute.id', 'Attribute.object_relation'), 'order' => array('Attribute.uuid ASC'), - 'enforceWarninglist' => $enforceWarninglist, 'flatten' => true ); + // copy over the parameters that have to deal with pagination or additional functionality to be executed + $control_params = array( + 'limit', 'page', 'enforceWarninglist' + ); + foreach ($control_params as $control_param) { + if (!empty($params[$control_param])) { + $csv_params[$control_param] = $params[$control_param]; + } + } + $csv_params = $this->__appendIncludesCSV($csv_params, !empty($params['includeContext'])); + $attributes = $this->Attribute->fetchAttributes($user, $csv_params, $continue); + $attributes = $this->__sanitiseCSVAttributes($attributes, !empty($params['includeContext']), !empty($params['ignore'])); + return $attributes; + } + + private function __appendIncludesCSV($params, $includeContext) { if ($includeContext) { $params['contain'] = array( 'Event' => array( @@ -1985,9 +2028,14 @@ class Event extends AppModel ); } $params['contain']['Object'] = array('fields' => array('id', 'uuid', 'name', 'meta-category')); - $attributes = $this->Attribute->fetchAttributes($user, $params); - if (empty($attributes)) { - return array(); + return $params; + } + + private function __sanitiseCSVAttributes($attributes, $includeContext, $ignore) + { + if (!empty($ignore)) { + $this->Log = ClassRegistry::init('Log'); + $attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true); } foreach ($attributes as &$attribute) { $this->__escapeCSVField($attribute['Attribute']['value']);