From 5c1a5fad751c00bfc4a18b2690b864fd1699ece3 Mon Sep 17 00:00:00 2001 From: Iglocska Date: Tue, 9 Feb 2016 15:43:09 +0100 Subject: [PATCH] Fixes to the event downloads / APIs - download event as JSON now has the option to include attachments - switched to using the restsearch api instead of the deprecated /events/xml API - added attachment inclusion to both restsearch apis - fixed some bugs with the API --- app/Controller/AttributesController.php | 44 +++++++++++++------------ app/Controller/EventsController.php | 28 ++++++++-------- app/View/Events/ajax/exportChoice.ctp | 2 +- app/View/Events/automation.ctp | 8 +++-- app/webroot/js/ajaxification.js | 2 +- 5 files changed, 45 insertions(+), 39 deletions(-) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index df3def602..a4a715a29 100755 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -1448,9 +1448,9 @@ class AttributesController extends AppController { // the last 4 fields accept the following operators: // && - you can use && between two search values to put a logical OR between them. for value, 1.1.1.1&&2.2.2.2 would find attributes with the value being either of the two. // ! - you can negate a search term. For example: google.com&&!mail would search for all attributes with value google.com but not ones that include mail. www.google.com would get returned, mail.google.com wouldn't. - public function restSearch($key='download', $value=false, $type=false, $category=false, $org=false, $tags=false, $from=false, $to=false, $last=false, $eventid=false) { + public function restSearch($key='download', $value=false, $type=false, $category=false, $org=false, $tags=false, $from=false, $to=false, $last=false, $eventid=false, $withAttachments=false) { if ($tags) $tags = str_replace(';', ':', $tags); - $simpleFalse = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to'); + $simpleFalse = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments'); foreach ($simpleFalse as $sF) { if (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false') ${$sF} = false; } @@ -1463,8 +1463,7 @@ class AttributesController extends AppController { if (!$user) { throw new UnauthorizedException('This authentication key is not authorized to be used for exports. Contact your administrator.'); } - $value = str_replace('|', '/', $value); - + if ($value) $value = str_replace('|', '/', $value); // request handler for POSTed queries. If the request is a post, the parameters (apart from the key) will be ignored and replaced by the terms defined in the posted json or xml object. // The correct format for both is a "request" root element, as shown by the examples below: // For Json: {"request":{"value": "7.7.7.7&&1.1.1.1","type":"ip-src"}} @@ -1484,7 +1483,7 @@ class AttributesController extends AppController { else ${$p} = null; } } - $simpleFalse = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid'); + $simpleFalse = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments'); foreach ($simpleFalse as $sF) { if (!is_array(${$sF}) && (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false')) ${$sF} = false; } @@ -1509,27 +1508,29 @@ class AttributesController extends AppController { $values = explode('&&', $value); $parameters = array('value', 'type', 'category', 'org', 'eventid'); foreach ($parameters as $k => $param) { - if (isset(${$parameters[$k]}) && ${$parameters[$k]}!=='null') { + if (isset(${$parameters[$k]}) && ${$parameters[$k]}!==false) { if (is_array(${$parameters[$k]})) $elements = ${$parameters[$k]}; else $elements = explode('&&', ${$parameters[$k]}); foreach($elements as $v) { + if (empty($v)) continue; if (substr($v, 0, 1) == '!') { if ($parameters[$k] === 'value' && preg_match('@^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(\d|[1-2]\d|3[0-2]))$@', substr($v, 1))) { $cidrresults = $this->Cidr->CIDR(substr($v, 1)); foreach ($cidrresults as $result) { $subcondition['AND'][] = array('Attribute.value NOT LIKE' => $result); } - } else { - if ($parameters[$k] === 'org') { + } else if ($parameters[$k] === 'org') { + // from here $found_orgs = $this->Attribute->Event->Org->find('all', array( 'recursive' => -1, 'conditions' => array('LOWER(name) LIKE' => '%' . strtolower(substr($v, 1)) . '%'), )); foreach ($found_orgs as $o) $subcondition['AND'][] = array('Event.orgc_id !=' => $o['Org']['id']); - } else { - $subcondition['AND'][] = array('Attribute.' . $parameters[$k] . ' NOT LIKE' => '%'.substr($v, 1).'%'); - } + } else if ($parameters[$k] === 'eventid') { + $subcondition['AND'][] = array('Attribute.event_id !=' => substr($v, 1)); + } else { + $subcondition['AND'][] = array('Attribute.' . $parameters[$k] . ' NOT LIKE' => '%'.substr($v, 1).'%'); } } else { if ($parameters[$k] === 'value' && preg_match('@^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(\d|[1-2]\d|3[0-2]))$@', substr($v, 1))) { @@ -1537,17 +1538,17 @@ class AttributesController extends AppController { foreach ($cidrresults as $result) { $subcondition['OR'][] = array('Attribute.value LIKE' => $result); } + } else if ($parameters[$k] === 'org') { + // from here + $found_orgs = $this->Attribute->Event->Org->find('all', array( + 'recursive' => -1, + 'conditions' => array('LOWER(name) LIKE' => '%' . strtolower($v) . '%'), + )); + foreach ($found_orgs as $o) $subcondition['OR'][] = array('Event.orgc_id' => $o['Org']['id']); + } else if ($parameters[$k] === 'eventid'){ + if (!empty($v)) $subcondition['OR'][] = array('Attribute.event_id' => $v); } else { - if ($parameters[$k] === 'org') { - // from here - $found_orgs = $this->Attribute->Event->Org->find('all', array( - 'recursive' => -1, - 'conditions' => array('LOWER(name) LIKE' => '%' . strtolower($v) . '%'), - )); - foreach ($found_orgs as $o) $subcondition['OR'][] = array('Event.orgc_id' => $o['Org']['id']); - } else { - if (!empty($v)) $subcondition['OR'][] = array('Attribute.' . $parameters[$k] . ' LIKE' => '%'.$v.'%'); - } + if (!empty($v)) $subcondition['OR'][] = array('Attribute.' . $parameters[$k] . ' LIKE' => '%'.$v.'%'); } } } @@ -1581,6 +1582,7 @@ class AttributesController extends AppController { $params = array( 'conditions' => $conditions, 'fields' => array('Attribute.*', 'Event.org_id', 'Event.distribution'), + 'withAttachments' => $withAttachments ); $results = $this->Attribute->fetchAttributes($this->Auth->user(), $params); $this->loadModel('Whitelist'); diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 47520c309..bdcd24e42 100755 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -2187,7 +2187,7 @@ class EventsController extends AppController { // the last 4 fields accept the following operators: // && - you can use && between two search values to put a logical OR between them. for value, 1.1.1.1&&2.2.2.2 would find attributes with the value being either of the two. // ! - you can negate a search term. For example: google.com&&!mail would search for all attributes with value google.com but not ones that include mail. www.google.com would get returned, mail.google.com wouldn't. - public function restSearch($key='download', $value=false, $type=false, $category=false, $org=false, $tags=false, $searchall=false, $from=false, $to=false, $last=false, $eventid=false) { + public function restSearch($key='download', $value=false, $type=false, $category=false, $org=false, $tags=false, $searchall=false, $from=false, $to=false, $last=false, $eventid=false, $withAttachments = false) { if ($key!='download') { $user = $this->checkAuthUser($key); } else { @@ -2211,13 +2211,13 @@ class EventsController extends AppController { } else { throw new BadRequestException('Either specify the search terms in the url, or POST a json array / xml (with the root element being "request" and specify the correct headers based on content type.'); } - $paramArray = array('value', 'type', 'category', 'org', 'tags', 'searchall', 'from', 'to', 'last', 'eventid'); + $paramArray = array('value', 'type', 'category', 'org', 'tags', 'searchall', 'from', 'to', 'last', 'eventid', 'withAttachments'); foreach ($paramArray as $p) { if (isset($data['request'][$p])) ${$p} = $data['request'][$p]; else ${$p} = null; } } - $simpleFalse = array('value' , 'type', 'category', 'org', 'tags', 'searchall', 'from', 'to', 'last', 'eventid'); + $simpleFalse = array('value' , 'type', 'category', 'org', 'tags', 'searchall', 'from', 'to', 'last', 'eventid', 'withAttachments'); foreach ($simpleFalse as $sF) { if (!is_array(${$sF}) && (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false')) ${$sF} = false; } @@ -2326,7 +2326,7 @@ class EventsController extends AppController { $final = ""; $final .= '' . PHP_EOL . '' . PHP_EOL; foreach ($eventIds as $currentEventId) { - $result = $this->__fetchEvent($currentEventId, null, $this->Auth->user()); + $result = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $currentEventId, 'includeAttachments' => $withAttachments)); if (!empty($result)) { $result = $this->Whitelist->removeWhitelistedFromArray($result, false); $final .= $converter->event2XML($result[0]) . PHP_EOL; @@ -2343,7 +2343,7 @@ class EventsController extends AppController { $temp = array(); $final = '{"response":['; foreach ($eventIds as $k => $currentEventId) { - $result = $this->__fetchEvent($currentEventId, null, $this->Auth->user()); + $result = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $currentEventId, 'includeAttachments' => $withAttachments)); $final .= $converter->event2JSON($result[0]); if ($k < count($eventIds) -1 ) $final .= ','; } @@ -2947,18 +2947,20 @@ class EventsController extends AppController { $event = $event[0]; $exports = array( 'xml' => array( - 'url' => '/events/xml/download/' . $id, + 'url' => '/events/restsearch/download/false/false/false/false/false/false/false/false/false/' . $id . '/false.xml', 'text' => 'MISP XML (metadata + all attributes)', 'requiresPublished' => false, 'checkbox' => true, 'checkbox_text' => 'Encode Attachments', - 'checkbox_set' => '/true' + 'checkbox_set' => '/events/restsearch/download/false/false/false/false/false/false/false/false/false/' . $id . '/true.xml' ), 'json' => array( - 'url' => '/events/view/' . $id . '.json', + 'url' => '/events/restsearch/download/false/false/false/false/false/false/false/false/false/' . $id . '/false.json', 'text' => 'MISP JSON (metadata + all attributes)', 'requiresPublished' => false, - 'checkbox' => false, + 'checkbox' => true, + 'checkbox_text' => 'Encode Attachments', + 'checkbox_set' => '/events/restsearch/download/false/false/false/false/false/false/false/false/false/' . $id . '/true.json' ), 'openIOC' => array( 'url' => '/events/downloadOpenIOCEvent/' . $id, @@ -2972,7 +2974,7 @@ class EventsController extends AppController { 'requiresPublished' => true, 'checkbox' => true, 'checkbox_text' => 'Include non-IDS marked attributes', - 'checkbox_set' => '/1' + 'checkbox_set' => '/events/csv/download/' . $id . '/1' ), 'stix_xml' => array( 'url' => '/events/stix/download/' . $id . '.xml', @@ -2980,7 +2982,7 @@ class EventsController extends AppController { 'requiresPublished' => true, 'checkbox' => true, 'checkbox_text' => 'Encode Attachments', - 'checkbox_set' => '/true' + 'checkbox_set' => '/events/stix/download/' . $id . '/true.xml' ), 'stix_json' => array( 'url' => '/events/stix/download/' . $id . '.json', @@ -2988,7 +2990,7 @@ class EventsController extends AppController { 'requiresPublished' => true, 'checkbox' => true, 'checkbox_text' => 'Encode Attachments', - 'checkbox_set' => '/true' + 'checkbox_set' => '/events/stix/download/' . $id . '/true.json' ), 'rpz' => array( 'url' => '/attributes/rpz/download/false/' . $id, @@ -3014,7 +3016,7 @@ class EventsController extends AppController { 'requiresPublished' => true, 'checkbox' => true, 'checkbox_text' => 'Include non-IDS marked attributes', - 'checkbox_set' => '/true' + 'checkbox_set' => '/attributes/text/download/all/false/' . $id . '/true' ), ); if ($event['Event']['published'] == 0) { diff --git a/app/View/Events/ajax/exportChoice.ctp b/app/View/Events/ajax/exportChoice.ctp index 0f9a869c6..4a656e6e2 100644 --- a/app/View/Events/ajax/exportChoice.ctp +++ b/app/View/Events/ajax/exportChoice.ctp @@ -10,7 +10,7 @@ echo h($export['checkbox_text']); ?> - +   diff --git a/app/View/Events/automation.ctp b/app/View/Events/automation.ctp index 0144e1e4e..9dc7dce2e 100644 --- a/app/View/Events/automation.ctp +++ b/app/View/Events/automation.ctp @@ -223,7 +223,7 @@ Use semicolons instead (the search will automatically search for colons instead)

To return an event with all of its attributes, relations, shadowAttributes, use the following syntax:
value: Search for the given value in the attributes' value field.
@@ -243,6 +243,7 @@ Use semicolons instead (the search will automatically search for colons instead) to: Events with the date set to a date before the one specified in the to field (format: 2015-02-15)
last: Events published within the last x amount of time, where x can be defined in days, hours, minutes (for example 5d or 12h or 30m)
eventid: The events that should be included / excluded from the search
+withAttachments: If set, encodes the attachments / zipped malware samples as base64 in the data field within each attribute

The keywords false or null should be used for optional empty parameters in the URL.

For example, to find any event with the term "red october" mentioned, use the following syntax (the example is shown as a POST request instead of a GET, which is highly recommended):

POST to:

@@ -266,11 +267,12 @@ Use semicolons instead (the search will automatically search for colons instead) from: Events with the date set to a date after the one specified in the from field (format: 2015-02-15)
to: Events with the date set to a date before the one specified in the to field (format: 2015-02-15)
last: Events published within the last x amount of time, where x can be defined in days, hours, minutes (for example 5d or 12h or 30m)
-eventid: The events that should be included / excluded from the search

+eventid: The events that should be included / excluded from the search
+withAttachments: If set, encodes the attachments / zipped malware samples as base64 in the data field within each attribute

The keywords false or null should be used for optional empty parameters in the URL.

value, type, category and org are optional. It is possible to search for several terms in each category by joining them with the '&&' operator. It is also possible to negate a term with the '!' operator. Please be aware the colons (:) cannot be used in the tag search. Use semicolons instead (the search will automatically search for colons instead). diff --git a/app/webroot/js/ajaxification.js b/app/webroot/js/ajaxification.js index 7abe63cb6..1cdb84988 100644 --- a/app/webroot/js/ajaxification.js +++ b/app/webroot/js/ajaxification.js @@ -1535,7 +1535,7 @@ function selectContainsOption(selectid, value) { function exportChoiceSelect(url, elementId, checkbox) { if (checkbox == 1) { if ($('#' + elementId + '_toggle').prop('checked')) { - url = url + $('#' + elementId + '_set').html(); + url = $('#' + elementId + '_set').html(); } } document.location.href = url;