Fixes to the event downloads / APIs

- download event as JSON now has the option to include attachments
- switched to using the restsearch api instead of the deprecated /events/xml API

- added attachment inclusion to both restsearch apis

- fixed some bugs with the API

app/Controller/AttributesController.php

@@ -1448,9 +1448,9 @@ class AttributesController extends AppController {
 	// the last 4 fields accept the following operators:
 	// && - you can use && between two search values to put a logical OR between them. for value, 1.1.1.1&&2.2.2.2 would find attributes with the value being either of the two.
 	// ! - you can negate a search term. For example: google.com&&!mail would search for all attributes with value google.com but not ones that include mail. www.google.com would get returned, mail.google.com wouldn't.
-	public function restSearch($key='download', $value=false, $type=false, $category=false, $org=false, $tags=false, $from=false, $to=false, $last=false, $eventid=false) {
+	public function restSearch($key='download', $value=false, $type=false, $category=false, $org=false, $tags=false, $from=false, $to=false, $last=false, $eventid=false, $withAttachments=false) {
 		if ($tags) $tags = str_replace(';', ':', $tags);
-		$simpleFalse = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to');
+		$simpleFalse = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments');
 		foreach ($simpleFalse as $sF) {
 			if (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false') ${$sF} = false;
 		}
@@ -1463,8 +1463,7 @@ class AttributesController extends AppController {
 		if (!$user) {
 			throw new UnauthorizedException('This authentication key is not authorized to be used for exports. Contact your administrator.');
 		}
-		$value = str_replace('|', '/', $value);
-
+		if ($value) $value = str_replace('|', '/', $value);
 		// request handler for POSTed queries. If the request is a post, the parameters (apart from the key) will be ignored and replaced by the terms defined in the posted json or xml object.
 		// The correct format for both is a "request" root element, as shown by the examples below:
 		// For Json: {"request":{"value": "7.7.7.7&&1.1.1.1","type":"ip-src"}}
@@ -1484,7 +1483,7 @@ class AttributesController extends AppController {
 				else ${$p} = null;
 			}
 		}
-		$simpleFalse = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid');
+		$simpleFalse = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments');
 		foreach ($simpleFalse as $sF) {
 			if (!is_array(${$sF}) && (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false')) ${$sF} = false;
 		}
@@ -1509,27 +1508,29 @@ class AttributesController extends AppController {
 		$values = explode('&&', $value);
 		$parameters = array('value', 'type', 'category', 'org', 'eventid');
 		foreach ($parameters as $k => $param) {
-			if (isset(${$parameters[$k]}) && ${$parameters[$k]}!=='null') {
+			if (isset(${$parameters[$k]}) && ${$parameters[$k]}!==false) {
 				if (is_array(${$parameters[$k]})) $elements = ${$parameters[$k]};
 				else $elements = explode('&&', ${$parameters[$k]});
 				foreach($elements as $v) {
+					if (empty($v)) continue;
 					if (substr($v, 0, 1) == '!') {
 						if ($parameters[$k] === 'value' && preg_match('@^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(\d|[1-2]\d|3[0-2]))$@', substr($v, 1))) {
 							$cidrresults = $this->Cidr->CIDR(substr($v, 1));
 							foreach ($cidrresults as $result) {
 								$subcondition['AND'][] = array('Attribute.value NOT LIKE' => $result);
 							}
-						} else {
-							if ($parameters[$k] === 'org') {
+						} else if ($parameters[$k] === 'org') {
+
 								// from here
 								$found_orgs = $this->Attribute->Event->Org->find('all', array(
 										'recursive' => -1,
 										'conditions' => array('LOWER(name) LIKE' => '%' . strtolower(substr($v, 1)) . '%'),
 								));
 								foreach ($found_orgs as $o) $subcondition['AND'][] = array('Event.orgc_id !=' => $o['Org']['id']);
-							} else {
-								$subcondition['AND'][] = array('Attribute.' . $parameters[$k] . ' NOT LIKE' => '%'.substr($v, 1).'%');
-							}
+						} else if ($parameters[$k] === 'eventid') {
+							$subcondition['AND'][] = array('Attribute.event_id !=' => substr($v, 1));
+						} else {
+							$subcondition['AND'][] = array('Attribute.' . $parameters[$k] . ' NOT LIKE' => '%'.substr($v, 1).'%');
 						}
 					} else {
 						if ($parameters[$k] === 'value' && preg_match('@^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(\d|[1-2]\d|3[0-2]))$@', substr($v, 1))) {
@@ -1537,17 +1538,17 @@ class AttributesController extends AppController {
 							foreach ($cidrresults as $result) {
 								$subcondition['OR'][] = array('Attribute.value LIKE' => $result);
 							}
+						} else if ($parameters[$k] === 'org') {
+							// from here
+							$found_orgs = $this->Attribute->Event->Org->find('all', array(
+									'recursive' => -1,
+									'conditions' => array('LOWER(name) LIKE' => '%' . strtolower($v) . '%'),
+							));
+							foreach ($found_orgs as $o) $subcondition['OR'][] = array('Event.orgc_id' => $o['Org']['id']);
+						} else if ($parameters[$k] === 'eventid'){
+							if (!empty($v)) $subcondition['OR'][] = array('Attribute.event_id' => $v);
 						} else {
-							if ($parameters[$k] === 'org') {
-								// from here
-								$found_orgs = $this->Attribute->Event->Org->find('all', array(
-										'recursive' => -1,
-										'conditions' => array('LOWER(name) LIKE' => '%' . strtolower($v) . '%'),
-								));
-								foreach ($found_orgs as $o) $subcondition['OR'][] = array('Event.orgc_id' => $o['Org']['id']);
-							} else {
-								if (!empty($v)) $subcondition['OR'][] = array('Attribute.' . $parameters[$k] . ' LIKE' => '%'.$v.'%');
-							}
+							if (!empty($v)) $subcondition['OR'][] = array('Attribute.' . $parameters[$k] . ' LIKE' => '%'.$v.'%');
 						}
 					}
 				}	
@@ -1581,6 +1582,7 @@ class AttributesController extends AppController {
 		$params = array(
 				'conditions' => $conditions,
 				'fields' => array('Attribute.*', 'Event.org_id', 'Event.distribution'),
+				'withAttachments' => $withAttachments
 		);
 		$results = $this->Attribute->fetchAttributes($this->Auth->user(), $params);
 		$this->loadModel('Whitelist');

app/Controller/EventsController.php

@@ -2187,7 +2187,7 @@ class EventsController extends AppController {
 	// the last 4 fields accept the following operators:
 	// && - you can use && between two search values to put a logical OR between them. for value, 1.1.1.1&&2.2.2.2 would find attributes with the value being either of the two.
 	// ! - you can negate a search term. For example: google.com&&!mail would search for all attributes with value google.com but not ones that include mail. www.google.com would get returned, mail.google.com wouldn't.
-	public function restSearch($key='download', $value=false, $type=false, $category=false, $org=false, $tags=false, $searchall=false, $from=false, $to=false, $last=false, $eventid=false) {
+	public function restSearch($key='download', $value=false, $type=false, $category=false, $org=false, $tags=false, $searchall=false, $from=false, $to=false, $last=false, $eventid=false, $withAttachments = false) {
 		if ($key!='download') {
 			$user = $this->checkAuthUser($key);
 		} else {
@@ -2211,13 +2211,13 @@ class EventsController extends AppController {
 			} else {
 				throw new BadRequestException('Either specify the search terms in the url, or POST a json array / xml (with the root element being "request" and specify the correct headers based on content type.');
 			}
-			$paramArray = array('value', 'type', 'category', 'org', 'tags', 'searchall', 'from', 'to', 'last', 'eventid');
+			$paramArray = array('value', 'type', 'category', 'org', 'tags', 'searchall', 'from', 'to', 'last', 'eventid', 'withAttachments');
 			foreach ($paramArray as $p) {
 				if (isset($data['request'][$p])) ${$p} = $data['request'][$p];
 				else ${$p} = null;
 			}
 		}
-		$simpleFalse = array('value' , 'type', 'category', 'org', 'tags', 'searchall', 'from', 'to', 'last', 'eventid');
+		$simpleFalse = array('value' , 'type', 'category', 'org', 'tags', 'searchall', 'from', 'to', 'last', 'eventid', 'withAttachments');
 		foreach ($simpleFalse as $sF) {
 			if (!is_array(${$sF}) && (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false')) ${$sF} = false;
 		}
@@ -2326,7 +2326,7 @@ class EventsController extends AppController {
 				$final = "";
 				$final .= '<?xml version="1.0" encoding="UTF-8"?>' . PHP_EOL . '<response>' . PHP_EOL;
 				foreach ($eventIds as $currentEventId) {
-					$result = $this->__fetchEvent($currentEventId, null, $this->Auth->user());
+					$result = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $currentEventId, 'includeAttachments' => $withAttachments));
 					if (!empty($result)) {
 						$result = $this->Whitelist->removeWhitelistedFromArray($result, false);
 						$final .= $converter->event2XML($result[0]) . PHP_EOL;
@@ -2343,7 +2343,7 @@ class EventsController extends AppController {
 				$temp = array();
 				$final = '{"response":[';
 				foreach ($eventIds as $k => $currentEventId) {
-					$result = $this->__fetchEvent($currentEventId, null, $this->Auth->user());
+					$result = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $currentEventId, 'includeAttachments' => $withAttachments));
 					$final .= $converter->event2JSON($result[0]);
 					if ($k < count($eventIds) -1 ) $final .= ',';
 				}
@@ -2947,18 +2947,20 @@ class EventsController extends AppController {
 		$event = $event[0];
 		$exports = array(
 			'xml' => array(
-					'url' => '/events/xml/download/' . $id,
+					'url' => '/events/restsearch/download/false/false/false/false/false/false/false/false/false/' . $id . '/false.xml',
 					'text' => 'MISP XML (metadata + all attributes)',
 					'requiresPublished' => false,
 					'checkbox' => true,
 					'checkbox_text' => 'Encode Attachments',
-					'checkbox_set' => '/true'
+					'checkbox_set' => '/events/restsearch/download/false/false/false/false/false/false/false/false/false/' . $id . '/true.xml'
 			),
 			'json' => array(
-					'url' => '/events/view/' . $id . '.json',
+					'url' => '/events/restsearch/download/false/false/false/false/false/false/false/false/false/' . $id . '/false.json',
 					'text' => 'MISP JSON (metadata + all attributes)',
 					'requiresPublished' => false,
-					'checkbox' => false,
+					'checkbox' => true,
+					'checkbox_text' => 'Encode Attachments',
+					'checkbox_set' => '/events/restsearch/download/false/false/false/false/false/false/false/false/false/' . $id . '/true.json'
 			),
 			'openIOC' => array(
 					'url' => '/events/downloadOpenIOCEvent/' . $id,
@@ -2972,7 +2974,7 @@ class EventsController extends AppController {
 					'requiresPublished' => true,
 					'checkbox' => true,
 					'checkbox_text' => 'Include non-IDS marked attributes',
-					'checkbox_set' => '/1'
+					'checkbox_set' => '/events/csv/download/' . $id . '/1'
 			),
 			'stix_xml' => array(
 					'url' => '/events/stix/download/' . $id . '.xml',
@@ -2980,7 +2982,7 @@ class EventsController extends AppController {
 					'requiresPublished' => true,
 					'checkbox' => true,
 					'checkbox_text' => 'Encode Attachments',
-					'checkbox_set' => '/true'
+					'checkbox_set' => '/events/stix/download/' . $id . '/true.xml'
 			),
 			'stix_json' => array(
 					'url' => '/events/stix/download/' . $id . '.json',
@@ -2988,7 +2990,7 @@ class EventsController extends AppController {
 					'requiresPublished' => true,
 					'checkbox' => true,
 					'checkbox_text' => 'Encode Attachments',
-					'checkbox_set' => '/true'
+					'checkbox_set' => '/events/stix/download/' . $id . '/true.json'
 			),
 			'rpz' => array(
 					'url' => '/attributes/rpz/download/false/' . $id,
@@ -3014,7 +3016,7 @@ class EventsController extends AppController {
 					'requiresPublished' => true,
 					'checkbox' => true,
 					'checkbox_text' => 'Include non-IDS marked attributes',
-					'checkbox_set' => '/true'
+					'checkbox_set' => '/attributes/text/download/all/false/' . $id . '/true'
 			),
 		);
 		if ($event['Event']['published'] == 0) {

app/View/Events/ajax/exportChoice.ctp

@@ -10,7 +10,7 @@
 						echo h($export['checkbox_text']);
 					?>
 						<input id = "<?php echo h($k) . '_toggle';?>" type="checkbox" style="align;vertical-align:top;margin-top:8px;">
-						<span id ="<?php echo h($k);?>_set" style="display:none;"><?php echo h($export['checkbox_set']); ?></span>
+						<span id ="<?php echo h($k);?>_set" style="display:none;"><?php if (isset($export['checkbox_set'])) echo h($export['checkbox_set']); ?></span>
 					<?php else: ?>
 						&nbsp;
 					<?php endif; ?>

app/View/Events/automation.ctp

@@ -223,7 +223,7 @@ Use semicolons instead (the search will automatically search for colons instead)
 <p>To return an event with all of its attributes, relations, shadowAttributes, use the following syntax:</p>
 <pre>
 <?php
-	echo $baseurl.'/events/restSearch/download/[value]/[type]/[category]/[org]/[tag]/[quickfilter]/[from]/[to]/[last]';
+	echo $baseurl.'/events/restSearch/download/[value]/[type]/[category]/[org]/[tag]/[quickfilter]/[from]/[to]/[last]/[event_id]/[withAttachments]';
 ?>
 </pre>
 <b>value</b>: Search for the given value in the attributes' value field.<br />
@@ -243,6 +243,7 @@ Use semicolons instead (the search will automatically search for colons instead)
 <b>to</b>: Events with the date set to a date before the one specified in the to field (format: 2015-02-15)<br />
 <b>last</b>: Events published within the last x amount of time, where x can be defined in days, hours, minutes (for example 5d or 12h or 30m)<br />
 <b>eventid</b>: The events that should be included / excluded from the search<br />
+<b>withAttachments</b>: If set, encodes the attachments / zipped malware samples as base64 in the data field within each attribute<br />
 <p>The keywords false or null should be used for optional empty parameters in the URL.</p>
 <p>For example, to find any event with the term "red october" mentioned, use the following syntax (the example is shown as a POST request instead of a GET, which is highly recommended):</p>
 <p>POST to:</p>
@@ -266,11 +267,12 @@ Use semicolons instead (the search will automatically search for colons instead)
 <b>from</b>: Events with the date set to a date after the one specified in the from field (format: 2015-02-15)<br />
 <b>to</b>: Events with the date set to a date before the one specified in the to field (format: 2015-02-15)<br />
 <b>last</b>: Events published within the last x amount of time, where x can be defined in days, hours, minutes (for example 5d or 12h or 30m)<br />
-<b>eventid</b>: The events that should be included / excluded from the search<br /><br />
+<b>eventid</b>: The events that should be included / excluded from the search<br />
+<b>withAttachments</b>: If set, encodes the attachments / zipped malware samples as base64 in the data field within each attribute<br /><br />
 <p>The keywords false or null should be used for optional empty parameters in the URL.</p>
 <pre>
 <?php
-	echo $baseurl.'/attributes/restSearch/download/[value]/[type]/[category]/[org]/[tag]/[from]/[to]/[last]/[eventid]';
+	echo $baseurl.'/attributes/restSearch/download/[value]/[type]/[category]/[org]/[tag]/[from]/[to]/[last]/[eventid]/[withAttachments]';
 ?>
 </pre>
 <p>value, type, category and org are optional. It is possible to search for several terms in each category by joining them with the '&amp;&amp;' operator. It is also possible to negate a term with the '!' operator. Please be aware the colons (:) cannot be used in the tag search. Use semicolons instead (the search will automatically search for colons instead).

app/webroot/js/ajaxification.js

@@ -1535,7 +1535,7 @@ function selectContainsOption(selectid, value) {
 function exportChoiceSelect(url, elementId, checkbox) {
 	if (checkbox == 1) {
 		if ($('#' + elementId + '_toggle').prop('checked')) {
-			url = url + $('#' + elementId + '_set').html();
+			url = $('#' + elementId + '_set').html();
 		}
 	}
 	document.location.href = url;