Further progress on the OpenIOC import

- works fine now, but a lot of data still gets discarded

app/Controller/Component/IOCImportComponent.php

@@ -14,7 +14,7 @@ class IOCImportComponent extends Component {
 		);
 
 	public function readXML($data, $id) {
-		ClassRegistry::init('Attribute');
+		//ClassRegistry::init('Attribute');
 		$event = array();
 		$attributes = array();
 		$fails = array();
@@ -23,9 +23,10 @@ class IOCImportComponent extends Component {
 		App::uses('Xml', 'Utility');
 
 		// now parse it
-		$xml = Xml::build($data);
-		$xmlArray = Xml::toArray($xml);
-
+		$xmlArray = json_decode(json_encode((array) simplexml_load_string($data)), 1);
+		$temp = $xmlArray;
+		$xmlArray = array();
+		$xmlArray['ioc'] = $temp;
 		// add an attribute that holds the full description of the imported report.
 		$attributes[] = array(
 				'event_id' => $id,
@@ -35,11 +36,13 @@ class IOCImportComponent extends Component {
 				'category' => 'Other',
 				'type' => 'comment'
 				);
+
+		// set the event info based on the import.
 		$event['info'] = $xmlArray['ioc']['short_description'] . PHP_EOL .'By ' . $xmlArray['ioc']['authored_by'];
 		$event['date'] = $xmlArray['ioc']['authored_date'];
-		$event['uuid'] = $xmlArray['ioc']['@id'];
+		$event['uuid'] = $xmlArray['ioc']['@attributes']['id'];
 		foreach ($xmlArray['ioc']['definition'] as $current) {
-			if($current['@operator'] == 'OR') {
+			if($current['@attributes']['operator'] == 'OR' && isset($current['IndicatorItem'])) {
 				foreach ($current['IndicatorItem'] as $ii) {
 					$temp = $this->__analyseIndicator($ii, $id);
 					$attributes[] = $temp;
@@ -48,26 +51,33 @@ class IOCImportComponent extends Component {
 				$fails[] = $current;
 			}
 		}
-		// Check the logical operators, if there are exactly 2 indicators within an AND operator, check if they can be built into an accepted composite attribute type
+
+		// Try to see if any of the AND-ed indicators can be salvaged and converted instead of being discarded
 		foreach ($xmlArray['ioc']['definition'] as $current) {
-			foreach ($current['Indicator'] as $ii) {
-				if (isset($ii['IndicatorItem']) && count($ii['IndicatorItem']) == 2) {
-					$att1 = $this->__analyseIndicator($ii['IndicatorItem'][0], $id);
-					$att2 = $this->__analyseIndicator($ii['IndicatorItem'][1], $id);
-					$attempt = $this->__convertToCompositeAttribute($att1, $att2, $ii['@id']);
+			foreach ($current['Indicator'] as $key => $value) {
+				// During the xml->array conversion, if there is only a single indicator, it will be build directly as a child of definition,
+				// instead of the first element of an array. Here we move the IndicatorItem one level down
+				if ($key === 'IndicatorItem') {
+					$key = 0;
+					$value = array ('IndicatorItem' => $value);
+				}
+				if (isset($value['IndicatorItem']) && count($value['IndicatorItem']) == 2) {
+					$att1 = $this->__analyseIndicator($value['IndicatorItem'][0], $id);
+					$att2 = $this->__analyseIndicator($value['IndicatorItem'][1], $id);
+					$attempt = $this->__convertToCompositeAttribute($att1, $att2, $current['@attributes']['id']);
 					if ($attempt) {
 						$attributes[] = $attempt;
-					} else {
-						$fails[] = $ii;
 					}
 				} else {
-					$fails[] = $ii;
+						$fails[] = $value;
 				}
+				// If it is the only indicator, jump straight to the IndicatorItem
 			}
 		}
+
 		// remove all the temporary attribute types used for the pairing and turn them all into "other"
 		foreach ($attributes as &$att) {
-			if (substr($att['type'], 0, 3) == 'temp') {
+			if (substr($att['type'], 0, 3) === 'temp') {
 				$temp = $this->__convertToOther($temp);
 			}
 		}
@@ -87,11 +97,11 @@ class IOCImportComponent extends Component {
 	private function __analyseIndicator($ii, $id) {
 		$attribute = array();
 		$attribute['event_id'] = $id;
-		$attribute['uuid'] = $ii['@id'];
-		$attribute['value'] = $ii['Content']['@'];
+		$attribute['uuid'] = $ii['@attributes']['id'];
+		$attribute['value'] = $ii['Content'];
 		$attribute['to_ids'] = false;
-		$attribute['search'] = $ii['Context']['@search'];
-		$temp = $this->__checkType($ii['Context']['@search']);
+		$attribute['search'] = $ii['Context']['@attributes']['search'];
+		$temp = $this->__checkType($ii['Context']['@attributes']['search']);
 		if (!$temp) return false;
 		$attribute['category'] = $temp[0];
 		$attribute['type'] = $temp[1];
@@ -127,7 +137,7 @@ class IOCImportComponent extends Component {
 					default:
 						$value = $att1['value'] . '|' . $att2['value'];
 				}
-				return array('type' => $pair[2], 'value' => $value, 'uuid' => $uuid, 'category' => $pair[3], 'event_id' => $att1['event_id']);
+				return array('type' => $pair[2], 'value' => $value, 'uuid' => $uuid, 'category' => $pair[3], 'event_id' => $att1['event_id'], 'to_ids' => false);
 			}
 			// Try the same thing above with the attributes reversed
 			if ($att2['type'] == $pair[0] && $att1['type'] == $pair[1]) {
@@ -142,7 +152,7 @@ class IOCImportComponent extends Component {
 					default:
 						$value = $att2['value'] . '|' . $att1['value'];
 				}
-				return array('type' => $pair[2], 'value' => $value, 'uuid' => $uuid, 'category' => $pair[3], 'event_id' => $att1['event_id']);
+				return array('type' => $pair[2], 'value' => $value, 'uuid' => $uuid, 'category' => $pair[3], 'event_id' => $att1['event_id'], 'to_ids' => false);
 			}
 		}
 		// If no match found, return false, it's not a valid composite attribute for MISP
@@ -234,10 +244,11 @@ class IOCImportComponent extends Component {
 	private function __saveFailedUuids($array, &$failedAttributes) {
 		foreach ($array as $current => $value) {
 			if (is_array($value)) {
-				if (isset($value['@id']) && isset($value['Context'])) {
+				if (isset($value['Context'])) {
 					array_push($failedAttributes, $value);
+				} else {
+					$this->__saveFailedUuids($value, $failedAttributes);
 				}
-				$this->__saveFailedUuids($value, $failedAttributes);
 			}
 		}
 	}

app/Controller/EventsController.php

@@ -254,6 +254,10 @@ class EventsController extends AppController {
 					$file = new File($this->data['Event']['submittedfile']['name']);
 					$ext = $file->ext();
 				}
+				$ioc = false;
+				if($this->data['Event']['submittedioc']['error'] != 4) {
+					$ioc = true;
+				}
 				if (isset($this->data['Event']['submittedioc'])) {
 					App::uses('File', 'Utility');
 					$file = new File($this->data['Event']['submittedfile']['name']);
@@ -281,10 +285,8 @@ class EventsController extends AppController {
 								$existingFlash = CakeSession::read('Message.flash');
 								$this->Session->setFlash(__('The event has been saved. ' . $existingFlash['message']));
 							}
-							if (isset($this->data['Event']['submittedioc'])) {
-								$this->render('showIOCResults');
-							} else {
-							$this->redirect(array('action' => 'view', $this->Event->getId()));
+							if (!$ioc) {
+								$this->redirect(array('action' => 'view', $this->Event->getId()));
 							}
 						}
 					} else {
@@ -1479,24 +1481,29 @@ class EventsController extends AppController {
 
 			// read XML
 			$event = $this->IOCImport->readXML($fileData, $id);
-			$this->loadModel('Attribute');
-			foreach ($event['Attribute'] as $attribute) {
-				$this->Attribute->create();
-				$this->Attribute->save($attribute);
+
+			// make some changes to have $saveEvent in the format that is needed to save the event together with its attributes
+			$fails = $event['Fails'];
+			$saveEvent['Attribute'] = $event['Attribute'];
+			// we've already stored these elsewhere, unset them so we can extract the event related data
+			unset($event['Attribute']);
+			unset($event['Fails']);
+			// save the event related data into $saveEvent['Event']
+			$saveEvent['Event'] = $event;
+			$saveEvent['Event']['id'] = $id;
+			$fieldList = array(
+						'Event' => array('info', 'uuid'),
+						'Attribute' => array('event_id', 'category', 'type', 'value', 'value1', 'value2', 'to_ids', 'uuid')
+						);
+
+			// Save it all
+			$saveResult = $this->Event->saveAssociated($saveEvent, array('validate' => true, 'fieldList' => $fieldList));
+
+			// set stuff for the view and render the showIOCResults view.
+			$this->set('attributes', $saveEvent['Attribute']);
+			if (isset($fails)) {
+				$this->set('fails', $fails);
 			}
-
-			$updateEvent = $this->Event->read(null, $id);
-			// update the DB to set the published flag
-			$fieldList = array('info', 'uuid');
-			$updateEvent['Event']['uuid'] = $event['uuid'];
-			//$updateEvent['Event']['date'] = $event['date'];
-			$updateEvent['Event']['info'] = $event['info'];
-			$this->Event->save($updateEvent, array('fieldList' => $fieldList));
-
-			//$this->Session->setFlash(__('Import complete. Indicators successfully added: ' . count($event['Attribute']) . '. Indicators that could not be added: ' . count($event['Fails']) . '. To see a of the Uuids of the failed indicators, click here.'));
-
-			$this->set('attributes', $event['Attribute']);
-			$this->set('fails', $event['Fails']);
 			$this->set('eventId', $id);
 			$this->render('showIOCResults');
 		}

app/View/Events/show_i_o_c_results.ctp

@@ -1,4 +1,7 @@
 <div class="index">
+	<div class="actions" style="float:right;">
+		<ul><li><?php echo $this->Html->link(__('View the event', true), array('action' => 'view', $eventId)); ?> </li></ul>
+	</div>
 	<h2>Results of the import: </h2>
 	<h3><?php echo count($attributes); ?> attributes created successfully, <?php echo count($fails); ?> indicators could not be mapped and saved. </h3>
 	<br /><br />
@@ -24,7 +27,7 @@ endforeach; ?>
 <?php
 endif;?>
 <?php
-if (0 != count($fails)):?>
+if (isset($fails)):?>
 	<br /><br />
 	<h4>Failed indicators:</h4>
 	<table cellpadding="0" cellspacing="0">
@@ -35,9 +38,9 @@ if (0 != count($fails)):?>
 	</tr><?php
 foreach ($fails as $fail): ?>
 	<tr>
-		<td class="short"><?php echo h($fail['@id']); ?>&nbsp;</td>
-		<td class="short"><?php echo h($fail['Context']['@search']); ?>&nbsp;</td>
-		<td class="short"><?php echo h($fail['Content']['@']); ?>&nbsp;</td>
+		<td class="short"><?php echo h($fail['@attributes']['id']); ?>&nbsp;</td>
+		<td class="short"><?php echo h($fail['Context']['@attributes']['search']); ?>&nbsp;</td>
+		<td class="short"><?php echo h($fail['Content']); ?>&nbsp;</td>
 	</tr><?php
 endforeach; ?>
 </table>