Some fixes to the OpenIOC import tool

- added support for SHA types
- fixed an issue that caused the import to fail with duplicate attributes (the list gets pruned now)
- fixed an issue where no supplied contextual fields would lead to empty attributes being created
- removed the requirement for the files to have the .ioc extension

VERSION.json

@@ -1 +1 @@
-{"major":2, "minor":3, "hotfix":109}
+{"major":2, "minor":3, "hotfix":111}

app/Controller/Component/IOCImportComponent.php

@@ -68,15 +68,21 @@ class IOCImportComponent extends Component {
 		// Since the tree created by simplexml is a bit of a pain to traverse (partially because of branches with 1 leaves and with several leaves ending up in a different format -
 		// $branch['leaf'] vs $branch[0]['leaf'] we convert it to an easier to deal with tree structure
 		// This tree also only contains the information that we care about.
-		$tree = array(
-				'uuid' => $xmlArray['ioc']['@attributes']['id'],
-				'info' =>  $xmlArray['ioc']['short_description'] . PHP_EOL . 'by ' . $xmlArray['ioc']['authored_by'],
-				'long_info' => $xmlArray['ioc']['description'],
-				'date' => $xmlArray['ioc']['authored_date'],
+		$tree = array(				
 				'type' => 'OR',
 				'branches' => array(),
 				'leaves' => array()
-			);
+		);
+		if (isset($xmlArray['ioc']['@attributes']['id'])) $tree['uuid'] = $xmlArray['ioc']['@attributes']['id'];
+		$temp = '';
+		if (isset($xmlArray['ioc']['short_description'])) {
+			$temp = $xmlArray['ioc']['short_description'];
+			if (isset($xmlArray['ioc']['authored_by'])) $temp .= PHP_EOL . 'by ' . $xmlArray['ioc']['authored_by'];
+		}
+		if ($temp !== '') $tree['info'] = $temp;
+		if (isset($xmlArray['ioc']['description'])) $tree['longinfo'] = $xmlArray['ioc']['description'];
+		if (isset($xmlArray['ioc']['authored_date'])) $tree['date'] = $xmlArray['ioc']['authored_date'];
+
 		$tree['branches'] = $this->__createRootNode($xmlArray);
 
 		// Once we're done, let's back the tree up for later use, so we can start shuffling things around and converting it to our own attribute format
@@ -93,10 +99,15 @@ class IOCImportComponent extends Component {
 		// attach the attributes to the event
 		$event['Attribute'] = $attributes;
 
+		$duplicateFilter = array();
 		// check if we have any attributes, if yes, add their UUIDs to our list of success-array
 		if (count ($event['Attribute']) > 0) {
-			foreach ($event['Attribute'] as $attribute) {
-				$this->saved_uuids[] = $attribute['uuid'];
+			foreach ($event['Attribute'] as $k => $attribute) {
+				$condensed = strtolower($attribute['value']) . $attribute['category'] . $attribute['type'];
+				if (!in_array($condensed, $duplicateFilter)) {
+					$this->saved_uuids[] = $attribute['uuid'];
+					$duplicateFilter[] = $condensed;
+				} else unset($event['Attribute'][$k]);
 			}
 		}
 
@@ -107,7 +118,7 @@ class IOCImportComponent extends Component {
 		// Add a special attribute that captures the basic data about the .ioc such as the ioc-s uuid, info, long info, author, etc.
 		// Define the fields used in the global iocinfo variable.
 		foreach ($this->iocinfo as $k => $v) {
-			$event['Attribute'][] = array('uuid' => String::uuid(), 'category' => 'Other', 'type' => 'comment', 'event_id' => $id, 'value' => $v . ': ' . $event[$v], 'to_ids' => false, 'distribution' => $this->distribution, 'comment' => 'OpenIOC import');
+			if (isset($event[$v])) $event['Attribute'][] = array('uuid' => String::uuid(), 'category' => 'Other', 'type' => 'comment', 'event_id' => $id, 'value' => $v . ': ' . $event[$v], 'to_ids' => false, 'distribution' => $this->distribution, 'comment' => 'OpenIOC import');
 		}
 
 		// attach the graph to the event
@@ -232,6 +243,12 @@ class IOCImportComponent extends Component {
 			case 'FileItem/Md5sum':
 				return array('Payload installation', 'md5');
 				break;
+			case 'FileItem/Sha1sum':
+				return array('Payload installation', 'sha1');
+				break;
+			case 'FileItem/Sha256sum':
+				return array('Payload installation', 'sha256');
+				break;
 			case 'TaskItem/sha1sum':
 				return array('Payload installation', 'sha1');
 				break;

app/Controller/EventsController.php

@@ -1047,10 +1047,6 @@ class EventsController extends AppController {
 					$file = new File($this->data['Event']['submittedioc']['name']);
 					$ext = $file->ext();
 				}
-				if (isset($this->data['Event']['submittedioc']) && ($ext != 'ioc') && $this->data['Event']['submittedioc']['size'] > 0 &&
-						is_uploaded_file($this->data['Event']['submittedioc']['tmp_name'])) {
-					$this->Session->setFlash(__('You may only upload OpenIOC ioc files.'));
-				}
 				if (isset($this->data['Event']['submittedioc'])) $this->_addIOCFile($id);
 
 				// redirect to the view of the newly created event
@@ -2180,7 +2176,6 @@ class EventsController extends AppController {
 			);
 			// Save it all
 			$saveResult = $this->Event->saveAssociated($saveEvent, array('validate' => true, 'fieldList' => $fieldList));
-			
 			// set stuff for the view and render the showIOCResults view.
 			$this->set('attributes', $saveEvent['Attribute']);
 			if (isset($fails)) {

app/View/Attributes/search.ctp

@@ -5,7 +5,7 @@
 		You can search for attributes based on contained expression within the value, event ID, submiting organisation, category and type. <br />For the value, event ID and organisation, you can enter several search terms by entering each term as a new line. To exclude things from a result, use the NOT operator (!) infront of the term.<br/><br />
 		<?php
 			echo $this->Form->input('keyword', array('type' => 'textarea', 'label' => 'Containing the following expressions', 'div' => 'clear', 'class' => 'input-xxlarge'));
-			echo $this->Form->input('keyword2', array('type' => 'textarea', 'label' => 'Being attributes of the following event IDs', 'div' => 'clear', 'class' => 'input-xxlarge'));
+			echo $this->Form->input('keyword2', array('type' => 'textarea', 'label' => 'Being attributes of the following event IDs or event UUIDs', 'div' => 'clear', 'class' => 'input-xxlarge'));
 			echo $this->Form->input('tags', array('type' => 'textarea', 'label' => 'Being an attribute of an event matching the following tags', 'div' => 'clear', 'class' => 'input-xxlarge'));
 			
 		?>