Resolved an issue that can lead to duplicate events showing up in MISP

- UUID uniqueness was previously not enforced
- changed the MYSQL.sql file to reflect the changes
- Added upgrade admin tool to remove duplicate events and make the database changes required
- Tweaked the tool for the attribute uuid fix so that it cannot created duplicate keys

- some minor fixes, such as automatically removing eventTag objects on event deletion

INSTALL/MYSQL.sql

@@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS `attributes` (
   `comment` text COLLATE utf8_bin NOT NULL,
   PRIMARY KEY (`id`),
   KEY `event_id` (`event_id`),
-  KEY `uuid` (`uuid`)
+  UNIQUE KEY `uuid` (`uuid`),
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
 
 -- --------------------------------------------------------
@@ -94,7 +94,7 @@ CREATE TABLE IF NOT EXISTS `events` (
   `threat_level_id` int(11) NOT NULL,
   `publish_timestamp` int(11) NOT NULL,
   PRIMARY KEY (`id`),
-  KEY `uuid` (`uuid`),
+  UNIQUE KEY `uuid` (`uuid`),
   FULLTEXT KEY `info` (`info`)
 ) ENGINE=MyISAM  DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
 

VERSION.json

@@ -1 +1 @@
-{"major":2, "minor":3, "hotfix":114}
+{"major":2, "minor":3, "hotfix":115}

app/Controller/AppController.php

@@ -379,6 +379,42 @@ class AppController extends Controller {
 		$this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
 	}
 	
+	public function removeDuplicateEvents() {
+		if (!$this->_isSiteAdmin()) throw new MethodNotAllowedException();
+		$this->LoadModel('Event');
+		$duplicates = $this->Event->find('all', array(
+				'fields' => array('Event.uuid', 'count(*) as occurance'),
+				'recursive' => -1,
+				'group' => array('Event.uuid HAVING COUNT(*) > 1'),
+		));
+		$counter = 0;
+		
+		// load this so we can remove the blacklist item that will be created, this is the one case when we do not want it.
+		if (Configure::read('MISP.enableEventBlacklisting')) $this->EventBlacklist = ClassRegistry::init('EventBlacklist');
+		
+		foreach ($duplicates as $duplicate) {
+			$events = $this->Event->find('all', array(
+					'recursive' => -1,
+					'conditions' => array('uuid' => $duplicate['Event']['uuid'])
+			));
+			foreach ($events as $k => $event) {
+				if ($k > 0) {
+					$uuid = $event['Event']['uuid'];
+					$this->Event->delete($event['Event']['id']);
+					$counter++;
+					// remove the blacklist entry that we just created with the event deletion, if the feature is enabled
+					// We do not want to block the UUID, since we just deleted a copy
+					if (Configure::read('MISP.enableEventBlacklisting')) {
+						$this->EventBlacklist->deleteAll(array('EventBlacklist.event_uuid' => $uuid));
+					}
+				}
+			}
+		}
+		$this->Server->updateDatabase('makeEventUUIDsUnique');
+		$this->Session->setFlash('Done. Removed ' . $counter . ' duplicate events.');
+		$this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
+	}
+	
 	public function updateDatabase($command) {
 		if (!$this->_isSiteAdmin()) throw new MethodNotAllowedException();
 		$this->loadModel('Server');

app/Model/Attribute.php

@@ -265,7 +265,7 @@ class Attribute extends AppModel {
 				//'last' => false, // Stop validation after this rule
 				//'on' => 'create', // Limit validation to 'create' or 'update' operations
 			),
-			'unique' => array(
+			'uniqueValue' => array(
 					'rule' => array('valueIsUnique'),
 					'message' => 'A similar attribute already exists for this event.',
 					//'allowEmpty' => false,
@@ -522,7 +522,7 @@ class Attribute extends AppModel {
 			$conditions['Attribute.id !='] = $this->data['Attribute']['id'];
 		}
 
-		$params = array('recursive' => 0,
+		$params = array('recursive' => -1,
 				'conditions' => $conditions,
 		);
 		if (0 != $this->find('count', $params)) {
@@ -1002,8 +1002,9 @@ class Attribute extends AppModel {
 			            	'AND' => array(
 			            		'Attribute.type !=' => $this->nonCorrelatingTypes,
 						)),
-			            'recursive' => 0,
-			    		//'contain' => 'Event',
+			            'recursive' => -1,
+			    		'fields' => array('Attribute.event_id', 'Attribute.id', 'Attribute.distribution'),
+			    		'contain' => array('Event' => array('fields' => array('Event.id', 'Event.date', 'Event.info', 'Event.org', 'Event.distribution'))),
 			            //'fields' => '', // we want to have the Attribute AND Event, so do not filter here
 			    );
 			    // search for the related attributes for that "value(1|2)"

app/Model/Event.php

@@ -323,6 +323,7 @@ class Event extends AppModel {
 		),
 		'EventTag' => array(
 				'className' => 'EventTag',
+				'dependent' => true,
 		)
 	);
 

app/Model/Server.php

@@ -1680,7 +1680,12 @@ class Server extends AppModel {
 				$sql = 'CREATE TABLE IF NOT EXISTS `event_blacklists` ( `id` int(11) NOT NULL AUTO_INCREMENT, `event_uuid` varchar(40) COLLATE utf8_bin NOT NULL, `created` datetime NOT NULL, PRIMARY KEY (`id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin ;';
 				break;
 			case 'makeAttributeUUIDsUnique':
-				$sql = 'ALTER TABLE `attributes` ADD UNIQUE (uuid);';
+				$indexCheck = "SELECT COUNT(1) IndexIsThere FROM INFORMATION_SCHEMA.STATISTICS WHERE table_schema=DATABASE() AND table_name='attributes' AND index_name LIKE 'uuid%'";
+				if ($this->query($indexCheck)[0][0]['IndexIsThere'] == '0') $sql = 'ALTER TABLE `attributes` ADD UNIQUE (uuid);';
+				break;
+			case 'makeEventUUIDsUnique':
+				$indexCheck = "SELECT COUNT(1) IndexIsThere FROM INFORMATION_SCHEMA.STATISTICS WHERE table_schema=DATABASE() AND table_name='events' AND index_name LIKE 'uuid%'";
+				if ($this->query($indexCheck)[0][0]['IndexIsThere'] == '0') $sql = 'ALTER TABLE `events` ADD UNIQUE (uuid);';
 				break;
 			default:
 				$this->Session->setFlash('Invalid command.');

app/View/Pages/administration.ctp

@@ -19,6 +19,7 @@ if (!$isSiteAdmin) exit();
 <li><a href="/servers/updateDatabase/extendServerOrganizationLength">Extend Organization length</a> (Hotfix 2.3.57: Increase the max length of the organization field when adding a new server connection.)</li>
 <li><a href="/servers/updateDatabase/convertLogFieldsToText">Convert log fields to text</a> (Hotfix 2.3.78: Some of the log fields that were varchar(255) ended up truncating the data. This function will change them to "text")</li>
 <li><a href="/servers/pruneDuplicateUUIDs">Fix duplicate UUIDs</a> (Hotfix 2.3.107: it was previously possible to get duplicate attribute UUIDs in the database, this script will remove all duplicates and ensure that duplicates will not be entered into the database in the future.)</li>
+<li><a href="/servers/removeDuplicateEvents">Remove dupicate events (with the same UUID)</a> (Hotfix 2.3.114: In some rare situations it could occur that a duplicate of an event was created on an instance, with the exact same uuid. This action will remove any such duplicates and make sure that this cannot happen again.)</li>
 </ul>
 </div>
 <?php