Update to attribute validation and the freetext import tool, fixes #742

- defanged URL type attributes are refanged on input
- admin script to do the same for all existing attributes

- admin tool doesn't recognise a word followed by a . as a url

VERSION.json

@@ -1 +1 @@
-{"major":2, "minor":3, "hotfix":173}
+{"major":2, "minor":3, "hotfix":174}

app/Controller/AttributesController.php

@@ -2225,4 +2225,32 @@ class AttributesController extends AppController {
 		$this->Session->setFlash('Removed ' . count($orphans) . ' attribute(s).');
 		$this->redirect('/pages/display/administration');
 	}
+	
+	public function updateAttributeValues($script) {
+		if (!$this->_isSiteAdmin() || !$this->request->is('post')) throw new MethodNotAllowedException('You are not authorised to do that.');
+		switch ($script) {
+			case 'urlSanitisation':
+				$replaceConditions = array(
+					array('search' => 'UPPER(Attribute.value1) LIKE', 'from' => 'hxxp', 'to' => 'http', 'ci' => true),
+					array('search' => 'Attribute.value1 LIKE', 'from' => '%[.]%', 'to' => '.', 'ci' => false),
+				);
+				break;
+			default:
+				throw new Exception('Invalid script.');
+		}
+		$counter = 0;
+		foreach ($replaceConditions as &$rC) {
+			$searchPattern = '%' . $rC['from'] . '%';
+			if ($rC['ci']) $seachPattern = '%' . strtoupper($rC['from']) . '%';
+			$attributes = $this->Attribute->find('all', array('conditions' => array($rC['search'] => $searchPattern), 'recursive' => -1));
+			foreach ($attributes as &$attribute) {
+				if ($rC['ci']) $attribute['Attribute']['value'] = str_ireplace($rC['from'], $rC['to'], $attribute['Attribute']['value']);
+				else $attribute['Attribute']['value'] = str_replace($rC['from'], $rC['to'], $attribute['Attribute']['value']);
+				$this->Attribute->save($attribute);
+				$counter++;
+			}
+		}
+		$this->Session->setFlash('Updated ' . $counter . ' attribute(s).');
+		$this->redirect('/pages/display/administration');
+	}
 }

app/Lib/Tools/ComplexTypeTool.php

@@ -128,7 +128,7 @@ class ComplexTypeTool {
 			} else {
 				// check if it is a URL
 				// Adding http:// infront of the input in case it was left off. github.com/MISP/MISP should still be counted as a valid link
-				if (filter_var($input2, FILTER_VALIDATE_URL) || filter_var('http://' . $input2, FILTER_VALIDATE_URL)) {
+				if (count($temp) < 2 && (filter_var($input2, FILTER_VALIDATE_URL) || filter_var('http://' . $input2, FILTER_VALIDATE_URL))) {
 					if (preg_match('/^https:\/\/www.virustotal.com\//i', $input2)) return array('types' => array('link'), 'to_ids' => true, 'default_type' => 'link', 'comment' => $comment, 'value' => $input2);
 					return array('types' => array('url'), 'to_ids' => true, 'default_type' => 'url', 'comment' => $comment, 'value' => $input2);
 				}

app/Model/Attribute.php

@@ -468,6 +468,8 @@ class Attribute extends AppModel {
 				$pieces = explode('|', $this->data['Attribute']['value']);
 				$this->data['Attribute']['value'] = $pieces[0] . '|' . strtolower($pieces[1]);
 				break;
+			case 'url':
+				$this->data['Attribute']['value'] = str_ireplace(array('hxxp', '[.]'), array('http', '.'), $this->data['Attribute']['value']);
 		}
 
 		// uppercase the following types

app/View/Pages/administration.ctp

@@ -20,6 +20,7 @@ if (!$isSiteAdmin) exit();
 <li><?php echo $this->Form->postLink('Remove dupicate events (with the same UUID)', $baseurl . '/servers/removeDuplicateEvents');?> (Hotfix 2.3.115: In some rare situations it could occur that a duplicate of an event was created on an instance, with the exact same uuid. This action will remove any such duplicates and make sure that this cannot happen again.)</li>
 <li><?php echo $this->Form->postLink('Prune orphaned attributes', $baseurl . '/attributes/pruneOrphanedAttributes');?> (In some rare occasions it can happen that you end up with some attributes in your database that do not belong to an event - for example during a race condition between an event insert and a delete. This tool will collect and delete any such orphaned attributes. If you ever run into an issue where you cannot add an attribute with a specific valid value, this is probably the reason.)</li>
 <li><?php echo $this->Form->postLink('Clean regex table of potentially malicious entries', $baseurl . '/regexp/cleanRegexModifiers');?> (Hotfix 2.3.160: Prior to this version it was possible for a user/admin with Regex permission to create a malicious regular expression that could be used to execute arbitrary code. Since this version it is no longer possible to input such expressions, but already existing malicious entries still have to be cleaned using this tool.)</li>
+<li><?php echo $this->Form->postLink('Remove url type attribute sanitisation', $baseurl . '/attributes/updateAttributeValues/urlSanitisation');?> (Hotfix 2.3.173: Sanitised URLs can cause issues with the NIDS exports and as of this version attributes will be modified on entry to correct this. To correct existing entries, run this script.)</li>
 </ul>
 </div>
 <?php