fix: missing new TLDs in free text import, solves #1149 (#1574)

* fix: missing new TLDs in free text import, solves #1149

app/Controller/EventsController.php

@@ -2795,7 +2795,9 @@ class EventsController extends AppController {
 		if ($this->request->is('post')) {
 			App::uses('ComplexTypeTool', 'Tools');
 			$complexTypeTool = new ComplexTypeTool();
-			$resultArray = $complexTypeTool->checkComplexRouter($this->request->data['Attribute']['value'], 'FreeText');
+			$this->loadModel('Warninglist');
+			$IANATLDentries = $this->Warninglist->getAllIANAEntries();
+			$resultArray = $complexTypeTool->checkComplexRouter($this->request->data['Attribute']['value'], 'FreeText', $IANATLDentries);
 			foreach ($resultArray as $key => $r) {
 				$temp = array();
 				foreach ($r['types'] as $type) {

app/Lib/Tools/ComplexTypeTool.php

@@ -10,7 +10,7 @@ class ComplexTypeTool {
 		'/\.+/' => '.'
 	);
 
-	public function checkComplexRouter($input, $type) {
+	public function checkComplexRouter($input, $type, $IANATLDentries) {
 		switch ($type) {
 			case 'File':
 				return $this->checkComplexFile($input);
@@ -19,7 +19,7 @@ class ComplexTypeTool {
 				return $this->checkComplexCnC($input);
 				break;
 			case 'FreeText':
-				return $this->checkFreeText($input);
+				return $this->checkFreeText($input, $IANATLDentries);
 				break;
 			default:
 				return false;
@@ -73,7 +73,7 @@ class ComplexTypeTool {
 		return array_values($array);
 	}
 
-	public function checkFreeText($input) {
+	public function checkFreeText($input, $IANATLDentries) {
 		$iocArray = preg_split("/\r\n|\n|\r|\s|\s+|,|;/", $input);
 		$quotedText = explode('"', $input);
 		foreach ($quotedText as $k => $temp) {
@@ -93,7 +93,7 @@ class ComplexTypeTool {
 				$ioc = trim($ioc, ',');
 				$ioc = preg_replace('/\p{C}+/u', '', $ioc);
 				if (empty($ioc)) continue;
-				$typeArray = $this->__resolveType($ioc);
+				$typeArray = $this->__resolveType($ioc, $IANATLDentries);
 				if ($typeArray === false) continue;
 				$temp = $typeArray;
 				if (!isset($temp['value'])) $temp['value'] = $ioc;
@@ -112,8 +112,9 @@ class ComplexTypeTool {
 		128 => array('single' => array('sha512'), 'composite' => array('filename|sha512'))
 	);
 
-	private function __resolveType($input) {
+	private function __resolveType($input, $IANATLDentries) {
 		$input = trim($input);
+		// check for composite (|) attributes
 		if (strpos($input, '|')) {
 			$compositeParts = explode('|', $input);
 			if (count($compositeParts) == 2) {
@@ -156,6 +157,16 @@ class ComplexTypeTool {
 		// check for domain name, hostname, filename
 		if (strpos($inputRefanged, '.') !== false) {
 			$temp = explode('.', $inputRefanged);
+			// check for the new TLDs as known by IANA (if the Warninglists are not empty)
+			if (!empty($IANATLDentries)) {
+				$stringEnd = $temp[count($temp)-1];
+				if (in_array($stringEnd, $IANATLDentries)) {
+					$types = array('filename', 'domain');
+					if (count($temp) > 2)
+						$types[] = 'url';
+					return array('types' => $types, 'to_ids' => true, 'default_type' => 'filename', 'merge_categories' => true);
+				}
+			}
 			// TODO: use a more flexible matching approach, like the one below (that still doesn't support non-ASCII domains)
 			//if (filter_var($input, FILTER_VALIDATE_URL)) {
 			if (preg_match('/^([-\pL\pN]+\.)+([a-z][a-z]|biz|cat|com|edu|gov|int|mil|net|org|pro|tel|aero|arpa|asia|coop|info|jobs|mobi|name|museum|travel)(:[0-9]{2,5})?$/iu', $inputRefanged)) {

app/Model/Warninglist.php

@@ -236,4 +236,26 @@ class Warninglist extends AppModel{
 		if (in_array($value, $listValues)) return true;
 		return false;
 	}
+
+	public function getAllIANAEntries() {
+		$result = $this->find('first', array(
+			'conditions' => array('Warninglist.name' => 'TLDs as known by IANA', 'enabled' => 1),
+			'recursive' => -1,
+			'contain' => array(
+				'WarninglistEntry' => array(
+					'fields' => array('WarninglistEntry.value')
+				)
+			)
+		));
+		if ((count($result))>0) {
+			return array_map(
+				function ($element) {
+					return strtolower($element['value']);
+				},
+				$result['WarninglistEntry']
+			);
+		} else {
+			return [];
+		}
+	}
 }

app/View/Events/resolved_attributes.ctp

@@ -29,7 +29,7 @@
 				<th>Similar Attributes</th>
 				<th>Category</th>
 				<th>Type</th>
-				<th>IDS<input type="checkbox" id="checkAll" style="margin:0px;margin-left:3px;"/></th>
+				<th>IDS<input type="checkbox" id="checkAll" style="margin:0;margin-left:3px;"/></th>
 				<th>Comment</th>
 				<th>Actions</th>
 		</tr>
@@ -55,7 +55,7 @@
 					echo $this->Form->input('Attribute' . $k . 'Value', array(
 							'label' => false,
 							'value' => h($item['value']),
-							'style' => 'padding:0px;height:20px;margin-bottom:0px;width:90%;',
+							'style' => 'padding:0;height:20px;margin-bottom:0;width:90%;',
 							'div' => false
 					));
 				?>
@@ -98,17 +98,24 @@
 
 					}
 				?>
-				<select id="<?php echo 'Attribute' . $k . 'Category'; ?>" style='padding:0px;height:20px;margin-bottom:0px;'>
+				<select id="<?php echo 'Attribute' . $k . 'Category'; ?>" style="padding:0;height:20px;margin-bottom:0;">
-					<?php
+				<?php
-						foreach ($typeCategoryMapping[$item['default_type']] as $category) {
+					$categoriesArray = $typeCategoryMapping[$item['default_type']];
-							if (isset($item['categories']) && !in_array($category, $item['categories'])) {
+					if (isset($item['merge_categories']) && $item['merge_categories'] === true) {
-								continue;
+						$categoriesArray = [];
-							}
+						foreach ($item['types'] as $type) {
-							echo '<option value="' . $category . '" ';
+							$categoriesArray = array_merge($categoriesArray, $typeCategoryMapping[$type]);
-							if ($category == $default) echo 'selected="selected"';
-							echo '>' . $category . '</option>';
 						}
-					?>
+					}
+					foreach ($categoriesArray as $category) {
+						if (isset($item['categories']) && !in_array($category, $item['categories'])) {
+							continue;
+						}
+						echo '<option value="' . $category . '" ';
+						if ($category == $default) echo 'selected="selected"';
+						echo '>' . $category . '</option>';
+					}
+				?>
 				</select>
 			</td>
 			<td class="short">