MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #8676 from JakubOnderka/security-domains-freetext 

Security domains freetext
pull/8677/head
Jakub Onderka 2022-10-18 13:03:35 +02:00 committed by GitHub
parent c500dff38c f7b9b82d2c
commit 6255effe4a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 84 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/Controller/EventsController.php
View File

@ -3958,6 +3958,7 @@ class EventsController extends AppController
$complexTypeTool = new ComplexTypeTool();
$this->loadModel('Warninglist');
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
$complexTypeTool->setSecurityVendorDomains($this->Warninglist->fetchSecurityVendorDomains());
if (!isset($this->request->data['Attribute'])) {
$this->request->data = array('Attribute' => $this->request->data);
}
@ -3968,6 +3969,9 @@ class EventsController extends AppController
$adhereToWarninglists = $this->request->data['Attribute']['adhereToWarninglists'];
}
$resultArray = $complexTypeTool->checkFreeText($this->request->data['Attribute']['value']);
foreach ($resultArray as &$attribute) {
$attribute['to_ids'] = $this->Event->Attribute->typeDefinitions[$attribute['default_type']]['to_ids'];
}
if ($this->_isRest()) {
// Keep this 'types' format for rest response, but it is not necessary for UI
foreach ($resultArray as $key => $r) {

82
app/Lib/Tools/ComplexTypeTool.php
View File

@ -40,7 +40,13 @@ class ComplexTypeTool
128 => ['single' => ['sha512'], 'composite' => ['filename|sha512']],
];
private $__tlds = null;
private $__tlds;
/**
* Hardcoded list if properly warninglist is not available
* @var string[]
*/
private $securityVendorDomains = ['virustotal.com', 'hybrid-analysis.com'];
public static function refangValue($value, $type)
{
@ -60,6 +66,14 @@ class ComplexTypeTool
}
}
public function setSecurityVendorDomains(array $securityVendorDomains)
{
if (empty($securityVendorDomains)) {
return; // if provided warninglist is empty, keep hardcoded domains
}
$this->securityVendorDomains = $securityVendorDomains;
}
public function checkComplexRouter($input, $type, $settings = array())
{
switch ($type) {
@ -188,6 +202,7 @@ class ComplexTypeTool
continue;
}
$resolvedResult = $this->__resolveType($element);
// Do not extract datetime from CSV
if ($resolvedResult) {
$iocArray[] = $resolvedResult;
}
@ -250,7 +265,6 @@ class ComplexTypeTool
if (filter_var($raw_input, FILTER_VALIDATE_IP)) {
return [
'types' => ['ip-dst', 'ip-src', 'ip-src/ip-dst'],
'to_ids' => true,
'default_type' => 'ip-dst',
'value' => $raw_input,
];
@ -296,7 +310,6 @@ class ComplexTypeTool
if (preg_match("#^([13][a-km-zA-HJ-NP-Z1-9]{25,34})|(bc|tb)1([023456789acdefghjklmnpqrstuvwxyz]{11,71})$#i", $input['raw'])) {
return [
'types' => ['btc'],
'to_ids' => true,
'default_type' => 'btc',
'value' => $input['raw'],
];
@ -311,7 +324,6 @@ class ComplexTypeTool
if (filter_var($input['refanged'], FILTER_VALIDATE_EMAIL)) {
return [
'types' => array('email', 'email-src', 'email-dst', 'target-email', 'whois-registrant-email'),
'to_ids' => true,
'default_type' => 'email-src',
'value' => $input['refanged'],
];
@ -324,7 +336,7 @@ class ComplexTypeTool
{
if (preg_match('#^as[0-9]+$#i', $input['raw'])) {
$input['raw'] = strtoupper($input['raw']);
return array('types' => array('AS'), 'to_ids' => false, 'default_type' => 'AS', 'value' => $input['raw']);
return array('types' => array('AS'), 'default_type' => 'AS', 'value' => $input['raw']);
}
return false;
}
@ -338,10 +350,10 @@ class ComplexTypeTool
if ($this->__resolveFilename($compositeParts[0])) {
$hash = $this->__resolveHash($compositeParts[1]);
if ($hash) {
return array('types' => $hash['composite'], 'to_ids' => true, 'default_type' => $hash['composite'][0], 'value' => $input['raw']);
return array('types' => $hash['composite'], 'default_type' => $hash['composite'][0], 'value' => $input['raw']);
}
if ($this->__resolveSsdeep($compositeParts[1])) {
return array('types' => array('filename|ssdeep'), 'to_ids' => true, 'default_type' => 'filename|ssdeep', 'value' => $input['raw']);
return array('types' => array('filename|ssdeep'), 'default_type' => 'filename|ssdeep', 'value' => $input['raw']);
}
}
}
@ -354,11 +366,11 @@ class ComplexTypeTool
if ($this->__checkForBTC($input)) {
$types[] = 'btc';
}
return array('types' => $types, 'to_ids' => true, 'default_type' => $types[0], 'value' => $input['raw']);
return array('types' => $types, 'default_type' => $types[0], 'value' => $input['raw']);
}
// ssdeep has a different pattern
if ($this->__resolveSsdeep($input['raw'])) {
return array('types' => array('ssdeep'), 'to_ids' => true, 'default_type' => 'ssdeep', 'value' => $input['raw']);
return array('types' => array('ssdeep'), 'default_type' => 'ssdeep', 'value' => $input['raw']);
}
return false;
}
@ -401,15 +413,15 @@ class ComplexTypeTool
if (preg_match("#^cve-[0-9]{4}-[0-9]{4,9}$#i", $input['raw'])) {
return [
'types' => ['vulnerability'],
'to_ids' => false,
'default_type' => 'vulnerability',
'value' => strtoupper($input['raw']), // 'CVE' must be uppercase
];
}
// Phone numbers - for automatic recognition, needs to start with + or include dashes
if ($input['raw'][0] === '+' || strpos($input['raw'], '-')) {
if (!preg_match('#^[0-9]{4}-[0-9]{2}-[0-9]{2}$#i', $input['raw']) && preg_match("#^(\+)?([0-9]{1,3}(\(0\))?)?[0-9\/\-]{5,}[0-9]$#i", $input['raw'])) {
return array('types' => array('phone-number', 'prtn', 'whois-registrant-phone'), 'to_ids' => false, 'default_type' => 'phone-number', 'value' => $input['raw']);
return array('types' => array('phone-number', 'prtn', 'whois-registrant-phone'), 'default_type' => 'phone-number', 'value' => $input['raw']);
}
}
return false;
@ -419,16 +431,15 @@ class ComplexTypeTool
{
if (filter_var($input['refanged_no_port'], FILTER_VALIDATE_IP)) {
if (isset($input['port'])) {
return array('types' => array('ip-dst|port', 'ip-src|port', 'ip-src|port/ip-dst|port'), 'to_ids' => true, 'default_type' => 'ip-dst|port', 'comment' => $input['comment'], 'value' => $input['refanged_no_port'] . '|' . $input['port']);
return array('types' => array('ip-dst|port', 'ip-src|port', 'ip-src|port/ip-dst|port'), 'default_type' => 'ip-dst|port', 'comment' => $input['comment'], 'value' => $input['refanged_no_port'] . '|' . $input['port']);
} else {
return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'to_ids' => true, 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
}
}
// IPv6 address that is considered as IP address with port
if (filter_var($input['refanged'], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) {
return [
'types' => ['ip-dst', 'ip-src', 'ip-src/ip-dst'],
'to_ids' => true,
'default_type' => 'ip-dst',
'comment' => '',
'value' => $input['refanged'],
@ -443,7 +454,6 @@ class ComplexTypeTool
$value = substr($input['refanged_no_port'], 1, -1); // remove brackets
return [
'types' => ['ip-dst|port', 'ip-src|port', 'ip-src|port/ip-dst|port'],
'to_ids' => true,
'default_type' => 'ip-dst|port',
'comment' => $input['comment'],
'value' => "$value|{$input['port']}",
@ -453,7 +463,7 @@ class ComplexTypeTool
if (strpos($input['refanged_no_port'], '/')) {
$temp = explode('/', $input['refanged_no_port']);
if (count($temp) === 2 && filter_var($temp[0], FILTER_VALIDATE_IP) && is_numeric($temp[1])) {
return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'to_ids' => true, 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
}
}
return false;
@ -473,25 +483,24 @@ class ComplexTypeTool
}
if ($domainDetection) {
if (count($temp) > 2) {
return array('types' => array('hostname', 'domain', 'url', 'filename'), 'to_ids' => true, 'default_type' => 'hostname', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
return array('types' => array('hostname', 'domain', 'url', 'filename'), 'default_type' => 'hostname', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
} else {
return array('types' => array('domain', 'filename'), 'to_ids' => true, 'default_type' => 'domain', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
return array('types' => array('domain', 'filename'), 'default_type' => 'domain', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
}
} else {
// check if it is a URL
// Adding http:// infront of the input in case it was left off. github.com/MISP/MISP should still be counted as a valid link
if (count($temp) > 1 && (filter_var($input['refanged_no_port'], FILTER_VALIDATE_URL) || filter_var('http://' . $input['refanged_no_port'], FILTER_VALIDATE_URL))) {
// Even though some domains are valid, we want to exclude them as they are known security vendors / etc
// TODO, replace that with the appropriate warninglist.
if (preg_match('/^(https:\/\/(www.)?virustotal.com\/|https:\/\/www\.hybrid-analysis\.com\/)/i', $input['refanged_no_port'])) {
return array('types' => array('link'), 'to_ids' => false, 'default_type' => 'link', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
if ($this->isLink($input['refanged_no_port'])) {
return array('types' => array('link'), 'default_type' => 'link', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
}
if (strpos($input['refanged_no_port'], '/')) {
return array('types' => array('url'), 'to_ids' => true, 'default_type' => 'url', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
return array('types' => array('url'), 'default_type' => 'url', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
}
}
if ($this->__resolveFilename($input['raw'])) {
return array('types' => array('filename'), 'to_ids' => true, 'default_type' => 'filename', 'value' => $input['raw']);
return array('types' => array('filename'), 'default_type' => 'filename', 'value' => $input['raw']);
}
}
}
@ -499,10 +508,10 @@ class ComplexTypeTool
$temp = explode('\\', $input['raw']);
if (strpos(end($temp), '.') || preg_match('/^.:/i', $temp[0])) {
if ($this->__resolveFilename(end($temp))) {
return array('types' => array('filename'), 'to_ids' => true, 'default_type' => 'filename', 'value' => $input['raw']);
return array('types' => array('filename'), 'default_type' => 'filename', 'value' => $input['raw']);
}
} else if (!empty($temp[0])) {
return array('types' => array('regkey'), 'to_ids' => false, 'default_type' => 'regkey', 'value' => $input['raw']);
return array('types' => array('regkey'), 'default_type' => 'regkey', 'value' => $input['raw']);
}
}
return false;
@ -553,6 +562,29 @@ class ComplexTypeTool
return isset($this->__tlds[strtolower($tld)]);
}
/**
* Check if URL should be considered as link attribute type
* @param string $value
* @return bool
*/
private function isLink($value)
{
if (!preg_match('/^https:\/\/([^\/]*)/i', $value, $matches)) {
return false;
}
$domainToCheck = '';
$domainParts = array_reverse(explode('.', strtolower($matches[1])));
foreach ($domainParts as $domainPart) {
$domainToCheck = $domainPart . $domainToCheck;
if (in_array($domainToCheck, $this->securityVendorDomains, true)) {
return true;
}
$domainToCheck = '.' . $domainToCheck;
}
return false;
}
private function __generateTLDList()
{
$tlds = array('biz', 'cat', 'com', 'edu', 'gov', 'int', 'mil', 'net', 'org', 'pro', 'tel', 'aero', 'arpa', 'asia', 'coop', 'info', 'jobs', 'mobi', 'name', 'museum', 'travel', 'onion');

1
app/Model/Event.php
View File

@ -5444,6 +5444,7 @@ class Event extends AppModel
}
$this->Warninglist = ClassRegistry::init('Warninglist');
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
$complexTypeTool->setSecurityVendorDomains($this->Warninglist->fetchSecurityVendorDomains());
$freetextResults = array_merge($freetextResults, $complexTypeTool->checkFreeText($value));
if (!empty($freetextResults)) {
foreach ($freetextResults as &$ft) {

1
app/Model/EventReport.php
View File

@ -738,6 +738,7 @@ class EventReport extends AppModel
$complexTypeTool = new ComplexTypeTool();
$this->Warninglist = ClassRegistry::init('Warninglist');
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
$complexTypeTool->setSecurityVendorDomains($this->Warninglist->fetchSecurityVendorDomains());
$complexTypeToolResult = $complexTypeTool->checkFreeText($report['EventReport']['content']);
$replacementResult = $this->transformFreeTextIntoReplacement($user, $report, $complexTypeToolResult);

1
app/Model/Feed.php
View File

@ -377,6 +377,7 @@ class Feed extends AppModel
$complexTypeTool = new ComplexTypeTool();
$this->Warninglist = ClassRegistry::init('Warninglist');
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
$complexTypeTool->setSecurityVendorDomains($this->Warninglist->fetchSecurityVendorDomains());
$settings = array();
if (!empty($feed['Feed']['settings']) && !is_array($feed['Feed']['settings'])) {
$feed['Feed']['settings'] = json_decode($feed['Feed']['settings'], true);

29
app/Model/Warninglist.php
View File

@ -727,22 +727,33 @@ class Warninglist extends AppModel
'conditions' => array('Warninglist.name' => self::TLDS),
'fields' => array('Warninglist.id')
));
$tlds = array();
if (!empty($tldLists)) {
$tlds = $this->WarninglistEntry->find('column', array(
'conditions' => array('WarninglistEntry.warninglist_id' => $tldLists),
'fields' => array('WarninglistEntry.value')
));
foreach ($tlds as $key => $value) {
$tlds[$key] = strtolower($value);
}
$tlds = [];
foreach ($tldLists as $warninglistId) {
$tlds = array_merge($tlds, $this->getWarninglistEntries($warninglistId));
}
$tlds = array_map('strtolower', $tlds);
if (!in_array('onion', $tlds, true)) {
$tlds[] = 'onion';
}
return $tlds;
}
/**
* @return array
*/
public function fetchSecurityVendorDomains()
{
$securityVendorList = $this->find('column', array(
'conditions' => array('Warninglist.name' => 'List of known domains used by automated malware analysis services & security vendors'),
'fields' => array('Warninglist.id')
));
$domains = [];
foreach ($securityVendorList as $warninglistId) {
$domains = array_merge($domains, $this->getWarninglistEntries($warninglistId));
}
return $domains;
}
/**
* @param array $attribute
* @param array|null $warninglists If null, all enabled warninglists will be used

3
app/Test/ComplexTypeToolTest.php
View File

@ -361,11 +361,9 @@ EOT;
$this->assertEquals('https://www.virustotal.com/example', $results[0]['value']);
$this->assertEquals('link', $results[0]['default_type']);
$this->assertFalse($results[0]['to_ids']);
$this->assertEquals('https://virustotal.com/example', $results[1]['value']);
$this->assertEquals('link', $results[1]['default_type']);
$this->assertFalse($results[1]['to_ids']);
}
public function testCheckFreeTextUrlHybridAnalysis(): void
@ -375,7 +373,6 @@ EOT;
$this->assertCount(1, $results);
$this->assertEquals('https://www.hybrid-analysis.com/example', $results[0]['value']);
$this->assertEquals('link', $results[0]['default_type']);
$this->assertFalse($results[0]['to_ids']);
}
// Issue https://github.com/MISP/MISP/issues/4908