MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

new: [freetext] Fetch security vendor domains from warninglist

pull/8676/head
Jakub Onderka 2022-10-03 21:12:23 +02:00
parent 4f19c0f47b
commit 3cc2b7b826
6 changed files with 63 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/Controller/EventsController.php
View File

@ -3958,6 +3958,7 @@ class EventsController extends AppController
$complexTypeTool = new ComplexTypeTool();
$this->loadModel('Warninglist');
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
$complexTypeTool->setSecurityVendorDomains($this->Warninglist->fetchSecurityVendorDomains());
if (!isset($this->request->data['Attribute'])) {
$this->request->data = array('Attribute' => $this->request->data);
}

42
app/Lib/Tools/ComplexTypeTool.php
View File

@ -40,7 +40,13 @@ class ComplexTypeTool
128 => ['single' => ['sha512'], 'composite' => ['filename|sha512']],
];
private $__tlds = null;
private $__tlds;
/**
* Hardcoded list if properly warninglist is not available
* @var string[]
*/
private $securityVendorDomains = ['virustotal.com', 'hybrid-analysis.com'];
public static function refangValue($value, $type)
{
@ -60,6 +66,14 @@ class ComplexTypeTool
}
}
public function setSecurityVendorDomains(array $securityVendorDomains)
{
if (empty($securityVendorDomains)) {
return; // if provided warninglist is empty, keep hardcoded domains
}
$this->securityVendorDomains = $securityVendorDomains;
}
public function checkComplexRouter($input, $type, $settings = array())
{
switch ($type) {
@ -478,8 +492,7 @@ class ComplexTypeTool
// Adding http:// infront of the input in case it was left off. github.com/MISP/MISP should still be counted as a valid link
if (count($temp) > 1 && (filter_var($input['refanged_no_port'], FILTER_VALIDATE_URL) || filter_var('http://' . $input['refanged_no_port'], FILTER_VALIDATE_URL))) {
// Even though some domains are valid, we want to exclude them as they are known security vendors / etc
// TODO, replace that with the appropriate warninglist.
if (preg_match('/^(https:\/\/(www.)?virustotal.com\/|https:\/\/www\.hybrid-analysis\.com\/)/i', $input['refanged_no_port'])) {
if ($this->isLink($input['refanged_no_port'])) {
return array('types' => array('link'), 'default_type' => 'link', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
}
if (strpos($input['refanged_no_port'], '/')) {
@ -549,6 +562,29 @@ class ComplexTypeTool
return isset($this->__tlds[strtolower($tld)]);
}
/**
* Check if URL should be considered as link attribute type
* @param string $value
* @return bool
*/
private function isLink($value)
{
if (!preg_match('/^https:\/\/([^\/]*)/i', $value, $matches)) {
return false;
}
$domainToCheck = '';
$domainParts = array_reverse(explode('.', strtolower($matches[1])));
foreach ($domainParts as $domainPart) {
$domainToCheck = $domainPart . $domainToCheck;
if (in_array($domainToCheck, $this->securityVendorDomains, true)) {
return true;
}
$domainToCheck = '.' . $domainToCheck;
}
return false;
}
private function __generateTLDList()
{
$tlds = array('biz', 'cat', 'com', 'edu', 'gov', 'int', 'mil', 'net', 'org', 'pro', 'tel', 'aero', 'arpa', 'asia', 'coop', 'info', 'jobs', 'mobi', 'name', 'museum', 'travel', 'onion');

1
app/Model/Event.php
View File

@ -5449,6 +5449,7 @@ class Event extends AppModel
}
$this->Warninglist = ClassRegistry::init('Warninglist');
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
$complexTypeTool->setSecurityVendorDomains($this->Warninglist->fetchSecurityVendorDomains());
$freetextResults = array_merge($freetextResults, $complexTypeTool->checkFreeText($value));
if (!empty($freetextResults)) {
foreach ($freetextResults as &$ft) {

1
app/Model/EventReport.php
View File

@ -738,6 +738,7 @@ class EventReport extends AppModel
$complexTypeTool = new ComplexTypeTool();
$this->Warninglist = ClassRegistry::init('Warninglist');
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
$complexTypeTool->setSecurityVendorDomains($this->Warninglist->fetchSecurityVendorDomains());
$complexTypeToolResult = $complexTypeTool->checkFreeText($report['EventReport']['content']);
$replacementResult = $this->transformFreeTextIntoReplacement($user, $report, $complexTypeToolResult);

1
app/Model/Feed.php
View File

@ -377,6 +377,7 @@ class Feed extends AppModel
$complexTypeTool = new ComplexTypeTool();
$this->Warninglist = ClassRegistry::init('Warninglist');
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
$complexTypeTool->setSecurityVendorDomains($this->Warninglist->fetchSecurityVendorDomains());
$settings = array();
if (!empty($feed['Feed']['settings']) && !is_array($feed['Feed']['settings'])) {
$feed['Feed']['settings'] = json_decode($feed['Feed']['settings'], true);

23
app/Model/Warninglist.php
View File

@ -733,9 +733,7 @@ class Warninglist extends AppModel
'conditions' => array('WarninglistEntry.warninglist_id' => $tldLists),
'fields' => array('WarninglistEntry.value')
));
foreach ($tlds as $key => $value) {
$tlds[$key] = strtolower($value);
}
$tlds = array_map('strtolower', $tlds);
}
if (!in_array('onion', $tlds, true)) {
$tlds[] = 'onion';
@ -743,6 +741,25 @@ class Warninglist extends AppModel
return $tlds;
}
/**
* @return array
*/
public function fetchSecurityVendorDomains()
{
$securityVendorList = $this->find('column', array(
'conditions' => array('Warninglist.name' => 'List of known domains used by automated malware analysis services & security vendors'),
'fields' => array('Warninglist.id')
));
if (!empty($securityVendorList)) {
return $this->WarninglistEntry->find('column', array(
'conditions' => array('WarninglistEntry.warninglist_id' => $securityVendorList),
'fields' => array('WarninglistEntry.value')
));
} else {
return [];
}
}
/**
* @param array $attribute
* @param array|null $warninglists If null, all enabled warninglists will be used