mirror of https://github.com/MISP/MISP
new: [freetext] Fetch security vendor domains from warninglist
parent
4f19c0f47b
commit
3cc2b7b826
|
@ -3958,6 +3958,7 @@ class EventsController extends AppController
|
||||||
$complexTypeTool = new ComplexTypeTool();
|
$complexTypeTool = new ComplexTypeTool();
|
||||||
$this->loadModel('Warninglist');
|
$this->loadModel('Warninglist');
|
||||||
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
|
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
|
||||||
|
$complexTypeTool->setSecurityVendorDomains($this->Warninglist->fetchSecurityVendorDomains());
|
||||||
if (!isset($this->request->data['Attribute'])) {
|
if (!isset($this->request->data['Attribute'])) {
|
||||||
$this->request->data = array('Attribute' => $this->request->data);
|
$this->request->data = array('Attribute' => $this->request->data);
|
||||||
}
|
}
|
||||||
|
|
|
@ -40,7 +40,13 @@ class ComplexTypeTool
|
||||||
128 => ['single' => ['sha512'], 'composite' => ['filename|sha512']],
|
128 => ['single' => ['sha512'], 'composite' => ['filename|sha512']],
|
||||||
];
|
];
|
||||||
|
|
||||||
private $__tlds = null;
|
private $__tlds;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Hardcoded list if properly warninglist is not available
|
||||||
|
* @var string[]
|
||||||
|
*/
|
||||||
|
private $securityVendorDomains = ['virustotal.com', 'hybrid-analysis.com'];
|
||||||
|
|
||||||
public static function refangValue($value, $type)
|
public static function refangValue($value, $type)
|
||||||
{
|
{
|
||||||
|
@ -60,6 +66,14 @@ class ComplexTypeTool
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function setSecurityVendorDomains(array $securityVendorDomains)
|
||||||
|
{
|
||||||
|
if (empty($securityVendorDomains)) {
|
||||||
|
return; // if provided warninglist is empty, keep hardcoded domains
|
||||||
|
}
|
||||||
|
$this->securityVendorDomains = $securityVendorDomains;
|
||||||
|
}
|
||||||
|
|
||||||
public function checkComplexRouter($input, $type, $settings = array())
|
public function checkComplexRouter($input, $type, $settings = array())
|
||||||
{
|
{
|
||||||
switch ($type) {
|
switch ($type) {
|
||||||
|
@ -478,8 +492,7 @@ class ComplexTypeTool
|
||||||
// Adding http:// infront of the input in case it was left off. github.com/MISP/MISP should still be counted as a valid link
|
// Adding http:// infront of the input in case it was left off. github.com/MISP/MISP should still be counted as a valid link
|
||||||
if (count($temp) > 1 && (filter_var($input['refanged_no_port'], FILTER_VALIDATE_URL) || filter_var('http://' . $input['refanged_no_port'], FILTER_VALIDATE_URL))) {
|
if (count($temp) > 1 && (filter_var($input['refanged_no_port'], FILTER_VALIDATE_URL) || filter_var('http://' . $input['refanged_no_port'], FILTER_VALIDATE_URL))) {
|
||||||
// Even though some domains are valid, we want to exclude them as they are known security vendors / etc
|
// Even though some domains are valid, we want to exclude them as they are known security vendors / etc
|
||||||
// TODO, replace that with the appropriate warninglist.
|
if ($this->isLink($input['refanged_no_port'])) {
|
||||||
if (preg_match('/^(https:\/\/(www.)?virustotal.com\/|https:\/\/www\.hybrid-analysis\.com\/)/i', $input['refanged_no_port'])) {
|
|
||||||
return array('types' => array('link'), 'default_type' => 'link', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
|
return array('types' => array('link'), 'default_type' => 'link', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
|
||||||
}
|
}
|
||||||
if (strpos($input['refanged_no_port'], '/')) {
|
if (strpos($input['refanged_no_port'], '/')) {
|
||||||
|
@ -549,6 +562,29 @@ class ComplexTypeTool
|
||||||
return isset($this->__tlds[strtolower($tld)]);
|
return isset($this->__tlds[strtolower($tld)]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if URL should be considered as link attribute type
|
||||||
|
* @param string $value
|
||||||
|
* @return bool
|
||||||
|
*/
|
||||||
|
private function isLink($value)
|
||||||
|
{
|
||||||
|
if (!preg_match('/^https:\/\/([^\/]*)/i', $value, $matches)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
$domainToCheck = '';
|
||||||
|
$domainParts = array_reverse(explode('.', strtolower($matches[1])));
|
||||||
|
foreach ($domainParts as $domainPart) {
|
||||||
|
$domainToCheck = $domainPart . $domainToCheck;
|
||||||
|
if (in_array($domainToCheck, $this->securityVendorDomains, true)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
$domainToCheck = '.' . $domainToCheck;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
private function __generateTLDList()
|
private function __generateTLDList()
|
||||||
{
|
{
|
||||||
$tlds = array('biz', 'cat', 'com', 'edu', 'gov', 'int', 'mil', 'net', 'org', 'pro', 'tel', 'aero', 'arpa', 'asia', 'coop', 'info', 'jobs', 'mobi', 'name', 'museum', 'travel', 'onion');
|
$tlds = array('biz', 'cat', 'com', 'edu', 'gov', 'int', 'mil', 'net', 'org', 'pro', 'tel', 'aero', 'arpa', 'asia', 'coop', 'info', 'jobs', 'mobi', 'name', 'museum', 'travel', 'onion');
|
||||||
|
|
|
@ -5449,6 +5449,7 @@ class Event extends AppModel
|
||||||
}
|
}
|
||||||
$this->Warninglist = ClassRegistry::init('Warninglist');
|
$this->Warninglist = ClassRegistry::init('Warninglist');
|
||||||
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
|
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
|
||||||
|
$complexTypeTool->setSecurityVendorDomains($this->Warninglist->fetchSecurityVendorDomains());
|
||||||
$freetextResults = array_merge($freetextResults, $complexTypeTool->checkFreeText($value));
|
$freetextResults = array_merge($freetextResults, $complexTypeTool->checkFreeText($value));
|
||||||
if (!empty($freetextResults)) {
|
if (!empty($freetextResults)) {
|
||||||
foreach ($freetextResults as &$ft) {
|
foreach ($freetextResults as &$ft) {
|
||||||
|
|
|
@ -738,6 +738,7 @@ class EventReport extends AppModel
|
||||||
$complexTypeTool = new ComplexTypeTool();
|
$complexTypeTool = new ComplexTypeTool();
|
||||||
$this->Warninglist = ClassRegistry::init('Warninglist');
|
$this->Warninglist = ClassRegistry::init('Warninglist');
|
||||||
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
|
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
|
||||||
|
$complexTypeTool->setSecurityVendorDomains($this->Warninglist->fetchSecurityVendorDomains());
|
||||||
|
|
||||||
$complexTypeToolResult = $complexTypeTool->checkFreeText($report['EventReport']['content']);
|
$complexTypeToolResult = $complexTypeTool->checkFreeText($report['EventReport']['content']);
|
||||||
$replacementResult = $this->transformFreeTextIntoReplacement($user, $report, $complexTypeToolResult);
|
$replacementResult = $this->transformFreeTextIntoReplacement($user, $report, $complexTypeToolResult);
|
||||||
|
|
|
@ -377,6 +377,7 @@ class Feed extends AppModel
|
||||||
$complexTypeTool = new ComplexTypeTool();
|
$complexTypeTool = new ComplexTypeTool();
|
||||||
$this->Warninglist = ClassRegistry::init('Warninglist');
|
$this->Warninglist = ClassRegistry::init('Warninglist');
|
||||||
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
|
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
|
||||||
|
$complexTypeTool->setSecurityVendorDomains($this->Warninglist->fetchSecurityVendorDomains());
|
||||||
$settings = array();
|
$settings = array();
|
||||||
if (!empty($feed['Feed']['settings']) && !is_array($feed['Feed']['settings'])) {
|
if (!empty($feed['Feed']['settings']) && !is_array($feed['Feed']['settings'])) {
|
||||||
$feed['Feed']['settings'] = json_decode($feed['Feed']['settings'], true);
|
$feed['Feed']['settings'] = json_decode($feed['Feed']['settings'], true);
|
||||||
|
|
|
@ -733,9 +733,7 @@ class Warninglist extends AppModel
|
||||||
'conditions' => array('WarninglistEntry.warninglist_id' => $tldLists),
|
'conditions' => array('WarninglistEntry.warninglist_id' => $tldLists),
|
||||||
'fields' => array('WarninglistEntry.value')
|
'fields' => array('WarninglistEntry.value')
|
||||||
));
|
));
|
||||||
foreach ($tlds as $key => $value) {
|
$tlds = array_map('strtolower', $tlds);
|
||||||
$tlds[$key] = strtolower($value);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (!in_array('onion', $tlds, true)) {
|
if (!in_array('onion', $tlds, true)) {
|
||||||
$tlds[] = 'onion';
|
$tlds[] = 'onion';
|
||||||
|
@ -743,6 +741,25 @@ class Warninglist extends AppModel
|
||||||
return $tlds;
|
return $tlds;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return array
|
||||||
|
*/
|
||||||
|
public function fetchSecurityVendorDomains()
|
||||||
|
{
|
||||||
|
$securityVendorList = $this->find('column', array(
|
||||||
|
'conditions' => array('Warninglist.name' => 'List of known domains used by automated malware analysis services & security vendors'),
|
||||||
|
'fields' => array('Warninglist.id')
|
||||||
|
));
|
||||||
|
if (!empty($securityVendorList)) {
|
||||||
|
return $this->WarninglistEntry->find('column', array(
|
||||||
|
'conditions' => array('WarninglistEntry.warninglist_id' => $securityVendorList),
|
||||||
|
'fields' => array('WarninglistEntry.value')
|
||||||
|
));
|
||||||
|
} else {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param array $attribute
|
* @param array $attribute
|
||||||
* @param array|null $warninglists If null, all enabled warninglists will be used
|
* @param array|null $warninglists If null, all enabled warninglists will be used
|
||||||
|
|
Loading…
Reference in New Issue