MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

fix: [internal] Advanced correlations

pull/8524/head
Jakub Onderka 2022-08-08 18:31:00 +02:00
parent ec209a98b9
commit 986e109f76
3 changed files with 91 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
app/Model/Correlation.php
View File

@ -3,6 +3,9 @@ App::uses('AppModel', 'Model');
/**
* @property Attribute $Attribute
* @method saveCorrelations
* @method runBeforeSaveCorrelation
* @method fetchRelatedEventIds
*/
class Correlation extends AppModel
{
@ -43,36 +46,27 @@ class Correlation extends AppModel
/** @var array */
private $exclusions;
/**
* Use old schema with `date` and `info` fields.
* @var bool
*/
private $oldSchema;
/** @var bool */
private $deadlockAvoidance;
/** @var bool */
private $advancedCorrelationEnabled;
/** @var array */
private $cidrListCache;
private $__correlationEngine = 'DefaultCorrelation';
protected $_config = [];
/** @var string */
private $__correlationEngine;
private $__tempContainCache = [];
public $OverCorrelatingValue = null;
/** @var OverCorrelatingValue */
public $OverCorrelatingValue;
public function __construct($id = false, $table = null, $ds = null)
{
parent::__construct($id, $table, $ds);
$this->__correlationEngine = $this->getCorrelationModelName();
$this->deadlockAvoidance = Configure::check('MISP.deadlock_avoidance') ? Configure::read('MISP.deadlock_avoidance') : false;
$deadlockAvoidance = Configure::check('MISP.deadlock_avoidance') ? Configure::read('MISP.deadlock_avoidance') : false;
// load the currently used correlation engine
$this->Behaviors->load($this->__correlationEngine . 'Correlation', ['deadlockAvoidance' => false]);
$this->Behaviors->load($this->__correlationEngine . 'Correlation', ['deadlockAvoidance' => $deadlockAvoidance]);
// getTableName() needs to be implemented by the engine - this points us to the table to be used
$this->useTable = $this->getTableName();
$this->advancedCorrelationEnabled = (bool)Configure::read('MISP.enable_advanced_correlations');
@ -289,6 +283,7 @@ class Correlation extends AppModel
* @param bool $full
* @param array|false $event
* @return array|bool|bool[]|mixed
* @throws Exception
*/
public function afterSaveCorrelation($a, $full = false, $event = false)
{
@ -335,7 +330,7 @@ class Correlation extends AppModel
return true;
}
$correlations = [];
foreach ($correlatingValues as $k => $cV) {
foreach ($correlatingValues as $cV) {
if ($cV === null) {
continue;
}
@ -346,6 +341,7 @@ class Correlation extends AppModel
'Attribute.value2' => $cV,
'NOT' => ['Attribute.type' => Attribute::PRIMARY_ONLY_CORRELATING_TYPES]
],
$extraConditions,
],
'NOT' => [
'Attribute.event_id' => $a['Attribute']['event_id'],
@ -850,7 +846,7 @@ class Correlation extends AppModel
/**
* @param array $user User array
* @param int $eventIds List of event IDs
* @param int $eventId List of event IDs
* @param array $sgids List of sharing group IDs
* @return array
*/
@ -871,27 +867,32 @@ class Correlation extends AppModel
return $data;
}
/**
* @param array $attribute
* @return array
*/
public function setCorrelationExclusion($attribute)
{
if (empty($this->__compositeTypes)) {
if (!isset($this->__compositeTypes)) {
$this->__compositeTypes = $this->Attribute->getCompositeTypes();
}
$values = [$attribute['value']];
if (in_array($attribute['type'], $this->__compositeTypes)) {
if (in_array($attribute['type'], $this->__compositeTypes, true)) {
$values = explode('|', $attribute['value']);
} else {
$values = [$attribute['value']];
}
if ($this->__preventExcludedCorrelations($values[0])) {
$attribute['correlation_exclusion'] = true;
}
if (!empty($values[1]) && $this->__preventExcludedCorrelations($values[1])) {
} elseif (!empty($values[1]) && $this->__preventExcludedCorrelations($values[1])) {
$attribute['correlation_exclusion'] = true;
}
if ($this->OverCorrelatingValue->checkValue($values[0])) {
$attribute['over_correlation'] = true;
}
if (!empty($values[1]) && $this->OverCorrelatingValue->checkValue($values[1])) {
} elseif (!empty($values[1]) && $this->OverCorrelatingValue->checkValue($values[1])) {
$attribute['over_correlation'] = true;
}
return $attribute;
}

29
app/Model/OverCorrelatingValue.php
View File

@ -1,6 +1,5 @@
<?php
App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
class OverCorrelatingValue extends AppModel
{
@ -10,9 +9,13 @@ class OverCorrelatingValue extends AppModel
'Containable'
);
public $validate = [
];
/**
* @param string $value
* @param int $count
* @return void
* @throws Exception
*/
public function block($value, $count = 0)
{
$this->unblock($value);
@ -25,15 +28,23 @@ class OverCorrelatingValue extends AppModel
);
}
/**
* @param string $value
* @return void
*/
public function unBlock($value)
{
$this->deleteAll(
[
'OverCorrelatingValue.value' => $value
]
],
false
);
}
/**
* @return int
*/
public function getLimit()
{
return Configure::check('MISP.correlation_limit') ? Configure::read('MISP.correlation_limit') : 20;
@ -55,15 +66,7 @@ class OverCorrelatingValue extends AppModel
public function checkValue($value)
{
$hit = $this->find('first', [
'recursive' => -1,
'conditions' => ['value' => $value],
'fields' => ['id']
]);
if (empty($hit)) {
return false;
}
return true;
return $this->hasAny(['value' => $value]);
}
public function generateOccurrencesRouter()

53
tests/testlive_comprehensive_local.py
View File

@ -24,13 +24,15 @@ key = os.environ["AUTH"]
urllib3.disable_warnings()
def create_simple_event():
def create_simple_event() -> MISPEvent:
event_uuid = str(uuid.uuid4())
event = MISPEvent()
event.info = 'This is a super simple test'
event.uuid = event_uuid
event.info = 'This is a super simple test ({})'.format(event_uuid.split('-')[0])
event.distribution = Distribution.your_organisation_only
event.threat_level_id = ThreatLevel.low
event.analysis = Analysis.completed
event.add_attribute('text', str(uuid.uuid4()))
event.add_attribute('text', event_uuid)
return event
@ -512,6 +514,51 @@ class TestComprehensive(unittest.TestCase):
for event in (first, second, third, four):
check_response(self.admin_misp_connector.delete_event(event))
def test_correlations(self):
first = create_simple_event()
first.add_attribute("ip-src", "10.0.0.1")
first = check_response(self.admin_misp_connector.add_event(first))
second = create_simple_event()
second.add_attribute("ip-src", "10.0.0.1")
second = check_response(self.admin_misp_connector.add_event(second))
# Reload to get event data with related events
first = check_response(self.admin_misp_connector.get_event(first))
try:
self.assertEqual(1, len(first.RelatedEvent), first.RelatedEvent)
self.assertEqual(1, len(second.RelatedEvent), second.RelatedEvent)
except:
raise
finally:
# Delete events
for event in (first, second):
check_response(self.admin_misp_connector.delete_event(event))
def test_advanced_correlations(self):
with MISPSetting(self.admin_misp_connector, {"MISP.enable_advanced_correlations": True}):
first = create_simple_event()
first.add_attribute("ip-src", "10.0.0.0/8")
first = check_response(self.admin_misp_connector.add_event(first))
second = create_simple_event()
second.add_attribute("ip-src", "10.0.0.1")
second = check_response(self.admin_misp_connector.add_event(second))
# Reload to get event data with related events
first = check_response(self.admin_misp_connector.get_event(first))
try:
self.assertEqual(1, len(first.RelatedEvent), first.RelatedEvent)
self.assertEqual(1, len(second.RelatedEvent), second.RelatedEvent)
except:
raise
finally:
# Delete events
for event in (first, second):
check_response(self.admin_misp_connector.delete_event(event))
def test_remove_orphaned_correlations(self):
result = self.admin_misp_connector._check_json_response(self.admin_misp_connector._prepare_request('GET', 'servers/removeOrphanedCorrelations'))
check_response(result)