mirror of https://github.com/MISP/MISP
new: [Correlations] refactor / rework
- moved to own controller and model - refactored several long incomprehensible functions - extracted reused tasks from functions and made them reusable - added a way to correlate individual values as opposed to attributes - Added top correlations indexpull/7374/head
iglocska
parent
65baee7267
commit
15fc60ebbf
|
|
@ -94,6 +94,9 @@ class ACLComponent extends Component
|
|||
'index' => [],
|
||||
'view' => []
|
||||
],
|
||||
'correlations' => [
|
||||
'top' => []
|
||||
],
|
||||
'dashboards' => array(
|
||||
'getForm' => array('*'),
|
||||
'index' => array('*'),
|
||||
|
|
|
|||
|
|
@ -43,9 +43,15 @@ class CorrelationExclusionsController extends AppController
|
|||
}
|
||||
}
|
||||
|
||||
public function add($user_id = false)
|
||||
public function add()
|
||||
{
|
||||
$params = [];
|
||||
$options = [
|
||||
'filters' => ['value', 'redirect']
|
||||
];
|
||||
$params = $this->IndexFilter->harvestParameters($options['filters']);
|
||||
if (!empty($params['value'])) {
|
||||
$this->request->data['CorrelationExclusion']['value'] = $params['value'];
|
||||
}
|
||||
$this->CRUD->add($params);
|
||||
if ($this->IndexFilter->isRest()) {
|
||||
return $this->restResponsePayload;
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ App::uses('AppController', 'Controller');
|
|||
*/
|
||||
class CorrelationsController extends AppController
|
||||
{
|
||||
public function top_correlations()
|
||||
public function top()
|
||||
{
|
||||
$options = [
|
||||
'filters' => ['value', 'quickFilter'],
|
||||
|
|
@ -32,28 +32,16 @@ class CorrelationsController extends AppController
|
|||
$data = $this->Correlation->find('all', $query);
|
||||
return $this->RestResponse->viewData($data, 'json');
|
||||
} else {
|
||||
if (empty($query['limit'])) {
|
||||
$query['limit'] = 20;
|
||||
}
|
||||
if (empty($query['page'])) {
|
||||
$query['page'] = 1;
|
||||
}
|
||||
$query['limit'] = empty($query['limit']) ? 20 : $query['limit'];
|
||||
$query['page'] = empty($query['page']) ? 1 : $query['page'];
|
||||
$this->paginate = $query;
|
||||
$data = $this->Correlation->find('all', $query);
|
||||
$this->set('data', $data);
|
||||
$this->set('title_for_layout', __('Top correlations index'));
|
||||
$this->set('menuData', [
|
||||
'menuList' => 'correlationExclusions',
|
||||
'menuItem' => 'top_correlations'
|
||||
'menuItem' => 'top'
|
||||
]);
|
||||
}
|
||||
if ($this->IndexFilter->isRest()) {
|
||||
return $this->restResponsePayload;
|
||||
}
|
||||
$this->set('title_for_layout', __('Correlation Exclusions index'));
|
||||
$this->set('menuData', [
|
||||
'menuList' => 'correlationExclusions',
|
||||
'menuItem' => 'index'
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -293,15 +293,18 @@ class Attribute extends AppModel
|
|||
)
|
||||
);
|
||||
|
||||
public $hasMany = array(
|
||||
'AttributeTag' => array(
|
||||
public $hasMany = [
|
||||
'AttributeTag' => [
|
||||
'dependent' => true
|
||||
),
|
||||
'Sighting' => array(
|
||||
],
|
||||
'Correlation' => [
|
||||
'dependent' => false
|
||||
],
|
||||
'Sighting' => [
|
||||
'className' => 'Sighting',
|
||||
'dependent' => true,
|
||||
)
|
||||
);
|
||||
]
|
||||
];
|
||||
|
||||
public $hashTypes = array(
|
||||
'md5' => array(
|
||||
|
|
@ -388,7 +391,10 @@ class Attribute extends AppModel
|
|||
}
|
||||
// update correlation...
|
||||
if (isset($this->data['Attribute']['deleted']) && $this->data['Attribute']['deleted']) {
|
||||
$this->__beforeSaveCorrelation($this->data['Attribute']);
|
||||
if (empty($this->Correlation)) {
|
||||
$this->Correlation = ClassRegistry::init('Correlation');
|
||||
}
|
||||
$this->Correlation->beforeSaveCorrelation($this->data['Attribute']);
|
||||
if (isset($this->data['Attribute']['event_id'])) {
|
||||
$this->__alterAttributeCount($this->data['Attribute']['event_id'], false);
|
||||
}
|
||||
|
|
@ -409,11 +415,11 @@ class Attribute extends AppModel
|
|||
$this->data['Attribute']['distribution'] != $this->old['Attribute']['distribution'] ||
|
||||
$this->data['Attribute']['sharing_group_id'] != $this->old['Attribute']['sharing_group_id']
|
||||
) {
|
||||
$this->__beforeSaveCorrelation($this->data['Attribute']);
|
||||
$this->__afterSaveCorrelation($this->data['Attribute'], false, $passedEvent);
|
||||
$this->Correlation->beforeSaveCorrelation($this->data['Attribute']);
|
||||
$this->Correlation->afterSaveCorrelation($this->data['Attribute'], false, $passedEvent);
|
||||
}
|
||||
} else {
|
||||
$this->__afterSaveCorrelation($this->data['Attribute'], false, $passedEvent);
|
||||
$this->Correlation->afterSaveCorrelation($this->data['Attribute'], false, $passedEvent);
|
||||
}
|
||||
}
|
||||
$result = true;
|
||||
|
|
@ -475,7 +481,7 @@ class Attribute extends AppModel
|
|||
$this->loadAttachmentTool()->delete($this->data['Attribute']['event_id'], $this->data['Attribute']['id']);
|
||||
}
|
||||
// update correlation..
|
||||
$this->__beforeDeleteCorrelation($this->data['Attribute']['id']);
|
||||
$this->Correlation->beforeDeleteCorrelation($this->data['Attribute']['id']);
|
||||
if (!empty($this->data['Attribute']['id'])) {
|
||||
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_attribute_notifications_enable')) {
|
||||
$pubSubTool = $this->getPubSubTool();
|
||||
|
|
@ -1618,398 +1624,6 @@ class Attribute extends AppModel
|
|||
return $imageData;
|
||||
}
|
||||
|
||||
public function __beforeSaveCorrelation($a)
|
||||
{
|
||||
// (update-only) clean up the relation of the old value: remove the existing relations related to that attribute, we DO have a reference, the id
|
||||
// ==> DELETE FROM correlations WHERE 1_attribute_id = $a_id OR attribute_id = $a_id; */
|
||||
// first check if it's an update
|
||||
if (isset($a['id'])) {
|
||||
$this->Correlation = ClassRegistry::init('Correlation');
|
||||
// FIXME : check that $a['id'] is checked correctly so that the user can't remove attributes he shouldn't
|
||||
$dummy = $this->Correlation->deleteAll(
|
||||
array('OR' => array(
|
||||
'Correlation.1_attribute_id' => $a['id'],
|
||||
'Correlation.attribute_id' => $a['id']))
|
||||
);
|
||||
}
|
||||
if ($a['type'] === 'ssdeep') {
|
||||
$this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep');
|
||||
$this->FuzzyCorrelateSsdeep->purge(null, $a['id']);
|
||||
}
|
||||
}
|
||||
|
||||
// using Alnitak's solution from http://stackoverflow.com/questions/594112/matching-an-ip-to-a-cidr-mask-in-php5
|
||||
private function __ipv4InCidr($ip, $cidr)
|
||||
{
|
||||
list($subnet, $bits) = explode('/', $cidr);
|
||||
$ip = ip2long($ip);
|
||||
$subnet = ip2long($subnet);
|
||||
$mask = -1 << (32 - $bits);
|
||||
$subnet &= $mask; # nb: in case the supplied subnet wasn't correctly aligned
|
||||
return ($ip & $mask) == $subnet;
|
||||
}
|
||||
|
||||
// Using solution from https://github.com/symfony/symfony/blob/master/src/Symfony/Component/HttpFoundation/IpUtils.php
|
||||
private function __ipv6InCidr($ip, $cidr)
|
||||
{
|
||||
list($address, $netmask) = explode('/', $cidr);
|
||||
|
||||
$bytesAddr = unpack('n*', inet_pton($address));
|
||||
$bytesTest = unpack('n*', inet_pton($ip));
|
||||
|
||||
for ($i = 1, $ceil = ceil($netmask / 16); $i <= $ceil; ++$i) {
|
||||
$left = $netmask - 16 * ($i - 1);
|
||||
$left = ($left <= 16) ? $left : 16;
|
||||
$mask = ~(0xffff >> $left) & 0xffff;
|
||||
if (($bytesAddr[$i] & $mask) != ($bytesTest[$i] & $mask)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function __cidrCorrelation($a)
|
||||
{
|
||||
$ipValues = array();
|
||||
$ip = $a['value1'];
|
||||
if (strpos($ip, '/') !== false) { // IP is CIDR
|
||||
list($networkIp, $mask) = explode('/', $ip);
|
||||
$ip_version = filter_var($networkIp, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) ? 4 : 6;
|
||||
|
||||
$conditions = array(
|
||||
'type' => array('ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port'),
|
||||
'value1 NOT LIKE' => '%/%', // do not return CIDR, just plain IPs
|
||||
'disable_correlation' => 0,
|
||||
'deleted' => 0,
|
||||
);
|
||||
|
||||
if (in_array($this->getDataSource()->config['datasource'], array('Database/Mysql', 'Database/MysqlObserver'))) {
|
||||
// Massive speed up for CIDR correlation. Instead of testing all in PHP, database can do that work much
|
||||
// faster. But these methods are just supported by MySQL.
|
||||
if ($ip_version === 4) {
|
||||
$startIp = ip2long($networkIp) & ((-1 << (32 - $mask)));
|
||||
$endIp = $startIp + pow(2, (32 - $mask)) - 1;
|
||||
// Just fetch IP address that fit in CIDR range.
|
||||
$conditions['INET_ATON(value1) BETWEEN ? AND ?'] = array($startIp, $endIp);
|
||||
|
||||
// Just fetch IPv4 address that starts with given prefix. This is fast, because value1 is indexed.
|
||||
// This optimisation is possible just to mask bigger than 8 bites.
|
||||
if ($mask >= 8) {
|
||||
$ipv4Parts = explode('.', $networkIp);
|
||||
$ipv4Parts = array_slice($ipv4Parts, 0, intval($mask / 8));
|
||||
$prefix = implode('.', $ipv4Parts);
|
||||
$conditions['value1 LIKE'] = $prefix . '%';
|
||||
}
|
||||
} else {
|
||||
$conditions[] = 'IS_IPV6(value1)';
|
||||
// Just fetch IPv6 address that starts with given prefix. This is fast, because value1 is indexed.
|
||||
if ($mask >= 16) {
|
||||
$ipv6Parts = explode(':', rtrim($networkIp, ':'));
|
||||
$ipv6Parts = array_slice($ipv6Parts, 0, intval($mask / 16));
|
||||
$prefix = implode(':', $ipv6Parts);
|
||||
$conditions['value1 LIKE'] = $prefix . '%';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$ipList = $this->find('column', array(
|
||||
'conditions' => $conditions,
|
||||
'fields' => ['Attribute.value1'],
|
||||
'unique' => true,
|
||||
'order' => false,
|
||||
));
|
||||
foreach ($ipList as $ipToCheck) {
|
||||
$ipToCheckVersion = filter_var($ipToCheck, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) ? 4 : 6;
|
||||
if ($ipToCheckVersion === $ip_version) {
|
||||
if ($ip_version === 4) {
|
||||
if ($this->__ipv4InCidr($ipToCheck, $ip)) {
|
||||
$ipValues[] = $ipToCheck;
|
||||
}
|
||||
} else {
|
||||
if ($this->__ipv6InCidr($ipToCheck, $ip)) {
|
||||
$ipValues[] = $ipToCheck;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
$ip_version = filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) ? 4 : 6;
|
||||
$cidrList = $this->getSetCIDRList();
|
||||
foreach ($cidrList as $cidr) {
|
||||
if (strpos($cidr, '.') !== false) {
|
||||
if ($ip_version === 4 && $this->__ipv4InCidr($ip, $cidr)) {
|
||||
$ipValues[] = $cidr;
|
||||
}
|
||||
} else {
|
||||
if ($ip_version === 6 && $this->__ipv6InCidr($ip, $cidr)) {
|
||||
$ipValues[] = $cidr;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
$extraConditions = array();
|
||||
if (!empty($ipValues)) {
|
||||
$extraConditions = array('OR' => array(
|
||||
'Attribute.value1' => $ipValues,
|
||||
'Attribute.value2' => $ipValues
|
||||
));
|
||||
}
|
||||
return $extraConditions;
|
||||
}
|
||||
|
||||
private function __preventExcludedCorrelations($a)
|
||||
{
|
||||
$value = $a['value1'];
|
||||
if (!empty($a['value2'])) {
|
||||
$value .= '|' . $a['value2'];
|
||||
}
|
||||
if (empty($this->exclusions)) {
|
||||
try {
|
||||
$redis = $this->setupRedisWithException();
|
||||
} catch (Exception $e) {
|
||||
$redisFail = true;
|
||||
}
|
||||
if (empty($redisFail)) {
|
||||
$this->Correlation = ClassRegistry::init('Correlation');
|
||||
$this->exclusions = $redis->sMembers('misp:correlation_exclusions');
|
||||
}
|
||||
}
|
||||
foreach ($this->exclusions as $exclusion) {
|
||||
if (!empty($exclusion)) {
|
||||
$firstChar = $exclusion[0];
|
||||
$lastChar = substr($exclusion, -1);
|
||||
if ($firstChar === '%' && $lastChar === '%') {
|
||||
$exclusion = substr($exclusion, 1, -1);
|
||||
if (strpos($value, $exclusion) !== false) {
|
||||
return true;
|
||||
}
|
||||
} else if ($firstChar === '%') {
|
||||
$exclusion = substr($exclusion, 1);
|
||||
if (substr($value, -strlen($exclusion)) === $exclusion) {
|
||||
return true;
|
||||
}
|
||||
} else if ($lastChar === '%') {
|
||||
$exclusion = substr($exclusion, 0, -1);
|
||||
if (substr($value, 0, strlen($exclusion)) === $exclusion) {
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
if ($value === $exclusion) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public function __afterSaveCorrelation($a, $full = false, $event = false)
|
||||
{
|
||||
if (!empty($a['disable_correlation']) || Configure::read('MISP.completely_disable_correlation')) {
|
||||
return true;
|
||||
}
|
||||
if ($this->__preventExcludedCorrelations($a)) {
|
||||
return true;
|
||||
}
|
||||
// Don't do any correlation if the type is a non correlating type
|
||||
if (in_array($a['type'], $this->nonCorrelatingTypes)) {
|
||||
return true;
|
||||
}
|
||||
if (!$event) {
|
||||
$event = $this->Event->find('first', array(
|
||||
'recursive' => -1,
|
||||
'fields' => array('Event.distribution', 'Event.id', 'Event.info', 'Event.org_id', 'Event.date', 'Event.sharing_group_id', 'Event.disable_correlation'),
|
||||
'conditions' => array('id' => $a['event_id']),
|
||||
'order' => array(),
|
||||
));
|
||||
}
|
||||
|
||||
if (!empty($event['Event']['disable_correlation']) && $event['Event']['disable_correlation']) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (Configure::read('MISP.enable_advanced_correlations') && in_array($a['type'], array('ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port'))) {
|
||||
$extraConditions = $this->__cidrCorrelation($a);
|
||||
} else if ($a['type'] === 'ssdeep' && function_exists('ssdeep_fuzzy_compare')) {
|
||||
$this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep');
|
||||
$fuzzyIds = $this->FuzzyCorrelateSsdeep->query_ssdeep_chunks($a['value1'], $a['id']);
|
||||
if (!empty($fuzzyIds)) {
|
||||
$ssdeepIds = $this->find('list', array(
|
||||
'recursive' => -1,
|
||||
'conditions' => array(
|
||||
'Attribute.type' => 'ssdeep',
|
||||
'Attribute.id' => $fuzzyIds
|
||||
),
|
||||
'fields' => array('Attribute.id', 'Attribute.value1')
|
||||
));
|
||||
$threshold = Configure::read('MISP.ssdeep_correlation_threshold') ?: 40;
|
||||
$attributeIds = array();
|
||||
foreach ($ssdeepIds as $attributeId => $v) {
|
||||
$ssdeep_value = ssdeep_fuzzy_compare($a['value1'], $v);
|
||||
if ($ssdeep_value >= $threshold) {
|
||||
$attributeIds[] = $attributeId;
|
||||
}
|
||||
}
|
||||
$extraConditions = array('Attribute.id' => $attributeIds);
|
||||
}
|
||||
}
|
||||
|
||||
$correlatingValues = array($a['value1']);
|
||||
if (!empty($a['value2']) && !in_array($a['type'], $this->primaryOnlyCorrelatingTypes)) {
|
||||
$correlatingValues[] = $a['value2'];
|
||||
}
|
||||
|
||||
$correlatingAttributes = array();
|
||||
foreach ($correlatingValues as $k => $cV) {
|
||||
$conditions = array(
|
||||
'OR' => array(
|
||||
'Attribute.value1' => $cV,
|
||||
'AND' => array(
|
||||
'Attribute.value2' => $cV,
|
||||
'NOT' => array('Attribute.type' => $this->primaryOnlyCorrelatingTypes)
|
||||
)
|
||||
),
|
||||
'NOT' => array(
|
||||
'Attribute.event_id' => $a['event_id'],
|
||||
'Attribute.type' => $this->nonCorrelatingTypes,
|
||||
),
|
||||
'Attribute.disable_correlation' => 0,
|
||||
'Event.disable_correlation' => 0,
|
||||
'(Attribute.deleted + 0)' => 0
|
||||
);
|
||||
if (!empty($extraConditions)) {
|
||||
$conditions['OR'][] = $extraConditions;
|
||||
}
|
||||
if ($full) {
|
||||
$conditions['Attribute.id > '] = $a['id'];
|
||||
}
|
||||
$correlatingAttributes[$k] = $this->find('all', array(
|
||||
'conditions' => $conditions,
|
||||
'recursive' => -1,
|
||||
'fields' => array(
|
||||
'Attribute.event_id',
|
||||
'Attribute.id',
|
||||
'Attribute.distribution',
|
||||
'Attribute.sharing_group_id',
|
||||
'Attribute.value1',
|
||||
'Attribute.value2',
|
||||
),
|
||||
'contain' => array('Event' => array('fields' => array('Event.id', 'Event.date', 'Event.info', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id'))),
|
||||
'order' => array(),
|
||||
));
|
||||
}
|
||||
$correlations = array();
|
||||
foreach ($correlatingAttributes as $k => $cA) {
|
||||
foreach ($cA as $corr) {
|
||||
if (Configure::read('MISP.deadlock_avoidance')) {
|
||||
$correlations[] = array(
|
||||
'value' => $k === 0 ? $corr['Attribute']['value1'] : $corr['Attribute']['value2'],
|
||||
'1_event_id' => $event['Event']['id'],
|
||||
'1_attribute_id' => $a['id'],
|
||||
'event_id' => $corr['Attribute']['event_id'],
|
||||
'attribute_id' => $corr['Attribute']['id'],
|
||||
'org_id' => $corr['Event']['org_id'],
|
||||
'distribution' => $corr['Event']['distribution'],
|
||||
'a_distribution' => $corr['Attribute']['distribution'],
|
||||
'sharing_group_id' => $corr['Event']['sharing_group_id'],
|
||||
'a_sharing_group_id' => $corr['Attribute']['sharing_group_id'],
|
||||
'date' => $corr['Event']['date'],
|
||||
'info' => $corr['Event']['info']
|
||||
);
|
||||
$correlations[] = array(
|
||||
'value' => $correlatingValues[$k],
|
||||
'1_event_id' => $corr['Event']['id'],
|
||||
'1_attribute_id' => $corr['Attribute']['id'],
|
||||
'event_id' => $a['event_id'],
|
||||
'attribute_id' => $a['id'],
|
||||
'org_id' => $event['Event']['org_id'],
|
||||
'distribution' => $event['Event']['distribution'],
|
||||
'a_distribution' => $a['distribution'],
|
||||
'sharing_group_id' => $event['Event']['sharing_group_id'],
|
||||
'a_sharing_group_id' => $a['sharing_group_id'],
|
||||
'date' => $event['Event']['date'],
|
||||
'info' => $event['Event']['info']
|
||||
);
|
||||
} else {
|
||||
$correlations[] = array(
|
||||
$k === 0 ? $corr['Attribute']['value1'] : $corr['Attribute']['value2'],
|
||||
$event['Event']['id'],
|
||||
$a['id'],
|
||||
$corr['Attribute']['event_id'],
|
||||
$corr['Attribute']['id'],
|
||||
$corr['Event']['org_id'],
|
||||
$corr['Event']['distribution'],
|
||||
$corr['Attribute']['distribution'],
|
||||
$corr['Event']['sharing_group_id'],
|
||||
$corr['Attribute']['sharing_group_id'],
|
||||
$corr['Event']['date'],
|
||||
$corr['Event']['info']
|
||||
);
|
||||
$correlations[] = array(
|
||||
$correlatingValues[$k],
|
||||
$corr['Event']['id'],
|
||||
$corr['Attribute']['id'],
|
||||
$a['event_id'],
|
||||
$a['id'],
|
||||
$event['Event']['org_id'],
|
||||
$event['Event']['distribution'],
|
||||
$a['distribution'],
|
||||
$event['Event']['sharing_group_id'],
|
||||
$a['sharing_group_id'],
|
||||
$event['Event']['date'],
|
||||
$event['Event']['info']
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (empty($correlations)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$fields = array(
|
||||
'value',
|
||||
'1_event_id',
|
||||
'1_attribute_id',
|
||||
'event_id',
|
||||
'attribute_id',
|
||||
'org_id',
|
||||
'distribution',
|
||||
'a_distribution',
|
||||
'sharing_group_id',
|
||||
'a_sharing_group_id',
|
||||
'date',
|
||||
'info'
|
||||
);
|
||||
if (Configure::read('MISP.deadlock_avoidance')) {
|
||||
$this->Correlation = ClassRegistry::init('Correlation');
|
||||
return $this->Correlation->saveMany($correlations, array(
|
||||
'atomic' => false,
|
||||
'callbacks' => false,
|
||||
'deep' => false,
|
||||
'validate' => false,
|
||||
'fieldList' => $fields,
|
||||
));
|
||||
} else {
|
||||
$db = $this->getDataSource();
|
||||
return $db->insertMulti('correlations', $fields, $correlations);
|
||||
}
|
||||
}
|
||||
|
||||
private function __beforeDeleteCorrelation($attribute_id)
|
||||
{
|
||||
$this->Correlation = ClassRegistry::init('Correlation');
|
||||
// When we remove an attribute we need to
|
||||
// - remove the existing relations related to that attribute, we DO have an id reference
|
||||
// ==> DELETE FROM correlations WHERE 1_attribute_id = $a_id OR attribute_id = $a_id;
|
||||
$dummy = $this->Correlation->deleteAll(
|
||||
array('OR' => array(
|
||||
'Correlation.1_attribute_id' => $attribute_id,
|
||||
'Correlation.attribute_id' => $attribute_id))
|
||||
);
|
||||
}
|
||||
|
||||
public function checkComposites()
|
||||
{
|
||||
$compositeTypes = $this->getCompositeTypes();
|
||||
|
|
@ -2622,7 +2236,7 @@ class Attribute extends AppModel
|
|||
'order' => [],
|
||||
]);
|
||||
foreach ($attributes as $attribute) {
|
||||
$this->__afterSaveCorrelation($attribute['Attribute'], $full, $event);
|
||||
$this->Correlation->afterSaveCorrelation($attribute['Attribute'], $full, $event);
|
||||
$attributeCount++;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,11 +5,24 @@ App::uses('RandomTool', 'Tools');
|
|||
class Correlation extends AppModel
|
||||
{
|
||||
|
||||
public $belongsTo = array(
|
||||
'Attribute' => [
|
||||
'className' => 'Attribute',
|
||||
'foreignKey' => 'attribute_id'
|
||||
],
|
||||
'Event' => array(
|
||||
'className' => 'Event',
|
||||
'foreignKey' => 'event_id'
|
||||
)
|
||||
);
|
||||
|
||||
public function correlateValueRouter($value)
|
||||
{
|
||||
if (Configure::read('MISP.background_jobs')) {
|
||||
$job = ClassRegistry::init('Job');
|
||||
$job->create();
|
||||
if (empty($this->Job)) {
|
||||
$this->Job = ClassRegistry::init('Job');
|
||||
}
|
||||
$this->Job->create();
|
||||
$data = array(
|
||||
'worker' => 'default',
|
||||
'job_type' => 'correlateValue',
|
||||
|
|
@ -20,74 +33,75 @@ class Correlation extends AppModel
|
|||
'org' => 0,
|
||||
'message' => 'Recorrelating',
|
||||
);
|
||||
$job->save($data);
|
||||
$jobId = $job->id;
|
||||
$this->Job->save($data);
|
||||
$jobId = $this->Job->id;
|
||||
$process_id = CakeResque::enqueue(
|
||||
'email',
|
||||
'EventShell',
|
||||
['correlateValue', $value, $jobId],
|
||||
true
|
||||
);
|
||||
$job->saveField('process_id', $process_id);
|
||||
$this->Job->saveField('process_id', $process_id);
|
||||
return true;
|
||||
} else {
|
||||
return $this->correlateValue($value, $jobId);
|
||||
return $this->correlateValue($value);
|
||||
}
|
||||
}
|
||||
|
||||
public function addAdvancedCorrelations($correlatingAttribute)
|
||||
private function __buildAdvancedCorrelationConditions($a)
|
||||
{
|
||||
$a = $correlatingAttribute['Attribute'];
|
||||
if (Configure::read('MISP.enable_advanced_correlations')) {
|
||||
if (in_array($a['type'], array('ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port'))) {
|
||||
return $this->Attribute->cidrCorrelation($a);
|
||||
} else if ($a['type'] === 'ssdeep' && function_exists('ssdeep_fuzzy_compare')) {
|
||||
if (isset($a['Attribute'])) {
|
||||
$a = $a['Attribute'];
|
||||
}
|
||||
$extraConditions = null;
|
||||
if (in_array($a['type'], array('ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port'))) {
|
||||
$extraConditions = $this->cidrCorrelation($a);
|
||||
} else if ($a['type'] === 'ssdeep' && function_exists('ssdeep_fuzzy_compare')) {
|
||||
if (empty($this->FuzzyCorrelateSsdeep)) {
|
||||
$this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep');
|
||||
$fuzzyIds = $this->FuzzyCorrelateSsdeep->query_ssdeep_chunks($a['value1'], $a['id']);
|
||||
if (!empty($fuzzyIds)) {
|
||||
$ssdeepIds = $this->find('list', array(
|
||||
'recursive' => -1,
|
||||
'conditions' => array(
|
||||
'Attribute.type' => 'ssdeep',
|
||||
'Attribute.id' => $fuzzyIds
|
||||
),
|
||||
'fields' => array('Attribute.id', 'Attribute.value1')
|
||||
));
|
||||
$threshold = Configure::read('MISP.ssdeep_correlation_threshold') ?: 40;
|
||||
$attributeIds = array();
|
||||
foreach ($ssdeepIds as $attributeId => $v) {
|
||||
$ssdeep_value = ssdeep_fuzzy_compare($a['value1'], $v);
|
||||
if ($ssdeep_value >= $threshold) {
|
||||
$attributeIds[] = $attributeId;
|
||||
}
|
||||
}
|
||||
$fuzzyIds = $this->FuzzyCorrelateSsdeep->query_ssdeep_chunks($a['value1'], $a['id']);
|
||||
if (!empty($fuzzyIds)) {
|
||||
$ssdeepIds = $this->Attribute->find('list', array(
|
||||
'recursive' => -1,
|
||||
'conditions' => array(
|
||||
'Attribute.type' => 'ssdeep',
|
||||
'Attribute.id' => $fuzzyIds
|
||||
),
|
||||
'fields' => array('Attribute.id', 'Attribute.value1')
|
||||
));
|
||||
$threshold = Configure::read('MISP.ssdeep_correlation_threshold') ?: 40;
|
||||
$attributeIds = array();
|
||||
foreach ($ssdeepIds as $attributeId => $v) {
|
||||
$ssdeep_value = ssdeep_fuzzy_compare($a['value1'], $v);
|
||||
if ($ssdeep_value >= $threshold) {
|
||||
$attributeIds[] = $attributeId;
|
||||
}
|
||||
return ['Attribute.id' => $attributeIds];
|
||||
}
|
||||
$extraConditions = ['Attribute.id' => $attributeIds];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function correlateValue($value, $jobId)
|
||||
private function __addAdvancedCorrelations($correlatingAttribute)
|
||||
{
|
||||
$valueConditions = [
|
||||
'Attribute.value1' => $exclusion['CorrelationExclusion']['value'],
|
||||
'AND' => [
|
||||
'Attribute.value2' => $exclusion['CorrelationExclusion']['value'],
|
||||
'NOT' => ['Attribute.type' => $this->Attribute->primaryOnlyCorrelatingTypes]
|
||||
]
|
||||
];
|
||||
$conditions = [
|
||||
'OR' => $valueConditions,
|
||||
'NOT' => [
|
||||
'Attribute.type' => $this->Attribute->nonCorrelatingTypes,
|
||||
if (empty(Configure::read('MISP.enable_advanced_correlations'))) {
|
||||
return [];
|
||||
}
|
||||
$extraConditions = $this->__buildAdvancedCorrelationConditions($correlatingAttribute);
|
||||
if (empty($extraConditions)) {
|
||||
return [];
|
||||
}
|
||||
return $this->Attribute->find('all', [
|
||||
'conditions' => [
|
||||
'AND' => $extraConditions,
|
||||
'NOT' => [
|
||||
'Attribute.type' => $this->Attribute->nonCorrelatingTypes,
|
||||
],
|
||||
'Attribute.disable_correlation' => 0,
|
||||
'Event.disable_correlation' => 0,
|
||||
'Attribute.deleted' => 0
|
||||
],
|
||||
'Attribute.disable_correlation' => 0,
|
||||
'Event.disable_correlation' => 0,
|
||||
'Attribute.deleted' => 0
|
||||
];
|
||||
$this->Attribute = ClassRegistry::init('Attribute');
|
||||
$correlatingAttributes[$k] = $this->Attribute->find('all', [
|
||||
'conditions' => $conditions,
|
||||
'recursive' => -1,
|
||||
'fields' => [
|
||||
'Attribute.event_id',
|
||||
|
|
@ -104,52 +118,427 @@ class Correlation extends AppModel
|
|||
],
|
||||
'order' => [],
|
||||
]);
|
||||
}
|
||||
|
||||
private function __getMatchingAttributes($value)
|
||||
{
|
||||
$conditions = [
|
||||
'OR' => [
|
||||
'Attribute.value1' => $value,
|
||||
'AND' => [
|
||||
'Attribute.value2' => $value,
|
||||
'NOT' => ['Attribute.type' => $this->Attribute->primaryOnlyCorrelatingTypes]
|
||||
]
|
||||
],
|
||||
'NOT' => [
|
||||
'Attribute.type' => $this->Attribute->nonCorrelatingTypes,
|
||||
],
|
||||
'Attribute.disable_correlation' => 0,
|
||||
'Event.disable_correlation' => 0,
|
||||
'Attribute.deleted' => 0
|
||||
];
|
||||
$correlatingAttributes = $this->Attribute->find('all', [
|
||||
'conditions' => $conditions,
|
||||
'recursive' => -1,
|
||||
'fields' => [
|
||||
'Attribute.event_id',
|
||||
'Attribute.id',
|
||||
'Attribute.type',
|
||||
'Attribute.distribution',
|
||||
'Attribute.sharing_group_id',
|
||||
'Attribute.value1',
|
||||
'Attribute.value2',
|
||||
],
|
||||
'contain' => [
|
||||
'Event' => [
|
||||
'fields' => ['Event.id', 'Event.date', 'Event.info', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id', 'Event.disable_correlation']
|
||||
]
|
||||
],
|
||||
'order' => [],
|
||||
]);
|
||||
return $correlatingAttributes;
|
||||
}
|
||||
|
||||
private function __addCorrelationEntry($value, $a, $b, $correlations)
|
||||
{
|
||||
if (
|
||||
$a['Attribute']['event_id'] !== $b['Attribute']['event_id']
|
||||
) {
|
||||
if (Configure::read('MISP.deadlock_avoidance')) {
|
||||
$correlations[] = [
|
||||
'value' => $value,
|
||||
'1_event_id' => $a['Event']['id'],
|
||||
'1_attribute_id' => $a['Attribute']['id'],
|
||||
'event_id' => $b['Attribute']['event_id'],
|
||||
'attribute_id' => $b['Attribute']['id'],
|
||||
'org_id' => $b['Event']['org_id'],
|
||||
'distribution' => $b['Event']['distribution'],
|
||||
'a_distribution' => $b['Attribute']['distribution'],
|
||||
'sharing_group_id' => $b['Event']['sharing_group_id'],
|
||||
'a_sharing_group_id' => $b['Attribute']['sharing_group_id'],
|
||||
'date' => $b['Event']['date'],
|
||||
'info' => $b['Event']['info']
|
||||
];
|
||||
} else {
|
||||
$correlations[] = [
|
||||
$value,
|
||||
$a['Event']['id'],
|
||||
$a['Attribute']['id'],
|
||||
$b['Attribute']['event_id'],
|
||||
$b['Attribute']['id'],
|
||||
$b['Event']['org_id'],
|
||||
$b['Event']['distribution'],
|
||||
$b['Attribute']['distribution'],
|
||||
$b['Event']['sharing_group_id'],
|
||||
$b['Attribute']['sharing_group_id'],
|
||||
$b['Event']['date'],
|
||||
$b['Event']['info']
|
||||
];
|
||||
}
|
||||
}
|
||||
return $correlations;
|
||||
}
|
||||
|
||||
public function correlateValue($value, $jobId = false)
|
||||
{
|
||||
$correlatingAttributes = $this->__getMatchingAttributes($value);
|
||||
$count = count($correlatingAttributes);
|
||||
$correlations = [];
|
||||
foreach ($correlatingAttributes as $k => $correlatingAttribute) {
|
||||
if (
|
||||
in_array($correlatingAttribute2['Attribute']['type'], $this->Attribute->nonCorrelatingTypes) ||
|
||||
!empty($correlatingAttribute['Event']['disable_correlation'])
|
||||
) {
|
||||
continue;
|
||||
if ($jobId) {
|
||||
if (empty($this->Job)) {
|
||||
$this->Job = ClassRegistry::init('Job');
|
||||
}
|
||||
foreach ($correlatingAttribute as $k2 => $correlatingAttribute2) {
|
||||
if (
|
||||
$correlatingAttribute['Attribute']['event_id'] === $correlatingAttribute2['Attribute']['event_id'] ||
|
||||
!empty($correlatingAttribute2['Event']['disable_correlation']) ||
|
||||
in_array($correlatingAttribute2['Attribute']['type'], $this->Attribute->nonCorrelatingTypes)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
$correlations[] = array(
|
||||
'value' => $value,
|
||||
'1_event_id' => $correlatingAttribute['Event']['id'],
|
||||
'1_attribute_id' => $correlatingAttribute['Attribute']['id'],
|
||||
'event_id' => $correlatingAttribute2['Attribute']['event_id'],
|
||||
'attribute_id' => $correlatingAttribute2['Attribute']['id'],
|
||||
'org_id' => $correlatingAttribute2['Event']['org_id'],
|
||||
'distribution' => $correlatingAttribute2['Event']['distribution'],
|
||||
'a_distribution' => $correlatingAttribute2['Attribute']['distribution'],
|
||||
'sharing_group_id' => $correlatingAttribute2['Event']['sharing_group_id'],
|
||||
'a_sharing_group_id' => $correlatingAttribute2['Attribute']['sharing_group_id'],
|
||||
'date' => $correlatingAttribute2['Event']['date'],
|
||||
'info' => $correlatingAttribute2['Event']['info']
|
||||
);
|
||||
$job = $this->Job->find('first', [
|
||||
'recursive' => -1,
|
||||
'conditions' => ['id' => $jobId]
|
||||
]);
|
||||
if (empty($job)) {
|
||||
$jobId = false;
|
||||
}
|
||||
$correlations = $this->addAdvancedCorrelations($correlatingAttribute);
|
||||
}
|
||||
foreach ($correlatingAttributes as $k => $correlatingAttribute) {
|
||||
foreach ($correlatingAttributes as $k2 => $correlatingAttribute2) {
|
||||
$correlations = $this->__addCorrelationEntry($value, $correlatingAttribute, $correlatingAttribute2, $correlations);
|
||||
}
|
||||
$extraCorrelations = $this->__addAdvancedCorrelations($correlatingAttribute);
|
||||
if (!empty($extraCorrelations)) {
|
||||
foreach ($extraCorrelations as $k3 => $extraCorrelation) {
|
||||
$correlations = $this->__addCorrelationEntry($value, $correlatingAttribute, $extraCorrelation, $correlations);
|
||||
$correlations = $this->__addCorrelationEntry($value, $extraCorrelation, $correlatingAttribute, $correlations);
|
||||
}
|
||||
}
|
||||
if ($jobId && $k % 100 === 0) {
|
||||
$job['Job']['progress'] = floor(100 * $k / $count);
|
||||
$job['Job']['date_modified'] = date("Y-m-d H:i:s");
|
||||
$job['Job']['message'] = __('Correlating Attributes based on value. %s attributes correlated out of %s.', $k, $count);
|
||||
$this->Job->save($job);
|
||||
}
|
||||
}
|
||||
return $this->__saveCorrelations($correlations);
|
||||
}
|
||||
|
||||
private function __saveCorrelations($correlations)
|
||||
{
|
||||
$fields = [
|
||||
'value', '1_event_id', '1_attribute_id', 'event_id', 'attribute_id', 'org_id',
|
||||
'distribution', 'a_distribution', 'sharing_group_id', 'a_sharing_group_id',
|
||||
'date', 'info'
|
||||
];
|
||||
if (Configure::read('MISP.deadlock_avoidance')) {
|
||||
return $this->saveMany($correlations, array(
|
||||
'atomic' => false,
|
||||
'callbacks' => false,
|
||||
'deep' => false,
|
||||
'validate' => false,
|
||||
'fieldList' => $fields,
|
||||
'fieldList' => $fields
|
||||
));
|
||||
} else {
|
||||
$db = $this->getDataSource();
|
||||
return $db->insertMulti('correlations', $fields, $correlations);
|
||||
}
|
||||
}
|
||||
|
||||
public function beforeSaveCorrelation($attribute)
|
||||
{
|
||||
// (update-only) clean up the relation of the old value: remove the existing relations related to that attribute, we DO have a reference, the id
|
||||
// ==> DELETE FROM correlations WHERE 1_attribute_id = $a_id OR attribute_id = $a_id; */
|
||||
// first check if it's an update
|
||||
if (isset($attribute['id'])) {
|
||||
// FIXME : check that $attribute['id'] is checked correctly so that the user can't remove attributes he shouldn't
|
||||
$dummy = $this->deleteAll(
|
||||
array('OR' => array(
|
||||
'Correlation.1_attribute_id' => $attribute['id'],
|
||||
'Correlation.attribute_id' => $attribute['id']))
|
||||
);
|
||||
}
|
||||
if ($attribute['type'] === 'ssdeep') {
|
||||
$this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep');
|
||||
$this->FuzzyCorrelateSsdeep->purge(null, $attribute['id']);
|
||||
}
|
||||
}
|
||||
|
||||
public function afterSaveCorrelation($a, $full = false, $event = false)
|
||||
{
|
||||
if (!empty($a['disable_correlation']) || Configure::read('MISP.completely_disable_correlation')) {
|
||||
return true;
|
||||
}
|
||||
if ($this->__preventExcludedCorrelations($a)) {
|
||||
return true;
|
||||
}
|
||||
// Don't do any correlation if the type is a non correlating type
|
||||
if (in_array($a['type'], $this->Attribute->nonCorrelatingTypes)) {
|
||||
return true;
|
||||
}
|
||||
if (!$event) {
|
||||
$event = $this->Attribute->Event->find('first', array(
|
||||
'recursive' => -1,
|
||||
'fields' => array('Event.distribution', 'Event.id', 'Event.info', 'Event.org_id', 'Event.date', 'Event.sharing_group_id', 'Event.disable_correlation'),
|
||||
'conditions' => array('id' => $a['event_id']),
|
||||
'order' => array(),
|
||||
));
|
||||
}
|
||||
|
||||
if (!empty($event['Event']['disable_correlation']) && $event['Event']['disable_correlation']) {
|
||||
return true;
|
||||
}
|
||||
// generate additional correlating attribute list based on the advanced correlations
|
||||
$extraConditions = $this->__buildAdvancedCorrelationConditions($a);
|
||||
$correlatingValues = array($a['value1']);
|
||||
if (!empty($a['value2']) && !in_array($a['type'], $this->primaryOnlyCorrelatingTypes)) {
|
||||
$correlatingValues[] = $a['value2'];
|
||||
}
|
||||
|
||||
$correlatingAttributes = [];
|
||||
foreach ($correlatingValues as $k => $cV) {
|
||||
$conditions = [
|
||||
'OR' => [
|
||||
'Attribute.value1' => $cV,
|
||||
'AND' => [
|
||||
'Attribute.value2' => $cV,
|
||||
'NOT' => ['Attribute.type' => $this->primaryOnlyCorrelatingTypes]
|
||||
]
|
||||
],
|
||||
'NOT' => [
|
||||
'Attribute.event_id' => $a['event_id'],
|
||||
'Attribute.type' => $this->nonCorrelatingTypes,
|
||||
],
|
||||
'Attribute.disable_correlation' => 0,
|
||||
'Event.disable_correlation' => 0,
|
||||
'Attribute.deleted' => 0
|
||||
];
|
||||
if (!empty($extraConditions)) {
|
||||
$conditions['OR'][] = $extraConditions;
|
||||
}
|
||||
if ($full) {
|
||||
$conditions['Attribute.id > '] = $a['id'];
|
||||
}
|
||||
$correlatingAttributes[$k] = $this->Attribute->find('all', array(
|
||||
'conditions' => $conditions,
|
||||
'recursive' => -1,
|
||||
'fields' => [
|
||||
'Attribute.event_id', 'Attribute.id', 'Attribute.distribution', 'Attribute.sharing_group_id',
|
||||
'Attribute.value1', 'Attribute.value2'
|
||||
],
|
||||
'contain' => ['Event.id', 'Event.date', 'Event.info', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id'],
|
||||
'order' => []
|
||||
));
|
||||
}
|
||||
$correlations = array();
|
||||
foreach ($correlatingAttributes as $k => $cA) {
|
||||
foreach ($cA as $corr) {
|
||||
$correlations = $this->__addCorrelationEntry(
|
||||
$k === 0 ? $corr['Attribute']['value1'] : $corr['Attribute']['value2'],
|
||||
['Attribute' => $a, 'Event' => $event['Event']],
|
||||
$corr,
|
||||
$correlations
|
||||
);
|
||||
$correlations = $this->__addCorrelationEntry(
|
||||
$correlatingValues[$k],
|
||||
$corr,
|
||||
['Attribute' => $a, 'Event' => $event['Event']],
|
||||
$correlations
|
||||
);
|
||||
}
|
||||
}
|
||||
return $this->__saveCorrelations($correlations);
|
||||
}
|
||||
|
||||
private function __preventExcludedCorrelations($a)
|
||||
{
|
||||
$value = $a['value1'];
|
||||
if (!empty($a['value2'])) {
|
||||
$value .= '|' . $a['value2'];
|
||||
}
|
||||
if (empty($this->exclusions)) {
|
||||
try {
|
||||
$redis = $this->setupRedisWithException();
|
||||
} catch (Exception $e) {
|
||||
$redisFail = true;
|
||||
}
|
||||
if (empty($redisFail)) {
|
||||
$this->Correlation = ClassRegistry::init('Correlation');
|
||||
$this->exclusions = $redis->sMembers('misp:correlation_exclusions');
|
||||
}
|
||||
}
|
||||
foreach ($this->exclusions as $exclusion) {
|
||||
if (!empty($exclusion)) {
|
||||
$firstChar = $exclusion[0];
|
||||
$lastChar = substr($exclusion, -1);
|
||||
if ($firstChar === '%' && $lastChar === '%') {
|
||||
$exclusion = substr($exclusion, 1, -1);
|
||||
if (strpos($value, $exclusion) !== false) {
|
||||
return true;
|
||||
}
|
||||
} else if ($firstChar === '%') {
|
||||
$exclusion = substr($exclusion, 1);
|
||||
if (substr($value, -strlen($exclusion)) === $exclusion) {
|
||||
return true;
|
||||
}
|
||||
} else if ($lastChar === '%') {
|
||||
$exclusion = substr($exclusion, 0, -1);
|
||||
if (substr($value, 0, strlen($exclusion)) === $exclusion) {
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
if ($value === $exclusion) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public function cidrCorrelation($a)
|
||||
{
|
||||
$ipValues = array();
|
||||
$ip = $a['value1'];
|
||||
if (strpos($ip, '/') !== false) { // IP is CIDR
|
||||
list($networkIp, $mask) = explode('/', $ip);
|
||||
$ip_version = filter_var($networkIp, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) ? 4 : 6;
|
||||
|
||||
$conditions = array(
|
||||
'type' => array('ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port'),
|
||||
'value1 NOT LIKE' => '%/%', // do not return CIDR, just plain IPs
|
||||
'disable_correlation' => 0,
|
||||
'deleted' => 0,
|
||||
);
|
||||
|
||||
if (in_array($this->getDataSource()->config['datasource'], array('Database/Mysql', 'Database/MysqlObserver'))) {
|
||||
// Massive speed up for CIDR correlation. Instead of testing all in PHP, database can do that work much
|
||||
// faster. But these methods are just supported by MySQL.
|
||||
if ($ip_version === 4) {
|
||||
$startIp = ip2long($networkIp) & ((-1 << (32 - $mask)));
|
||||
$endIp = $startIp + pow(2, (32 - $mask)) - 1;
|
||||
// Just fetch IP address that fit in CIDR range.
|
||||
$conditions['INET_ATON(value1) BETWEEN ? AND ?'] = array($startIp, $endIp);
|
||||
|
||||
// Just fetch IPv4 address that starts with given prefix. This is fast, because value1 is indexed.
|
||||
// This optimisation is possible just to mask bigger than 8 bits.
|
||||
if ($mask >= 8) {
|
||||
$ipv4Parts = explode('.', $networkIp);
|
||||
$ipv4Parts = array_slice($ipv4Parts, 0, intval($mask / 8));
|
||||
$prefix = implode('.', $ipv4Parts);
|
||||
$conditions['value1 LIKE'] = $prefix . '%';
|
||||
}
|
||||
} else {
|
||||
$conditions[] = 'IS_IPV6(value1)';
|
||||
// Just fetch IPv6 address that starts with given prefix. This is fast, because value1 is indexed.
|
||||
if ($mask >= 16) {
|
||||
$ipv6Parts = explode(':', rtrim($networkIp, ':'));
|
||||
$ipv6Parts = array_slice($ipv6Parts, 0, intval($mask / 16));
|
||||
$prefix = implode(':', $ipv6Parts);
|
||||
$conditions['value1 LIKE'] = $prefix . '%';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$ipList = $this->Attribute->find('column', array(
|
||||
'conditions' => $conditions,
|
||||
'fields' => ['Attribute.value1'],
|
||||
'unique' => true,
|
||||
'order' => false,
|
||||
));
|
||||
foreach ($ipList as $ipToCheck) {
|
||||
$ipToCheckVersion = filter_var($ipToCheck, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) ? 4 : 6;
|
||||
if ($ipToCheckVersion === $ip_version) {
|
||||
if ($ip_version === 4) {
|
||||
if ($this->__ipv4InCidr($ipToCheck, $ip)) {
|
||||
$ipValues[] = $ipToCheck;
|
||||
}
|
||||
} else {
|
||||
if ($this->__ipv6InCidr($ipToCheck, $ip)) {
|
||||
$ipValues[] = $ipToCheck;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
$ip_version = filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) ? 4 : 6;
|
||||
$cidrList = $this->Attribute->getSetCIDRList();
|
||||
foreach ($cidrList as $cidr) {
|
||||
if (strpos($cidr, '.') !== false) {
|
||||
if ($ip_version === 4 && $this->__ipv4InCidr($ip, $cidr)) {
|
||||
$ipValues[] = $cidr;
|
||||
}
|
||||
} else {
|
||||
if ($ip_version === 6 && $this->__ipv6InCidr($ip, $cidr)) {
|
||||
$ipValues[] = $cidr;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
$extraConditions = array();
|
||||
if (!empty($ipValues)) {
|
||||
$extraConditions = array('OR' => array(
|
||||
'Attribute.value1' => $ipValues,
|
||||
'Attribute.value2' => $ipValues
|
||||
));
|
||||
}
|
||||
return $extraConditions;
|
||||
}
|
||||
|
||||
public function beforeDeleteCorrelation($attribute_id)
|
||||
{
|
||||
// When we remove an attribute we need to
|
||||
// - remove the existing relations related to that attribute, we DO have an id reference
|
||||
// ==> DELETE FROM correlations WHERE 1_attribute_id = $a_id OR attribute_id = $a_id;
|
||||
$dummy = $this->deleteAll([
|
||||
'OR' => [
|
||||
'Correlation.1_attribute_id' => $attribute_id,
|
||||
'Correlation.attribute_id' => $attribute_id
|
||||
]
|
||||
]);
|
||||
}
|
||||
|
||||
// using Alnitak's solution from http://stackoverflow.com/questions/594112/matching-an-ip-to-a-cidr-mask-in-php5
|
||||
private function __ipv4InCidr($ip, $cidr)
|
||||
{
|
||||
list($subnet, $bits) = explode('/', $cidr);
|
||||
$ip = ip2long($ip);
|
||||
$subnet = ip2long($subnet);
|
||||
$mask = -1 << (32 - $bits);
|
||||
$subnet &= $mask; # nb: in case the supplied subnet wasn't correctly aligned
|
||||
return ($ip & $mask) == $subnet;
|
||||
}
|
||||
|
||||
// Using solution from https://github.com/symfony/symfony/blob/master/src/Symfony/Component/HttpFoundation/IpUtils.php
|
||||
private function __ipv6InCidr($ip, $cidr)
|
||||
{
|
||||
list($address, $netmask) = explode('/', $cidr);
|
||||
|
||||
$bytesAddr = unpack('n*', inet_pton($address));
|
||||
$bytesTest = unpack('n*', inet_pton($ip));
|
||||
|
||||
for ($i = 1, $ceil = ceil($netmask / 16); $i <= $ceil; ++$i) {
|
||||
$left = $netmask - 16 * ($i - 1);
|
||||
$left = ($left <= 16) ? $left : 16;
|
||||
$mask = ~(0xffff >> $left) & 0xffff;
|
||||
if (($bytesAddr[$i] & $mask) != ($bytesTest[$i] & $mask)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,20 @@ class CorrelationExclusion extends AppModel
|
|||
$this->cacheValues();
|
||||
}
|
||||
|
||||
public function beforeDelete($cascade = true)
|
||||
{
|
||||
$exclusion = $this->find('first', [
|
||||
'recursive' => -1,
|
||||
'conditions' => [
|
||||
'id' => $this->id
|
||||
]
|
||||
]);
|
||||
$this->Correlation = ClassRegistry::init('Correlation');
|
||||
if (!empty($exclusion)) {
|
||||
$this->Correlation->correlateValueRouter($exclusion['CorrelationExclusion']['value']);
|
||||
}
|
||||
}
|
||||
|
||||
public function afterDelete()
|
||||
{
|
||||
$this->cacheValues();
|
||||
|
|
|
|||
Loading…
Reference in New Issue