MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #7400 from JakubOnderka/after-save-correlation-fix 

After save correlation fix
pull/7403/head
Jakub Onderka 2021-05-10 08:43:42 +02:00 committed by GitHub
parent fcc1996a2e 99070b73ef
commit 2233d40b2e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 38 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/Controller/AttributesController.php
View File

@ -2964,7 +2964,7 @@ class AttributesController extends AppController
if ($attribute['Attribute']['disable_correlation']) {
$attribute['Attribute']['disable_correlation'] = 0;
$this->Attribute->save($attribute);
$this->Attribute->__afterSaveCorrelation($attribute['Attribute'], false, $attribute);
ClassRegistry::init('Correlation')->afterSaveCorrelation($attribute['Attribute'], false, $attribute);
} else {
$attribute['Attribute']['disable_correlation'] = 1;
$this->Attribute->save($attribute);

102
app/Model/Correlation.php
View File

@ -2,11 +2,13 @@
App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
/**
* @property Attribute $Attribute
*/
class Correlation extends AppModel
{
public $cache_name = 'misp:top_correlations';
public $cache_age = 'misp:top_correlations_age';
const CACHE_NAME = 'misp:top_correlations',
CACHE_AGE = 'misp:top_correlations_age';
public $belongsTo = array(
'Attribute' => [
@ -19,6 +21,8 @@ class Correlation extends AppModel
)
);
private $exclusions = [];
public function correlateValueRouter($value)
{
if (Configure::read('MISP.background_jobs')) {
@ -57,7 +61,7 @@ class Correlation extends AppModel
$a = $a['Attribute'];
}
$extraConditions = null;
if (in_array($a['type'], array('ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port'))) {
if (in_array($a['type'], ['ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port'], true)) {
$extraConditions = $this->cidrCorrelation($a);
} else if ($a['type'] === 'ssdeep' && function_exists('ssdeep_fuzzy_compare')) {
$extraConditions = $this->ssdeepCorrelation($a);
@ -199,21 +203,18 @@ class Correlation extends AppModel
}
}
foreach ($correlatingAttributes as $k => $correlatingAttribute) {
foreach ($correlatingAttributes as $k2 => $correlatingAttribute2) {
foreach ($correlatingAttributes as $correlatingAttribute2) {
$correlations = $this->__addCorrelationEntry($value, $correlatingAttribute, $correlatingAttribute2, $correlations);
}
$extraCorrelations = $this->__addAdvancedCorrelations($correlatingAttribute);
if (!empty($extraCorrelations)) {
foreach ($extraCorrelations as $k3 => $extraCorrelation) {
foreach ($extraCorrelations as $extraCorrelation) {
$correlations = $this->__addCorrelationEntry($value, $correlatingAttribute, $extraCorrelation, $correlations);
//$correlations = $this->__addCorrelationEntry($value, $extraCorrelation, $correlatingAttribute, $correlations);
}
}
if ($jobId && $k % 100 === 0) {
$job['Job']['progress'] = floor(100 * $k / $count);
$job['Job']['date_modified'] = date("Y-m-d H:i:s");
$job['Job']['message'] = __('Correlating Attributes based on value. %s attributes correlated out of %s.', $k, $count);
$this->Job->save($job);
$this->Job->saveProgress($jobId, __('Correlating Attributes based on value. %s attributes correlated out of %s.', $k, $count), floor(100 * $k / $count));
}
}
return $this->__saveCorrelations($correlations);
@ -289,7 +290,7 @@ class Correlation extends AppModel
// generate additional correlating attribute list based on the advanced correlations
$extraConditions = $this->__buildAdvancedCorrelationConditions($a);
$correlatingValues = array($a['value1']);
if (!empty($a['value2']) && !in_array($a['type'], $this->Attribute->primaryOnlyCorrelatingTypes)) {
if (!empty($a['value2']) && !in_array($a['type'], $this->Attribute->primaryOnlyCorrelatingTypes, true)) {
$correlatingValues[] = $a['value2'];
}
@ -356,13 +357,10 @@ class Correlation extends AppModel
}
if (empty($this->exclusions)) {
try {
$this->redis = $this->setupRedisWithException();
$redis = $this->setupRedisWithException();
$this->exclusions = $redis->sMembers('misp:correlation_exclusions');
} catch (Exception $e) {
$redisFail = true;
}
if (empty($redisFail)) {
$this->Correlation = ClassRegistry::init('Correlation');
$this->exclusions = $this->redis->sMembers('misp:correlation_exclusions');
return false;
}
}
foreach ($this->exclusions as $exclusion) {
@ -394,7 +392,7 @@ class Correlation extends AppModel
return false;
}
public function ssdeepCorrelation($a)
private function ssdeepCorrelation($a)
{
if (empty($this->FuzzyCorrelateSsdeep)) {
$this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep');
@ -422,7 +420,7 @@ class Correlation extends AppModel
return false;
}
public function cidrCorrelation($a)
private function cidrCorrelation($a)
{
$ipValues = array();
$ip = $a['value1'];
@ -437,7 +435,7 @@ class Correlation extends AppModel
'deleted' => 0,
);
if (in_array($this->getDataSource()->config['datasource'], array('Database/Mysql', 'Database/MysqlObserver'))) {
if (in_array($this->getDataSource()->config['datasource'], ['Database/Mysql', 'Database/MysqlObserver'])) {
// Massive speed up for CIDR correlation. Instead of testing all in PHP, database can do that work much
// faster. But these methods are just supported by MySQL.
if ($ip_version === 4) {
@ -446,31 +444,6 @@ class Correlation extends AppModel
// Just fetch IP address that fit in CIDR range.
$conditions['INET_ATON(value1) BETWEEN ? AND ?'] = array($startIp, $endIp);
// Just fetch IPv4 address that starts with given prefix. This is fast, because value1 is indexed.
// This optimisation is possible just to mask bigger than 8 bites.
if ($mask >= 8) {
$ipv4Parts = explode('.', $networkIp);
$ipv4Parts = array_slice($ipv4Parts, 0, intval($mask / 8));
$prefix = implode('.', $ipv4Parts);
$conditions['value1 LIKE'] = $prefix . '%';
}
} else {
$conditions[] = 'IS_IPV6(value1)';
// Just fetch IPv6 address that starts with given prefix. This is fast, because value1 is indexed.
if ($mask >= 16) {
$ipv6Parts = explode(':', rtrim($networkIp, ':'));
$ipv6Parts = array_slice($ipv6Parts, 0, intval($mask / 16));
$prefix = implode(':', $ipv6Parts);
$conditions['value1 LIKE'] = $prefix . '%';
}
}// Massive speed up for CIDR correlation. Instead of testing all in PHP, database can do that work much
// faster. But these methods are just supported by MySQL.
if ($ip_version === 4) {
$startIp = ip2long($networkIp) & ((-1 << (32 - $mask)));
$endIp = $startIp + pow(2, (32 - $mask)) - 1;
// Just fetch IP address that fit in CIDR range.
$conditions['INET_ATON(value1) BETWEEN ? AND ?'] = array($startIp, $endIp);
// Just fetch IPv4 address that starts with given prefix. This is fast, because value1 is indexed.
// This optimisation is possible just to mask bigger than 8 bites.
if ($mask >= 8) {
@ -580,6 +553,10 @@ class Correlation extends AppModel
return true;
}
/**
* @return int|bool
* @throws Exception
*/
public function generateTopCorrelationsRouter()
{
if (Configure::read('MISP.background_jobs')) {
@ -606,7 +583,7 @@ class Correlation extends AppModel
true
);
$this->Job->saveField('process_id', $process_id);
return true;
return $jobId;
} else {
return $this->generateTopCorrelations();
}
@ -615,11 +592,10 @@ class Correlation extends AppModel
public function generateTopCorrelations($jobId = false)
{
try {
$this->redis = $this->setupRedisWithException();
$redis = $this->setupRedisWithException();
} catch (Exception $e) {
throw new NotFoundException(__('No redis connection found.'));
}
$mem_initial = memory_get_usage();
$max_id = $this->find('first', [
'fields' => ['MAX(id) AS max_id'],
'recursive' => -1
@ -641,54 +617,50 @@ class Correlation extends AppModel
}
$max_id = $max_id[0]['max_id'];
$this->redis->del($this->cache_name);
$this->redis->set($this->cache_age, time());
$redis->del(self::CACHE_NAME);
$redis->set(self::CACHE_AGE, time());
$chunk_size = 1000000;
$max = ceil($max_id / $chunk_size);
for ($i = 0; $i < $max; $i++) {
$correlations = $this->find('column', [
'recursive' => -1,
'fields' => ['value'],
'conditions' => [
'id >' => ($i * $chunk_size),
'id >' => $i * $chunk_size,
'id <=' => (($i + 1) * $chunk_size)
]
]);
$newElements = count($correlations);
$correlations = array_count_values($correlations);
$pipeline = $this->redis->pipeline();
$pipeline = $redis->pipeline();
foreach ($correlations as $correlation => $count) {
$pipeline->zadd($this->cache_name, ['INCR'], $count, $correlation);
$pipeline->zadd(self::CACHE_NAME, ['INCR'], $count, $correlation);
}
$pipeline->exec();
if ($jobId) {
$job['Job']['progress'] = floor(100 * $i / $max);
$job['Job']['date_modified'] = date("Y-m-d H:i:s");
$job['Job']['message'] = __('Generating top correlations. Processed %s IDs.', ($i * $chunk_size) + $newElements);
$this->Job->save($job);
$this->Job->saveProgress($jobId, __('Generating top correlations. Processed %s IDs.', ($i * $chunk_size) + $newElements), floor(100 * $i / $max));
return $jobId;
}
}
return true;
}
public function findTop($query)
public function findTop(array $query)
{
try {
$this->redis = $this->setupRedisWithException();
$redis = $this->setupRedisWithException();
} catch (Exception $e) {
return false;
}
$start = $query['limit'] * ($query['page'] -1);
$end = $query['limit'] * ($query['page']);
$list = $this->redis->zRevRange($this->cache_name, $start, $end, true);
$end = $query['limit'] * $query['page'];
$list = $redis->zRevRange(self::CACHE_NAME, $start, $end, true);
$results = [];
foreach ($list as $value => $count) {
$results[] = [
'Correlation' => [
'value' => $value,
'count' => $count,
'excluded' => $this->redis->sismember('misp:correlation_exclusions', $value)
'excluded' => $this->__preventExcludedCorrelations(['value1' => $value]),
]
];
}
@ -698,10 +670,10 @@ class Correlation extends AppModel
public function getTopTime()
{
try {
$this->redis = $this->setupRedisWithException();
$redis = $this->setupRedisWithException();
} catch (Exception $e) {
return false;
}
return $this->redis->get($this->cache_age);
return $redis->get(self::CACHE_AGE);
}
}