MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'develop' of github.com:MISP/MISP into develop

pull/8537/head
iglocska 2022-08-11 14:00:03 +02:00
parent 0b0b62991a 579aba6a86
commit 54821e6297
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
9 changed files with 509 additions and 380 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
app/Model/AppModel.php
View File

@ -82,7 +82,8 @@ class AppModel extends Model
69 => false, 70 => false, 71 => true, 72 => true, 73 => false, 74 => false, 69 => false, 70 => false, 71 => true, 72 => true, 73 => false, 74 => false,
75 => false, 76 => true, 77 => false, 78 => false, 79 => false, 80 => false, 75 => false, 76 => true, 77 => false, 78 => false, 79 => false, 80 => false,
81 => false, 82 => false, 83 => false, 84 => false, 85 => false, 86 => false, 81 => false, 82 => false, 83 => false, 84 => false, 85 => false, 86 => false,
87 => false, 88 => false, 89 => false, 90 => false, 91 => false, 92 => false 87 => false, 88 => false, 89 => false, 90 => false, 91 => false, 92 => false,
93 => false,
); );
const ADVANCED_UPDATES_DESCRIPTION = array( const ADVANCED_UPDATES_DESCRIPTION = array(
@ -1834,6 +1835,22 @@ class AppModel extends Model
case 92: case 92:
$sqlArray[] = "ALTER TABLE users ADD `last_api_access` INT(11) DEFAULT 0;"; $sqlArray[] = "ALTER TABLE users ADD `last_api_access` INT(11) DEFAULT 0;";
break; break;
case 93:
$this->__dropIndex('default_correlations', 'distribution');
$this->__dropIndex('default_correlations', 'object_distribution');
$this->__dropIndex('default_correlations', 'event_distribution');
$this->__dropIndex('default_correlations', 'sharing_group_id');
$this->__dropIndex('default_correlations', 'object_sharing_group_id');
$this->__dropIndex('default_correlations', 'event_sharing_group_id');
$this->__dropIndex('default_correlations', 'org_id');
$this->__dropIndex('default_correlations', '1_distribution');
$this->__dropIndex('default_correlations', '1_object_distribution');
$this->__dropIndex('default_correlations', '1_event_distribution');
$this->__dropIndex('default_correlations', '1_sharing_group_id');
$this->__dropIndex('default_correlations', '1_object_sharing_group_id');
$this->__dropIndex('default_correlations', '1_event_sharing_group_id');
$this->__dropIndex('default_correlations', '1_org_id');
break;
case 'fixNonEmptySharingGroupID': case 'fixNonEmptySharingGroupID':
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';

157
app/Model/Behavior/DefaultCorrelationBehavior.php
View File

@ -1,17 +1,14 @@
<?php <?php
App::uses('AppModel', 'Model'); App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
/** /**
* Default correlation behaviour * Default correlation behaviour
*/ */
class DefaultCorrelationBehavior extends ModelBehavior class DefaultCorrelationBehavior extends ModelBehavior
{ {
const TABLE_NAME = 'default_correlations';
private $__tableName = 'default_correlations'; const CONFIG = [
private $__config = [
'AttributeFetcher' => [ 'AttributeFetcher' => [
'fields' => [ 'fields' => [
'Attribute.event_id', 'Attribute.event_id',
@ -44,26 +41,29 @@ class DefaultCorrelationBehavior extends ModelBehavior
] ]
]; ];
public $Correlation = null; /** @var Correlation */
public $Correlation;
private $deadlockAvoidance = false; private $deadlockAvoidance = false;
public function setup(Model $Model, $settings = []) { public function setup(Model $Model, $settings = [])
$Model->useTable = $this->__tableName; {
$Model->useTable = self::TABLE_NAME;
$this->Correlation = $Model; $this->Correlation = $Model;
$this->deadlockAvoidance = $settings['deadlockAvoidance']; $this->deadlockAvoidance = $settings['deadlockAvoidance'];
} }
public function getTableName(Model $Model) public function getTableName(Model $Model)
{ {
return $this->__tableName; return self::TABLE_NAME;
} }
public function createCorrelationEntry(Model $Model, $value, $a, $b) { public function createCorrelationEntry(Model $Model, $value, $a, $b)
$value_id = $this->Correlation->CorrelationValue->getValueId($value); {
$valueId = $this->Correlation->CorrelationValue->getValueId($value);
if ($this->deadlockAvoidance) { if ($this->deadlockAvoidance) {
return [ return [
'value_id' => $value_id, 'value_id' => $valueId,
'1_event_id' => $a['Event']['id'], '1_event_id' => $a['Event']['id'],
'1_object_id' => $a['Attribute']['object_id'], '1_object_id' => $a['Attribute']['object_id'],
'1_attribute_id' => $a['Attribute']['id'], '1_attribute_id' => $a['Attribute']['id'],
@ -87,7 +87,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
]; ];
} else { } else {
return [ return [
(int) $value_id, (int) $valueId,
(int) $a['Event']['id'], (int) $a['Event']['id'],
(int) $a['Attribute']['object_id'], (int) $a['Attribute']['object_id'],
(int) $a['Attribute']['id'], (int) $a['Attribute']['id'],
@ -182,20 +182,28 @@ class DefaultCorrelationBehavior extends ModelBehavior
public function getContainRules(Model $Model, $filter = null) public function getContainRules(Model $Model, $filter = null)
{ {
if (empty($filter)) { if (empty($filter)) {
return $this->__config['AttributeFetcher']['contain']; return self::CONFIG['AttributeFetcher']['contain'];
} else { } else {
return empty($this->__config['AttributeFetcher']['contain'][$filter]) ? false : $this->__config['AttributeFetcher']['contain'][$filter]; return empty(self::CONFIG['AttributeFetcher']['contain'][$filter]) ? false : self::CONFIG['AttributeFetcher']['contain'][$filter];
} }
} }
public function getFieldRules(Model $Model) public function getFieldRules(Model $Model)
{ {
return $this->__config['AttributeFetcher']['fields']; return self::CONFIG['AttributeFetcher']['fields'];
} }
private function __collectCorrelations($user, $id, $sgids, $primary) /**
* Fetch correlations for given event.
* @param array $user
* @param int $eventId
* @param array $sgids
* @param bool $primary
* @return array
*/
private function __collectCorrelations(array $user, $eventId, $sgids, $primary)
{ {
$max_correlations = Configure::read('MISP.max_correlations_per_event') ?: 5000; $maxCorrelations = Configure::read('MISP.max_correlations_per_event') ?: 5000;
$source = $primary ? '' : '1_'; $source = $primary ? '' : '1_';
$prefix = $primary ? '1_' : ''; $prefix = $primary ? '1_' : '';
$correlations = $this->Correlation->find('all', array( $correlations = $this->Correlation->find('all', array(
@ -214,7 +222,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
], ],
'conditions' => [ 'conditions' => [
'OR' => [ 'OR' => [
$source . 'event_id' => $id $source . 'event_id' => $eventId
], ],
'AND' => [ 'AND' => [
[ [
@ -235,46 +243,50 @@ class DefaultCorrelationBehavior extends ModelBehavior
] ]
], ],
'order' => false, 'order' => false,
'limit' => $max_correlations 'limit' => $maxCorrelations
)); ));
foreach ($correlations as $k => &$correlation) { foreach ($correlations as $k => $correlation) {
if (!$this->checkCorrelationACL($user, $correlation['Correlation'], $sgids, $prefix)) { if (!$this->checkCorrelationACL($user, $correlation['Correlation'], $sgids, $prefix)) {
unset($correlations[$k]); unset($correlations[$k]);
} }
} }
$correlations = array_values($correlations);
return $correlations; return $correlations;
} }
/**
* @param Correlation $Model
* @param array $user
* @param int $id Event ID
* @param array $sgids
* @return array
*/
public function runGetAttributesRelatedToEvent(Model $Model, $user, $id, $sgids) public function runGetAttributesRelatedToEvent(Model $Model, $user, $id, $sgids)
{ {
$temp_correlations = $this->__collectCorrelations($user, $id, $sgids, false);
$temp_correlations_1 = $this->__collectCorrelations($user, $id, $sgids, true);
$correlations = []; $correlations = [];
$event_ids = []; $eventIds = [];
foreach ($temp_correlations as $temp_correlation) { foreach ($this->__collectCorrelations($user, $id, $sgids, false) as $correlation) {
$correlations[] = [ $correlations[] = [
'id' => $temp_correlation['Correlation']['event_id'], 'id' => $correlation['Correlation']['event_id'],
'attribute_id' => $temp_correlation['Correlation']['attribute_id'], 'attribute_id' => $correlation['Correlation']['attribute_id'],
'parent_id' => $temp_correlation['Correlation']['1_attribute_id'], 'parent_id' => $correlation['Correlation']['1_attribute_id'],
'value' => $temp_correlation['CorrelationValue']['value'] 'value' => $correlation['CorrelationValue']['value']
]; ];
$event_ids[$temp_correlation['Correlation']['event_id']] = true; $eventIds[$correlation['Correlation']['event_id']] = true;
} }
foreach ($temp_correlations_1 as $temp_correlation) { foreach ($this->__collectCorrelations($user, $id, $sgids, true) as $correlation) {
$correlations[] = [ $correlations[] = [
'id' => $temp_correlation['Correlation']['1_event_id'], 'id' => $correlation['Correlation']['1_event_id'],
'attribute_id' => $temp_correlation['Correlation']['1_attribute_id'], 'attribute_id' => $correlation['Correlation']['1_attribute_id'],
'parent_id' => $temp_correlation['Correlation']['attribute_id'], 'parent_id' => $correlation['Correlation']['attribute_id'],
'value' => $temp_correlation['CorrelationValue']['value'] 'value' => $correlation['CorrelationValue']['value']
]; ];
$event_ids[$temp_correlation['Correlation']['1_event_id']] = true; $eventIds[$correlation['Correlation']['1_event_id']] = true;
} }
if (empty($correlations)) { if (empty($correlations)) {
return []; return [];
} }
$conditions = $Model->Event->createEventConditions($user); $conditions = $Model->Event->createEventConditions($user);
$conditions['Event.id'] = array_keys($event_ids); $conditions['Event.id'] = array_keys($eventIds);
$events = $Model->Event->find('all', [ $events = $Model->Event->find('all', [
'recursive' => -1, 'recursive' => -1,
'conditions' => $conditions, 'conditions' => $conditions,
@ -289,9 +301,9 @@ class DefaultCorrelationBehavior extends ModelBehavior
continue; continue;
} }
$event = $events[$eventId]; $event = $events[$eventId];
$correlation['org_id'] = $events[$eventId]['orgc_id']; $correlation['org_id'] = $event['orgc_id'];
$correlation['info'] = $events[$eventId]['info']; $correlation['info'] = $event['info'];
$correlation['date'] = $events[$eventId]['date']; $correlation['date'] = $event['date'];
$parentId = $correlation['parent_id']; $parentId = $correlation['parent_id'];
unset($correlation['parent_id']); unset($correlation['parent_id']);
$relatedAttributes[$parentId][] = $correlation; $relatedAttributes[$parentId][] = $correlation;
@ -389,7 +401,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
]); ]);
if (!empty($includeEventData)) { if (!empty($includeEventData)) {
$results = []; $results = [];
foreach ($relatedAttributes as $k => $attribute) { foreach ($relatedAttributes as $attribute) {
$temp = $attribute['Attribute']; $temp = $attribute['Attribute'];
$temp['Event'] = $attribute['Event']; $temp['Event'] = $attribute['Event'];
$results[] = $temp; $results[] = $temp;
@ -411,18 +423,27 @@ class DefaultCorrelationBehavior extends ModelBehavior
// ii. Event has a sharing group that the user is accessible to view // ii. Event has a sharing group that the user is accessible to view
// b. Attribute: // b. Attribute:
// i. Attribute has a distribution of 5 (inheritance of the event, for this the event check has to pass anyway) // i. Attribute has a distribution of 5 (inheritance of the event, for this the event check has to pass anyway)
// ii. Atttibute has a distribution between 1-3 (community only, connected communities, all orgs) // ii. Attribute has a distribution between 1-3 (community only, connected communities, all orgs)
// iii. Attribute has a sharing group that the user is accessible to view // iii. Attribute has a sharing group that the user is accessible to view
$primaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, $sgids, true); $primaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, $sgids, true);
$secondaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, $sgids, false); $secondaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, $sgids, false);
return array_unique(array_merge($primaryEventIds,$secondaryEventIds)); return array_unique(array_merge($primaryEventIds,$secondaryEventIds), SORT_REGULAR);
} }
/**
* @param Model $Model
* @param array $user
* @param int $eventId
* @param array $sgids
* @param bool $primary
* @return array|int[]
*/
private function __filterRelatedEvents(Model $Model, array $user, int $eventId, array $sgids, bool $primary) private function __filterRelatedEvents(Model $Model, array $user, int $eventId, array $sgids, bool $primary)
{ {
$current = $primary ? '' : '1_'; $current = $primary ? '' : '1_';
$prefix = $primary ? '1_' : ''; $prefix = $primary ? '1_' : '';
if (empty($user['Role']['perm_site_admin'])) {
$correlations = $Model->find('all', [ $correlations = $Model->find('all', [
'recursive' => -1, 'recursive' => -1,
'fields' => [ 'fields' => [
@ -439,35 +460,44 @@ class DefaultCorrelationBehavior extends ModelBehavior
'conditions' => [ 'conditions' => [
$current . 'event_id' => $eventId $current . 'event_id' => $eventId
], ],
'unique' => true,
]); ]);
$eventIds = []; $eventIds = [];
if (empty($user['Role']['perm_site_admin'])) { foreach ($correlations as $correlation) {
foreach ($correlations as $k => $correlation) {
// if we have already added this event as a valid target, no need to check again.
if (isset($eventIds[$correlation['Correlation'][$prefix . 'event_id']])) {
continue;
}
$correlation = $correlation['Correlation']; $correlation = $correlation['Correlation'];
if (!$this->checkCorrelationACL($user, $correlation, $sgids, $prefix)) { // if we have already added this event as a valid target, no need to check again.
unset($correlations[$k]); if (isset($eventIds[$correlation[$prefix . 'event_id']])) {
continue; continue;
} }
if ($this->checkCorrelationACL($user, $correlation, $sgids, $prefix)) {
$eventIds[$correlation[$prefix . 'event_id']] = true; $eventIds[$correlation[$prefix . 'event_id']] = true;
} }
return array_keys($eventIds);
} else {
$eventIds = Hash::extract($correlations, '{n}.Correlation.' . $prefix . 'event_id');
return $eventIds;
} }
return array_keys($eventIds);
} }
private function checkCorrelationACL($user, $correlation, $sgids, $prefix) return $Model->find('column', [
'fields' => [$prefix . 'event_id'],
'conditions' => [
$current . 'event_id' => $eventId
],
'unique' => true,
]);
}
/**
* @param array $user
* @param array $correlation
* @param array $sgids
* @param string $prefix
* @return bool
*/
private function checkCorrelationACL(array $user, $correlation, $sgids, $prefix)
{ {
if ($user['Role']['perm_site_admin']) { if ($user['Role']['perm_site_admin']) {
return true; return true;
} }
// check if user can see the event // Check if user can see the event
if (isset($correlation['Correlation'])) { if (isset($correlation['Correlation'])) {
$correlation = $correlation['Correlation']; $correlation = $correlation['Correlation'];
} }
@ -484,7 +514,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
return false; return false;
} }
//check if the user can see the object, if we're looking at an object attribute // Check if the user can see the object, if we're looking at an object attribute
if ( if (
$correlation[$prefix . 'object_id'] && $correlation[$prefix . 'object_id'] &&
( (
@ -499,7 +529,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
return false; return false;
} }
//check if the user can see the attribute // Check if the user can see the attribute
if ( if (
( (
$correlation[$prefix . 'distribution'] == 0 || $correlation[$prefix . 'distribution'] == 0 ||
@ -551,7 +581,8 @@ class DefaultCorrelationBehavior extends ModelBehavior
$Model->updateAll( $Model->updateAll(
$side, $side,
[ [
$updateFields[$k] => (int)$data['id']] $updateFields[$k] => (int)$data['id']
]
); );
} }
} }

82
app/Model/Correlation.php
View File

@ -1,9 +1,15 @@
<?php <?php
App::uses('AppModel', 'Model'); App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
/** /**
* @property Attribute $Attribute * @property Attribute $Attribute
* @property Event $Event
* @property CorrelationValue $CorrelationValue
* @method saveCorrelations(array $correlations)
* @method runBeforeSaveCorrelation
* @method fetchRelatedEventIds(array $user, int $eventId, array $sgids)
* @method getFieldRules
* @method getContainRules($filter = null)
*/ */
class Correlation extends AppModel class Correlation extends AppModel
{ {
@ -44,36 +50,27 @@ class Correlation extends AppModel
/** @var array */ /** @var array */
private $exclusions; private $exclusions;
/**
* Use old schema with `date` and `info` fields.
* @var bool
*/
private $oldSchema;
/** @var bool */
private $deadlockAvoidance;
/** @var bool */ /** @var bool */
private $advancedCorrelationEnabled; private $advancedCorrelationEnabled;
/** @var array */ /** @var array */
private $cidrListCache; private $cidrListCache;
private $__correlationEngine = 'DefaultCorrelation'; /** @var string */
private $__correlationEngine;
protected $_config = [];
private $__tempContainCache = []; private $__tempContainCache = [];
public $OverCorrelatingValue = null; /** @var OverCorrelatingValue */
public $OverCorrelatingValue;
public function __construct($id = false, $table = null, $ds = null) public function __construct($id = false, $table = null, $ds = null)
{ {
parent::__construct($id, $table, $ds); parent::__construct($id, $table, $ds);
$this->__correlationEngine = $this->getCorrelationModelName(); $this->__correlationEngine = $this->getCorrelationModelName();
$this->deadlockAvoidance = Configure::check('MISP.deadlock_avoidance') ? Configure::read('MISP.deadlock_avoidance') : false; $deadlockAvoidance = Configure::check('MISP.deadlock_avoidance') ? Configure::read('MISP.deadlock_avoidance') : false;
// load the currently used correlation engine // load the currently used correlation engine
$this->Behaviors->load($this->__correlationEngine . 'Correlation', ['deadlockAvoidance' => false]); $this->Behaviors->load($this->__correlationEngine . 'Correlation', ['deadlockAvoidance' => $deadlockAvoidance]);
// getTableName() needs to be implemented by the engine - this points us to the table to be used // getTableName() needs to be implemented by the engine - this points us to the table to be used
$this->useTable = $this->getTableName(); $this->useTable = $this->getTableName();
$this->advancedCorrelationEnabled = (bool)Configure::read('MISP.enable_advanced_correlations'); $this->advancedCorrelationEnabled = (bool)Configure::read('MISP.enable_advanced_correlations');
@ -290,6 +287,7 @@ class Correlation extends AppModel
* @param bool $full * @param bool $full
* @param array|false $event * @param array|false $event
* @return array|bool|bool[]|mixed * @return array|bool|bool[]|mixed
* @throws Exception
*/ */
public function afterSaveCorrelation($a, $full = false, $event = false) public function afterSaveCorrelation($a, $full = false, $event = false)
{ {
@ -336,7 +334,7 @@ class Correlation extends AppModel
return true; return true;
} }
$correlations = []; $correlations = [];
foreach ($correlatingValues as $k => $cV) { foreach ($correlatingValues as $cV) {
if ($cV === null) { if ($cV === null) {
continue; continue;
} }
@ -347,6 +345,7 @@ class Correlation extends AppModel
'Attribute.value2' => $cV, 'Attribute.value2' => $cV,
'NOT' => ['Attribute.type' => Attribute::PRIMARY_ONLY_CORRELATING_TYPES] 'NOT' => ['Attribute.type' => Attribute::PRIMARY_ONLY_CORRELATING_TYPES]
], ],
$extraConditions,
], ],
'NOT' => [ 'NOT' => [
'Attribute.event_id' => $a['Attribute']['event_id'], 'Attribute.event_id' => $a['Attribute']['event_id'],
@ -851,7 +850,7 @@ class Correlation extends AppModel
/** /**
* @param array $user User array * @param array $user User array
* @param int $eventIds List of event IDs * @param int $eventId List of event IDs
* @param array $sgids List of sharing group IDs * @param array $sgids List of sharing group IDs
* @return array * @return array
*/ */
@ -872,28 +871,55 @@ class Correlation extends AppModel
return $data; return $data;
} }
public function setCorrelationExclusion($attribute) /**
* @param array $attributes
* @return array
*/
public function attachCorrelationExclusion(array $attributes)
{ {
if (empty($this->__compositeTypes)) { if (!isset($this->__compositeTypes)) {
$this->__compositeTypes = $this->Attribute->getCompositeTypes(); $this->__compositeTypes = $this->Attribute->getCompositeTypes();
} }
$values = [$attribute['value']];
if (in_array($attribute['type'], $this->__compositeTypes)) { $valuesToCheck = [];
foreach ($attributes as &$attribute) {
if (in_array($attribute['type'], $this->__compositeTypes, true)) {
$values = explode('|', $attribute['value']); $values = explode('|', $attribute['value']);
$valuesToCheck[$values[0]] = true;
$valuesToCheck[$values[1]] = true;
} else {
$values = [$attribute['value']];
$valuesToCheck[$values[0]] = true;
} }
if ($this->__preventExcludedCorrelations($values[0])) { if ($this->__preventExcludedCorrelations($values[0])) {
$attribute['correlation_exclusion'] = true; $attribute['correlation_exclusion'] = true;
} } elseif (!empty($values[1]) && $this->__preventExcludedCorrelations($values[1])) {
if (!empty($values[1]) && $this->__preventExcludedCorrelations($values[1])) {
$attribute['correlation_exclusion'] = true; $attribute['correlation_exclusion'] = true;
} }
if ($this->OverCorrelatingValue->checkValue($values[0])) { }
$overCorrelatingValues = array_flip($this->OverCorrelatingValue->find('column', [
'conditions' => ['value' => array_keys($valuesToCheck)],
'fields' => ['value'],
]));
unset($valuesToCheck);
foreach ($attributes as &$attribute) {
if (in_array($attribute['type'], $this->__compositeTypes, true)) {
$values = explode('|', $attribute['value']);
} else {
$values = [$attribute['value']];
}
if (isset($overCorrelatingValues[$values[0]])) {
$attribute['over_correlation'] = true;
} elseif (!empty($values[1]) && isset($overCorrelatingValues[$values[1]])) {
$attribute['over_correlation'] = true; $attribute['over_correlation'] = true;
} }
if (!empty($values[1]) && $this->OverCorrelatingValue->checkValue($values[1])) {
$attribute['over_correlation'] = true;
} }
return $attribute;
return $attributes;
} }
public function collectMetrics() public function collectMetrics()

18
app/Model/CorrelationValue.php
View File

@ -1,24 +1,21 @@
<?php <?php
App::uses('AppModel', 'Model'); App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
class CorrelationValue extends AppModel class CorrelationValue extends AppModel
{ {
public $recursive = -1; public $recursive = -1;
public $actsAs = array( /**
'Containable' * @param string $value
); * @return int
*/
public $validate = [
];
public function getValueId($value) public function getValueId($value)
{ {
// index is 191 long, missing the existing value lookup can lead to a duplicate entry // index is 191 long, missing the existing value lookup can lead to a duplicate entry
$value = mb_substr($value, 0, 191); $value = mb_substr($value, 0, 191);
$existingValue = $this->find('first', [ $existingValue = $this->find('first', [
'recursive' => -1, 'recursive' => -1,
'fields' => ['id'],
'conditions' => [ 'conditions' => [
'value' => $value 'value' => $value
] ]
@ -31,22 +28,23 @@ class CorrelationValue extends AppModel
} catch (Exception $e) { } catch (Exception $e) {
$existingValue = $this->find('first', [ $existingValue = $this->find('first', [
'recursive' => -1, 'recursive' => -1,
'fields' => ['id'],
'conditions' => [ 'conditions' => [
'value' => $value 'value' => $value
] ]
]); ]);
return $existingValue['ExistingValue']['id']; return $existingValue['CorrelationValue']['id'];
} }
} else { } else {
return $existingValue['CorrelationValue']['id']; return $existingValue['CorrelationValue']['id'];
} }
return false;
} }
public function getValue($id) public function getValue($id)
{ {
$existingValue = $this->find('first', [ $existingValue = $this->find('first', [
'recursive' => -1, 'recursive' => -1,
'fields' => ['value'],
'conditions' => [ 'conditions' => [
'id' => $id 'id' => $id
] ]

5
app/Model/Event.php
View File

@ -608,7 +608,7 @@ class Event extends AppModel
return $events; return $events;
} }
public function getRelatedEventCount($user, $eventId, $sgids) public function getRelatedEventCount(array $user, $eventId, $sgids)
{ {
if (!isset($sgids) || empty($sgids)) { if (!isset($sgids) || empty($sgids)) {
$sgids = array(-1); $sgids = array(-1);
@ -2018,6 +2018,8 @@ class Event extends AppModel
$event['Attribute'] = $this->__attachSharingGroups($event['Attribute'], $sharingGroupData); $event['Attribute'] = $this->__attachSharingGroups($event['Attribute'], $sharingGroupData);
} }
$event['Attribute'] = $this->Attribute->Correlation->attachCorrelationExclusion($event['Attribute']);
// move all object attributes to a temporary container // move all object attributes to a temporary container
$tempObjectAttributeContainer = array(); $tempObjectAttributeContainer = array();
foreach ($event['Attribute'] as $key => &$attribute) { foreach ($event['Attribute'] as $key => &$attribute) {
@ -2025,7 +2027,6 @@ class Event extends AppModel
unset($event['Attribute'][$key]); unset($event['Attribute'][$key]);
continue; continue;
} }
$attribute = $this->Attribute->Correlation->setCorrelationExclusion($attribute);
if ($attribute['category'] === 'Financial fraud') { if ($attribute['category'] === 'Financial fraud') {
$attribute = $this->Attribute->attachValidationWarnings($attribute); $attribute = $this->Attribute->attachValidationWarnings($attribute);
} }

29
app/Model/OverCorrelatingValue.php
View File

@ -1,6 +1,5 @@
<?php <?php
App::uses('AppModel', 'Model'); App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
class OverCorrelatingValue extends AppModel class OverCorrelatingValue extends AppModel
{ {
@ -10,9 +9,13 @@ class OverCorrelatingValue extends AppModel
'Containable' 'Containable'
); );
public $validate = [
];
/**
* @param string $value
* @param int $count
* @return void
* @throws Exception
*/
public function block($value, $count = 0) public function block($value, $count = 0)
{ {
$this->unblock($value); $this->unblock($value);
@ -25,15 +28,23 @@ class OverCorrelatingValue extends AppModel
); );
} }
/**
* @param string $value
* @return void
*/
public function unBlock($value) public function unBlock($value)
{ {
$this->deleteAll( $this->deleteAll(
[ [
'OverCorrelatingValue.value' => $value 'OverCorrelatingValue.value' => $value
] ],
false
); );
} }
/**
* @return int
*/
public function getLimit() public function getLimit()
{ {
return Configure::check('MISP.correlation_limit') ? Configure::read('MISP.correlation_limit') : 20; return Configure::check('MISP.correlation_limit') ? Configure::read('MISP.correlation_limit') : 20;
@ -55,15 +66,7 @@ class OverCorrelatingValue extends AppModel
public function checkValue($value) public function checkValue($value)
{ {
$hit = $this->find('first', [ return $this->hasAny(['value' => $value]);
'recursive' => -1,
'conditions' => ['value' => $value],
'fields' => ['id']
]);
if (empty($hit)) {
return false;
}
return true;
} }
public function generateOccurrencesRouter() public function generateOccurrencesRouter()

2
app/Model/Server.php
View File

@ -3213,7 +3213,7 @@ class Server extends AppModel
private function getDatabaseIndexes($database, $table) private function getDatabaseIndexes($database, $table)
{ {
$sqlTableIndex = sprintf( $sqlTableIndex = sprintf(
"SELECT DISTINCT TABLE_NAME, COLUMN_NAME, NON_UNIQUE FROM information_schema.statistics WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s';", "SELECT DISTINCT TABLE_NAME, COLUMN_NAME, NON_UNIQUE FROM information_schema.statistics WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s' ORDER BY COLUMN_NAME;",
$database, $database,
$table $table
); );

470
db_schema.json
View File

@ -1010,28 +1010,6 @@
"column_type": "int(11)", "column_type": "int(11)",
"column_default": null, "column_default": null,
"extra": "" "extra": ""
},
{
"column_name": "date",
"is_nullable": "NO",
"data_type": "date",
"character_maximum_length": null,
"numeric_precision": null,
"collation_name": null,
"column_type": "date",
"column_default": null,
"extra": ""
},
{
"column_name": "info",
"is_nullable": "NO",
"data_type": "text",
"character_maximum_length": "65535",
"numeric_precision": null,
"collation_name": "utf8mb3_bin",
"column_type": "text",
"column_default": null,
"extra": ""
} }
], ],
"correlation_exclusions": [ "correlation_exclusions": [
@ -1094,13 +1072,13 @@
}, },
{ {
"column_name": "value", "column_name": "value",
"is_nullable": "YES", "is_nullable": "NO",
"data_type": "text", "data_type": "varchar",
"character_maximum_length": "65535", "character_maximum_length": "191",
"numeric_precision": null, "numeric_precision": null,
"collation_name": "utf8mb4_unicode_ci", "collation_name": "utf8mb4_unicode_ci",
"column_type": "text", "column_type": "varchar(191)",
"column_default": "NULL", "column_default": null,
"extra": "" "extra": ""
} }
], ],
@ -2897,6 +2875,17 @@
"column_default": null, "column_default": null,
"extra": "auto_increment" "extra": "auto_increment"
}, },
{
"column_name": "uuid",
"is_nullable": "NO",
"data_type": "varchar",
"character_maximum_length": "255",
"numeric_precision": null,
"collation_name": "utf8mb3_bin",
"column_type": "varchar(255)",
"column_default": "''",
"extra": ""
},
{ {
"column_name": "collection_uuid", "column_name": "collection_uuid",
"is_nullable": "NO", "is_nullable": "NO",
@ -2996,17 +2985,6 @@
"column_default": "0", "column_default": "0",
"extra": "" "extra": ""
}, },
{
"column_name": "uuid",
"is_nullable": "NO",
"data_type": "varchar",
"character_maximum_length": "255",
"numeric_precision": null,
"collation_name": "utf8mb3_bin",
"column_type": "varchar(255)",
"column_default": "''",
"extra": ""
},
{ {
"column_name": "distribution", "column_name": "distribution",
"is_nullable": "NO", "is_nullable": "NO",
@ -4179,13 +4157,13 @@
}, },
{ {
"column_name": "comment", "column_name": "comment",
"is_nullable": "YES", "is_nullable": "NO",
"data_type": "text", "data_type": "text",
"character_maximum_length": "65535", "character_maximum_length": "65535",
"numeric_precision": null, "numeric_precision": null,
"collation_name": "utf8mb3_unicode_ci", "collation_name": "utf8mb3_unicode_ci",
"column_type": "text", "column_type": "text",
"column_default": "NULL", "column_default": null,
"extra": "" "extra": ""
}, },
{ {
@ -4862,13 +4840,13 @@
}, },
{ {
"column_name": "comment", "column_name": "comment",
"is_nullable": "NO", "is_nullable": "YES",
"data_type": "text", "data_type": "text",
"character_maximum_length": "65535", "character_maximum_length": "65535",
"numeric_precision": null, "numeric_precision": null,
"collation_name": "utf8mb3_unicode_ci", "collation_name": "utf8mb3_unicode_ci",
"column_type": "text", "column_type": "text",
"column_default": null, "column_default": "NULL",
"extra": "" "extra": ""
} }
], ],
@ -5080,11 +5058,11 @@
{ {
"column_name": "body", "column_name": "body",
"is_nullable": "YES", "is_nullable": "YES",
"data_type": "longtext", "data_type": "text",
"character_maximum_length": "4294967295", "character_maximum_length": "65535",
"numeric_precision": null, "numeric_precision": null,
"collation_name": "utf8mb3_general_ci", "collation_name": "utf8mb3_general_ci",
"column_type": "longtext", "column_type": "text",
"column_default": "NULL", "column_default": "NULL",
"extra": "" "extra": ""
}, },
@ -5497,6 +5475,17 @@
"column_default": "0", "column_default": "0",
"extra": "" "extra": ""
}, },
{
"column_name": "perm_decaying",
"is_nullable": "NO",
"data_type": "tinyint",
"character_maximum_length": null,
"numeric_precision": "3",
"collation_name": null,
"column_type": "tinyint(1)",
"column_default": "0",
"extra": ""
},
{ {
"column_name": "enforce_rate_limit", "column_name": "enforce_rate_limit",
"is_nullable": "NO", "is_nullable": "NO",
@ -5519,17 +5508,6 @@
"column_default": "0", "column_default": "0",
"extra": "" "extra": ""
}, },
{
"column_name": "perm_decaying",
"is_nullable": "NO",
"data_type": "tinyint",
"character_maximum_length": null,
"numeric_precision": "3",
"collation_name": null,
"column_type": "tinyint(1)",
"column_default": "0",
"extra": ""
},
{ {
"column_name": "perm_galaxy_editor", "column_name": "perm_galaxy_editor",
"is_nullable": "NO", "is_nullable": "NO",
@ -5965,13 +5943,13 @@
}, },
{ {
"column_name": "comment", "column_name": "comment",
"is_nullable": "YES", "is_nullable": "NO",
"data_type": "text", "data_type": "text",
"character_maximum_length": "65535", "character_maximum_length": "65535",
"numeric_precision": null, "numeric_precision": null,
"collation_name": "utf8mb3_unicode_ci", "collation_name": "utf8mb3_unicode_ci",
"column_type": "text", "column_type": "text",
"column_default": "NULL", "column_default": null,
"extra": "" "extra": ""
}, },
{ {
@ -6761,6 +6739,30 @@
"extra": "" "extra": ""
} }
], ],
"system_settings": [
{
"column_name": "setting",
"is_nullable": "NO",
"data_type": "varchar",
"character_maximum_length": "255",
"numeric_precision": null,
"collation_name": "utf8mb4_unicode_ci",
"column_type": "varchar(255)",
"column_default": null,
"extra": ""
},
{
"column_name": "value",
"is_nullable": "NO",
"data_type": "blob",
"character_maximum_length": "65535",
"numeric_precision": null,
"collation_name": null,
"column_type": "blob",
"column_default": null,
"extra": ""
}
],
"tags": [ "tags": [
{ {
"column_name": "id", "column_name": "id",
@ -8166,6 +8168,17 @@
"column_type": "text", "column_type": "text",
"column_default": "NULL", "column_default": "NULL",
"extra": "" "extra": ""
},
{
"column_name": "last_api_access",
"is_nullable": "YES",
"data_type": "int",
"character_maximum_length": null,
"numeric_precision": "10",
"collation_name": null,
"column_type": "int(11)",
"column_default": "0",
"extra": ""
} }
], ],
"user_settings": [ "user_settings": [
@ -8590,99 +8603,99 @@
}, },
"indexes": { "indexes": {
"admin_settings": { "admin_settings": {
"id": true "id": true,
"setting": true
}, },
"allowedlist": { "allowedlist": {
"id": true "id": true
}, },
"attachment_scans": { "attachment_scans": {
"attribute_id": false,
"id": true, "id": true,
"type": false, "type": false
"attribute_id": false
}, },
"attributes": { "attributes": {
"id": true,
"uuid": false,
"event_id": false,
"sharing_group_id": false,
"type": false,
"category": false, "category": false,
"value1": false, "event_id": false,
"value2": false, "first_seen": false,
"id": true,
"last_seen": false,
"object_id": false, "object_id": false,
"object_relation": false, "object_relation": false,
"deleted": false, "sharing_group_id": false,
"first_seen": false, "timestamp": false,
"last_seen": false, "type": false,
"timestamp": false "uuid": true,
"value1": false,
"value2": false
}, },
"attribute_tags": { "attribute_tags": {
"id": true,
"attribute_id": false, "attribute_id": false,
"event_id": false, "event_id": false,
"id": true,
"tag_id": false "tag_id": false
}, },
"audit_logs": { "audit_logs": {
"id": true,
"event_id": false, "event_id": false,
"id": true,
"model_id": false "model_id": false
}, },
"auth_keys": { "auth_keys": {
"id": true,
"authkey_start": false,
"authkey_end": false, "authkey_end": false,
"authkey_start": false,
"created": false, "created": false,
"expiration": false, "expiration": false,
"id": true,
"user_id": false "user_id": false
}, },
"bruteforces": [], "bruteforces": [],
"cake_sessions": { "cake_sessions": {
"id": true, "expires": false,
"expires": false "id": true
}, },
"cerebrates": { "cerebrates": {
"id": true, "id": true,
"url": false, "org_id": false,
"org_id": false "url": false
}, },
"correlations": { "correlations": {
"id": true, "1_attribute_id": false,
"event_id": false,
"1_event_id": false, "1_event_id": false,
"attribute_id": false, "attribute_id": false,
"1_attribute_id": false "event_id": false,
"id": true
}, },
"correlation_exclusions": { "correlation_exclusions": {
"id": true, "id": true,
"value": false "value": true
}, },
"correlation_values": { "correlation_values": {
"id": true, "id": true,
"value": true "value": true
}, },
"cryptographic_keys": { "cryptographic_keys": {
"fingerprint": false,
"id": true, "id": true,
"uuid": false,
"type": false,
"parent_id": false, "parent_id": false,
"parent_type": false, "parent_type": false,
"fingerprint": false "type": false,
"uuid": false
}, },
"dashboards": { "dashboards": {
"id": true, "id": true,
"name": false, "name": false,
"uuid": false,
"user_id": false,
"restrict_to_org_id": false, "restrict_to_org_id": false,
"restrict_to_permission_flag": false "restrict_to_permission_flag": false,
"user_id": false,
"uuid": false
}, },
"decaying_models": { "decaying_models": {
"all_orgs": false,
"enabled": false,
"id": true, "id": true,
"uuid": false,
"name": false, "name": false,
"org_id": false, "org_id": false,
"enabled": false, "uuid": false,
"all_orgs": false,
"version": false "version": false
}, },
"decaying_model_mappings": { "decaying_model_mappings": {
@ -8690,76 +8703,62 @@
"model_id": false "model_id": false
}, },
"default_correlations": { "default_correlations": {
"id": true,
"attribute_id": false,
"1_attribute_id": false, "1_attribute_id": false,
"value_id": false,
"event_id": false,
"object_id": false,
"org_id": false,
"distribution": false,
"object_distribution": false,
"event_distribution": false,
"sharing_group_id": false,
"object_sharing_group_id": false,
"event_sharing_group_id": false,
"1_event_id": false, "1_event_id": false,
"1_object_id": false, "1_object_id": false,
"1_org_id": false, "attribute_id": false,
"1_distribution": false, "event_id": false,
"1_object_distribution": false, "id": true,
"1_event_distribution": false, "object_id": false,
"1_sharing_group_id": false, "value_id": false
"1_object_sharing_group_id": false,
"1_event_sharing_group_id": false
}, },
"events": { "events": {
"extends_uuid": false,
"id": true, "id": true,
"uuid": true,
"info": false, "info": false,
"sharing_group_id": false,
"org_id": false,
"orgc_id": false, "orgc_id": false,
"extends_uuid": false "org_id": false,
"sharing_group_id": false,
"uuid": true
}, },
"event_blocklists": { "event_blocklists": {
"id": true, "event_orgc": false,
"event_uuid": false, "event_uuid": true,
"event_orgc": false "id": true
}, },
"event_delegations": { "event_delegations": {
"event_id": false,
"id": true, "id": true,
"org_id": false, "org_id": false
"event_id": false
}, },
"event_graph": { "event_graph": {
"id": true,
"event_id": false, "event_id": false,
"user_id": false, "id": true,
"org_id": false, "org_id": false,
"timestamp": false "timestamp": false,
"user_id": false
}, },
"event_locks": { "event_locks": {
"id": true,
"event_id": false, "event_id": false,
"user_id": false, "id": true,
"timestamp": false "timestamp": false,
"user_id": false
}, },
"event_reports": { "event_reports": {
"event_id": false,
"id": true, "id": true,
"uuid": true,
"name": false, "name": false,
"event_id": false "uuid": true
}, },
"event_tags": { "event_tags": {
"id": true,
"event_id": false, "event_id": false,
"id": true,
"tag_id": false "tag_id": false
}, },
"favourite_tags": { "favourite_tags": {
"id": true, "id": true,
"user_id": false, "tag_id": false,
"tag_id": false "user_id": false
}, },
"feeds": { "feeds": {
"id": true, "id": true,
@ -8767,66 +8766,66 @@
"orgc_id": false "orgc_id": false
}, },
"fuzzy_correlate_ssdeep": { "fuzzy_correlate_ssdeep": {
"id": true, "attribute_id": false,
"chunk": false, "chunk": false,
"attribute_id": false "id": true
}, },
"galaxies": { "galaxies": {
"id": true, "id": true,
"name": false, "name": false,
"uuid": false, "namespace": false,
"type": false, "type": false,
"namespace": false "uuid": true
}, },
"galaxy_clusters": { "galaxy_clusters": {
"id": true, "collection_uuid": false,
"value": false, "default": false,
"extends_uuid": false,
"extends_version": false,
"galaxy_id": false, "galaxy_id": false,
"version": false, "id": true,
"orgc_id": false,
"org_id": false,
"sharing_group_id": false,
"tag_name": false, "tag_name": false,
"type": false, "type": false,
"uuid": false, "uuid": false,
"collection_uuid": false, "value": false,
"org_id": false, "version": false
"orgc_id": false,
"sharing_group_id": false,
"extends_uuid": false,
"extends_version": false,
"default": false
}, },
"galaxy_cluster_blocklists": { "galaxy_cluster_blocklists": {
"id": true, "cluster_orgc": false,
"cluster_uuid": false, "cluster_uuid": false,
"cluster_orgc": false "id": true
}, },
"galaxy_cluster_relations": { "galaxy_cluster_relations": {
"id": true, "default": false,
"galaxy_cluster_id": false, "galaxy_cluster_id": false,
"galaxy_cluster_uuid": false,
"id": true,
"referenced_galaxy_cluster_id": false, "referenced_galaxy_cluster_id": false,
"referenced_galaxy_cluster_type": false, "referenced_galaxy_cluster_type": false,
"galaxy_cluster_uuid": false, "sharing_group_id": false
"sharing_group_id": false,
"default": false
}, },
"galaxy_cluster_relation_tags": { "galaxy_cluster_relation_tags": {
"id": true,
"galaxy_cluster_relation_id": false, "galaxy_cluster_relation_id": false,
"id": true,
"tag_id": false "tag_id": false
}, },
"galaxy_elements": { "galaxy_elements": {
"galaxy_cluster_id": false,
"id": true, "id": true,
"key": false, "key": false,
"value": false, "value": false
"galaxy_cluster_id": false
}, },
"inbox": { "inbox": {
"id": true, "id": true,
"ip": false,
"timestamp": false,
"title": false, "title": false,
"type": false, "type": false,
"uuid": false,
"user_agent_sha256": false, "user_agent_sha256": false,
"ip": false, "uuid": false
"timestamp": false
}, },
"jobs": { "jobs": {
"id": true "id": true
@ -8838,9 +8837,9 @@
"id": true "id": true
}, },
"noticelists": { "noticelists": {
"geographical_area": false,
"id": true, "id": true,
"name": false, "name": false
"geographical_area": false
}, },
"noticelist_entries": { "noticelist_entries": {
"id": true, "id": true,
@ -8852,32 +8851,33 @@
"type": false "type": false
}, },
"no_acl_correlations": { "no_acl_correlations": {
"id": true,
"attribute_id": false,
"1_attribute_id": false, "1_attribute_id": false,
"value_id": false, "1_event_id": false,
"attribute_id": false,
"event_id": false, "event_id": false,
"1_event_id": false "id": true,
"value_id": false
}, },
"objects": { "objects": {
"distribution": false,
"event_id": false,
"first_seen": false,
"id": true, "id": true,
"last_seen": false,
"meta-category": false,
"name": false, "name": false,
"sharing_group_id": false,
"template_uuid": false, "template_uuid": false,
"template_version": false, "template_version": false,
"meta-category": false,
"event_id": false,
"uuid": false,
"timestamp": false, "timestamp": false,
"distribution": false, "uuid": true
"sharing_group_id": false,
"first_seen": false,
"last_seen": false
}, },
"object_references": { "object_references": {
"event_id": false,
"id": true, "id": true,
"object_id": false, "object_id": false,
"referenced_id": false, "referenced_id": false,
"event_id": false "uuid": true
}, },
"object_relationships": { "object_relationships": {
"id": true, "id": true,
@ -8885,11 +8885,11 @@
}, },
"object_templates": { "object_templates": {
"id": true, "id": true,
"user_id": false, "meta-category": false,
"org_id": false,
"uuid": false,
"name": false, "name": false,
"meta-category": false "org_id": false,
"user_id": false,
"uuid": false
}, },
"object_template_elements": { "object_template_elements": {
"id": true, "id": true,
@ -8898,16 +8898,18 @@
}, },
"organisations": { "organisations": {
"id": true, "id": true,
"uuid": false, "name": true,
"name": false "uuid": true
}, },
"org_blocklists": { "org_blocklists": {
"id": true "id": true,
"org_name": false,
"org_uuid": true
}, },
"over_correlating_values": { "over_correlating_values": {
"id": true, "id": true,
"value": true, "occurrence": false,
"occurrence": false "value": true
}, },
"posts": { "posts": {
"id": true, "id": true,
@ -8920,8 +8922,8 @@
"rest_client_histories": { "rest_client_histories": {
"id": true, "id": true,
"org_id": false, "org_id": false,
"user_id": false, "timestamp": false,
"timestamp": false "user_id": false
}, },
"roles": { "roles": {
"id": true "id": true
@ -8929,46 +8931,47 @@
"servers": { "servers": {
"id": true, "id": true,
"org_id": false, "org_id": false,
"remote_org_id": false, "priority": false,
"priority": false "remote_org_id": false
}, },
"shadow_attributes": { "shadow_attributes": {
"id": true,
"event_id": false,
"event_uuid": false,
"event_org_id": false,
"uuid": false,
"old_id": false,
"value1": false,
"value2": false,
"type": false,
"category": false, "category": false,
"event_id": false,
"event_org_id": false,
"event_uuid": false,
"first_seen": false, "first_seen": false,
"last_seen": false "id": true,
"last_seen": false,
"old_id": false,
"type": false,
"uuid": false,
"value1": false,
"value2": false
}, },
"shadow_attribute_correlations": { "shadow_attribute_correlations": {
"id": true, "1_event_id": false,
"org_id": false, "1_shadow_attribute_id": false,
"attribute_id": false, "attribute_id": false,
"a_sharing_group_id": false, "a_sharing_group_id": false,
"event_id": false, "event_id": false,
"1_event_id": false, "id": true,
"sharing_group_id": false, "org_id": false,
"1_shadow_attribute_id": false "sharing_group_id": false
}, },
"sharing_groups": { "sharing_groups": {
"id": true, "id": true,
"uuid": true, "name": true,
"organisation_uuid": false,
"org_id": false, "org_id": false,
"sync_user_id": false, "sync_user_id": false,
"organisation_uuid": false "uuid": true
}, },
"sharing_group_blueprints": { "sharing_group_blueprints": {
"id": true, "id": true,
"uuid": false,
"name": false, "name": false,
"org_id": false, "org_id": false,
"sharing_group_id": false "sharing_group_id": false,
"uuid": false
}, },
"sharing_group_orgs": { "sharing_group_orgs": {
"id": true, "id": true,
@ -8981,38 +8984,41 @@
"sharing_group_id": false "sharing_group_id": false
}, },
"sightingdbs": { "sightingdbs": {
"host": false,
"id": true, "id": true,
"name": false, "name": false,
"owner": false, "owner": false,
"host": false,
"port": false "port": false
}, },
"sightingdb_orgs": { "sightingdb_orgs": {
"id": true, "id": true,
"sightingdb_id": false, "org_id": false,
"org_id": false "sightingdb_id": false
}, },
"sightings": { "sightings": {
"id": true,
"attribute_id": false, "attribute_id": false,
"event_id": false, "event_id": false,
"id": true,
"org_id": false, "org_id": false,
"uuid": false,
"source": false, "source": false,
"type": false "type": false,
"uuid": true
},
"system_settings": {
"setting": true
}, },
"tags": { "tags": {
"id": true, "id": true,
"name": false, "name": true,
"numerical_value": false,
"org_id": false, "org_id": false,
"user_id": false, "user_id": false
"numerical_value": false
}, },
"tag_collections": { "tag_collections": {
"id": true, "id": true,
"uuid": false, "org_id": false,
"user_id": false, "user_id": false,
"org_id": false "uuid": false
}, },
"tag_collection_tags": { "tag_collection_tags": {
"id": true, "id": true,
@ -9027,13 +9033,13 @@
}, },
"taxonomy_entries": { "taxonomy_entries": {
"id": true, "id": true,
"taxonomy_predicate_id": false, "numerical_value": false,
"numerical_value": false "taxonomy_predicate_id": false
}, },
"taxonomy_predicates": { "taxonomy_predicates": {
"id": true, "id": true,
"taxonomy_id": false, "numerical_value": false,
"numerical_value": false "taxonomy_id": false
}, },
"templates": { "templates": {
"id": true "id": true
@ -9054,26 +9060,26 @@
"id": true "id": true
}, },
"threads": { "threads": {
"id": true,
"user_id": false,
"event_id": false, "event_id": false,
"id": true,
"org_id": false, "org_id": false,
"sharing_group_id": false "sharing_group_id": false,
"user_id": false
}, },
"threat_levels": { "threat_levels": {
"id": true "id": true
}, },
"users": { "users": {
"email": true,
"id": true, "id": true,
"sub": true,
"email": false,
"org_id": false, "org_id": false,
"server_id": false "server_id": false,
"sub": true
}, },
"user_settings": { "user_settings": {
"id": true, "id": true,
"user_id": false, "setting": false,
"setting": false "user_id": false
}, },
"warninglists": { "warninglists": {
"id": true "id": true
@ -9087,17 +9093,17 @@
}, },
"workflows": { "workflows": {
"id": true, "id": true,
"uuid": false,
"name": false, "name": false,
"timestamp": false, "timestamp": false,
"trigger_id": false "trigger_id": false,
"uuid": false
}, },
"workflow_blueprints": { "workflow_blueprints": {
"id": true, "id": true,
"uuid": false,
"name": false, "name": false,
"timestamp": false "timestamp": false,
"uuid": false
} }
}, },
"db_version": "92" "db_version": "93"
} }

53
tests/testlive_comprehensive_local.py
View File

@ -24,13 +24,15 @@ key = os.environ["AUTH"]
urllib3.disable_warnings() urllib3.disable_warnings()
def create_simple_event(): def create_simple_event() -> MISPEvent:
event_uuid = str(uuid.uuid4())
event = MISPEvent() event = MISPEvent()
event.info = 'This is a super simple test' event.uuid = event_uuid
event.info = 'This is a super simple test ({})'.format(event_uuid.split('-')[0])
event.distribution = Distribution.your_organisation_only event.distribution = Distribution.your_organisation_only
event.threat_level_id = ThreatLevel.low event.threat_level_id = ThreatLevel.low
event.analysis = Analysis.completed event.analysis = Analysis.completed
event.add_attribute('text', str(uuid.uuid4())) event.add_attribute('text', event_uuid)
return event return event
@ -512,6 +514,51 @@ class TestComprehensive(unittest.TestCase):
for event in (first, second, third, four): for event in (first, second, third, four):
check_response(self.admin_misp_connector.delete_event(event)) check_response(self.admin_misp_connector.delete_event(event))
def test_correlations(self):
first = create_simple_event()
first.add_attribute("ip-src", "10.0.0.1")
first = check_response(self.admin_misp_connector.add_event(first))
second = create_simple_event()
second.add_attribute("ip-src", "10.0.0.1")
second = check_response(self.admin_misp_connector.add_event(second))
# Reload to get event data with related events
first = check_response(self.admin_misp_connector.get_event(first))
try:
self.assertEqual(1, len(first.RelatedEvent), first.RelatedEvent)
self.assertEqual(1, len(second.RelatedEvent), second.RelatedEvent)
except:
raise
finally:
# Delete events
for event in (first, second):
check_response(self.admin_misp_connector.delete_event(event))
def test_advanced_correlations(self):
with MISPSetting(self.admin_misp_connector, {"MISP.enable_advanced_correlations": True}):
first = create_simple_event()
first.add_attribute("ip-src", "10.0.0.0/8")
first = check_response(self.admin_misp_connector.add_event(first))
second = create_simple_event()
second.add_attribute("ip-src", "10.0.0.1")
second = check_response(self.admin_misp_connector.add_event(second))
# Reload to get event data with related events
first = check_response(self.admin_misp_connector.get_event(first))
try:
self.assertEqual(1, len(first.RelatedEvent), first.RelatedEvent)
self.assertEqual(1, len(second.RelatedEvent), second.RelatedEvent)
except:
raise
finally:
# Delete events
for event in (first, second):
check_response(self.admin_misp_connector.delete_event(event))
def test_remove_orphaned_correlations(self): def test_remove_orphaned_correlations(self):
result = self.admin_misp_connector._check_json_response(self.admin_misp_connector._prepare_request('GET', 'servers/removeOrphanedCorrelations')) result = self.admin_misp_connector._check_json_response(self.admin_misp_connector._prepare_request('GET', 'servers/removeOrphanedCorrelations'))
check_response(result) check_response(result)