From b53d8c828dc911eac2849d0493c39ed2085120f2 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 8 Aug 2022 17:02:52 +0200 Subject: [PATCH 1/7] chg: [internal] Cleanup code for new correlation engine --- .../Behavior/DefaultCorrelationBehavior.php | 115 ++++++++++-------- app/Model/Correlation.php | 1 - app/Model/CorrelationValue.php | 18 ++- 3 files changed, 75 insertions(+), 59 deletions(-) diff --git a/app/Model/Behavior/DefaultCorrelationBehavior.php b/app/Model/Behavior/DefaultCorrelationBehavior.php index d1ae2b0f0..b6f1a62b6 100644 --- a/app/Model/Behavior/DefaultCorrelationBehavior.php +++ b/app/Model/Behavior/DefaultCorrelationBehavior.php @@ -1,17 +1,14 @@ [ 'fields' => [ 'Attribute.event_id', @@ -44,26 +41,29 @@ class DefaultCorrelationBehavior extends ModelBehavior ] ]; - public $Correlation = null; + /** @var Correlation */ + public $Correlation; private $deadlockAvoidance = false; - public function setup(Model $Model, $settings = []) { - $Model->useTable = $this->__tableName; + public function setup(Model $Model, $settings = []) + { + $Model->useTable = self::TABLE_NAME; $this->Correlation = $Model; $this->deadlockAvoidance = $settings['deadlockAvoidance']; } public function getTableName(Model $Model) { - return $this->__tableName; + return self::TABLE_NAME; } - public function createCorrelationEntry(Model $Model, $value, $a, $b) { - $value_id = $this->Correlation->CorrelationValue->getValueId($value); + public function createCorrelationEntry(Model $Model, $value, $a, $b) + { + $valueId = $this->Correlation->CorrelationValue->getValueId($value); if ($this->deadlockAvoidance) { return [ - 'value_id' => $value_id, + 'value_id' => $valueId, '1_event_id' => $a['Event']['id'], '1_object_id' => $a['Attribute']['object_id'], '1_attribute_id' => $a['Attribute']['id'], @@ -87,7 +87,7 @@ class DefaultCorrelationBehavior extends ModelBehavior ]; } else { return [ - (int) $value_id, + (int) $valueId, (int) $a['Event']['id'], (int) $a['Attribute']['object_id'], (int) $a['Attribute']['id'], @@ -182,20 +182,28 @@ class DefaultCorrelationBehavior extends ModelBehavior public function getContainRules(Model $Model, $filter = null) { if (empty($filter)) { - return $this->__config['AttributeFetcher']['contain']; + return self::CONFIG['AttributeFetcher']['contain']; } else { - return empty($this->__config['AttributeFetcher']['contain'][$filter]) ? false : $this->__config['AttributeFetcher']['contain'][$filter]; + return empty(self::CONFIG['AttributeFetcher']['contain'][$filter]) ? false : self::CONFIG['AttributeFetcher']['contain'][$filter]; } } public function getFieldRules(Model $Model) { - return $this->__config['AttributeFetcher']['fields']; + return self::CONFIG['AttributeFetcher']['fields']; } - private function __collectCorrelations($user, $id, $sgids, $primary) + /** + * Fetch correalations for given event. + * @param array $user + * @param int $eventId + * @param array $sgids + * @param bool $primary + * @return array + */ + private function __collectCorrelations(array $user, $eventId, $sgids, $primary) { - $max_correlations = Configure::read('MISP.max_correlations_per_event') ?: 5000; + $maxCorrelations = Configure::read('MISP.max_correlations_per_event') ?: 5000; $source = $primary ? '' : '1_'; $prefix = $primary ? '1_' : ''; $correlations = $this->Correlation->find('all', array( @@ -214,7 +222,7 @@ class DefaultCorrelationBehavior extends ModelBehavior ], 'conditions' => [ 'OR' => [ - $source . 'event_id' => $id + $source . 'event_id' => $eventId ], 'AND' => [ [ @@ -235,46 +243,50 @@ class DefaultCorrelationBehavior extends ModelBehavior ] ], 'order' => false, - 'limit' => $max_correlations + 'limit' => $maxCorrelations )); - foreach ($correlations as $k => &$correlation) { + foreach ($correlations as $k => $correlation) { if (!$this->checkCorrelationACL($user, $correlation['Correlation'], $sgids, $prefix)) { unset($correlations[$k]); } } - $correlations = array_values($correlations); return $correlations; } + /** + * @param Model $Model + * @param array $user + * @param int $id Event ID + * @param array $sgids + * @return array + */ public function runGetAttributesRelatedToEvent(Model $Model, $user, $id, $sgids) { - $temp_correlations = $this->__collectCorrelations($user, $id, $sgids, false); - $temp_correlations_1 = $this->__collectCorrelations($user, $id, $sgids, true); $correlations = []; - $event_ids = []; - foreach ($temp_correlations as $temp_correlation) { + $eventIds = []; + foreach ($this->__collectCorrelations($user, $id, $sgids, false) as $correlation) { $correlations[] = [ - 'id' => $temp_correlation['Correlation']['event_id'], - 'attribute_id' => $temp_correlation['Correlation']['attribute_id'], - 'parent_id' => $temp_correlation['Correlation']['1_attribute_id'], - 'value' => $temp_correlation['CorrelationValue']['value'] + 'id' => $correlation['Correlation']['event_id'], + 'attribute_id' => $correlation['Correlation']['attribute_id'], + 'parent_id' => $correlation['Correlation']['1_attribute_id'], + 'value' => $correlation['CorrelationValue']['value'] ]; - $event_ids[$temp_correlation['Correlation']['event_id']] = true; + $eventIds[$correlation['Correlation']['event_id']] = true; } - foreach ($temp_correlations_1 as $temp_correlation) { + foreach ($this->__collectCorrelations($user, $id, $sgids, true) as $correlation) { $correlations[] = [ - 'id' => $temp_correlation['Correlation']['1_event_id'], - 'attribute_id' => $temp_correlation['Correlation']['1_attribute_id'], - 'parent_id' => $temp_correlation['Correlation']['attribute_id'], - 'value' => $temp_correlation['CorrelationValue']['value'] + 'id' => $correlation['Correlation']['1_event_id'], + 'attribute_id' => $correlation['Correlation']['1_attribute_id'], + 'parent_id' => $correlation['Correlation']['attribute_id'], + 'value' => $correlation['CorrelationValue']['value'] ]; - $event_ids[$temp_correlation['Correlation']['1_event_id']] = true; + $eventIds[$correlation['Correlation']['1_event_id']] = true; } if (empty($correlations)) { return []; } $conditions = $Model->Event->createEventConditions($user); - $conditions['Event.id'] = array_keys($event_ids); + $conditions['Event.id'] = array_keys($eventIds); $events = $Model->Event->find('all', [ 'recursive' => -1, 'conditions' => $conditions, @@ -289,9 +301,9 @@ class DefaultCorrelationBehavior extends ModelBehavior continue; } $event = $events[$eventId]; - $correlation['org_id'] = $events[$eventId]['orgc_id']; - $correlation['info'] = $events[$eventId]['info']; - $correlation['date'] = $events[$eventId]['date']; + $correlation['org_id'] = $event['orgc_id']; + $correlation['info'] = $event['info']; + $correlation['date'] = $event['date']; $parentId = $correlation['parent_id']; unset($correlation['parent_id']); $relatedAttributes[$parentId][] = $correlation; @@ -389,7 +401,7 @@ class DefaultCorrelationBehavior extends ModelBehavior ]); if (!empty($includeEventData)) { $results = []; - foreach ($relatedAttributes as $k => $attribute) { + foreach ($relatedAttributes as $attribute) { $temp = $attribute['Attribute']; $temp['Event'] = $attribute['Event']; $results[] = $temp; @@ -416,7 +428,6 @@ class DefaultCorrelationBehavior extends ModelBehavior $primaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, $sgids, true); $secondaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, $sgids, false); return array_unique(array_merge($primaryEventIds,$secondaryEventIds)); - } private function __filterRelatedEvents(Model $Model, array $user, int $eventId, array $sgids, bool $primary) @@ -462,12 +473,19 @@ class DefaultCorrelationBehavior extends ModelBehavior } } - private function checkCorrelationACL($user, $correlation, $sgids, $prefix) + /** + * @param array $user + * @param array $correlation + * @param array $sgids + * @param string $prefix + * @return bool + */ + private function checkCorrelationACL(array $user, $correlation, $sgids, $prefix) { if ($user['Role']['perm_site_admin']) { return true; } - // check if user can see the event + // Check if user can see the event if (isset($correlation['Correlation'])) { $correlation = $correlation['Correlation']; } @@ -484,7 +502,7 @@ class DefaultCorrelationBehavior extends ModelBehavior return false; } - //check if the user can see the object, if we're looking at an object attribute + // Check if the user can see the object, if we're looking at an object attribute if ( $correlation[$prefix . 'object_id'] && ( @@ -499,7 +517,7 @@ class DefaultCorrelationBehavior extends ModelBehavior return false; } - //check if the user can see the attribute + // Check if the user can see the attribute if ( ( $correlation[$prefix . 'distribution'] == 0 || @@ -551,7 +569,8 @@ class DefaultCorrelationBehavior extends ModelBehavior $Model->updateAll( $side, [ - $updateFields[$k] => (int)$data['id']] + $updateFields[$k] => (int)$data['id'] + ] ); } } diff --git a/app/Model/Correlation.php b/app/Model/Correlation.php index d4f62de65..64e789ee3 100644 --- a/app/Model/Correlation.php +++ b/app/Model/Correlation.php @@ -1,6 +1,5 @@ find('first', [ 'recursive' => -1, + 'fields' => ['id'], 'conditions' => [ 'value' => $value ] @@ -31,22 +28,23 @@ class CorrelationValue extends AppModel } catch (Exception $e) { $existingValue = $this->find('first', [ 'recursive' => -1, + 'fields' => ['id'], 'conditions' => [ 'value' => $value ] ]); - return $existingValue['ExistingValue']['id']; + return $existingValue['CorrelationValue']['id']; } } else { return $existingValue['CorrelationValue']['id']; } - return false; } public function getValue($id) { $existingValue = $this->find('first', [ 'recursive' => -1, + 'fields' => ['value'], 'conditions' => [ 'id' => $id ] From ec209a98b9df721297c767f8ae2f1e5e8d78b82f Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 8 Aug 2022 17:03:25 +0200 Subject: [PATCH 2/7] chg: [internal] Drop unnecessary indexes from default_correlations table --- app/Model/AppModel.php | 19 ++++++++++++++++++- db_schema.json | 33 +++++---------------------------- 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index cfd8b1de9..018979088 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -82,7 +82,8 @@ class AppModel extends Model 69 => false, 70 => false, 71 => true, 72 => true, 73 => false, 74 => false, 75 => false, 76 => true, 77 => false, 78 => false, 79 => false, 80 => false, 81 => false, 82 => false, 83 => false, 84 => false, 85 => false, 86 => false, - 87 => false, 88 => false, 89 => false, 90 => false, 91 => false, 92 => false + 87 => false, 88 => false, 89 => false, 90 => false, 91 => false, 92 => false, + 93 => false, ); const ADVANCED_UPDATES_DESCRIPTION = array( @@ -1834,6 +1835,22 @@ class AppModel extends Model case 92: $sqlArray[] = "ALTER TABLE users ADD `last_api_access` INT(11) DEFAULT 0;"; break; + case 93: + $this->__dropIndex('default_correlations', 'distribution'); + $this->__dropIndex('default_correlations', 'object_distribution'); + $this->__dropIndex('default_correlations', 'event_distribution'); + $this->__dropIndex('default_correlations', 'sharing_group_id'); + $this->__dropIndex('default_correlations', 'object_sharing_group_id'); + $this->__dropIndex('default_correlations', 'event_sharing_group_id'); + $this->__dropIndex('default_correlations', 'org_id'); + $this->__dropIndex('default_correlations', '1_distribution'); + $this->__dropIndex('default_correlations', '1_object_distribution'); + $this->__dropIndex('default_correlations', '1_event_distribution'); + $this->__dropIndex('default_correlations', '1_sharing_group_id'); + $this->__dropIndex('default_correlations', '1_object_sharing_group_id'); + $this->__dropIndex('default_correlations', '1_event_sharing_group_id'); + $this->__dropIndex('default_correlations', '1_org_id'); + break; case 'fixNonEmptySharingGroupID': $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; diff --git a/db_schema.json b/db_schema.json index d4add988e..17d53d9bd 100644 --- a/db_schema.json +++ b/db_schema.json @@ -8690,37 +8690,14 @@ "model_id": false }, "default_correlations": { - "id": true, - "attribute_id": false, "1_attribute_id": false, - "value_id": false, - "event_id": false, - "object_id": false, - "org_id": false, - "distribution": false, - "object_distribution": false, - "event_distribution": false, - "sharing_group_id": false, - "object_sharing_group_id": false, - "event_sharing_group_id": false, "1_event_id": false, "1_object_id": false, - "1_org_id": false, - "1_distribution": false, - "1_object_distribution": false, - "1_event_distribution": false, - "1_sharing_group_id": false, - "1_object_sharing_group_id": false, - "1_event_sharing_group_id": false - }, - "events": { + "attribute_id": false, + "event_id": false, + "object_id": false, "id": true, - "uuid": true, - "info": false, - "sharing_group_id": false, - "org_id": false, - "orgc_id": false, - "extends_uuid": false + "value_id": false }, "event_blocklists": { "id": true, @@ -9099,5 +9076,5 @@ "timestamp": false } }, - "db_version": "92" + "db_version": "93" } From 986e109f76407d397b72b4ed34fc286a771c16a6 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 8 Aug 2022 18:31:00 +0200 Subject: [PATCH 3/7] fix: [internal] Advanced correlations --- app/Model/Correlation.php | 49 +++++++++++++------------ app/Model/OverCorrelatingValue.php | 29 ++++++++------- tests/testlive_comprehensive_local.py | 53 +++++++++++++++++++++++++-- 3 files changed, 91 insertions(+), 40 deletions(-) diff --git a/app/Model/Correlation.php b/app/Model/Correlation.php index 64e789ee3..5c607be44 100644 --- a/app/Model/Correlation.php +++ b/app/Model/Correlation.php @@ -3,6 +3,9 @@ App::uses('AppModel', 'Model'); /** * @property Attribute $Attribute + * @method saveCorrelations + * @method runBeforeSaveCorrelation + * @method fetchRelatedEventIds */ class Correlation extends AppModel { @@ -43,36 +46,27 @@ class Correlation extends AppModel /** @var array */ private $exclusions; - /** - * Use old schema with `date` and `info` fields. - * @var bool - */ - private $oldSchema; - - /** @var bool */ - private $deadlockAvoidance; - /** @var bool */ private $advancedCorrelationEnabled; /** @var array */ private $cidrListCache; - private $__correlationEngine = 'DefaultCorrelation'; - - protected $_config = []; + /** @var string */ + private $__correlationEngine; private $__tempContainCache = []; - public $OverCorrelatingValue = null; + /** @var OverCorrelatingValue */ + public $OverCorrelatingValue; public function __construct($id = false, $table = null, $ds = null) { parent::__construct($id, $table, $ds); $this->__correlationEngine = $this->getCorrelationModelName(); - $this->deadlockAvoidance = Configure::check('MISP.deadlock_avoidance') ? Configure::read('MISP.deadlock_avoidance') : false; + $deadlockAvoidance = Configure::check('MISP.deadlock_avoidance') ? Configure::read('MISP.deadlock_avoidance') : false; // load the currently used correlation engine - $this->Behaviors->load($this->__correlationEngine . 'Correlation', ['deadlockAvoidance' => false]); + $this->Behaviors->load($this->__correlationEngine . 'Correlation', ['deadlockAvoidance' => $deadlockAvoidance]); // getTableName() needs to be implemented by the engine - this points us to the table to be used $this->useTable = $this->getTableName(); $this->advancedCorrelationEnabled = (bool)Configure::read('MISP.enable_advanced_correlations'); @@ -289,6 +283,7 @@ class Correlation extends AppModel * @param bool $full * @param array|false $event * @return array|bool|bool[]|mixed + * @throws Exception */ public function afterSaveCorrelation($a, $full = false, $event = false) { @@ -335,7 +330,7 @@ class Correlation extends AppModel return true; } $correlations = []; - foreach ($correlatingValues as $k => $cV) { + foreach ($correlatingValues as $cV) { if ($cV === null) { continue; } @@ -346,6 +341,7 @@ class Correlation extends AppModel 'Attribute.value2' => $cV, 'NOT' => ['Attribute.type' => Attribute::PRIMARY_ONLY_CORRELATING_TYPES] ], + $extraConditions, ], 'NOT' => [ 'Attribute.event_id' => $a['Attribute']['event_id'], @@ -850,7 +846,7 @@ class Correlation extends AppModel /** * @param array $user User array - * @param int $eventIds List of event IDs + * @param int $eventId List of event IDs * @param array $sgids List of sharing group IDs * @return array */ @@ -871,27 +867,32 @@ class Correlation extends AppModel return $data; } + /** + * @param array $attribute + * @return array + */ public function setCorrelationExclusion($attribute) { - if (empty($this->__compositeTypes)) { + if (!isset($this->__compositeTypes)) { $this->__compositeTypes = $this->Attribute->getCompositeTypes(); } - $values = [$attribute['value']]; - if (in_array($attribute['type'], $this->__compositeTypes)) { + if (in_array($attribute['type'], $this->__compositeTypes, true)) { $values = explode('|', $attribute['value']); + } else { + $values = [$attribute['value']]; } if ($this->__preventExcludedCorrelations($values[0])) { $attribute['correlation_exclusion'] = true; - } - if (!empty($values[1]) && $this->__preventExcludedCorrelations($values[1])) { + } elseif (!empty($values[1]) && $this->__preventExcludedCorrelations($values[1])) { $attribute['correlation_exclusion'] = true; } + if ($this->OverCorrelatingValue->checkValue($values[0])) { $attribute['over_correlation'] = true; - } - if (!empty($values[1]) && $this->OverCorrelatingValue->checkValue($values[1])) { + } elseif (!empty($values[1]) && $this->OverCorrelatingValue->checkValue($values[1])) { $attribute['over_correlation'] = true; } + return $attribute; } diff --git a/app/Model/OverCorrelatingValue.php b/app/Model/OverCorrelatingValue.php index c3af295ab..066076a7b 100644 --- a/app/Model/OverCorrelatingValue.php +++ b/app/Model/OverCorrelatingValue.php @@ -1,6 +1,5 @@ unblock($value); @@ -25,15 +28,23 @@ class OverCorrelatingValue extends AppModel ); } + /** + * @param string $value + * @return void + */ public function unBlock($value) { $this->deleteAll( [ 'OverCorrelatingValue.value' => $value - ] + ], + false ); } + /** + * @return int + */ public function getLimit() { return Configure::check('MISP.correlation_limit') ? Configure::read('MISP.correlation_limit') : 20; @@ -55,15 +66,7 @@ class OverCorrelatingValue extends AppModel public function checkValue($value) { - $hit = $this->find('first', [ - 'recursive' => -1, - 'conditions' => ['value' => $value], - 'fields' => ['id'] - ]); - if (empty($hit)) { - return false; - } - return true; + return $this->hasAny(['value' => $value]); } public function generateOccurrencesRouter() diff --git a/tests/testlive_comprehensive_local.py b/tests/testlive_comprehensive_local.py index 3163ebf35..e1b6359a2 100644 --- a/tests/testlive_comprehensive_local.py +++ b/tests/testlive_comprehensive_local.py @@ -24,13 +24,15 @@ key = os.environ["AUTH"] urllib3.disable_warnings() -def create_simple_event(): +def create_simple_event() -> MISPEvent: + event_uuid = str(uuid.uuid4()) event = MISPEvent() - event.info = 'This is a super simple test' + event.uuid = event_uuid + event.info = 'This is a super simple test ({})'.format(event_uuid.split('-')[0]) event.distribution = Distribution.your_organisation_only event.threat_level_id = ThreatLevel.low event.analysis = Analysis.completed - event.add_attribute('text', str(uuid.uuid4())) + event.add_attribute('text', event_uuid) return event @@ -512,6 +514,51 @@ class TestComprehensive(unittest.TestCase): for event in (first, second, third, four): check_response(self.admin_misp_connector.delete_event(event)) + def test_correlations(self): + first = create_simple_event() + first.add_attribute("ip-src", "10.0.0.1") + first = check_response(self.admin_misp_connector.add_event(first)) + + second = create_simple_event() + second.add_attribute("ip-src", "10.0.0.1") + second = check_response(self.admin_misp_connector.add_event(second)) + + # Reload to get event data with related events + first = check_response(self.admin_misp_connector.get_event(first)) + + try: + self.assertEqual(1, len(first.RelatedEvent), first.RelatedEvent) + self.assertEqual(1, len(second.RelatedEvent), second.RelatedEvent) + except: + raise + finally: + # Delete events + for event in (first, second): + check_response(self.admin_misp_connector.delete_event(event)) + + def test_advanced_correlations(self): + with MISPSetting(self.admin_misp_connector, {"MISP.enable_advanced_correlations": True}): + first = create_simple_event() + first.add_attribute("ip-src", "10.0.0.0/8") + first = check_response(self.admin_misp_connector.add_event(first)) + + second = create_simple_event() + second.add_attribute("ip-src", "10.0.0.1") + second = check_response(self.admin_misp_connector.add_event(second)) + + # Reload to get event data with related events + first = check_response(self.admin_misp_connector.get_event(first)) + + try: + self.assertEqual(1, len(first.RelatedEvent), first.RelatedEvent) + self.assertEqual(1, len(second.RelatedEvent), second.RelatedEvent) + except: + raise + finally: + # Delete events + for event in (first, second): + check_response(self.admin_misp_connector.delete_event(event)) + def test_remove_orphaned_correlations(self): result = self.admin_misp_connector._check_json_response(self.admin_misp_connector._prepare_request('GET', 'servers/removeOrphanedCorrelations')) check_response(result) From 14501e8a78763e11de0fcedea8b78535eeb255d0 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Tue, 9 Aug 2022 12:16:44 +0200 Subject: [PATCH 4/7] chg: [internal] Use less SQL queries for event fetching --- .../Behavior/DefaultCorrelationBehavior.php | 3 +- app/Model/Correlation.php | 56 +++++++++++++------ app/Model/Event.php | 3 +- 3 files changed, 43 insertions(+), 19 deletions(-) diff --git a/app/Model/Behavior/DefaultCorrelationBehavior.php b/app/Model/Behavior/DefaultCorrelationBehavior.php index b6f1a62b6..d090b930f 100644 --- a/app/Model/Behavior/DefaultCorrelationBehavior.php +++ b/app/Model/Behavior/DefaultCorrelationBehavior.php @@ -423,7 +423,7 @@ class DefaultCorrelationBehavior extends ModelBehavior // ii. Event has a sharing group that the user is accessible to view // b. Attribute: // i. Attribute has a distribution of 5 (inheritance of the event, for this the event check has to pass anyway) - // ii. Atttibute has a distribution between 1-3 (community only, connected communities, all orgs) + // ii. Attribute has a distribution between 1-3 (community only, connected communities, all orgs) // iii. Attribute has a sharing group that the user is accessible to view $primaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, $sgids, true); $secondaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, $sgids, false); @@ -450,7 +450,6 @@ class DefaultCorrelationBehavior extends ModelBehavior 'conditions' => [ $current . 'event_id' => $eventId ], - 'unique' => true, ]); $eventIds = []; if (empty($user['Role']['perm_site_admin'])) { diff --git a/app/Model/Correlation.php b/app/Model/Correlation.php index 5c607be44..0e21d1826 100644 --- a/app/Model/Correlation.php +++ b/app/Model/Correlation.php @@ -6,6 +6,8 @@ App::uses('AppModel', 'Model'); * @method saveCorrelations * @method runBeforeSaveCorrelation * @method fetchRelatedEventIds + * @method getFieldRules + * @method getContainRules */ class Correlation extends AppModel { @@ -868,32 +870,54 @@ class Correlation extends AppModel } /** - * @param array $attribute + * @param array $attributes * @return array */ - public function setCorrelationExclusion($attribute) + public function attachCorrelationExclusion(array $attributes) { if (!isset($this->__compositeTypes)) { $this->__compositeTypes = $this->Attribute->getCompositeTypes(); } - if (in_array($attribute['type'], $this->__compositeTypes, true)) { - $values = explode('|', $attribute['value']); - } else { - $values = [$attribute['value']]; - } - if ($this->__preventExcludedCorrelations($values[0])) { - $attribute['correlation_exclusion'] = true; - } elseif (!empty($values[1]) && $this->__preventExcludedCorrelations($values[1])) { - $attribute['correlation_exclusion'] = true; + + $valuesToCheck = []; + foreach ($attributes as &$attribute) { + if (in_array($attribute['type'], $this->__compositeTypes, true)) { + $values = explode('|', $attribute['value']); + $valuesToCheck[$values[0]] = true; + $valuesToCheck[$values[1]] = true; + } else { + $values = [$attribute['value']]; + $valuesToCheck[$values[0]] = true; + } + + if ($this->__preventExcludedCorrelations($values[0])) { + $attribute['correlation_exclusion'] = true; + } elseif (!empty($values[1]) && $this->__preventExcludedCorrelations($values[1])) { + $attribute['correlation_exclusion'] = true; + } } - if ($this->OverCorrelatingValue->checkValue($values[0])) { - $attribute['over_correlation'] = true; - } elseif (!empty($values[1]) && $this->OverCorrelatingValue->checkValue($values[1])) { - $attribute['over_correlation'] = true; + $overCorrelatingValues = array_flip($this->OverCorrelatingValue->find('column', [ + 'conditions' => ['value' => array_keys($valuesToCheck)], + 'fields' => ['value'], + ])); + unset($valuesToCheck); + + foreach ($attributes as &$attribute) { + if (in_array($attribute['type'], $this->__compositeTypes, true)) { + $values = explode('|', $attribute['value']); + } else { + $values = [$attribute['value']]; + } + + if (isset($overCorrelatingValues[$values[0]])) { + $attribute['over_correlation'] = true; + } elseif (!empty($values[1]) && isset($overCorrelatingValues[$values[1]])) { + $attribute['over_correlation'] = true; + } } - return $attribute; + return $attributes; } public function collectMetrics() diff --git a/app/Model/Event.php b/app/Model/Event.php index 1ca0a61be..b8cbf75c3 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -2018,6 +2018,8 @@ class Event extends AppModel $event['Attribute'] = $this->__attachSharingGroups($event['Attribute'], $sharingGroupData); } + $event['Attribute'] = $this->Attribute->Correlation->attachCorrelationExclusion($event['Attribute']); + // move all object attributes to a temporary container $tempObjectAttributeContainer = array(); foreach ($event['Attribute'] as $key => &$attribute) { @@ -2025,7 +2027,6 @@ class Event extends AppModel unset($event['Attribute'][$key]); continue; } - $attribute = $this->Attribute->Correlation->setCorrelationExclusion($attribute); if ($attribute['category'] === 'Financial fraud') { $attribute = $this->Attribute->attachValidationWarnings($attribute); } From 48c8a7eab3ce17acbb2f29783f9ba93887574ec2 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Tue, 9 Aug 2022 15:20:53 +0200 Subject: [PATCH 5/7] chg: [internal] Code cleanup --- .../Behavior/DefaultCorrelationBehavior.php | 83 +++++++++++-------- app/Model/Correlation.php | 8 +- app/Model/Event.php | 2 +- 3 files changed, 54 insertions(+), 39 deletions(-) diff --git a/app/Model/Behavior/DefaultCorrelationBehavior.php b/app/Model/Behavior/DefaultCorrelationBehavior.php index d090b930f..be0e83307 100644 --- a/app/Model/Behavior/DefaultCorrelationBehavior.php +++ b/app/Model/Behavior/DefaultCorrelationBehavior.php @@ -194,7 +194,7 @@ class DefaultCorrelationBehavior extends ModelBehavior } /** - * Fetch correalations for given event. + * Fetch correlations for given event. * @param array $user * @param int $eventId * @param array $sgids @@ -254,7 +254,7 @@ class DefaultCorrelationBehavior extends ModelBehavior } /** - * @param Model $Model + * @param Correlation $Model * @param array $user * @param int $id Event ID * @param array $sgids @@ -427,49 +427,62 @@ class DefaultCorrelationBehavior extends ModelBehavior // iii. Attribute has a sharing group that the user is accessible to view $primaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, $sgids, true); $secondaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, $sgids, false); - return array_unique(array_merge($primaryEventIds,$secondaryEventIds)); + return array_unique(array_merge($primaryEventIds,$secondaryEventIds), SORT_REGULAR); } + /** + * @param Model $Model + * @param array $user + * @param int $eventId + * @param array $sgids + * @param bool $primary + * @return array|int[] + */ private function __filterRelatedEvents(Model $Model, array $user, int $eventId, array $sgids, bool $primary) { $current = $primary ? '' : '1_'; $prefix = $primary ? '1_' : ''; - $correlations = $Model->find('all', [ - 'recursive' => -1, - 'fields' => [ - $prefix . 'org_id', - $prefix . 'event_id', - $prefix . 'event_distribution', - $prefix . 'event_sharing_group_id', - $prefix . 'object_id', - $prefix . 'object_distribution', - $prefix . 'object_sharing_group_id', - $prefix . 'distribution', - $prefix . 'sharing_group_id' - ], + + if (empty($user['Role']['perm_site_admin'])) { + $correlations = $Model->find('all', [ + 'recursive' => -1, + 'fields' => [ + $prefix . 'org_id', + $prefix . 'event_id', + $prefix . 'event_distribution', + $prefix . 'event_sharing_group_id', + $prefix . 'object_id', + $prefix . 'object_distribution', + $prefix . 'object_sharing_group_id', + $prefix . 'distribution', + $prefix . 'sharing_group_id' + ], + 'conditions' => [ + $current . 'event_id' => $eventId + ], + ]); + + $eventIds = []; + foreach ($correlations as $correlation) { + $correlation = $correlation['Correlation']; + // if we have already added this event as a valid target, no need to check again. + if (isset($eventIds[$correlation[$prefix . 'event_id']])) { + continue; + } + if ($this->checkCorrelationACL($user, $correlation, $sgids, $prefix)) { + $eventIds[$correlation[$prefix . 'event_id']] = true; + } + } + return array_keys($eventIds); + } + + return $Model->find('column', [ + 'fields' => [$prefix . 'event_id'], 'conditions' => [ $current . 'event_id' => $eventId ], + 'unique' => true, ]); - $eventIds = []; - if (empty($user['Role']['perm_site_admin'])) { - foreach ($correlations as $k => $correlation) { - // if we have already added this event as a valid target, no need to check again. - if (isset($eventIds[$correlation['Correlation'][$prefix . 'event_id']])) { - continue; - } - $correlation = $correlation['Correlation']; - if (!$this->checkCorrelationACL($user, $correlation, $sgids, $prefix)) { - unset($correlations[$k]); - continue; - } - $eventIds[$correlation[$prefix . 'event_id']] = true; - } - return array_keys($eventIds); - } else { - $eventIds = Hash::extract($correlations, '{n}.Correlation.' . $prefix . 'event_id'); - return $eventIds; - } } /** diff --git a/app/Model/Correlation.php b/app/Model/Correlation.php index 0e21d1826..29071b16e 100644 --- a/app/Model/Correlation.php +++ b/app/Model/Correlation.php @@ -3,11 +3,13 @@ App::uses('AppModel', 'Model'); /** * @property Attribute $Attribute - * @method saveCorrelations + * @property Event $Event + * @property CorrelationValue $CorrelationValue + * @method saveCorrelations(array $correlations) * @method runBeforeSaveCorrelation - * @method fetchRelatedEventIds + * @method fetchRelatedEventIds(array $user, int $eventId, array $sgids) * @method getFieldRules - * @method getContainRules + * @method getContainRules($filter = null) */ class Correlation extends AppModel { diff --git a/app/Model/Event.php b/app/Model/Event.php index b8cbf75c3..e139a4a60 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -608,7 +608,7 @@ class Event extends AppModel return $events; } - public function getRelatedEventCount($user, $eventId, $sgids) + public function getRelatedEventCount(array $user, $eventId, $sgids) { if (!isset($sgids) || empty($sgids)) { $sgids = array(-1); From 47c716e207dc6bea904a55fa7f4afb9390fd3c0f Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Thu, 11 Aug 2022 12:58:54 +0200 Subject: [PATCH 6/7] chg: [internal] Order columns by name when generating db_schema.json --- app/Model/Server.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Model/Server.php b/app/Model/Server.php index 136bdef94..eebb058ad 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -3213,7 +3213,7 @@ class Server extends AppModel private function getDatabaseIndexes($database, $table) { $sqlTableIndex = sprintf( - "SELECT DISTINCT TABLE_NAME, COLUMN_NAME, NON_UNIQUE FROM information_schema.statistics WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s';", + "SELECT DISTINCT TABLE_NAME, COLUMN_NAME, NON_UNIQUE FROM information_schema.statistics WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s' ORDER BY COLUMN_NAME;", $database, $table ); From ec82933fdff6a404ab2f3a28cf87835c8e78c287 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Thu, 11 Aug 2022 13:11:09 +0200 Subject: [PATCH 7/7] fix: [db_schema]: Order column names --- db_schema.json | 449 ++++++++++++++++++++++++++----------------------- 1 file changed, 239 insertions(+), 210 deletions(-) diff --git a/db_schema.json b/db_schema.json index 17d53d9bd..10371914d 100644 --- a/db_schema.json +++ b/db_schema.json @@ -1010,28 +1010,6 @@ "column_type": "int(11)", "column_default": null, "extra": "" - }, - { - "column_name": "date", - "is_nullable": "NO", - "data_type": "date", - "character_maximum_length": null, - "numeric_precision": null, - "collation_name": null, - "column_type": "date", - "column_default": null, - "extra": "" - }, - { - "column_name": "info", - "is_nullable": "NO", - "data_type": "text", - "character_maximum_length": "65535", - "numeric_precision": null, - "collation_name": "utf8mb3_bin", - "column_type": "text", - "column_default": null, - "extra": "" } ], "correlation_exclusions": [ @@ -1094,13 +1072,13 @@ }, { "column_name": "value", - "is_nullable": "YES", - "data_type": "text", - "character_maximum_length": "65535", + "is_nullable": "NO", + "data_type": "varchar", + "character_maximum_length": "191", "numeric_precision": null, "collation_name": "utf8mb4_unicode_ci", - "column_type": "text", - "column_default": "NULL", + "column_type": "varchar(191)", + "column_default": null, "extra": "" } ], @@ -2897,6 +2875,17 @@ "column_default": null, "extra": "auto_increment" }, + { + "column_name": "uuid", + "is_nullable": "NO", + "data_type": "varchar", + "character_maximum_length": "255", + "numeric_precision": null, + "collation_name": "utf8mb3_bin", + "column_type": "varchar(255)", + "column_default": "''", + "extra": "" + }, { "column_name": "collection_uuid", "is_nullable": "NO", @@ -2996,17 +2985,6 @@ "column_default": "0", "extra": "" }, - { - "column_name": "uuid", - "is_nullable": "NO", - "data_type": "varchar", - "character_maximum_length": "255", - "numeric_precision": null, - "collation_name": "utf8mb3_bin", - "column_type": "varchar(255)", - "column_default": "''", - "extra": "" - }, { "column_name": "distribution", "is_nullable": "NO", @@ -4179,13 +4157,13 @@ }, { "column_name": "comment", - "is_nullable": "YES", + "is_nullable": "NO", "data_type": "text", "character_maximum_length": "65535", "numeric_precision": null, "collation_name": "utf8mb3_unicode_ci", "column_type": "text", - "column_default": "NULL", + "column_default": null, "extra": "" }, { @@ -4862,13 +4840,13 @@ }, { "column_name": "comment", - "is_nullable": "NO", + "is_nullable": "YES", "data_type": "text", "character_maximum_length": "65535", "numeric_precision": null, "collation_name": "utf8mb3_unicode_ci", "column_type": "text", - "column_default": null, + "column_default": "NULL", "extra": "" } ], @@ -5080,11 +5058,11 @@ { "column_name": "body", "is_nullable": "YES", - "data_type": "longtext", - "character_maximum_length": "4294967295", + "data_type": "text", + "character_maximum_length": "65535", "numeric_precision": null, "collation_name": "utf8mb3_general_ci", - "column_type": "longtext", + "column_type": "text", "column_default": "NULL", "extra": "" }, @@ -5497,6 +5475,17 @@ "column_default": "0", "extra": "" }, + { + "column_name": "perm_decaying", + "is_nullable": "NO", + "data_type": "tinyint", + "character_maximum_length": null, + "numeric_precision": "3", + "collation_name": null, + "column_type": "tinyint(1)", + "column_default": "0", + "extra": "" + }, { "column_name": "enforce_rate_limit", "is_nullable": "NO", @@ -5519,17 +5508,6 @@ "column_default": "0", "extra": "" }, - { - "column_name": "perm_decaying", - "is_nullable": "NO", - "data_type": "tinyint", - "character_maximum_length": null, - "numeric_precision": "3", - "collation_name": null, - "column_type": "tinyint(1)", - "column_default": "0", - "extra": "" - }, { "column_name": "perm_galaxy_editor", "is_nullable": "NO", @@ -5965,13 +5943,13 @@ }, { "column_name": "comment", - "is_nullable": "YES", + "is_nullable": "NO", "data_type": "text", "character_maximum_length": "65535", "numeric_precision": null, "collation_name": "utf8mb3_unicode_ci", "column_type": "text", - "column_default": "NULL", + "column_default": null, "extra": "" }, { @@ -6761,6 +6739,30 @@ "extra": "" } ], + "system_settings": [ + { + "column_name": "setting", + "is_nullable": "NO", + "data_type": "varchar", + "character_maximum_length": "255", + "numeric_precision": null, + "collation_name": "utf8mb4_unicode_ci", + "column_type": "varchar(255)", + "column_default": null, + "extra": "" + }, + { + "column_name": "value", + "is_nullable": "NO", + "data_type": "blob", + "character_maximum_length": "65535", + "numeric_precision": null, + "collation_name": null, + "column_type": "blob", + "column_default": null, + "extra": "" + } + ], "tags": [ { "column_name": "id", @@ -8166,6 +8168,17 @@ "column_type": "text", "column_default": "NULL", "extra": "" + }, + { + "column_name": "last_api_access", + "is_nullable": "YES", + "data_type": "int", + "character_maximum_length": null, + "numeric_precision": "10", + "collation_name": null, + "column_type": "int(11)", + "column_default": "0", + "extra": "" } ], "user_settings": [ @@ -8590,99 +8603,99 @@ }, "indexes": { "admin_settings": { - "id": true + "id": true, + "setting": true }, "allowedlist": { "id": true }, "attachment_scans": { + "attribute_id": false, "id": true, - "type": false, - "attribute_id": false + "type": false }, "attributes": { - "id": true, - "uuid": false, - "event_id": false, - "sharing_group_id": false, - "type": false, "category": false, - "value1": false, - "value2": false, + "event_id": false, + "first_seen": false, + "id": true, + "last_seen": false, "object_id": false, "object_relation": false, - "deleted": false, - "first_seen": false, - "last_seen": false, - "timestamp": false + "sharing_group_id": false, + "timestamp": false, + "type": false, + "uuid": true, + "value1": false, + "value2": false }, "attribute_tags": { - "id": true, "attribute_id": false, "event_id": false, + "id": true, "tag_id": false }, "audit_logs": { - "id": true, "event_id": false, + "id": true, "model_id": false }, "auth_keys": { - "id": true, - "authkey_start": false, "authkey_end": false, + "authkey_start": false, "created": false, "expiration": false, + "id": true, "user_id": false }, "bruteforces": [], "cake_sessions": { - "id": true, - "expires": false + "expires": false, + "id": true }, "cerebrates": { "id": true, - "url": false, - "org_id": false + "org_id": false, + "url": false }, "correlations": { - "id": true, - "event_id": false, + "1_attribute_id": false, "1_event_id": false, "attribute_id": false, - "1_attribute_id": false + "event_id": false, + "id": true }, "correlation_exclusions": { "id": true, - "value": false + "value": true }, "correlation_values": { "id": true, "value": true }, "cryptographic_keys": { + "fingerprint": false, "id": true, - "uuid": false, - "type": false, "parent_id": false, "parent_type": false, - "fingerprint": false + "type": false, + "uuid": false }, "dashboards": { "id": true, "name": false, - "uuid": false, - "user_id": false, "restrict_to_org_id": false, - "restrict_to_permission_flag": false + "restrict_to_permission_flag": false, + "user_id": false, + "uuid": false }, "decaying_models": { + "all_orgs": false, + "enabled": false, "id": true, - "uuid": false, "name": false, "org_id": false, - "enabled": false, - "all_orgs": false, + "uuid": false, "version": false }, "decaying_model_mappings": { @@ -8695,48 +8708,57 @@ "1_object_id": false, "attribute_id": false, "event_id": false, - "object_id": false, "id": true, + "object_id": false, "value_id": false }, - "event_blocklists": { + "events": { + "extends_uuid": false, "id": true, - "event_uuid": false, - "event_orgc": false + "info": false, + "orgc_id": false, + "org_id": false, + "sharing_group_id": false, + "uuid": true + }, + "event_blocklists": { + "event_orgc": false, + "event_uuid": true, + "id": true }, "event_delegations": { + "event_id": false, "id": true, - "org_id": false, - "event_id": false + "org_id": false }, "event_graph": { - "id": true, "event_id": false, - "user_id": false, + "id": true, "org_id": false, - "timestamp": false + "timestamp": false, + "user_id": false }, "event_locks": { - "id": true, "event_id": false, - "user_id": false, - "timestamp": false + "id": true, + "timestamp": false, + "user_id": false }, "event_reports": { + "event_id": false, "id": true, - "uuid": true, "name": false, - "event_id": false + "uuid": true }, "event_tags": { - "id": true, "event_id": false, + "id": true, "tag_id": false }, "favourite_tags": { "id": true, - "user_id": false, - "tag_id": false + "tag_id": false, + "user_id": false }, "feeds": { "id": true, @@ -8744,66 +8766,66 @@ "orgc_id": false }, "fuzzy_correlate_ssdeep": { - "id": true, + "attribute_id": false, "chunk": false, - "attribute_id": false + "id": true }, "galaxies": { "id": true, "name": false, - "uuid": false, + "namespace": false, "type": false, - "namespace": false + "uuid": true }, "galaxy_clusters": { - "id": true, - "value": false, + "collection_uuid": false, + "default": false, + "extends_uuid": false, + "extends_version": false, "galaxy_id": false, - "version": false, + "id": true, + "orgc_id": false, + "org_id": false, + "sharing_group_id": false, "tag_name": false, "type": false, "uuid": false, - "collection_uuid": false, - "org_id": false, - "orgc_id": false, - "sharing_group_id": false, - "extends_uuid": false, - "extends_version": false, - "default": false + "value": false, + "version": false }, "galaxy_cluster_blocklists": { - "id": true, + "cluster_orgc": false, "cluster_uuid": false, - "cluster_orgc": false + "id": true }, "galaxy_cluster_relations": { - "id": true, + "default": false, "galaxy_cluster_id": false, + "galaxy_cluster_uuid": false, + "id": true, "referenced_galaxy_cluster_id": false, "referenced_galaxy_cluster_type": false, - "galaxy_cluster_uuid": false, - "sharing_group_id": false, - "default": false + "sharing_group_id": false }, "galaxy_cluster_relation_tags": { - "id": true, "galaxy_cluster_relation_id": false, + "id": true, "tag_id": false }, "galaxy_elements": { + "galaxy_cluster_id": false, "id": true, "key": false, - "value": false, - "galaxy_cluster_id": false + "value": false }, "inbox": { "id": true, + "ip": false, + "timestamp": false, "title": false, "type": false, - "uuid": false, "user_agent_sha256": false, - "ip": false, - "timestamp": false + "uuid": false }, "jobs": { "id": true @@ -8815,9 +8837,9 @@ "id": true }, "noticelists": { + "geographical_area": false, "id": true, - "name": false, - "geographical_area": false + "name": false }, "noticelist_entries": { "id": true, @@ -8829,32 +8851,33 @@ "type": false }, "no_acl_correlations": { - "id": true, - "attribute_id": false, "1_attribute_id": false, - "value_id": false, + "1_event_id": false, + "attribute_id": false, "event_id": false, - "1_event_id": false + "id": true, + "value_id": false }, "objects": { + "distribution": false, + "event_id": false, + "first_seen": false, "id": true, + "last_seen": false, + "meta-category": false, "name": false, + "sharing_group_id": false, "template_uuid": false, "template_version": false, - "meta-category": false, - "event_id": false, - "uuid": false, "timestamp": false, - "distribution": false, - "sharing_group_id": false, - "first_seen": false, - "last_seen": false + "uuid": true }, "object_references": { + "event_id": false, "id": true, "object_id": false, "referenced_id": false, - "event_id": false + "uuid": true }, "object_relationships": { "id": true, @@ -8862,11 +8885,11 @@ }, "object_templates": { "id": true, - "user_id": false, - "org_id": false, - "uuid": false, + "meta-category": false, "name": false, - "meta-category": false + "org_id": false, + "user_id": false, + "uuid": false }, "object_template_elements": { "id": true, @@ -8875,16 +8898,18 @@ }, "organisations": { "id": true, - "uuid": false, - "name": false + "name": true, + "uuid": true }, "org_blocklists": { - "id": true + "id": true, + "org_name": false, + "org_uuid": true }, "over_correlating_values": { "id": true, - "value": true, - "occurrence": false + "occurrence": false, + "value": true }, "posts": { "id": true, @@ -8897,8 +8922,8 @@ "rest_client_histories": { "id": true, "org_id": false, - "user_id": false, - "timestamp": false + "timestamp": false, + "user_id": false }, "roles": { "id": true @@ -8906,46 +8931,47 @@ "servers": { "id": true, "org_id": false, - "remote_org_id": false, - "priority": false + "priority": false, + "remote_org_id": false }, "shadow_attributes": { - "id": true, - "event_id": false, - "event_uuid": false, - "event_org_id": false, - "uuid": false, - "old_id": false, - "value1": false, - "value2": false, - "type": false, "category": false, + "event_id": false, + "event_org_id": false, + "event_uuid": false, "first_seen": false, - "last_seen": false + "id": true, + "last_seen": false, + "old_id": false, + "type": false, + "uuid": false, + "value1": false, + "value2": false }, "shadow_attribute_correlations": { - "id": true, - "org_id": false, + "1_event_id": false, + "1_shadow_attribute_id": false, "attribute_id": false, "a_sharing_group_id": false, "event_id": false, - "1_event_id": false, - "sharing_group_id": false, - "1_shadow_attribute_id": false + "id": true, + "org_id": false, + "sharing_group_id": false }, "sharing_groups": { "id": true, - "uuid": true, + "name": true, + "organisation_uuid": false, "org_id": false, "sync_user_id": false, - "organisation_uuid": false + "uuid": true }, "sharing_group_blueprints": { "id": true, - "uuid": false, "name": false, "org_id": false, - "sharing_group_id": false + "sharing_group_id": false, + "uuid": false }, "sharing_group_orgs": { "id": true, @@ -8958,38 +8984,41 @@ "sharing_group_id": false }, "sightingdbs": { + "host": false, "id": true, "name": false, "owner": false, - "host": false, "port": false }, "sightingdb_orgs": { "id": true, - "sightingdb_id": false, - "org_id": false + "org_id": false, + "sightingdb_id": false }, "sightings": { - "id": true, "attribute_id": false, "event_id": false, + "id": true, "org_id": false, - "uuid": false, "source": false, - "type": false + "type": false, + "uuid": true + }, + "system_settings": { + "setting": true }, "tags": { "id": true, - "name": false, + "name": true, + "numerical_value": false, "org_id": false, - "user_id": false, - "numerical_value": false + "user_id": false }, "tag_collections": { "id": true, - "uuid": false, + "org_id": false, "user_id": false, - "org_id": false + "uuid": false }, "tag_collection_tags": { "id": true, @@ -9004,13 +9033,13 @@ }, "taxonomy_entries": { "id": true, - "taxonomy_predicate_id": false, - "numerical_value": false + "numerical_value": false, + "taxonomy_predicate_id": false }, "taxonomy_predicates": { "id": true, - "taxonomy_id": false, - "numerical_value": false + "numerical_value": false, + "taxonomy_id": false }, "templates": { "id": true @@ -9031,26 +9060,26 @@ "id": true }, "threads": { - "id": true, - "user_id": false, "event_id": false, + "id": true, "org_id": false, - "sharing_group_id": false + "sharing_group_id": false, + "user_id": false }, "threat_levels": { "id": true }, "users": { + "email": true, "id": true, - "sub": true, - "email": false, "org_id": false, - "server_id": false + "server_id": false, + "sub": true }, "user_settings": { "id": true, - "user_id": false, - "setting": false + "setting": false, + "user_id": false }, "warninglists": { "id": true @@ -9064,17 +9093,17 @@ }, "workflows": { "id": true, - "uuid": false, "name": false, "timestamp": false, - "trigger_id": false + "trigger_id": false, + "uuid": false }, "workflow_blueprints": { "id": true, - "uuid": false, "name": false, - "timestamp": false + "timestamp": false, + "uuid": false } }, "db_version": "93" -} +} \ No newline at end of file