MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

new: [UI] Filtering attributes by correlated event ID

pull/8317/head
Jakub Onderka 2022-04-29 14:36:06 +02:00
parent 28a58dbb61
commit d7bdc32c1f
4 changed files with 130 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
app/Controller/EventsController.php
View File

@ -31,7 +31,7 @@ class EventsController extends AppController
'sort', 'direction', 'focus', 'extended', 'overrideLimit', 'filterColumnsOverwrite', 'attributeFilter', 'page',
'searchFor', 'proposal', 'correlation', 'warning', 'deleted', 'includeRelatedTags', 'includeDecayScore', 'distribution',
'taggedAttributes', 'galaxyAttachedAttributes', 'objectType', 'attributeType', 'feed', 'server', 'toIDS',
'sighting', 'includeSightingdb', 'warninglistId'
'sighting', 'includeSightingdb', 'warninglistId', 'correlationId',
);
// private
@ -52,6 +52,7 @@ class EventsController extends AppController
'taggedAttributes' => '',
'galaxyAttachedAttributes' => '',
'warninglistId' => '',
'correlationId' => '',
);
// private
@ -1277,7 +1278,7 @@ class EventsController extends AppController
'fetchFullClusters' => false,
'includeAllTags' => true,
'includeGranularCorrelations' => true,
'includeEventCorrelations' => false,
'includeEventCorrelations' => true, // event correlations are need for filtering
'noEventReports' => true, // event reports for view are loaded dynamically
'noSightings' => true,
'includeServerCorrelations' => $filters['includeServerCorrelations'] ?? 1.
@ -1672,7 +1673,7 @@ class EventsController extends AppController
private function __eventViewCommon(array $user)
{
$this->set('defaultFilteringRules', self::DEFAULT_FILTERING_RULE);
$this->set('typeGroups', array_keys($this->Event->Attribute->typeGroupings));
$this->set('typeGroups', array_keys(Attribute::TYPE_GROUPINGS));
$orgTable = $this->Event->Orgc->find('list', array(
'fields' => array('Orgc.id', 'Orgc.name')

10
app/Model/Attribute.php
View File

@ -192,11 +192,11 @@ class Attribute extends AppModel
// for example, IP addresses, domain names, hostnames and e-mail addresses are network related attribute types
// whilst filenames and hashes are file related attribute types
// This helps generate quick filtering for the event view, but we may reuse this and enhance it in the future for other uses (such as the API?)
public $typeGroupings = array(
'file' => array('attachment', 'pattern-in-file', 'filename-pattern', 'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|pehash', 'malware-sample', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'x509-fingerprint-md5'),
'network' => array('ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port', 'mac-address', 'mac-eui-64', 'hostname', 'hostname|port', 'domain', 'domain|ip', 'email-dst', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'bro', 'zeek', 'pattern-in-traffic', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256','ja3-fingerprint-md5', 'jarm-fingerprint', 'favicon-mmh3', 'hassh-md5', 'hasshserver-md5', 'community-id'),
'financial' => array('btc', 'xmr', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'phone-number')
);
const TYPE_GROUPINGS = [
'file' => ['attachment', 'pattern-in-file', 'filename-pattern', 'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|pehash', 'malware-sample', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'x509-fingerprint-md5'],
'network' => ['ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port', 'mac-address', 'mac-eui-64', 'hostname', 'hostname|port', 'domain', 'domain|ip', 'email-dst', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'bro', 'zeek', 'pattern-in-traffic', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256','ja3-fingerprint-md5', 'jarm-fingerprint', 'favicon-mmh3', 'hassh-md5', 'hasshserver-md5', 'community-id'],
'financial' => ['btc', 'xmr', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'phone-number']
];
private $__fTool = false;

118
app/Model/Event.php
View File

@ -4842,12 +4842,24 @@ class Event extends AppModel
/* correlation */
if ($filterType['correlation'] == 0) { // `both`
// pass, do not consider as `both` is selected
} else if (in_array($attribute['id'], $correlatedAttributes)) { // `include only`
} else if (isset($correlatedAttributes[$attribute['id']])) { // `include only`
$include = $include && ($filterType['correlation'] == 1);
} else { // `exclude`
$include = $include && ($filterType['correlation'] == 2);
}
if ($filterType['correlationId'] && $include) {
$include = false;
if (isset($correlatedAttributes[$attribute['id']])) {
foreach ($correlatedAttributes[$attribute['id']] as $correlation) {
if (in_array($correlation['id'], $filterType['correlationId'])) {
$include = true;
break;
}
}
}
}
/* feed */
if ($filterType['feed'] == 0) { // `both`
// pass, do not consider as `both` is selected
@ -4877,11 +4889,11 @@ class Event extends AppModel
/* TypeGroupings */
if (
$filterType['attributeFilter'] != 'all'
&& isset($this->Attribute->typeGroupings[$filterType['attributeFilter']])
&& !in_array($attribute['type'], $this->Attribute->typeGroupings[$filterType['attributeFilter']])
$filterType['attributeFilter'] !== 'all'
&& isset(Attribute::TYPE_GROUPINGS[$filterType['attributeFilter']])
&& !in_array($attribute['type'], Attribute::TYPE_GROUPINGS[$filterType['attributeFilter']], true)
) {
$include = false;
return null;
}
if ($filterType['warning'] == 0) { // `both`
@ -4892,7 +4904,7 @@ class Event extends AppModel
$include = $include && ($filterType['warning'] == 2);
}
if ($filterType['warninglistId']) {
if ($filterType['warninglistId'] && $include) {
$include = false;
if (isset($attribute['warnings'])) {
foreach ($attribute['warnings'] as $warning) {
@ -4922,6 +4934,12 @@ class Event extends AppModel
return $this->__prepareGenericForView($attribute);
}
/**
* @param array $proposal
* @param array $correlatedShadowAttributes
* @param array $filterType
* @return array|null
*/
private function __prepareProposalForView($proposal, $correlatedShadowAttributes, $filterType = false)
{
if ($proposal['proposal_to_delete']) {
@ -4937,12 +4955,24 @@ class Event extends AppModel
/* correlation */
if ($filterType['correlation'] == 0) { // `both`
// pass, do not consider as `both` is selected
} else if (in_array($proposal['id'], $correlatedShadowAttributes)) { // `include only`
} else if (isset($correlatedShadowAttributes[$proposal['id']])) { // `include only`
$include = $include && ($filterType['correlation'] == 1);
} else { // `exclude`
$include = $include && ($filterType['correlation'] == 2);
}
if ($filterType['correlationId'] && $include) {
$include = false;
if (isset($correlatedShadowAttributes[$proposal['id']])) {
foreach ($correlatedShadowAttributes[$proposal['id']] as $correlation) {
if (in_array($correlation['id'], $filterType['correlationId'])) {
$include = true;
break;
}
}
}
}
/* feed */
if ($filterType['feed'] == 0) { // `both`
// pass, do not consider as `both` is selected
@ -4963,11 +4993,11 @@ class Event extends AppModel
/* TypeGroupings */
if (
$filterType['attributeFilter'] != 'all'
&& isset($this->Attribute->typeGroupings[$filterType['attributeFilter']])
&& !in_array($proposal['type'], $this->Attribute->typeGroupings[$filterType['attributeFilter']])
$filterType['attributeFilter'] !== 'all'
&& isset(Attribute::TYPE_GROUPINGS[$filterType['attributeFilter']])
&& !in_array($proposal['type'], Attribute::TYPE_GROUPINGS[$filterType['attributeFilter']], true)
) {
$include = false;
return null;
}
/* warning */
@ -4996,7 +5026,7 @@ class Event extends AppModel
) {
$object['category'] = $object['meta-category'];
$include = empty($filterType['attributeFilter']) || $filterType['attributeFilter'] == 'object' || $filterType['attributeFilter'] == 'all' || $object['meta-category'] === $filterType['attributeFilter'];
$include = empty($filterType['attributeFilter']) || $filterType['attributeFilter'] === 'all' || $filterType['attributeFilter'] === 'object' || $object['meta-category'] === $filterType['attributeFilter'];
if (!$include) {
return null;
@ -5028,6 +5058,7 @@ class Event extends AppModel
|| $filterType['feed'] != 0
|| $filterType['server'] != 0
|| $filterType['warninglistId'] !== null
|| $filterType['correlationId'] !== null
) {
$include = $this->__checkObjectByFilter($object, $filterType, $correlatedAttributes, $correlatedShadowAttributes, $sightingsData);
if (!$include) {
@ -5038,6 +5069,14 @@ class Event extends AppModel
return $object;
}
/**
* @param array $object
* @param array $filterType
* @param array $correlatedAttributes
* @param array $correlatedShadowAttributes
* @param array $sightingsData
* @return bool
*/
private function __checkObjectByFilter($object, $filterType, $correlatedAttributes, $correlatedShadowAttributes, $sightingsData)
{
if (empty($object['Attribute'])) { // reject empty object
@ -5049,7 +5088,7 @@ class Event extends AppModel
// pass, do not consider as `both` is selected
} else if ($filterType['proposal'] == 1 || $filterType['proposal'] == 2) {
$flagKeep = false;
foreach ($object['Attribute'] as $k => $attribute) { // check if object contains at least 1 proposal
foreach ($object['Attribute'] as $attribute) { // check if object contains at least 1 proposal
if (!empty($attribute['ShadowAttribute'])) {
$flagKeep = ($filterType['proposal'] == 1); // keep if proposal are included
break;
@ -5096,7 +5135,37 @@ class Event extends AppModel
foreach ($attribute['warnings'] as $warning) {
if (in_array($warning['warninglist_id'], $filterType['warninglistId'])) {
$flagKeep = true;
break;
break 2;
}
}
}
}
if (!$flagKeep) {
return false;
}
}
if ($filterType['correlationId']) {
$flagKeep = false;
// check if object contains at least one attribute that is correlating with given event ID
foreach ($object['Attribute'] as $attribute) {
if (isset($correlatedAttributes[$attribute['id']])) {
foreach ($correlatedAttributes[$attribute['id']] as $correlation) {
if (in_array($correlation['id'], $filterType['correlationId'])) {
$flagKeep = true;
break 2;
}
}
}
if (!empty($attribute['ShadowAttribute'])) {
foreach ($attribute['ShadowAttribute'] as $shadowAttribute) {
if (isset($correlatedShadowAttributes[$shadowAttribute['id']])) {
foreach ($correlatedShadowAttributes[$shadowAttribute['id']] as $correlation) {
if (in_array($correlation['id'], $filterType['correlationId'])) {
$flagKeep = true;
break 2;
}
}
}
}
}
@ -5112,14 +5181,14 @@ class Event extends AppModel
} else if ($filterType['correlation'] == 1 || $filterType['correlation'] == 2) {
$flagKeep = false;
foreach ($object['Attribute'] as $attribute) { // check if object contains at least 1 warning
if (in_array($attribute['id'], $correlatedAttributes)) {
if (isset($correlatedAttributes[$attribute['id']])) {
$flagKeep = ($filterType['correlation'] == 1); // keep if correlations are included
} else {
$flagKeep = ($filterType['correlation'] == 2); // keep if correlations are excluded
}
if (!$flagKeep && !empty($attribute['ShadowAttribute'])) {
foreach ($attribute['ShadowAttribute'] as $shadowAttribute) {
if (in_array($shadowAttribute['id'], $correlatedShadowAttributes)) {
if (isset($correlatedShadowAttributes[$shadowAttribute['id']])) {
$flagKeep = ($filterType['correlation'] == 1); // keep if correlations are included
break;
}
@ -5139,7 +5208,7 @@ class Event extends AppModel
// pass, do not consider as `both` is selected
} else if ($filterType['sighting'] == 1 || $filterType['sighting'] == 2) {
$flagKeep = false;
foreach ($object['Attribute'] as $k => $attribute) { // check if object contains at least 1 warning
foreach ($object['Attribute'] as $attribute) { // check if object contains at least 1 warning
if (isset($sightingsData['data'][$attribute['id']])) {
$flagKeep = ($filterType['sighting'] == 1); // keep if server are included
} else {
@ -5167,7 +5236,7 @@ class Event extends AppModel
// pass, do not consider as `both` is selected
} else if ($filterType['feed'] == 1 || $filterType['feed'] == 2) {
$flagKeep = false;
foreach ($object['Attribute'] as $k => $attribute) { // check if object contains at least 1 warning
foreach ($object['Attribute'] as $attribute) { // check if object contains at least 1 warning
if (!empty($attribute['Feed'])) {
$flagKeep = ($filterType['feed'] == 1); // keep if feed are included
} else {
@ -5195,7 +5264,7 @@ class Event extends AppModel
// pass, do not consider as `both` is selected
} else if ($filterType['server'] == 1 || $filterType['server'] == 2) {
$flagKeep = false;
foreach ($object['Attribute'] as $k => $attribute) { // check if object contains at least 1 warning
foreach ($object['Attribute'] as $attribute) { // check if object contains at least 1 warning
if (!empty($attribute['Server'])) {
$flagKeep = ($filterType['server'] == 1); // keep if server are included
} else {
@ -5220,6 +5289,10 @@ class Event extends AppModel
return true;
}
/**
* @param array $object
* @return array
*/
private function __prepareGenericForView($object)
{
if ($this->Attribute->isImage($object)) {
@ -5271,14 +5344,15 @@ class Event extends AppModel
'feed' => isset($passedArgs['feed']) ? $passedArgs['feed'] : 0,
'server' => isset($passedArgs['server']) ? $passedArgs['server'] : 0,
'warninglistId' => isset($passedArgs['warninglistId']) ? (is_array($passedArgs['warninglistId']) ? $passedArgs['warninglistId'] : [$passedArgs['warninglistId']]) : null,
'correlationId' => isset($passedArgs['correlationId']) ? (is_array($passedArgs['correlationId']) ? $passedArgs['correlationId'] : [$passedArgs['correlationId']]) : null,
);
// update proposal, correlation and warning accordingly
if (in_array($filterType['attributeFilter'], array('proposal', 'correlation', 'warning'))) {
if (in_array($filterType['attributeFilter'], array('proposal', 'correlation', 'warning'), true)) {
$filterType[$filterType['attributeFilter']] = 1;
}
$correlatedAttributes = isset($event['RelatedAttribute']) ? array_keys($event['RelatedAttribute']) : array();
$correlatedShadowAttributes = isset($event['RelatedShadowAttribute']) ? array_keys($event['RelatedShadowAttribute']) : array();
$correlatedAttributes = isset($event['RelatedAttribute']) ? $event['RelatedAttribute'] : [];
$correlatedShadowAttributes = isset($event['RelatedShadowAttribute']) ? $event['RelatedShadowAttribute'] : [];
$objects = array();
if (isset($event['Attribute'])) {

25
app/View/Elements/Events/View/eventFilteringQueryBuilder.ctp
View File

@ -4,6 +4,13 @@ foreach ($event['warnings'] as $id => $name) {
$warninglistsValues[] = [(int)$id => h($name)];
}
$warninglistsValues = json_encode($warninglistsValues, JSON_UNESCAPED_UNICODE);
$relatedEventsValues = [];
foreach ($event['RelatedEvent'] as $relatedEvent) {
$relatedEventsValues[] = [(int)$relatedEvent["Event"]["id"] => "#{$relatedEvent["Event"]["id"]} " . h($relatedEvent["Event"]["info"])];
}
$relatedEventsValues = json_encode($relatedEventsValues, JSON_UNESCAPED_UNICODE);
?>
<div id="eventFilteringQBWrapper" style="padding: 5px; display: none; border: 1px solid #dddddd; border-bottom: 0;">
<div id="eventFilteringQB" style="overflow-y: auto; padding-right: 5px; resize: vertical; max-height: 750px; height: 400px;"></div>
@ -82,6 +89,17 @@ function triggerEventFilteringTool(hide) {
2: "Exclude correlation"
}
},
{
"input": "select",
"type": "string",
"operators": [
"equal",
],
"unique": true,
"id": "correlationId",
"label": "Correlations with event",
"values": <?= $relatedEventsValues ?>
},
{
"input": "radio",
"type": "integer",
@ -320,6 +338,13 @@ function triggerEventFilteringTool(hide) {
value: <?php echo isset($filters['correlation']) ? h($filters['correlation']) : 0; ?>
},
<?php endif; ?>
<?php if (empty($advancedFilteringActiveRules) || isset($advancedFilteringActiveRules['correlationId'])): ?>
{
field: 'correlationId',
id: 'correlationId',
value: <?= isset($filters['correlationId']) ? json_encode($filters['correlationId']) : "''"; ?>
},
<?php endif; ?>
<?php if (empty($advancedFilteringActiveRules) || isset($advancedFilteringActiveRules['warning'])): ?>
{
field: 'warning',