MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #8592 from JakubOnderka/context-export-cleanup 

fix: [internal] Cleanup code for context exporter
pull/8600/head
Jakub Onderka 2022-09-20 18:01:28 +02:00 committed by GitHub
parent d3c26da69c 5a2da23b87
commit 9f21554d7a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 112 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/Lib/Export/AttackExport.php
View File

@ -127,7 +127,7 @@ class AttackExport
$result['galaxyId'] = $this->__galaxy_id;
$matrixGalaxies = $this->__GalaxyModel->getAllowedMatrixGalaxies();
$result['matrixGalaxies'] = $matrixGalaxies;
return json_encode($result);
return JsonTool::encode($result);
}
public function separator()

116
app/Lib/Export/ContextExport.php
View File

@ -1,8 +1,6 @@
<?php
class ContextExport
{
private $__attack_export_tool = null;
public $additional_params = [
'flatten' => 1,
'includeEventTags' => 1,
@ -14,19 +12,39 @@ class ContextExport
'includeEventCorrelations' => false,
];
private $__eventTags = [];
/** @var array Tag name => Galaxy */
private $__eventGalaxies = [];
private $__aggregatedTags = [];
private $__aggregatedClusters = [];
private $__taxonomyFetched = [];
private $__galaxyFetched = [];
private $__passedOptions = [];
public $non_restrictive_export = true;
public $renderView = 'context_view';
/** @var AttackExport */
private $AttackExport;
/** @var Taxonomy */
private $Taxonomy;
/** @var Galaxy */
private $Galaxy;
public function header($options = array())
{
$this->Taxonomy = ClassRegistry::init('Taxonomy');
$this->Galaxy = ClassRegistry::init('Galaxy');
App::uses('AttackExport', 'Export');
$this->AttackExport = new AttackExport();
$this->__passedOptions = $options;
return '';
}
public function handler($data, $options = array())
{
$this->__aggregate($data, Hash::extract($data, 'EventTag.{n}.Tag'));
@ -35,32 +53,20 @@ class ContextExport
$this->__aggregate($attribute, Hash::extract($attribute, 'AttributeTag.{n}.Tag'));
}
}
$this->__attack_export_tool->handler($data, $options);
return '';
}
public function header($options = array())
{
$this->__TaxonomyModel = ClassRegistry::init('Taxonomy');
$this->__GalaxyModel = ClassRegistry::init('Galaxy');
App::uses('AttackExport', 'Export');
$this->__attack_export_tool = new AttackExport();
$this->__attack_export_tool->handler($options);
$this->__passedOptions = $options;
$this->AttackExport->handler($data, $options);
return '';
}
public function footer()
{
$attackFinal = $this->__attack_export_tool->footer();
$attackFinal = $this->AttackExport->footer();
$this->__aggregateTagsPerTaxonomy();
$this->__aggregateClustersPerGalaxy();
$attackData = json_decode($attackFinal, true);
$attackData = $attackFinal === '' ? [] : JsonTool::decode($attackFinal);
if (!empty($this->__passedOptions['filters']['staticHtml'])) {
$attackData['static'] = true;
}
return json_encode([
return JsonTool::encode([
'attackData' => $attackData,
'tags' => $this->__aggregatedTags,
'clusters' => $this->__aggregatedClusters,
@ -69,17 +75,15 @@ class ContextExport
public function separator()
{
$this->__attack_export_tool->separator();
return '';
}
private function __aggregate($entity, $tags)
private function __aggregate(array $entity, array $tags)
{
if (!empty($entity['Galaxy'])) {
foreach ($entity['Galaxy'] as $galaxy) {
foreach ($galaxy['GalaxyCluster'] as $galaxyCluster) {
$this->__eventGalaxies[$galaxyCluster['tag_name']] = $galaxyCluster;
$this->fetchGalaxyForTag($galaxyCluster['tag_name']);
}
}
}
@ -94,42 +98,50 @@ class ContextExport
}
}
private function fetchTaxonomyForTag($tagname)
/**
* @param string $tagName
* @return void
* @throws RedisException
*/
private function fetchTaxonomyForTag($tagName)
{
$splits = $this->__TaxonomyModel->splitTagToComponents($tagname);
$splits = $this->Taxonomy->splitTagToComponents($tagName);
if ($splits === null) {
return; // tag is not taxonomy tag
}
if (!isset($this->__taxonomyFetched[$splits['namespace']])) {
$fetchedTaxonomy = $this->__TaxonomyModel->getTaxonomyForTag($tagname, false, true);
$fetchedTaxonomy = $this->Taxonomy->getTaxonomyForTag($tagName, false, true);
if (!empty($fetchedTaxonomy)) {
$this->__taxonomyFetched[$splits['namespace']]['Taxonomy'] = $fetchedTaxonomy['Taxonomy'];
$this->__taxonomyFetched[$splits['namespace']]['TaxonomyPredicate'] = [];
$fetched = [
'Taxonomy' => $fetchedTaxonomy['Taxonomy'],
'TaxonomyPredicate' => [],
];
foreach ($fetchedTaxonomy['TaxonomyPredicate'] as $predicate) {
$this->__taxonomyFetched[$splits['namespace']]['TaxonomyPredicate'][$predicate['value']] = $predicate;
$fetched['TaxonomyPredicate'][$predicate['value']] = $predicate;
if (!empty($predicate['TaxonomyEntry'])) {
$this->__taxonomyFetched[$splits['namespace']]['TaxonomyPredicate'][$predicate['value']]['TaxonomyEntry'] = [];
$fetched['TaxonomyPredicate'][$predicate['value']]['TaxonomyEntry'] = [];
foreach ($predicate['TaxonomyEntry'] as $entry) {
$this->__taxonomyFetched[$splits['namespace']]['TaxonomyPredicate'][$predicate['value']]['TaxonomyEntry'][$entry['value']] = $entry;
$fetched['TaxonomyPredicate'][$predicate['value']]['TaxonomyEntry'][$entry['value']] = $entry;
}
}
}
$this->__taxonomyFetched[$splits['namespace']] = $fetched;
} else {
// Do not try to fetch non existing taxonomy again
$this->__taxonomyFetched[$splits['namespace']] = false;
}
}
}
private function fetchGalaxyForTag($tagname)
{
$splits = $this->__TaxonomyModel->splitTagToComponents($tagname);
$galaxy = $this->__GalaxyModel->find('first', array(
'recursive' => -1,
'conditions' => array('Galaxy.type' => $splits['predicate'])
));
$this->__galaxyFetched[$splits['predicate']] = $galaxy;
}
private function __aggregateTagsPerTaxonomy()
{
ksort($this->__eventTags);
foreach ($this->__eventTags as $tagname => $tagData) {
$splits = $this->__TaxonomyModel->splitTagToComponents($tagname);
$splits = $this->Taxonomy->splitTagToComponents($tagname);
if ($splits === null) {
$this->__aggregatedTags['Custom Tags'][]['Tag'] = $tagData;
continue;
}
$taxonomy = [];
if (!empty($this->__taxonomyFetched[$splits['namespace']])) {
$taxonomy = $this->__taxonomyFetched[$splits['namespace']];
@ -137,7 +149,7 @@ class ContextExport
if (!empty($taxonomy['TaxonomyPredicate'][$splits['predicate']])) {
$predicate = $taxonomy['TaxonomyPredicate'][$splits['predicate']];
$entry = null;
if (!empty($splits['value'])) {
if (!empty($splits['value']) && isset($predicate['TaxonomyEntry'][$splits['value']])) {
$entry = $predicate['TaxonomyEntry'][$splits['value']];
}
unset($predicate['TaxonomyEntry']);
@ -155,12 +167,24 @@ class ContextExport
private function __aggregateClustersPerGalaxy()
{
$galaxyTypes = [];
foreach ($this->__eventGalaxies as $tagName => $foo) {
$splits = $this->Taxonomy->splitTagToComponents($tagName);
$galaxyTypes[$splits['predicate']] = true;
}
$fetchedGalaxies = $this->Galaxy->find('all', [
'recursive' => -1,
'conditions' => array('Galaxy.type' => array_keys($galaxyTypes)),
]);
$fetchedGalaxies = array_column(array_column($fetchedGalaxies, 'Galaxy'), null, 'type');
ksort($this->__eventGalaxies);
foreach ($this->__eventGalaxies as $tagname => $cluster) {
$splits = $this->__TaxonomyModel->splitTagToComponents($tagname);
$galaxy = $this->__galaxyFetched[$splits['predicate']];
foreach ($this->__eventGalaxies as $tagName => $cluster) {
$splits = $this->Taxonomy->splitTagToComponents($tagName);
$galaxy = $fetchedGalaxies[$splits['predicate']];
$this->__aggregatedClusters[$splits['predicate']][] = [
'Galaxy' => $galaxy['Galaxy'],
'Galaxy' => $galaxy,
'GalaxyCluster' => $cluster,
];
}

2
app/Model/Taxonomy.php
View File

@ -753,7 +753,7 @@ class Taxonomy extends AppModel
/**
* @param string $tag
* @return array|null
* @return array|null Returns null if tag is not in taxonomy format
*/
public function splitTagToComponents($tag)
{

11
app/Model/User.php
View File

@ -1774,8 +1774,8 @@ class User extends AppModel
$filters['includeEventCorrelations'] = !empty($periodicSettings['include_correlations']);
$filters['includeGranularCorrelations'] = !empty($periodicSettings['include_correlations']);
$filters['noSightings'] = true;
$filters['includeGalaxy'] = false;
$events = $this->__getEventsForFilters($user, $filters);
$filters['fetchFullClusters'] = false;
$events = $this->Event->fetchEvent($user, $filters);
$elementCounter = 0;
$renderView = false;
@ -1883,13 +1883,6 @@ class User extends AppModel
);
}
private function __getEventsForFilters(array $user, array $filters): array
{
$this->Event = ClassRegistry::init('Event');
$events = $this->Event->fetchEvent($user, $filters);
return $events;
}
public function prepareEmailTemplate(string $period='daily'): SendEmailTemplate
{
$subject = sprintf('[%s MISP] %s %s', Configure::read('MISP.org'), Inflector::humanize($period), __('Notification - %s', (new DateTime())->format('Y-m-d')));

80
app/View/Emails/notification_common.ctp
View File

@ -62,11 +62,10 @@ foreach ($events as $event) {
$event_report_number += count($event['EventReport']);
$proposal_number += count($event['ShadowAttribute']);
foreach ($event['EventTag'] as $event_tag) {
$tag = $event_tag['Tag'];
if (!empty($unique_tag_per_event[$tag['name']])) {
if (isset($unique_tag_per_event[$tag['name']])) {
continue; // Only one instance of tag per event
}
$unique_tag_per_event[$tag['name']] = true;
@ -75,9 +74,9 @@ foreach ($events as $event) {
$all_tag_amount[$tag['name']] = 0;
$tag_color_mapping[$tag['name']] = $tag['colour'];
}
$all_tag_amount[$tag['name']] += 1;
$all_tag_amount[$tag['name']]++;
if (!empty($tag['is_galaxy']) && substr($tag['name'], 0, strlen($mitre_galaxy_tag_prefix)) === $mitre_galaxy_tag_prefix) {
if ($tag['is_galaxy'] && substr($tag['name'], 0, strlen($mitre_galaxy_tag_prefix)) === $mitre_galaxy_tag_prefix) {
$technique = substr($tag['name'], strlen($mitre_galaxy_tag_prefix), strlen($tag['name']) - strlen($mitre_galaxy_tag_prefix) - 1);
$mitre_attack_techniques[$technique] = $event_tag;
}
@ -92,12 +91,12 @@ foreach ($events as $event) {
if (empty($attribute_types[$attribute['type']])) {
$attribute_types[$attribute['type']] = 0;
}
$attribute_types[$attribute['type']] += 1;
$attribute_types[$attribute['type']]++;
foreach ($attribute['AttributeTag'] as $attribute_tag) {
$tag = $attribute_tag['Tag'];
if (!empty($unique_tag_per_event[$tag['name']])) {
if (isset($unique_tag_per_event[$tag['name']])) {
continue; // Only one instance of tag per event
}
$unique_tag_per_event[$tag['name']] = true;
@ -106,9 +105,9 @@ foreach ($events as $event) {
$all_tag_amount[$tag['name']] = 0;
$tag_color_mapping[$tag['name']] = $tag['colour'];
}
$all_tag_amount[$tag['name']] += 1;
$all_tag_amount[$tag['name']]++;
if (!empty($tag['is_galaxy']) && substr($tag['name'], 0, strlen($mitre_galaxy_tag_prefix)) === $mitre_galaxy_tag_prefix) {
if ($tag['is_galaxy'] && substr($tag['name'], 0, strlen($mitre_galaxy_tag_prefix)) === $mitre_galaxy_tag_prefix) {
$technique = substr($tag['name'], strlen($mitre_galaxy_tag_prefix), strlen($tag['name']) - strlen($mitre_galaxy_tag_prefix) - 1);
$mitre_attack_techniques[$technique] = $attribute_tag;
}
@ -119,19 +118,19 @@ foreach ($events as $event) {
if (empty($object_types[$object['name']])) {
$object_types[$object['name']] = 0;
}
$object_types[$object['name']] += 1;
$object_types[$object['name']]++;
$attribute_number += count($object['Attribute']);
foreach ($object['Attribute'] as $attribute) {
if (empty($attribute_types[$attribute['type']])) {
$attribute_types[$attribute['type']] = 0;
}
$attribute_types[$attribute['type']] += 1;
$attribute_types[$attribute['type']]++;
foreach ($attribute['AttributeTag'] as $attribute_tag) {
$tag = $attribute_tag['Tag'];
if (!empty($unique_tag_per_event[$tag['name']])) {
if (isset($unique_tag_per_event[$tag['name']])) {
continue; // Only one instance of tag per event
}
$unique_tag_per_event[$tag['name']] = true;
@ -140,9 +139,9 @@ foreach ($events as $event) {
$all_tag_amount[$tag['name']] = 0;
$tag_color_mapping[$tag['name']] = $tag['colour'];
}
$all_tag_amount[$tag['name']] += 1;
$all_tag_amount[$tag['name']]++;
if (!empty($tag['is_galaxy']) && substr($tag['name'], 0, strlen($mitre_galaxy_tag_prefix)) === $mitre_galaxy_tag_prefix) {
if ($tag['is_galaxy'] && substr($tag['name'], 0, strlen($mitre_galaxy_tag_prefix)) === $mitre_galaxy_tag_prefix) {
$technique = substr($tag['name'], strlen($mitre_galaxy_tag_prefix), strlen($tag['name']) - strlen($mitre_galaxy_tag_prefix) - 1);
$mitre_attack_techniques[$technique] = $attribute_tag;
}
@ -170,7 +169,7 @@ foreach ($events as $event) {
foreach ($related_attributes as $related_attribute) {
$correlation_id = sprintf('%s-%s', $related_attribute['attribute_id'], $attribute_id);
$reversed_correlation_id = sprintf('%s-%s', $attribute_id, $related_attribute['attribute_id']);
$has_correlation_been_processed = !empty($processed_correlations[$correlation_id]); // We already added the correlation the other way around
$has_correlation_been_processed = isset($processed_correlations[$correlation_id]); // We already added the correlation the other way around
if ($has_attribute_been_modified_since_last_period && !$has_correlation_been_processed) {
$source_event = $event['Event'];
$source_event['Orgc'] = $event['Orgc'];
@ -199,7 +198,7 @@ if (!function_exists('findAndBuildTag')) {
}
}
$unique_tag_number = count(array_keys($all_tag_amount));
$unique_tag_number = count($all_tag_amount);
arsort($attribute_types);
arsort($object_types);
@ -208,7 +207,6 @@ arsort($mitre_attack_techniques);
array_splice($attribute_types, 10);
array_splice($object_types, 10);
array_splice($all_tag_amount, 10);
array_splice($mitre_attack_techniques, 10);
?>
@ -218,7 +216,7 @@ array_splice($mitre_attack_techniques, 10);
<?php if ($this->fetch('table-overview')) : ?>
<?= $this->fetch('table-overview'); ?>
<?php else : ?>
<?php else: ?>
<div class="panel">
<div class="panel-header">
<?= __('Data at a glance') ?>
@ -233,13 +231,11 @@ array_splice($mitre_attack_techniques, 10);
<tr>
<td><?= __('Summary for dates') ?></td>
<td>
<?=
sprintf('<strong>%s</strong> (Week %s) ➞ <strong>%s</strong> (Week %s)',
<?= __('<strong>%s</strong> (Week %s) ➞ <strong>%s</strong> (Week %s)',
$start_date->format('M d, o'),
$start_date->format('W'),
$now->format('M d, o'),
$now->format('W'),
$start_date->format('M d, o')
$now->format('W')
)
?>
</td>
@ -287,7 +283,7 @@ array_splice($mitre_attack_techniques, 10);
<?php if ($this->fetch('detailed-summary-full')) : ?>
<?= $this->fetch('detailed-summary-full'); ?>
<?php else : ?>
<?php else: ?>
<div class="panel">
<div class="panel-header">
<?= __('Detailed summary') ?>
@ -295,9 +291,9 @@ array_splice($mitre_attack_techniques, 10);
<div class="panel-body">
<?php if ($this->fetch('detailed-summary-mitre-attack')) : ?>
<?= $this->fetch('detailed-summary-mitre-attack'); ?>
<?php else : ?>
<?php else: ?>
<?php if (!empty($mitre_attack_techniques)) : ?>
<h4><?= __('Top 10 Mitre Att&ck techniques') ?></h4>
<h4><?= __('Top 10 MITRE ATT&CK techniques') ?></h4>
<ul>
<?php foreach ($mitre_attack_techniques as $technique => $tag) : ?>
<li>
@ -316,7 +312,7 @@ array_splice($mitre_attack_techniques, 10);
<?php if ($this->fetch('detailed-summary-type')) : ?>
<?= $this->fetch('detailed-summary-type'); ?>
<?php else : ?>
<?php else: ?>
<?php if (!empty($attribute_types)) : ?>
<h4><?= __('Top 10 Attribute types') ?></h4>
<ul>
@ -351,10 +347,10 @@ array_splice($mitre_attack_techniques, 10);
<?php if ($this->fetch('detailed-summary-tags')) : ?>
<?= $this->fetch('detailed-summary-tags'); ?>
<?php else : ?>
<?php else: ?>
<h4><?= __('Top 10 Tags') ?></h4>
<ul>
<?php foreach ($all_tag_amount as $tag_name => $amount) : ?>
<?php array_splice($all_tag_amount, 10); foreach ($all_tag_amount as $tag_name => $amount) : ?>
<li>
<span class="tag" style="background-color: #999; color: #fff; border-radius: 9px; padding: 2px 8px;">
<?= $amount ?>
@ -367,7 +363,7 @@ array_splice($mitre_attack_techniques, 10);
<?php if ($this->fetch('detailed-summary-events')) : ?>
<?= $this->fetch('detailed-summary-events'); ?>
<?php else : ?>
<?php else: ?>
<?php if (!empty($events)) : ?>
<h4><?= __('Event list') ?> <small style="color: #999999;"><?= sprintf(' (%s)', count($events)) ?></small></h4>
<table class="table table-condensed">
@ -424,7 +420,7 @@ array_splice($mitre_attack_techniques, 10);
</tr>
<?php endforeach; ?>
</table>
<?php else : ?>
<?php else: ?>
&nbsp;
<?php endif; ?>
</td>
@ -434,7 +430,7 @@ array_splice($mitre_attack_techniques, 10);
<?php endforeach; ?>
</tbody>
</table>
<?php else : ?>
<?php else: ?>
<p><?= __('No events.') ?></p>
<?php endif; ?>
<?php if (count($events) > $vars['event_table_max_event_count']) : ?>
@ -452,7 +448,7 @@ array_splice($mitre_attack_techniques, 10);
<?php if ($this->fetch('detailed-summary-correlations')) : ?>
<?= $this->fetch('detailed-summary-correlations'); ?>
<?php else : ?>
<?php else: ?>
<?php if (!empty($new_correlations)) : ?>
<h4><?= __('New correlations') ?><small style="color: #999999;"><?= sprintf(' (%s)', count($new_correlations)) ?></small></h4>
<div>
@ -465,8 +461,8 @@ array_splice($mitre_attack_techniques, 10);
<a href="<?= sprintf('%s/events/view/%s', $baseurl, h($correlation['source_event']['id'])) ?>"><?= h($correlation['source_event']['info']) ?></a>
</span>
<span class="org-date">
<span><?= h($correlation['source_event']['date']) ?></span>
<span><?= h($correlation['source_event']['Orgc']['name']) ?></span>
<span><?= h($correlation['source_event']['date']) ?></span>
<span><?= h($correlation['source_event']['Orgc']['name']) ?></span>
</span>
</span>
</span>
@ -481,8 +477,8 @@ array_splice($mitre_attack_techniques, 10);
<a href="<?= sprintf('%s/events/view/%s', $baseurl, h($correlation['target_event']['id'])) ?>"><?= h($correlation['target_event']['info']) ?></a>
</span>
<span class="org-date">
<span><?= h($correlation['target_event']['date']) ?></span>
<span><?= h($correlation['target_event']['Orgc']['name']) ?></span>
<span><?= h($correlation['target_event']['date']) ?></span>
<span><?= h($correlation['target_event']['Orgc']['name']) ?></span>
</span>
</span>
</span>
@ -492,16 +488,16 @@ array_splice($mitre_attack_techniques, 10);
<table class="table table-xcondensed">
<thead>
<tr>
<th><?= __('Event1.info') ?></th>
<th><?= __('First event info') ?></th>
<th><?= __('Value') ?></th>
<th><?= __('Event2.info') ?></th>
<th><?= __('Second event info') ?></th>
</tr>
</thead>
<tbody>
<?php foreach (array_slice($new_correlations, 0, $vars['correlation_table_max_count']) as $correlation): ?>
<tr>
<td><a href="<?= sprintf('%s/events/view/%s', $baseurl, h($correlation['source_event']['id'])) ?>"><?= h($correlation['source_event']['info']) ?></a></td>
<td><b><?= h($correlation['attribute_value']) ?></b></td>
<td><b><?= h($correlation['attribute_value']) ?></b></td>
<td><a href="<?= sprintf('%s/events/view/%s', $baseurl, h($correlation['target_event']['id'])) ?>"><?= h($correlation['target_event']['info']) ?></a></td>
</tr>
<?php endforeach; ?>
@ -519,8 +515,8 @@ array_splice($mitre_attack_techniques, 10);
?>
<?php endif; ?>
</div>
<?php endif ; ?>
<?php endif ; ?>
<?php endif; ?>
<?php endif; ?>
</div>
</div>
<?php endif; // detailed-summary-full
@ -528,7 +524,7 @@ array_splice($mitre_attack_techniques, 10);
<?php if ($this->fetch('trending-summary')) : ?>
<?= $this->fetch('trending-summary'); ?>
<?php else : ?>
<?php else: ?>
<div class="panel">
<div class="panel-header">
<?= __('Tag trendings') ?>
@ -541,7 +537,7 @@ array_splice($mitre_attack_techniques, 10);
<?php if ($this->fetch('aggregated-context')) : ?>
<?= $this->fetch('aggregated-context'); ?>
<?php else : ?>
<?php else: ?>
<div class="panel">
<div class="panel-header">
<?= __('Context summary') ?>