mirror of https://github.com/MISP/MISP
new: [event:trendsForTags] Added feature to generate trends based on tags for the provided event filters
Sami Mokaddem
parent
6c3a5f5dd0
commit
35a91893f0
|
|
@ -7621,4 +7621,130 @@ class Event extends AppModel
|
|||
$kafkaPubTool->publishJson($kafkaTopic, $fullEvent[0], 'publish');
|
||||
}
|
||||
}
|
||||
|
||||
public function getTrendsForTags(array $user, array $eventFilters=[], int $baseDayRange, int $rollingWindows=3, $tagFilterPrefixes=null): array
|
||||
{
|
||||
$fullDayNumber = $baseDayRange + $baseDayRange * $rollingWindows;
|
||||
$fullRange = $this->resolveTimeDelta($fullDayNumber . 'd');
|
||||
$eventFilters['last'] = $fullRange . 'd';
|
||||
$eventFilters['order'] = 'timestamp DESC';
|
||||
$events = $this->fetchEvent($user, $eventFilters);
|
||||
$clusteredTags = $this->__clusterTagsForRollingWindow($events, $baseDayRange, $rollingWindows, $tagFilterPrefixes);
|
||||
$trendAnalysis = $this->__computeTrendAnalysis($clusteredTags);
|
||||
return [
|
||||
'clustered_tags' => $trendAnalysis,
|
||||
'clustered_events' => $clusteredTags['eventNumberPerRollingWindow'],
|
||||
'all_tags' => $clusteredTags['allTagsPerPrefix'],
|
||||
'all_timestamps' => array_keys($clusteredTags['eventNumberPerRollingWindow']),
|
||||
];
|
||||
}
|
||||
|
||||
private function __computeTrendAnalysis(array $clusteredTags): array
|
||||
{
|
||||
$tagsPerRollingWindow = $clusteredTags['tagsPerRollingWindow'];
|
||||
$eventNumberPerRollingWindow = $clusteredTags['eventNumberPerRollingWindow'];
|
||||
$trendAnalysis = [];
|
||||
$allTimestamps = array_keys($tagsPerRollingWindow);
|
||||
$allTags = [];
|
||||
foreach ($allTimestamps as $i => $timestamp) {
|
||||
$trendAnalysis[$timestamp] = [];
|
||||
$tags = $tagsPerRollingWindow[$timestamp];
|
||||
$nextTimestamp = isset($allTimestamps[$i+1]) ? $allTimestamps[$i+1] : false;
|
||||
$previousTimestamp = isset($allTimestamps[$i-1]) ? $allTimestamps[$i-1] : false;
|
||||
foreach ($tags as $tag => $amount) {
|
||||
$rawChange = 0;
|
||||
$percentChange = 0;
|
||||
if (!empty($nextTimestamp)) {
|
||||
$nextAmount = !empty($tagsPerRollingWindow[$nextTimestamp][$tag]) ? $tagsPerRollingWindow[$nextTimestamp][$tag] : 0;
|
||||
$rawChange = $amount - $nextAmount;
|
||||
$percentChange = 100*$rawChange/$amount;
|
||||
}
|
||||
$allTags[$tag] = true;
|
||||
$trendAnalysis[$timestamp][$tag] = [
|
||||
'occurence' => round($amount / $eventNumberPerRollingWindow[$timestamp], 2),
|
||||
'raw_change' => $rawChange,
|
||||
'percent_change' => $percentChange,
|
||||
'change_sign' => $rawChange > 0 ? 1 : ($rawChange < 0 ? -1 : 0),
|
||||
];
|
||||
}
|
||||
if (!empty($previousTimestamp)) {
|
||||
foreach (array_keys($trendAnalysis[$timestamp]) as $tag) {
|
||||
if (empty($trendAnalysis[$previousTimestamp][$tag])) {
|
||||
$trendAnalysis[$previousTimestamp][$tag] = [
|
||||
'occurence' => 0,
|
||||
'raw_change' => -$amount,
|
||||
'percent_change' => 100 * (-$amount / $amount),
|
||||
'change_sign' => -$amount > 0 ? 1 : (-$amount < 0 ? -1 : 0),
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return $trendAnalysis;
|
||||
}
|
||||
|
||||
private function __clusterTagsForRollingWindow(array $events, int $baseDayRange, int $rollingWindows=3, $tagFilterPrefixes=null): array
|
||||
{
|
||||
$fullDayNumber = $baseDayRange + $baseDayRange * $rollingWindows;
|
||||
$tagsPerRollingWindow = [];
|
||||
$eventNumberPerRollingWindow = [];
|
||||
$timestampRollingWindow = [];
|
||||
for ($i=0; $i <= $fullDayNumber; $i += $baseDayRange) {
|
||||
$timestamp = $this->resolveTimeDelta($i . 'd');
|
||||
$timestampRollingWindow[] = $timestamp;
|
||||
$tagsPerRollingWindow[$timestamp] = [];
|
||||
}
|
||||
$tagsPerRollingWindow = array_map(function() { return []; }, array_flip(array_slice($timestampRollingWindow, 1)));
|
||||
$eventNumberPerRollingWindow = array_map(function() { return 0; }, array_flip(array_slice($timestampRollingWindow, 1)));
|
||||
$allTagsPerPrefix = [];
|
||||
|
||||
foreach ($events as $event) {
|
||||
$allTags = $this->extractAllTagNames($event);
|
||||
$rollingTimestamps = $this->__getTimestampFromRollingWindow($event['Event']['timestamp'], $timestampRollingWindow);
|
||||
$filteredTags = array_filter($allTags, function($tag) use ($tagFilterPrefixes, &$allTagsPerPrefix) {
|
||||
if (is_null($tagFilterPrefixes)) {
|
||||
return true;
|
||||
} else {
|
||||
foreach ($tagFilterPrefixes as $tagPrefix) {
|
||||
if(substr($tag, 0, strlen($tagPrefix)) === $tagPrefix) {
|
||||
$allTagsPerPrefix[$tagPrefix][$tag] = true;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
});
|
||||
foreach ($filteredTags as $tag) {
|
||||
if (empty($tagsPerRollingWindow[$rollingTimestamps['current']][$tag])) {
|
||||
$tagsPerRollingWindow[$rollingTimestamps['current']][$tag] = 0;
|
||||
}
|
||||
$tagsPerRollingWindow[$rollingTimestamps['current']][$tag] += 1;
|
||||
}
|
||||
$eventNumberPerRollingWindow[$rollingTimestamps['current']] += 1;
|
||||
}
|
||||
return [
|
||||
'tagsPerRollingWindow' => $tagsPerRollingWindow,
|
||||
'eventNumberPerRollingWindow' => $eventNumberPerRollingWindow,
|
||||
'allTagsPerPrefix' => array_map(function($clusteredTags) {
|
||||
return array_keys($clusteredTags);
|
||||
}, $allTagsPerPrefix),
|
||||
];
|
||||
}
|
||||
|
||||
private function __getTimestampFromRollingWindow(int $eventTimestamp, array $rollingWindow): array
|
||||
{
|
||||
$i = 0;
|
||||
if (count($rollingWindow) > 2) {
|
||||
for ($i=0; $i < count($rollingWindow)-1; $i++) {
|
||||
if ($eventTimestamp >= $rollingWindow[$i]) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return [
|
||||
'previous' => isset($rollingWindow[$i - 1]) ? $rollingWindow[$i - 1] : null,
|
||||
'current' => $rollingWindow[$i],
|
||||
'next' => isset($rollingWindow[$i + 1]) ? $rollingWindow[$i + 1] : null,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,194 @@
|
|||
<?php
|
||||
$clusteredTags = $trendAnalysis['clustered_tags'];
|
||||
$clusteredEvents = $trendAnalysis['clustered_events'];
|
||||
$allTags = $trendAnalysis['all_tags'];
|
||||
$allTimestamps = $trendAnalysis['all_timestamps'];
|
||||
$currentPeriod = $allTimestamps[0];
|
||||
$previousPeriod = $allTimestamps[1];
|
||||
$previousPeriod2 = $allTimestamps[2];
|
||||
|
||||
$COLOR_PALETTE = ['#eecc66', '#ee99aa', '#6699cc', '#997700', '#994455', '#dddddd'];
|
||||
|
||||
// $pieChartData = array_map(function(array $tagMetrics) {
|
||||
// return $tagMetrics['occurence'];
|
||||
// }, $clusteredTags[$currentPeriod]);
|
||||
// arsort($pieChartData);
|
||||
// $topPieChartData = array_slice($pieChartData, 0, 5);
|
||||
|
||||
$trendIconMapping = [
|
||||
1 => '▲',
|
||||
-1 => '▼',
|
||||
0 => '⮞',
|
||||
'?' => '',
|
||||
];
|
||||
$trendColorMapping = [
|
||||
1 => '#b94a48',
|
||||
-1 => '#468847',
|
||||
0 => '#3a87ad',
|
||||
'?' => '#999999',
|
||||
];
|
||||
$now = new DateTime();
|
||||
$currentPeriodDate = DateTime::createFromFormat('U', $currentPeriod);
|
||||
$previousPeriodDate = DateTime::createFromFormat('U', $previousPeriod);
|
||||
$previousPeriod2Date = DateTime::createFromFormat('U', $previousPeriod2);
|
||||
|
||||
if (!function_exists('reduceTag')) {
|
||||
function reduceTag(string $tagname, int $reductionLength=1): string
|
||||
{
|
||||
$re = '/^(?<namespace>[a-z0-9_-]+)(:(?<predicate>[a-z0-9_-]+)="(?<value>[^"]+)"$)?(:(?<predicate2>[a-z0-9_-]+))?/';
|
||||
$matches = [];
|
||||
preg_match($re, $tagname, $matches);
|
||||
if (!empty($matches['predicate2'])) {
|
||||
return $reductionLength == 0 ? $tagname : $matches['predicate2'];
|
||||
} else if (!empty($matches['value'])) {
|
||||
return $reductionLength == 0 ? $tagname : (
|
||||
$reductionLength == 1 ? sprintf('%s="%s"', $matches['predicate'], $matches['value']) : $matches['value']
|
||||
);
|
||||
} else if (!empty($matches['namespace'])) {
|
||||
return $matches['namespace'];
|
||||
} else {
|
||||
return $tagname;
|
||||
}
|
||||
}
|
||||
}
|
||||
?>
|
||||
|
||||
<h2><?= __('Tag trendings') ?></h2>
|
||||
|
||||
<div style="display: flex;">
|
||||
<div>
|
||||
<table class="table table-condensed" style="max-width: 400px;">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><?= __('Period duration') ?></td>
|
||||
<td><?= __('%s days', $currentPeriodDate->diff($now)->format('%a')); ?></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><?= __('Current period') ?></td>
|
||||
<td><?= sprintf('%s', $currentPeriodDate->format('M d, o. (\W\e\e\k W)')); ?></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><?= __('Previous period') ?></td>
|
||||
<td><?= sprintf('%s', $previousPeriodDate->format('M d, o. (\W\e\e\k W)')); ?></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><?= __('Period-2') ?></td>
|
||||
<td><?= sprintf('%s', $previousPeriod2Date->format('M d, o. (\W\e\e\k W)')); ?></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<?php if (!empty($allTags)): ?>
|
||||
<table class="table table-condensed no-border">
|
||||
<thead>
|
||||
<tr>
|
||||
<th></th>
|
||||
<th>
|
||||
<span>
|
||||
<div><?= __('Period-2') ?></div>
|
||||
<div style="font-weight: normal;"><?= __('%s events', h($clusteredEvents[$previousPeriod2])) ?></div>
|
||||
</span>
|
||||
<table>
|
||||
<thead style="font-size: small;">
|
||||
<tr>
|
||||
<td style="min-width: 20px;">#</td>
|
||||
<td style="min-width: 15px;">⥮</td>
|
||||
<td>%</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
</thead>
|
||||
</table>
|
||||
</th>
|
||||
<th>
|
||||
<span>
|
||||
<div><?= __('Previous period') ?></div>
|
||||
<div style="font-weight: normal;"><?= __('%s events', h($clusteredEvents[$previousPeriod])) ?></div>
|
||||
</span>
|
||||
<table>
|
||||
<thead style="font-size: small;">
|
||||
<tr>
|
||||
<td style="min-width: 20px;">#</td>
|
||||
<td style="min-width: 15px;">⥮</td>
|
||||
<td>%</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
</thead>
|
||||
</table>
|
||||
</th>
|
||||
<th>
|
||||
<span>
|
||||
<div><?= __('Current period') ?></div>
|
||||
<div style="font-weight: normal;"><?= __('%s events', h($clusteredEvents[$currentPeriod])) ?></div>
|
||||
</span>
|
||||
<table>
|
||||
<thead style="font-size: small;">
|
||||
<tr>
|
||||
<td style="min-width: 20px;">#</td>
|
||||
<td style="min-width: 15px;">⥮</td>
|
||||
<td>%</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
</thead>
|
||||
</table>
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<?php foreach ($tagFilterPrefixes as $tagPrefix) : ?>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td colspan="4">
|
||||
<h5><?= __('Taxonomy: %s', sprintf('<code>%s</code>', h($tagPrefix))) ?></h5>
|
||||
</td>
|
||||
</tr>
|
||||
<?php foreach ($allTags[$tagPrefix] as $tagName) : ?>
|
||||
<tr>
|
||||
<td style="padding-left: 15px;"><code style="color: black;"><?= h(reduceTag($tagName, count(explode(':', $tagPrefix)))) ?></code></td>
|
||||
<td>
|
||||
<table class="table-condensed no-border">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><?= h($clusteredTags[$previousPeriod2][$tagName]['occurence'] ?? '-') ?></td>
|
||||
<td><?= h($clusteredTags[$previousPeriod2][$tagName]['raw_change'] ?? '-') ?></td>
|
||||
<td><?= h($clusteredTags[$previousPeriod2][$tagName]['percent_change'] ?? '-') ?>%</td>
|
||||
<td style="font-size: large; color: <?= $trendColorMapping[$clusteredTags[$previousPeriod2][$tagName]['change_sign'] ?? '?'] ?>"><?= $trendIconMapping[$clusteredTags[$previousPeriod2][$tagName]['change_sign'] ?? '?'] ?></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</td>
|
||||
<td>
|
||||
<table class="table-condensed no-border">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><?= h($clusteredTags[$previousPeriod][$tagName]['occurence'] ?? '-') ?></td>
|
||||
<td><?= h($clusteredTags[$previousPeriod][$tagName]['raw_change'] ?? '-') ?></td>
|
||||
<td><?= h($clusteredTags[$previousPeriod][$tagName]['percent_change'] ?? '-') ?>%</td>
|
||||
<td style="font-size: large; color: <?= $trendColorMapping[$clusteredTags[$previousPeriod][$tagName]['change_sign'] ?? '?'] ?>"><?= $trendIconMapping[$clusteredTags[$previousPeriod][$tagName]['change_sign'] ?? '?'] ?></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</td>
|
||||
<td>
|
||||
<table class="table-condensed no-border">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><?= h($clusteredTags[$currentPeriod][$tagName]['occurence'] ?? '-') ?></td>
|
||||
<td><?= h($clusteredTags[$currentPeriod][$tagName]['raw_change'] ?? '-') ?></td>
|
||||
<td><?= h($clusteredTags[$currentPeriod][$tagName]['percent_change'] ?? '-') ?>%</td>
|
||||
<td style="font-size: large; color: <?= $trendColorMapping[$clusteredTags[$currentPeriod][$tagName]['change_sign'] ?? '?'] ?>"><?= $trendIconMapping[$clusteredTags[$currentPeriod][$tagName]['change_sign'] ?? '?'] ?></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
<?php endforeach; ?>
|
||||
</tbody>
|
||||
<?php endforeach; ?>
|
||||
</table>
|
||||
<?php endif; ?>
|
||||
|
||||
<style>
|
||||
</style>
|
||||
Loading…
Reference in New Issue