MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [decaying:simulation] Added support of base_score computation, 

various UI improvements and different method to compute scores
pull/5032/head
mokaddem 2019-07-17 16:14:24 +02:00
parent fdf7161dc0
commit 4817c38ac3
No known key found for this signature in database
GPG Key ID: 164C473F627A06FA
6 changed files with 282 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
app/Controller/DecayingModelController.php
View File

@ -16,7 +16,7 @@ class DecayingModelController extends AppController
public function update($force=false)
{
if (!$this->_isSiteAdmin()) {
throw new MethodNotAllowedException(_('You are not authorised to edit it.'));
throw new MethodNotAllowedException(__('You are not authorised to edit it.'));
}
if ($this->request->is('post')) {
@ -30,7 +30,7 @@ class DecayingModelController extends AppController
// return $this->RestResponse->viewData($message, $this->response->type());
}
} else {
throw new MethodNotAllowedException(_("This method is not allowed"));
throw new MethodNotAllowedException(__("This method is not allowed"));
}
}
@ -42,7 +42,7 @@ class DecayingModelController extends AppController
$decaying_model = $this->DecayingModel->checkAuthorisation($this->Auth->user(), $id, true);
if (!$this->_isSiteAdmin() && !$decModel) {
throw new MethodNotAllowedException(_('No Decaying Model with the provided ID exists, or you are not authorised to edit it.'));
throw new MethodNotAllowedException(__('No Decaying Model with the provided ID exists, or you are not authorised to edit it.'));
}
$this->set('mayModify', true);
$this->set('id', $id);
@ -71,7 +71,7 @@ class DecayingModelController extends AppController
}
if (empty($this->request->data['DecayingModel']['name'])) {
throw new MethodNotAllowedException(_("The model must have a name"));
throw new MethodNotAllowedException(__("The model must have a name"));
}
if ($this->DecayingModel->save($this->request->data)) {
@ -80,7 +80,7 @@ class DecayingModelController extends AppController
$response = array('data' => $saved, 'action' => 'add');
return $this->RestResponse->viewData($response, $this->response->type());
} else {
$this->Flash->success(_('The model has been saved.'));
$this->Flash->success(__('The model has been saved.'));
$this->redirect(array('action' => 'index'));
}
}
@ -93,7 +93,7 @@ class DecayingModelController extends AppController
{
$decayingModel = $this->DecayingModel->checkAuthorisation($this->Auth->user(), $id);
if (!$this->_isSiteAdmin() && !$decModel) {
throw new NotFoundException(_('No Decaying Model with the provided ID exists, or you are not authorised to edit it.'));
throw new NotFoundException(__('No Decaying Model with the provided ID exists, or you are not authorised to edit it.'));
}
$this->set('mayModify', true);
@ -104,21 +104,21 @@ class DecayingModelController extends AppController
$this->request->data['DecayingModel']['parameters'] = array();
} else {
if (!isset($this->request->data['DecayingModel']['parameters']['tau'])) {
$this->Flash->error(_('Invalid parameter `tau`.'));
$this->Flash->error(__('Invalid parameter `tau`.'));
return false;
}
if (!isset($this->request->data['DecayingModel']['parameters']['delta'])) {
$this->Flash->error(_('Invalid parameter `delta`.'));
$this->Flash->error(__('Invalid parameter `delta`.'));
return false;
}
if (!isset($this->request->data['DecayingModel']['parameters']['threshold'])) {
$this->Flash->error(_('Invalid parameter `threshold`.'));
$this->Flash->error(__('Invalid parameter `threshold`.'));
return false;
}
if (isset($this->request->data['DecayingModel']['parameters']['base_score_config']) && $this->request->data['DecayingModel']['parameters']['base_score_config'] != '') {
$encoded = json_decode($this->data['DecayingModel']['parameters']['base_score_config'], true);
if ($encoded === null) {
$this->Flash->error(_('Invalid parameter `base_score_config`.'));
$this->Flash->error(__('Invalid parameter `base_score_config`.'));
return false;
}
$this->request->data['DecayingModel']['parameters']['base_score_config'] = $encoded;
@ -135,7 +135,7 @@ class DecayingModelController extends AppController
$response = array('data' => $saved, 'action' => 'edit');
return $this->RestResponse->viewData($response, $this->response->type());
} else {
$this->Flash->success(_('The model has been saved.'));
$this->Flash->success(__('The model has been saved.'));
$this->redirect(array('action' => 'index'));
}
}
@ -153,14 +153,14 @@ class DecayingModelController extends AppController
if ($this->request->is('post')) {
$decayingModel = $this->DecayingModel->checkAuthorisation($this->Auth->user(), $id);
if (!$this->_isSiteAdmin() && !$decModel) {
throw new MethodNotAllowedException(_('No Decaying Model with the provided ID exists, or you are not authorised to edit it.'));
throw new MethodNotAllowedException(__('No Decaying Model with the provided ID exists, or you are not authorised to edit it.'));
}
if ($this->DecayingModel->delete($id, true)) {
$this->Flash->success(_('Decaying Model deleted.'));
$this->Flash->success(__('Decaying Model deleted.'));
$this->redirect(array('action' => 'index'));
} else {
$this->Flash->error(_('The Decaying Model could not be deleted.'));
$this->Flash->error(__('The Decaying Model could not be deleted.'));
$this->redirect(array('action' => 'index'));
}
}
@ -212,7 +212,7 @@ class DecayingModelController extends AppController
{
$decaying_model = $this->DecayingModel->checkAuthorisation($this->Auth->user(), $model_id);
if (!$decaying_model) {
throw new NotFoundException(_('No Decaying Model with the provided ID exists, or you are not authorised to edit it.'));
throw new NotFoundException(__('No Decaying Model with the provided ID exists, or you are not authorised to edit it.'));
}
if (isset($this->request->params['named']['attribute_id'])) {
$this->set('attribute_id', $this->request->params['named']['attribute_id']);
@ -229,7 +229,7 @@ class DecayingModelController extends AppController
$body = $this->request->data['decayingToolRestSearch']['filters'];
$decoded_body = json_decode($body, true);
if (is_null($decoded_body)) {
throw new Exception(_("Error Processing Request, can't parse the body"));
throw new Exception(__("Error Processing Request, can't parse the body"));
}
$this->request->data = $decoded_body;
$paramArray = array(
@ -356,7 +356,12 @@ class DecayingModelController extends AppController
public function decayingToolComputeSimulation($model_id, $attribute_id)
{
$score_overtime = $this->RestResponse->viewData($this->DecayingModel->getScoreOvertime($this->Auth->user(), $model_id, $attribute_id), $this->response->type());
return $score_overtime;
if (!$this->request->is('ajax')) {
throw new MethodNotAllowedException(__("This method is only accessible via AJAX."));
}
// contain score overtime, sightings, and base_score computation
$results = $this->DecayingModel->getScoreOvertime($this->Auth->user(), $model_id, $attribute_id);
return $this->RestResponse->viewData($results, $this->response->type());
}
}

111
app/Model/DecayingModel.php
View File

@ -234,28 +234,97 @@ class DecayingModel extends AppModel
);
}
// compute the current score for the provided atribute with the provided model
public function computeCurrentScore($model, $attribute)
// get effective taxonomy ratio based on taxonomies attached to the attribute
private function _getRatioScore($model, $tags)
{
$ratioScore = array();
$taxonomy_base_ratio = $model['DecayingModel']['parameters']['base_score_config'];
$total_score = 0.0;
foreach ($tags as $tag) {
$namespace_predicate = explode(':', $tag['Tag']['name'])[0];
$total_score += floatval($taxonomy_base_ratio[$namespace_predicate]);
}
foreach ($tags as $i => $tag) {
$namespace_predicate = explode(':', $tag['Tag']['name'])[0];
$ratioScore[$namespace_predicate] = floatval($taxonomy_base_ratio[$namespace_predicate]) / $total_score;
}
return $ratioScore;
}
// return attribute tag with event tag matching the namespace+predicate overridden
private function getPrioritizedTag($attribute)
{
$tags = array();
$overridden_tags = array();
$temp_mapping = array();
foreach ($attribute['EventTag'] as $i => $tag) {
$tags[] = $tag;
$namespace_predicate = explode('=', $tag['Tag']['name'])[0];
$temp_mapping[$namespace_predicate] = $i;
}
foreach ($attribute['AttributeTag'] as $tag) {
$namespace_predicate = explode('=', $tag['Tag']['name'])[0];
if (isset($temp_mapping[$namespace_predicate])) { // need to override event tag
$overridden_tags[] = array(
'EventTag' => $tags[$temp_mapping[$namespace_predicate]],
'AttributeTag' => $tag
);
$tags[$temp_mapping[$namespace_predicate]] = $tag;
} else {
$tags[] = $tag;
}
}
return array('tags' => $tags, 'overridden' => $overridden_tags);
}
public function computeBasescore($model, $attribute)
{
$temp = $this->getPrioritizedTag($attribute);
$tags = $temp['tags'];
$overridden_tags = $temp['overridden'];
$taxonomy_effective_ratios = $this->_getRatioScore($model, $tags);
$base_score = 0;
foreach ($tags as $k => $tag) {
$taxonomy = explode(':', $tag['Tag']['name'])[0];
$base_score += $taxonomy_effective_ratios[$taxonomy] * $tag['Tag']['numerical_value'];
}
return array('base_score' => $base_score, 'overridden' => $overridden_tags, 'tags' => $tags, 'taxonomy_effective_ratios' => $taxonomy_effective_ratios);
}
// compute the current score for the provided atribute with the provided model
public function computeScore($model, $elapsed_time, $base_score)
{
if ($elapsed_time < 0) {
return 0;
}
$delta = $model['DecayingModel']['parameters']['delta'];
$tau = $model['DecayingModel']['parameters']['tau']*24*60*60;
$score = $base_score * (1 - pow($elapsed_time / $tau, 1 / $delta));
return $score < 0 ? 0 : $score;
}
public function computeCurrentScore($model, $attribute, $base_score = false, $last_sighting_timestamp = false)
{
if ($base_score === false) {
$base_score = $this->computeBasescore($model, $attribute)['base_score'];
}
if ($last_sighting_timestamp === false) {
$last_sighting_timestamp = $this->Sighting->listSightings($user, $attribute_id, 'attribute', false, 0, true)[0]['Sighting']['date_sighting'];
}
$timestamp = time();
return $this->computeScore($model, $timestamp - $last_sighting_timestamp, $base_score);
}
// compute the score at the given timestamp for the provided atribute with the provided model
public function computeScore($model, $attribute, $timestamp, $last_sighting_timestamp)
public function computeScoreForTimestamp($model, $attribute, $timestamp, $last_sighting_timestamp, $base_score)
{
$t = $timestamp - $last_sighting_timestamp;
if ($t < 0) {
return 0;
}
$base_score = 100;
$delta = $model['DecayingModel']['parameters']['delta'];
$tau = $model['DecayingModel']['parameters']['tau']*24*60*60;
$score = $base_score * (1 - pow($t / $tau, 1 / $delta));
// debug($timestamp);
// debug($last_sighting_timestamp);
// debug($tau);
// debug($t);
// debug($score);
return $score < 0 ? 0 : $score;
}
@ -270,12 +339,24 @@ class DecayingModel extends AppModel
{
$this->Attribute = ClassRegistry::init('Attribute');
$attribute = $this->Attribute->fetchAttributesSimple($user, array(
'conditions' => array('id' => $attribute_id)
'conditions' => array('id' => $attribute_id),
'contain' => array('AttributeTag' => array('Tag'))
));
if (empty($attribute)) {
throw new NotFoundException(__('Attribute not found'));
} else {
$attribute = $attribute[0];
$tagConditions = array('EventTag.event_id' => $attribute['Attribute']['event_id']);
$temp = $this->Attribute->Event->EventTag->find('all', array(
'recursive' => -1,
'contain' => array('Tag'),
'conditions' => $tagConditions
));
foreach ($temp as $tag) {
$tag['EventTag']['Tag'] = $tag['Tag'];
unset($tag['Tag']);
$attribute['EventTag'][] = $tag['EventTag'];
}
}
$model = $this->checkAuthorisation($user, $model_id, true);
if ($model === false) {
@ -295,6 +376,8 @@ class DecayingModel extends AppModel
// get end time
$end_time = $sightings[count($sightings)-1]['Sighting']['date_sighting'] + $model['DecayingModel']['parameters']['tau']*24*60*60;
$end_time = $this->round_timestamp_to_hour($end_time);
$base_score_config = $this->computeBasescore($model, $attribute);
$base_score = $base_score_config['base_score'];
// generate time span from oldest timestamp to last decay, resolution is hours
$score_overtime = array();
@ -305,7 +388,8 @@ class DecayingModel extends AppModel
$sighting_index = $this->get_closest_sighting($sightings, $t, $sighting_index);
$last_sighting = $this->round_timestamp_to_hour($sightings[$sighting_index]['Sighting']['date_sighting']);
$sightings[$sighting_index]['Sighting']['rounded_timestamp'] = $last_sighting;
$score_overtime[$t] = $this->computeScore($model, $attribute, $t, $last_sighting);
$elapsed_time = $t - $last_sighting;
$score_overtime[$t] = $this->computeScore($model, $elapsed_time, $base_score);
}
$csv = 'date,value' . PHP_EOL;
foreach ($score_overtime as $t => $v) {
@ -313,7 +397,10 @@ class DecayingModel extends AppModel
}
return array(
'csv' => $csv,
'sightings' => $sightings
'sightings' => $sightings,
'base_score_config' => $base_score_config,
'last_sighting' => $sightings[count($sightings)-1],
'current_score' => $this->computeCurrentScore($model, $attribute, $base_score, $sightings[count($sightings)-1]['Sighting']['date_sighting'])
);
}

21
app/View/DecayingModel/decaying_tool_basescore.ctp
View File

@ -97,26 +97,7 @@
</table>
<h3><?php echo __('Computation steps') ?></h3>
<div id="computation_help_container" style="margin-bottom: 5px; border: 1px solid #dddddd; border-radius: 4px; text-align: center; background-color: white;">
<table class="table histogram-legendH4">
<thead>
<tr>
<th rowspan="2" style="vertical-align: middle;"><?php echo __('Tag') ?></th>
<th colspan="3"><?php echo __('Computation') ?></th>
<th rowspan="2" style="vertical-align: middle;"><?php echo __('Result') ?></th>
</tr>
<tr>
<th style="padding: 0px; width: 90px;"><?php echo __('Taxonomy ratio') ?></th>
<th></th>
<th style="padding: 0px; width: 90px;"><?php echo __('Tag value') ?></th>
</tr>
</thead>
<tbody id="computation_help_container_body">
<tr><td></td><td></td><td></td><td></td><td></td></tr>
</tbody>
</table>
<b id="pick_notice"><?php echo __('Pick an Attribute') ?></b>
</div>
<?php echo $this->element('DecayingModels/View/basescore_computation_steps'); ?>
</div>
<span class="btn btn-primary" onclick="applyBaseScoreConfig();"><i class="fas fa-wrench"> Apply base score</i></span>
</div>

38
app/View/DecayingModel/decaying_tool_simulation.ctp
View File

@ -1,7 +1,7 @@
<div id="simulationContainer">
<div style="padding: 15px; height: 90vh; display: flex; flex-direction: column;">
<div style="height: 40%; display: flex">
<div style="width: 30%; display: flex; flex-direction: column;">
<div style="width: 20%; display: flex; flex-direction: column;">
<div class="panel-container" style="display: flex; flex-direction: column; flex-grow: 1">
<div style="display: flex;">
<select id="select_model_to_simulate" onchange="$('#select_model_to_simulate_infobox').popover('show'); refreshSimulation()" style="flex-grow: 1;">
@ -53,9 +53,24 @@
</div>
</div>
<div style="width: 70%; display: flex;">
<div class="panel-container" style="flex-grow: 1;">
<div id="chart-decay-simulation-container" style="width: 100%; height: 100%; position: relative">
<div style="width: 80%; display: flex;">
<div class="panel-container" style="flex-grow: 1; display: flex;">
<div id="basescore-simulation-container" style="width: 30%; height: 100%;">
<h4><?php echo __('Basescore') ?></h4>
<div style="overflow: auto;">
<?php echo $this->element('DecayingModels/View/basescore_computation_steps'); ?>
</div>
<h4><?php echo __('Current score') ?></h4>
<div style="margin-left: 4px; margin-bottom: 5px;" class="input-prepend">
<span class="add-on" style="width: 100px;"><?php echo __('Last Sighting'); ?></span>
<input id="simulation-sighting" type="text" value="100" class="span2" disabled>
</div>
<div style="margin-left: 4px; margin-bottom: 0px;" class="input-prepend">
<span class="add-on" style="width: 100px;"><?php echo __('Current score'); ?></span>
<input id="simulation-current-score" type="text" value="100" class="span2" disabled>
</div>
</div>
<div id="chart-decay-simulation-container" style="width: 70%; height: 100%; position: relative">
<div id="simulation_chart" style="height: 100%; overflow: hidden;"></div>
</div>
</div>
@ -134,21 +149,36 @@ function doSimulation(clicked, attribute_id) {
$(clicked).addClass('success');
var model_id = $('#select_model_to_simulate').val();
var simulation_chart = $('#simulation_chart').data('DecayingSimulation');
var simulation_table = $('#basescore-simulation-container #computation_help_container_body ').data('BasescoreComputationTable');
if (simulation_chart === undefined) {
simulation_chart = $('#simulation_chart').decayingSimulation({});
simulation_table = $('#basescore-simulation-container #computation_help_container_body ').basescoreComputationTable({});
}
$.ajax({
beforeSend:function() {
simulation_chart.toggleLoading(true);
simulation_table.toggleLoading(true);
},
success:function (data, textStatus) {
simulation_chart.update(data, models[model_id]);
simulation_table.update(data, models[model_id]);
$('#simulation-sighting')
.val(
d3.time.format("%c")(new Date(parseInt(data.last_sighting.Sighting.date_sighting)*1000))
);
$('#simulation-sighting').parent().tooltip({
title: 'From ' + data.last_sighting.Organisation.name,
placement: 'right'
});
$('#simulation-current-score')
.val(data.current_score.toFixed(0))
},
error:function() {
showMessage('fail', '<?php echo __('Failed to perform the simulation') ?>');
},
complete:function() {
simulation_chart.toggleLoading(false);
simulation_table.toggleLoading(false);
},
type:'get',
cache: false,

20
app/View/Elements/DecayingModels/View/basescore_computation_steps.ctp Normal file
View File

@ -0,0 +1,20 @@
<div id="computation_help_container" style="margin-bottom: 5px; border: 1px solid #dddddd; border-radius: 4px; text-align: center; background-color: white;">
<table class="table histogram-legendH4">
<thead>
<tr>
<th rowspan="2" style="vertical-align: middle;"><?php echo __('Tag') ?></th>
<th colspan="3"><?php echo __('Computation') ?></th>
<th rowspan="2" style="vertical-align: middle;"><?php echo __('Result') ?></th>
</tr>
<tr>
<th style="padding: 0px; width: 90px;"><?php echo __('Taxonomy ratio') ?></th>
<th></th>
<th style="padding: 0px; width: 90px;"><?php echo __('Tag value') ?></th>
</tr>
</thead>
<tbody id="computation_help_container_body">
<tr><td></td><td></td><td></td><td></td><td></td></tr>
</tbody>
</table>
<b id="pick_notice"><?php echo __('Pick an Attribute') ?></b>
</div>

108
app/webroot/js/decayingModelSimulation.js
View File

@ -112,6 +112,7 @@
},
update: function(data, model) {
this.raw_data = data;
this.chart_data = data.csv;
this.sightings = data.sightings;
this.model = model;
@ -122,7 +123,7 @@
_draw: function() {
var that = this;
this.x.domain(d3.extent(this.chart_data, function(d) { return d.date; }))
this.y.domain([0, d3.max(this.chart_data, function(d) { return d.value; })])
// this.y.domain([0, d3.max(this.chart_data, function(d) { return d.value; })])
this.xAxis = this.svg.select('.axis-x')
.call(d3.svg.axis().scale(this.x).orient('bottom'));
@ -283,7 +284,7 @@
_generate_tooltip: function(datum) {
var formated_date = d3.time.format("%e %B @ %H:%M")(datum.date);
var html = 'Sighting on ' + formated_date;
var html = 'Sighting on ' + formated_date + ' by ' + datum.org;
return html;
},
@ -301,7 +302,7 @@
}
this.sightings_data = this.sightings.map(function(d) {
var sighting = d.Sighting;
return { timestamp: sighting.rounded_timestamp, date: new Date(sighting.rounded_timestamp*1000), value : 100.0 };
return { timestamp: sighting.rounded_timestamp, date: new Date(sighting.rounded_timestamp*1000), value : that.raw_data.base_score_config.base_score, org: d.Organisation.name };
});
}
}
@ -328,3 +329,104 @@
$.fn.decayingSimulation.constructor = DecayingSimulation;
})
);
(function(factory) {
"use strict";
if (typeof define === 'function' && define.amd) {
define(['jquery'], factory);
} else if (window.jQuery && !window.jQuery.fn.BasescoreComputationTable) {
factory(window.jQuery);
}
}
(function($) {
'use strict';
var BasescoreComputationTable = function(container, options, data) {
this.container_id = '#' + container.id;
this.$container = $(container);
this._validateOptions(options);
var default_options = {
};
this.options = $.extend(true, {}, default_options, options);
this._init();
if (data !== undefined) {
this.update(data)
}
};
BasescoreComputationTable.prototype = {
constructor: BasescoreComputationTable,
_validateOptions: function(options) {
},
_init: function() {
var that = this;
this.$loadingContainer = $('<div id="loadingSimulationContainer" style="background: #ffffff9f"><span class="fa fa-spinner fa-spin" style="font-size: xx-large;"></span></div>').css({
position: 'absolute',
left: '0',
right: '0',
top: '0',
bottom: '0',
display: 'flex',
'align-items': 'center',
'justify-content': 'center'
}).hide();
this.tooltip_container = d3.select('body').append('div')
.classed('tooltip', true)
.style('opacity', 0)
.style('padding', '3px')
.style('background-color', '#000')
.style('color', 'white')
.style('border-radius', '5px')
.style('display', 'none');
this.$container.append(this.$loadingContainer);
},
update: function(data, model) {
},
_draw: function() {
},
toggleLoading: function(state) {
if (state === undefined) {
this.$loadingContainer.toggle();
} else if(state) {
this.$loadingContainer.show();
} else {
this.$loadingContainer.hide();
}
this.$container;
},
_parseDataset: function() {
}
}
$.BasescoreComputationTable = BasescoreComputationTable;
$.fn.basescoreComputationTable = function(options) {
var pickedArgs = arguments;
var $elements = this.each(function() {
var $this = $(this),
inst = $this.data('BasescoreComputationTable');
options = ((typeof options === 'object') ? options : {});
if ((!inst) && (typeof options !== 'string')) {
$this.data('BasescoreComputationTable', new BasescoreComputationTable(this, options));
} else {
if (typeof options === 'string') {
inst[options].apply(inst, Array.prototype.slice.call(pickerArgs, 1));
}
}
});
return $elements.length == 1 ? $elements.data('BasescoreComputationTable') : $elements;
};
$.fn.basescoreComputationTable.constructor = BasescoreComputationTable;
})
);