MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

new: Added event enrichment functionality 

- select and run a set of enrichments on all applicable attributes of the event
- exposed to the API
- exposed to the command line tool
- adheres to attribute distributions
pull/3215/head
iglocska 2018-04-24 16:41:09 +02:00
parent 42a5aaf5e6
commit 2af8bfec4e
8 changed files with 215 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
app/Console/Command/EventShell.php
View File

@ -479,4 +479,42 @@ class EventShell extends AppShell
$log->createLogEntry($user, 'publish', 'Event', $id, 'Event (' . $id . '): published.', 'published () => (1)');
}
public function enrichment() {
file_put_contents('/var/www/MISP4/app/tmp/test', "0");
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
die('Usage: ' . $this->Server->command_line_functions['enrichment'] . PHP_EOL);
}
$userId = $this->args[0];
$user = $this->User->getAuthUser($userId);
if (empty($user)) die('Invalid user.');
$eventId = $this->args[1];
$modules = $this->args[2];
try {
$modules = json_decode($modules);
} catch (Exception $e) {
die('Invalid module JSON');
}
if (!empty($this->args[3])) {
$jobId = $this->args[3];
} else {
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'enrichment',
'job_input' => 'Event: ' . $eventId . ' modules: ' . $modules,
'status' => 0,
'retries' => 0,
'org' => $user['Organisation']['name'],
'message' => 'Enriching event.',
);
$this->Job->save($data);
$jobId = $this->Job->id;
}
$options = array(
'user' => $user,
'event_id' => $eventId,
'modules' => $modules
);
$result = $this->Event->enrichment($options);
}
}

38
app/Controller/EventsController.php
View File

@ -4877,4 +4877,42 @@ class EventsController extends AppController {
$this->set('event', $event);
}
}
public function enrichEvent($id) {
if (Validation::uuid($id)) {
$conditions = array('Event.uuid' => $id);
} else {
$conditions = array('Event.id' => $id);
}
$event = $this->Event->find('first', array('conditions' => $conditions, 'recursive' => -1));
if (empty($event) || (!$this->_isSiteAdmin() && ($this->Auth->user('org_id') != $event['Event']['orgc_id'] || !$this->userRole['perm_modify']))) {
throw new MethodNotAllowedException('Invalid Event');
}
if ($this->request->is('post')) {
$modules = array();
foreach ($this->request->data['Event'] as $module => $enabled) {
if ($enabled) $modules[] = $module;
}
$result = $this->Event->enrichmentRouter(array(
'user' => $this->Auth->user(),
'event_id' => $event['Event']['id'],
'modules' => $modules
));
if ($this->_isRest()) {
} else {
if ($result === true) {
$result = __('Enrichment task queued for background processing. Check back later to see the results.');
}
$this->Session->setFlash($result);
$this->redirect('/events/view/' . $id);
}
} else {
$this->loadModel('Module');
$modules = $this->Module->getEnabledModules($this->Auth->user(), 'expansion');
$this->layout = 'ajax';
$this->set('modules', $modules);
$this->render('ajax/enrich_event');
}
}
}

4
app/Model/Attribute.php
View File

@ -961,10 +961,10 @@ class Attribute extends AppModel {
break;
case 'hostname':
case 'domain':
if (preg_match("#^[A-Z0-9.\-_]+\.[A-Z0-9\-]{2,}$#i", $value)) {
if (preg_match("#^[A-Z0-9.\-_]+\.[A-Z0-9\-]{2,}[\.]?$#i", $value)) {
$returnValue = true;
} else {
$returnValue = 'Domain name has an invalid format. Please double check the value or select type "other".';
$returnValue = ucfirst($type) . ' name has an invalid format. Please double check the value or select type "other".';
}
break;
case 'hostname|port':

94
app/Model/Event.php
View File

@ -4164,4 +4164,98 @@ class Event extends AppModel {
return $response;
}
}
public function enrichmentRouter($options) {
if (Configure::read('MISP.background_jobs')) {
$job = ClassRegistry::init('Job');
$job->create();
$this->ResqueStatus = new ResqueStatus\ResqueStatus(Resque::redis());
$workers = $this->ResqueStatus->getWorkers();
$workerType = 'default';
foreach ($workers as $worker) {
if ($worker['queue'] === 'prio') {
$workerType = 'prio';
}
}
$data = array(
'worker' => $workerType,
'job_type' => 'enrichment',
'job_input' => 'Event ID: ' . $options['event_id'] . ' modules: ' . json_encode($options['modules']),
'status' => 0,
'retries' => 0,
'org_id' => $options['user']['org_id'],
'org' => $options['user']['Organisation']['name'],
'message' => 'Enriching event.',
);
$job->save($data);
$jobId = $job->id;
$process_id = CakeResque::enqueue(
'prio',
'EventShell',
array('enrichment', $options['user']['id'], $options['event_id'], json_encode($options['modules']), $jobId),
true
);
$job->saveField('process_id', $process_id);
return true;
} else {
$result = $this->enrichment($options);
return __('#' . $result . ' attributes have been created during the enrichment process.');
}
}
public function enrichment($params) {
$option_fields = array('user', 'event_id', 'modules');
foreach ($option_fields as $option_field) {
if (empty($params[$option_field])) {
throw new MethodNotAllowedException(__('%s not set', $params[$option_field]));
}
}
$event = $this->fetchEvent($params['user'], array('eventid' => $params['event_id'], 'includeAttachments' => 1, 'flatten' => 1));
$this->Module = ClassRegistry::init('Module');
$enabledModules = $this->Module->getEnabledModules($params['user']);
if (empty($enabledModules)) return true;
$options = array();
foreach ($enabledModules['modules'] as $k => $temp) {
if (isset($temp['meta']['config'])) {
$settings = array();
foreach ($temp['meta']['config'] as $conf) {
$settings[$conf] = Configure::read('Plugin.Enrichment_' . $temp['name'] . '_' . $conf);
}
$enabledModules['modules'][$k]['config'] = $settings;
}
}
if (empty($event)) throw new MethodNotAllowedException('Invalid event.');
$attributes_added = 0;
foreach ($event[0]['Attribute'] as $attribute) {
foreach ($enabledModules['modules'] as $module) {
if (in_array($module['name'], $params['modules'])) {
if (in_array($attribute['type'], $module['mispattributes']['input'])) {
$data = array('module' => $module['name'], $attribute['type'] => $attribute['value'], 'event_id' => $attribute['event_id'], 'attribute_uuid' => $attribute['uuid']);
if (!empty($module['config'])) $data['config'] = $module['config'];
$data = json_encode($data);
$result = $this->Module->queryModuleServer('/query', $data, false, 'Enrichment');
if (!$result) throw new MethodNotAllowedException($type . ' service not reachable.');
if (isset($result['error'])) $this->Session->setFlash($result['error']);
if (!is_array($result)) throw new Exception($result);
$attributes = $this->handleModuleResult($result, $attribute['event_id']);
foreach ($attributes as $a) {
$this->Attribute->create();
$a['distribution'] = $attribute['distribution'];
$a['sharing_group_id'] = $attribute['sharing_group_id'];
$comment = 'Attribute #' . $attribute['id'] . ' enriched by ' . $module['name'] . '.';
if (!empty($a['comment'])) {
$a['comment'] .= PHP_EOL . $comment;
} else {
$a['comment'] = $comment;
}
$a['type'] = empty($a['default_type']) ? $a['types'][0] : $a['default_type'];
$result = $this->Attribute->save($a);
if ($result) $attributes_added++;
}
}
}
}
}
return $attributes_added;
}
}

2
app/Model/Module.php
View File

@ -89,7 +89,7 @@ class Module extends AppModel {
$modules = $this->getModules($type, $moduleFamily);
if (is_array($modules)) {
foreach ($modules['modules'] as $k => $module) {
if (!Configure::read('Plugin.' . $moduleFamily . '_' . $module['name'] . '_enabled') || ($type && in_array(strtolower($type), $module['meta']['module-type']))) {
if (!Configure::read('Plugin.' . $moduleFamily . '_' . $module['name'] . '_enabled') || ($type && !in_array(strtolower($type), $module['meta']['module-type']))) {
unset($modules['modules'][$k]);
continue;
}

9
app/Model/Server.php
View File

@ -43,11 +43,7 @@ class Server extends AppModel {
'url' => array( // TODO add extra validation to refuse multiple time the same url from the same org
'url' => array(
'rule' => array('url'),
'message' => 'Please enter a valid base-url.',
//'allowEmpty' => false,
//'required' => false,
//'last' => false, // Stop validation after this rule
//'on' => 'create', // Limit validation to 'create' or 'update' operations
'message' => 'Please enter a valid base-url.'
)
),
'authkey' => array(
@ -113,7 +109,8 @@ class Server extends AppModel {
'pull' => 'MISP/app/Console/cake Server pull [user_id] [server_id] [full|update]',
'push' => 'MISP/app/Console/cake Server push [user_id] [server_id]',
'cacheFeed' => 'MISP/app/Console/cake Server cacheFeed [user_id] [feed_id|all|csv|text|misp]',
'fetchFeed' => 'MISP/app/Console/cake Server fetchFeed [user_id] [feed_id|all|csv|text|misp]'
'fetchFeed' => 'MISP/app/Console/cake Server fetchFeed [user_id] [feed_id|all|csv|text|misp]',
'enrichment' => 'MISP/app/Console/cake Event enrichEvent [user_id] [event_id] [json_encoded_module_list]'
);
public $serverSettings = array(

1
app/View/Elements/side_menu.ctp
View File

@ -42,6 +42,7 @@
<?php if ($menuItem === 'populateFromtemplate'): ?>
<li class="active"><a href="<?php echo $baseurl;?>/templates/populateEventFromTemplate/<?php echo $template_id . '/' . h($event['Event']['id']); ?>"><?php echo __('Populate From Template');?></a></li>
<?php endif; ?>
<li id='lienrichEvent'><a href="#" onClick="genericPopup('<?php echo $baseurl?>/events/enrichEvent/<?php echo h($event['Event']['id']); ?>', '#confirmation_box');" style="cursor:pointer;"><?php echo __('Enrich event');?></a></li>
<li id='merge'><a href="<?php echo $baseurl;?>/events/merge/<?php echo h($event['Event']['id']);?>"><?php echo __('Merge attributes from…');?></a></li>
<?php endif; ?>
<?php if (($isSiteAdmin && (!isset($mayModify) || !$mayModify)) || (!isset($mayModify) || !$mayModify)): ?>

38
app/View/Events/ajax/enrich_event.ctp Normal file
View File

@ -0,0 +1,38 @@
<div class="confirmation">
<legend><?php echo __('Enrich Event'); ?></legend>
<div style="padding-left:5px;padding-right:5px;padding-bottom:5px;">
<p><?php echo __('Select the enrichments you wish to run');?></p>
<?php
echo $this->Form->create('', array('style' => 'margin-bottom:0px;'));
foreach ($modules['modules'] as $module) {
echo $this->Form->input($module['name'], array('type' => 'checkbox', 'label' => h($module['name'])));
}
?>
<table>
<tr>
<td style="vertical-align:top">
<?php
echo $this->Form->submit('Enrich', array('class' => 'btn btn-primary'));
?>
</td>
<td style="width:540px;">
</td>
<td style="vertical-align:top;">
<span role="button" tabindex="0" aria-label="<?php echo __('Cancel');?>" title="<?php echo __('Cancel');?>" class="btn btn-inverse" id="PromptNoButton" onClick="cancelPrompt();"><?php echo __('Cancel');?></span>
</td>
</tr>
</table>
<?php
echo $this->Form->end();
?>
</div>
</div>
<script type="text/javascript">
$(document).ready(function() {
resizePopoverBody();
});
$(window).resize(function() {
resizePopoverBody();
});
</script>