MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Freetext import tool 

Added freetext import tool
pull/274/head^2
iglocska 2014-07-10 17:02:19 +02:00
parent 029ef252a2
commit 893ef5a129
18 changed files with 312 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
app/Controller/EventsController.php
View File

@ -2279,4 +2279,90 @@ class EventsController extends AppController {
return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Tag could not be removed.')),'status'=>200));
}
}
public function freeTextImport($id) {
if (!$this->userRole['perm_add']) {
throw new MethodNotAllowedException('Event not found or you don\'t have permissions to create attributes');
}
$event = $this->Event->find('first', array(
'conditions' => array('Event.id' => $id),
'fields' => array('id', 'orgc'),
'recursive' => -1
));
if (!$this->_isSiteAdmin() && !empty($event) && $event['Event']['orgc'] != $this->Auth->user('org')) throw new MethodNotAllowedException('Event not found or you don\'t have permissions to create attributes');
$this->set('event_id', $id);
if ($this->request->is('get')) {
$this->layout = 'ajax';
$this->request->data['Attribute']['event_id'] = $id;
}
if ($this->request->is('post')) {
App::uses('ComplexTypeTool', 'Tools');
$complexTypeTool = new ComplexTypeTool();
$resultArray = $complexTypeTool->checkComplexRouter($this->request->data['Attribute']['value'], 'FreeText');
foreach ($resultArray as &$r) {
$temp = array();
foreach ($r['types'] as $type) {
$temp[$type] = $type;
}
$r['types'] = $temp;
}
$typeCategoryMapping = array();
foreach ($this->Event->Attribute->categoryDefinitions as $k => $cat) {
foreach ($cat['types'] as $type) {
$typeCategoryMapping[$type][$k] = $k;
}
}
$defaultCategories = array(
'md5' => 'Payload delivery',
'sha1' => 'Payload delivery',
'sha256' => 'Payload delivery',
'regkey' => 'Persistence mechanism',
'filename' => 'Payload delivery',
'ip-src' => 'Network activity',
'ip-dst' => 'Network activity',
'hostname' => 'Network activity',
'domain' => 'Network activity',
'url' => 'Network activity',
'link' => 'Network activity',
'email-src' => 'Payload delivery',
'email-dst' => 'Payload delivery',
'text' => 'Other',
);
$this->set('defaultCategories', $defaultCategories);
$this->set('typeCategoryMapping', $typeCategoryMapping);
$this->set('resultArray', $resultArray);
//$this->autoRender = false;
$this->render('free_text_results');
}
}
public function saveFreeText($id) {
if ($this->request->is('post')) {
$event = $this->Event->find('first', array(
'conditions' => array('id' => $id),
'recursive' => -1,
'fields' => array('orgc', 'id', 'distribution'),
));
$saved = 0;
$failed = 0;
foreach ($this->request->data['Attribute'] as $k => $attribute) {
if ($attribute['save'] == '1') {
$this->Event->Attribute->create();
$attribute['distribution'] = $event['Event']['distribution'];
$attribute['comment'] = 'Imported via the freetext import.';
$attribute['event_id'] = $id;
if ($this->Event->Attribute->save($attribute)) {
$saved++;
} else {
$failed++;
}
}
}
$this->Session->setFlash($saved . ' attributes created. ' . $failed . ' attributes could not be saved. This may be due to attributes with similar values already existing.');
$this->redirect(array('controller' => 'events', 'action' => 'view', $id));
} else {
throw new MethodNotAllowedException();
}
}
}

71
app/Lib/Tools/ComplexTypeTool.php
View File

@ -9,6 +9,9 @@ class ComplexTypeTool {
case 'CnC':
return $this->checkComplexCnC($input);
break;
case 'FreeText':
return $this->checkFreetext($input);
break;
default:
return false;
}
@ -55,4 +58,72 @@ class ComplexTypeTool {
if (!preg_match("#\n#", $input)) return array('type' => 'url', 'value' => $input);
return array('type' => 'other', 'value' => $input);
}
public function checkFreeText($input) {
$iocArray = preg_split("/[\n,]+/", $input);
$resultArray = array();
foreach ($iocArray as $ioc) {
$ioc = trim($ioc);
$typeArray = $this->__resolveType($ioc);
$temp = $typeArray;
$temp['value'] = $ioc;
$resultArray[] = $temp;
}
return $resultArray;
}
private function __resolveType($input) {
$result = array();
$input = strtolower($input);
// check for hashes
if (strlen($input) == 32 && preg_match("#[0-9a-f]{32}$#", $input)) return array('types' => array('md5'), 'to_ids' => true, 'default_type' => 'md5');
if (strlen($input) == 40 && preg_match("#[0-9a-f]{40}$#", $input)) return array('types' => array('sha1'), 'to_ids' => true, 'default_type' => 'sha1');
if (strlen($input) == 64 && preg_match("#[0-9a-f]{64}$#", $input)) return array('types' => array('sha256'), 'to_ids' => true, 'default_type' => 'sha256');
// check for IP
if (filter_var($input, FILTER_VALIDATE_IP)) return array('types' => array('ip-dst', 'ip-src'), 'to_ids' => true, 'default_type' => 'ip-dst');
// check for domain name, hostname, filename
if (strpos($input, '.') !== false) {
$extra = '';
$temp = explode('.', $input);
if (strpos($temp[0], ':')) {
$extra = '([a-z0-9]+):\/\/';
}
// check if it is a URL
if (filter_var($input, FILTER_VALIDATE_URL)) {
return array('types' => array('url'), 'to_ids' => true, 'default_type' => 'url');
}
//if (filter_var($input, FILTER_VALIDATE_URL)) {
if (preg_match('/^([-\pL\pN]+\.)+([a-z][a-z]|biz|cat|com|edu|gov|int|mil|net|org|pro|tel|aero|arpa|asia|coop|info|jobs|mobi|name|museum|travel)$/u', $input)) {
if (count($temp) > 2) {
return array('types' => array('hostname', 'domain'), 'to_ids' => true, 'default_type' => 'hostname');
} else {
return array('types' => array('domain'), 'to_ids' => true, 'default_type' => 'domain');
}
} else {
if (!preg_match('/[?:<>|\\*:\/@]/', $input)) {
return array('types' => array('filename'), 'to_ids' => true, 'default_type' => 'filename');
}
}
}
if (strpos($input, '\\') !== false) {
$temp = explode('\\', $input);
if (strpos($temp[count($temp)-1], '.')) {
if (!preg_match('/[?:<>|\\*:\/]/', $temp[count($temp)-1])) {
return array('types' => array('filename'), 'category' => 'Payload installation', 'to_ids' => false, 'default_type' => 'filename');
}
} else {
return array('types' => array('regkey'), 'to_ids' => false, 'default_type' => 'regkey');
}
}
if (strpos($input, '@') !== false) {
if (filter_var($input, FILTER_VALIDATE_EMAIL)) return array('types' => array('email-src', 'email-dst'), 'to_ids' => true, 'default_type' => 'email-src');
}
return array('types' => array('text'), 'category' => 'Other', 'to_ids' => false, 'default_type' => 'text');
}
}

2
app/View/Attributes/add.ctp
View File

@ -2,8 +2,8 @@
<?php
echo $this->Form->create('Attribute', array('id'));
?>
<legend><?php echo __('Add Attribute'); ?></legend>
<fieldset>
<legend><?php echo __('Add Attribute'); ?></legend>
<div id="formWarning" class="message ajaxMessage"></div>
<div class="add_attribute_fields">
<?php

3
app/View/Attributes/ajax/attributeEditMassForm.ctp
View File

@ -1,10 +1,9 @@
<? echo $this->Html->script('ajaxification');?>
<div class="attributes">
<?php
echo $this->Form->create('Attribute', array('action' => 'editSelected'));
?>
<legend><?php echo __('Mass Edit Attributes'); ?></legend>
<fieldset>
<legend><?php echo __('Mass Edit Attributes'); ?></legend>
<div id="formWarning" class="message ajaxMessage"></div>
<div class="add_attribute_fields">
<?php

4
app/View/Elements/eventattribute.ctp
View File

@ -1,5 +1,4 @@
<?php
echo $this->Html->script('ajaxification');
$mayModify = ($isSiteAdmin || ($isAclModify && $event['Event']['user_id'] == $me['id'] && $event['Event']['orgc'] == $me['org']) || ($isAclModifyOrg && $event['Event']['orgc'] == $me['org']));
$mayPublish = ($isAclPublish && $event['Event']['orgc'] == $me['org']);
$pageCount = intval($objectCount / 50);
@ -64,7 +63,8 @@
</div>
<?php if ($mayModify): ?>
<div class="tabMenu tabMenuToolsBlock noPrint">
<span id="create-button" title="Populate using a template" class="icon-list-alt useCursorPointer" onClick="getTemplateChoicePopup(<?php echo $event['Event']['id']; ?>);"></span>
<span id="create-button" title="Populate using a template" class="icon-list-alt useCursorPointer" onClick="getPopup(<?php echo $event['Event']['id']; ?>, 'templates', 'templateChoices');"></span>
<span id="freetext-button" title="Populate using the freetext import tool" class="icon-exclamation-sign useCursorPointer" onClick="getPopup(<?php echo $event['Event']['id']; ?>, 'events', 'freeTextImport');"></span>
</div>
<?php endif; ?>
<table class="table table-striped table-condensed">

50
app/View/Events/free_text_import.ctp Normal file
View File

@ -0,0 +1,50 @@
<div class="freetext">
<?php
echo $this->Form->create('Attribute', array('id'));
?>
<fieldset>
<legend><?php echo __('Freetext Import Tool'); ?></legend>
<div class="add_attribute_fields">
<p>Paste a list of IOCs into the field below for automatic detection.</p>
<?php
echo $this->Form->hidden('event_id');
echo $this->Form->input('value', array(
'type' => 'textarea',
'error' => array('escape' => false),
'div' => 'input clear',
'class' => 'input-xxlarge',
'label' => false
));
?>
<div class="input clear"></div>
</div>
</fieldset>
<p style="color:red;font-weight:bold;display:none;" id="warning-message">Warning: You are about to share data that is of a classified nature (Attribution / targeting data). Make sure that you are authorised to share this.</p>
<div class="overlay_spacing">
<table>
<tr>
<td style="vertical-align:top">
<button id="submitButton" class="btn btn-primary">Submit</button>
</td>
<td style="width:540px;">
<p style="color:red;font-weight:bold;display:none;text-align:center" id="warning-message">Warning: You are about to share data that is of a classified nature (Attribution / targeting data). Make sure that you are authorised to share this.</p>
</td>
<td style="vertical-align:top;">
<span class="btn btn-inverse" id="cancel_attribute_add">Cancel</span>
</td>
</tr>
</table>
</div>
<?php
echo $this->Form->end();
?>
</div>
<script type="text/javascript">
$(document).ready(function() {
$('#cancel_attribute_add').click(function() {
cancelPopoverForm();
});
});
</script>
<?php echo $this->Js->writeBuffer(); // Write cached scripts

88
app/View/Events/free_text_results.ctp Normal file
View File

@ -0,0 +1,88 @@
<div class="index">
<h2>Freetext Import Results</h2>
<p>Below you can see the attributes that are to be created based on the results of the free-text import. Make sure that the categories and the types are correct, often several options will be offered based on an inconclusive automatic resolution. </p>
<table class="table table-striped table-hover table-condensed">
<tr>
<th>Value</th>
<th>Category</th>
<th>Type</th>
<th>IDS</th>
<th>Actions</th>
</tr>
<?php
echo $this->Form->create('Attribute', array('url' => '/events/saveFreeText/' . $event_id));
foreach ($resultArray as $k => $item):
?>
<tr id="row_<?php echo $k; ?>">
<?php
echo $this->Form->input('Attribute.' . $k . '.save', array(
'label' => false,
'style' => 'display:none;',
'value' => 1,
));
echo $this->Form->input('Attribute.' . $k . '.value', array(
'label' => false,
'type' => 'hidden',
'value' => $item['value'],
));
?>
<td><?php echo h($item['value']); ?></td>
<td class="short">
<?php
if (!isset($item['category'])) {
$default = array_search($defaultCategories[$item['default_type']], $typeCategoryMapping[$item['default_type']]);
} else {
$default = array_search($item['category'], $typeCategoryMapping[$item['default_type']]);
}
echo $this->Form->input('Attribute.' . $k . '.category', array(
'label' => false,
'style' => 'padding:0px;height:20px;margin-bottom:0px;',
'options' => $typeCategoryMapping[$item['default_type']],
'value' => $default,
));
?>
</td>
<td class="short">
<?php
if (count($item['types']) == 1) {
echo h($item['default_type']);
echo $this->Form->input('Attribute.' . $k . '.type', array(
'label' => false,
'type' => 'hidden',
'value' => $item['default_type'],
));
} else {
echo $this->Form->input('Attribute.' . $k . '.type', array(
'label' => false,
'style' => 'padding:0px;height:20px;margin-bottom:0px;',
'options' => $item['types'],
'value' => $item['default_type'],
));
}
?>
</td>
<td class="short">
<?php
echo $this->Form->input('Attribute.' . $k . '.to_ids', array(
'label' => false,
'type' => 'checkbox',
'checked' => $item['to_ids'],
));
?>
</td>
<td class="action short">
<span class="icon-remove pointer" onClick="freetextRemoveRow('<?php echo $k; ?>');"></span>
</td>
</tr>
<?php
endforeach;
?>
</table>
<?php
echo $this->Form->button('Submit', array('class' => 'btn btn-inverse'));
echo $this->Form->end();
?>
</div>
<?php
echo $this->element('side_menu', array('menuList' => 'regexp', 'menuItem' => 'index'));
?>

1
app/View/Events/view.ctp
View File

@ -3,7 +3,6 @@ $mayModify = (($isAclModify && $event['Event']['user_id'] == $me['id'] && $event
$mayPublish = ($isAclPublish && $event['Event']['orgc'] == $me['org']);
?>
<?php
echo $this->Html->script('ajaxification');
echo $this->element('side_menu', array('menuList' => 'event', 'menuItem' => 'viewEvent', 'mayModify' => $mayModify, 'mayPublish' => $mayPublish));
?>
<div class="events view">

1
app/View/Layouts/default.ctp
View File

@ -65,6 +65,7 @@
echo $this->Html->script('bootstrap-datepicker');
echo $this->Html->script('bootstrap-colorpicker.min');
echo $this->Html->script('main');
echo $this->Html->script('ajaxification');
?>
</div>
<div id = "ajax_success_container" class="ajax_container">

1
app/View/Layouts/iframe.ctp
View File

@ -20,4 +20,5 @@
echo $this->Html->css('bootstrap');
echo $this->Html->css('main');
echo $this->Html->script('jquery-2.1.0.min');
echo $this->Html->script('ajaxification');
echo $content_for_layout; ?>

3
app/View/ShadowAttributes/add.ctp
View File

@ -1,8 +1,7 @@
<? echo $this->Html->script('ajaxification');?>
<div class="shadow_attributes <? if (!$ajax) echo 'form';?>">
<?php echo $this->Form->create('ShadowAttribute');?>
<legend><?php echo __('Add Proposal'); ?></legend>
<fieldset>
<legend><?php echo __('Add Proposal'); ?></legend>
<div id="formWarning" class="message ajaxMessage"></div>
<div class="add_attribute_fields">
<?php

1
app/View/Templates/add.ctp
View File

@ -1,6 +1,5 @@
<div class="templates form">
<?php
echo $this->Html->script('ajaxification');
echo $this->Form->create('Template');
?>
<fieldset>

1
app/View/Templates/edit.ctp
View File

@ -1,6 +1,5 @@
<div class="templates form">
<?php
echo $this->Html->script('ajaxification');
echo $this->Form->create('Template');
?>
<fieldset>

1
app/View/Templates/populate_event_from_template.ctp
View File

@ -1,4 +1,3 @@
<?php echo $this->Html->script('ajaxification'); ?>
<div class="populate_from_template form">
<?php echo $this->Form->create('', array('type' => 'file'));?>
<fieldset>

3
app/View/Templates/upload_file.ctp
View File

@ -1,5 +1,4 @@
<?php
echo $this->Html->script('ajaxification');
<?php
if ($batch == 'yes') {
$buttonText = 'Upload Files';
$multiple = true;

3
app/View/Templates/view.ctp
View File

@ -1,7 +1,4 @@
<div class="templates view">
<?php
echo $this->Html->script('ajaxification');
?>
<h2><?php echo __('Template');?></h2>
<dl>
<dt><?php echo __('Id'); ?></dt>

1
app/View/Users/memberslist.ctp
View File

@ -1,4 +1,3 @@
<?php echo $this->Html->script('ajaxification'); ?>
<div class="users index">
<h2>Members</h2>
<table class="table table-striped table-condensed table-bordered" style="width:300px;">

11
app/webroot/js/ajaxification.js
View File

@ -742,7 +742,7 @@ function templateElementFileCategoryChange(category) {
}
}
function getTemplateChoicePopup(id) {
function getPopup(id, context, target) {
$("#gray_out").fadeIn();
$.ajax({
beforeSend: function (XMLHttpRequest) {
@ -755,7 +755,8 @@ function getTemplateChoicePopup(id) {
$("#popover_form").html(data);
$("#popover_form").fadeIn();
},
url:"/templates/templateChoices/" + id,
url:"/" + context + "/" + target + "/" + id,
//url:"/templates/templateChoices/" + id,
});
}
@ -856,3 +857,9 @@ function templateDeleteFileBubble(filename, tmp_name, element_id, context, batch
function templateFileUploadTriggerBrowse(id) {
$('#upload_' + id + '_file').click();
}
function freetextRemoveRow(id) {
$('#row_' + id).hide();
$('#Attribute' + id + 'Save').attr("value", "0");
}