mirror of https://github.com/MISP/MISP
chg: Refactor of the Bro export
Iglocska
parent
85879e735c
commit
59ecf40f42
|
|
@ -317,7 +317,8 @@ class EventShell extends AppShell
|
|||
} else {
|
||||
$zipname = DS . 'misp.bro.' . $user['Organisation']['name'] . '.intel.zip';
|
||||
}
|
||||
$tmpZipname = DS . "bro_export_tmp.zip";
|
||||
$random_component = $this->Event->generateRandomFileName();
|
||||
$tmpZipname = DS . "bro_export_tmp_" . $random_component . ".zip";
|
||||
$zip = new File($dir->pwd() . $tmpZipname);
|
||||
foreach ($types as $k => $type) {
|
||||
$final = $this->Attribute->bro($user, $type);
|
||||
|
|
@ -332,7 +333,7 @@ class EventShell extends AppShell
|
|||
|
||||
$execRetval = '';
|
||||
$execOutput = array();
|
||||
exec('zip -gj ' . $zip->path . ' ' . $dir->pwd() . '/' . $filename,
|
||||
exec('zip -gj ' . $zip->path . ' ' . $dir->pwd() . '/' . $filename,
|
||||
$execOutput, $execRetval);
|
||||
if ($execRetval != 0) { // not EXIT_SUCCESS
|
||||
throw new Exception('An error has occured while attempting to zip the intel files.');
|
||||
|
|
|
|||
|
|
@ -1871,7 +1871,11 @@ class AttributesController extends AppController {
|
|||
}
|
||||
}
|
||||
$this->response->type('txt'); // set the content type
|
||||
$this->header('Content-Disposition: download; filename="misp.' . $type . '.intel"');
|
||||
$filename = 'misp.' . $type . '.intel';
|
||||
if ($eventId) {
|
||||
$filename = 'misp.' . $type . '.event_' . $eventId . '.intel';
|
||||
}
|
||||
$this->header('Content-Disposition: download; filename="' . $filename . '"');
|
||||
$this->layout = 'text/default';
|
||||
$attributes = array("#fields indicator\tindicator_type\tmeta.source\tmeta.url\tmeta.do_notice\tmeta.if_in");
|
||||
$attributes = array_merge($attributes, $this->Attribute->bro($this->Auth->user(), $type, $tags, $eventId, $allowNonIDS, $from, $to, $last));
|
||||
|
|
|
|||
|
|
@ -1719,7 +1719,6 @@ class EventsController extends AppController {
|
|||
if ($last) $last = $this->Event->resolveTimeDelta($last);
|
||||
// backwards compatibility, swap key and format
|
||||
if ($format != 'snort' && $format != 'suricata') {
|
||||
$key = $format;
|
||||
$format = 'suricata'; // default format
|
||||
}
|
||||
$this->response->type('txt'); // set the content type
|
||||
|
|
@ -3142,10 +3141,10 @@ class EventsController extends AppController {
|
|||
'checkbox' => false,
|
||||
),
|
||||
'bro' => array(
|
||||
'url' => '/events/bro/download/' . $id,
|
||||
'text' => 'Download Bro rules',
|
||||
'requiresPublished' => true,
|
||||
'checkbox' => false,
|
||||
'url' => '/attributes/bro/download/all/false/' . $id,
|
||||
'text' => 'Download Bro rules',
|
||||
'requiresPublished' => true,
|
||||
'checkbox' => false
|
||||
),
|
||||
'text' => array(
|
||||
'url' => '/attributes/text/download/all/false/' . $id,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,176 @@
|
|||
<?php
|
||||
|
||||
class BroExport {
|
||||
|
||||
public $rules = array();
|
||||
|
||||
// mapping from misp attribute type to the bro intel type
|
||||
// alternative mechanisms are:
|
||||
// - alternate: array containing a detection regex and a replacement bro type
|
||||
// - composite: for composite misp attributes (domain|ip), use the provided bro type if the second value is queried
|
||||
// - replace: run a replacement regex on the value before generating the bro rule
|
||||
private $mapping = array(
|
||||
'ip-dst' => array('brotype' => 'ADDR', 'alternate' => array('#/#', 'SUBNET')),
|
||||
'ip-src' => array('brotype' => 'ADDR', 'alternate' => array('#/#', 'SUBNET')),
|
||||
'email-src' => array('brotype' => 'EMAIL'),
|
||||
'email-dst' => array('brotype' => 'EMAIL'),
|
||||
'email-attachment' => array('brotype' => 'FILE_NAME'),
|
||||
'filename' => array('brotype' => 'FILE_NAME'),
|
||||
'hostname' => array('brotype' => 'DOMAIN'),
|
||||
'domain' => array('brotype' => 'DOMAIN'),
|
||||
'domain|ip' => array('brotype' => 'DOMAIN', 'composite' => 'ADDR'),
|
||||
'url' => array('brotype' => 'URL', 'replace' => array('#^https?://#', '')),
|
||||
'user-agent' => array('brotype' => 'SOFTWARE'),
|
||||
'md5' => array('brotype' => 'FILE_HASH'),
|
||||
'malware-sample' => array('brotype' => 'FILE_NAME', 'composite' => 'FILE_HASH'),
|
||||
'filename|md5' => array('brotype' => 'FILE_NAME', 'composite' => 'FILE_HASH'),
|
||||
'sha1' => array('brotype' => 'FILE_HASH'),
|
||||
'filename|sha1' => array('brotype' => 'FILE_NAME', 'composite' => 'FILE_HASH'),
|
||||
'sha256' => array('brotype' => 'FILE_HASH'),
|
||||
'filename|sha256' => array('brotype' => 'FILE_NAME', 'composite' => 'FILE_HASH'),
|
||||
'x509-fingerprint-sha1' => array('brotype' => 'CERT_HASH'),
|
||||
);
|
||||
|
||||
// export group to misp type mapping
|
||||
// the mapped type is in an array format, first value being the misp type, second being the value field used
|
||||
private $__mispTypes = array(
|
||||
'ip' => array(
|
||||
array('ip-src', 1),
|
||||
array('ip-dst', 1),
|
||||
array('domain|ip', 2)
|
||||
),
|
||||
'url' => array(
|
||||
array('url', 1)
|
||||
),
|
||||
'domain' => array(
|
||||
array('hostname', 1),
|
||||
array('domain', 1),
|
||||
array('domain|ip', 1)
|
||||
),
|
||||
'email' => array(
|
||||
array('email-src', 1),
|
||||
array('email-dst', 1)
|
||||
),
|
||||
'filename' => array(
|
||||
array('filename', 1),
|
||||
array('email-attachment', 1),
|
||||
array('attachment', 1),
|
||||
array('filename|md5', 1),
|
||||
array('filename|sha1', 1),
|
||||
array('filename|sha256', 1),
|
||||
array('malware-sample', 1)
|
||||
),
|
||||
'filehash' => array(
|
||||
array('md5', 1),
|
||||
array('sha1', 1),
|
||||
array('sha256', 1),
|
||||
array('filename|md5', 2),
|
||||
array('filename|sha1', 2),
|
||||
array('filename|sha256', 2),
|
||||
array('malware-sample', 2)
|
||||
),
|
||||
'certhash' => array(
|
||||
array('x509-fingerprint-sha1', 1)
|
||||
),
|
||||
'software' => array(
|
||||
array('user-agent', 1)
|
||||
)
|
||||
);
|
||||
|
||||
private $whitelist = null;
|
||||
|
||||
public function export($items, $orgs, $valueField, $intel) {
|
||||
$this->Whitelist = ClassRegistry::init('Whitelist');
|
||||
$this->whitelist = $this->Whitelist->getBlockedValues();
|
||||
//For bro format organisation
|
||||
$orgsName = array();
|
||||
// generate the rules
|
||||
foreach ($items as $item) {
|
||||
if (!isset($orgs[$item['Event']['orgc_id']])) {
|
||||
continue;
|
||||
} else {
|
||||
$orgName = $orgs[$item['Event']['orgc_id']];
|
||||
}
|
||||
$ruleFormatReference = Configure::read('MISP.baseurl') . '/events/view/' . $item['Event']['id'];
|
||||
$ruleFormat = "%s\t%s\t" . $orgName . "\t" . $ruleFormatReference . "\t%s\t%s";
|
||||
$rule = $this->__generateRule($item['Attribute'], $ruleFormat, $valueField);
|
||||
if (!empty($rule)) {
|
||||
if (!in_array($rule, $intel)) {
|
||||
$intel[] = $rule;
|
||||
}
|
||||
}
|
||||
}
|
||||
return $intel;
|
||||
}
|
||||
|
||||
private function __generateRule($attribute, $ruleFormat, $valueField) {
|
||||
if (isset($this->mapping[$attribute['type']])) {
|
||||
$brotype = $this->mapping[$attribute['type']]['brotype'];
|
||||
$overruled = $this->checkWhitelist($attribute['value']);
|
||||
if (isset($this->mapping[$attribute['type']]['alternate'])) {
|
||||
if (preg_match($this->mapping[$attribute['type']]['alternate'][0], $attribute['value'])) {
|
||||
$brotype = $this->mapping[$attribute['type']]['alternate'][1];
|
||||
}
|
||||
}
|
||||
if ($valueField == 2 && isset($this->mapping[$attribute['type']]['composite'])) {
|
||||
$brotype = $this->mapping[$attribute['type']]['composite'];
|
||||
}
|
||||
$attribute['value'] = $this->replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
||||
if (isset($this->mapping[$attribute['type']]['replace'])) {
|
||||
$attribute['value'] = preg_replace(
|
||||
$this->mapping[$attribute['type']]['replace'][0],
|
||||
$this->mapping[$attribute['type']]['replace'][1],
|
||||
$attribute['value']
|
||||
);
|
||||
}
|
||||
return sprintf($ruleFormat,
|
||||
($overruled) ? '#OVERRULED BY WHITELIST# ' :
|
||||
$attribute['value'], // value - for composite values only the relevant element is taken
|
||||
'Intel:' . $brotype, // type
|
||||
'T', // meta.do_notice
|
||||
'-' // meta.if_in
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces characters that are not allowed in a signature.
|
||||
* example: " is converted to |22|
|
||||
* @param unknown_type $value
|
||||
*/
|
||||
public static function replaceIllegalChars($value) {
|
||||
$replace_pairs = array(
|
||||
'|' => '|7c|', // Needs to stay on top !
|
||||
'"' => '|22|',
|
||||
';' => '|3b|',
|
||||
':' => '|3a|',
|
||||
'\\' => '|5c|',
|
||||
'0x' => '|30 78|'
|
||||
);
|
||||
return strtr($value, $replace_pairs);
|
||||
}
|
||||
|
||||
public function checkWhitelist($value) {
|
||||
foreach ($this->whitelist as $wlitem) {
|
||||
if (preg_match($wlitem, $value)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public function getMispTypes($type) {
|
||||
$mispTypes = array();
|
||||
if ($type !== 'all') {
|
||||
if (isset($this->__mispTypes[$type])) {
|
||||
$mispTypes = $this->__mispTypes[$type];
|
||||
}
|
||||
} else {
|
||||
foreach ($this->__mispTypes as $mispType) {
|
||||
$mispTypes = array_merge($mispTypes, $mispType);
|
||||
}
|
||||
}
|
||||
return $mispTypes;
|
||||
}
|
||||
}
|
||||
|
|
@ -1412,7 +1412,7 @@ class Attribute extends AppModel {
|
|||
return $values;
|
||||
}
|
||||
|
||||
function bro($user, $type, $tags = false, $eventId = false, $from = false, $to = false, $last = false){
|
||||
function bro($user, $type, $tags = false, $eventId = false, $from = false, $to = false, $last = false) {
|
||||
//restricting to non-private or same org if the user is not a site-admin.
|
||||
$conditions['AND'] = array('Attribute.to_ids =' => 1, 'Event.published =' => 1);
|
||||
if ($from) $conditions['AND']['Event.date >='] = $from;
|
||||
|
|
@ -1438,113 +1438,31 @@ class Attribute extends AppModel {
|
|||
$conditions['AND'][] = $temp;
|
||||
}
|
||||
|
||||
$mispTypes = $this->getMispTypes($type);
|
||||
|
||||
App::uses('BroExport', 'Export');
|
||||
$export = new BroExport();
|
||||
$mispTypes = $export->getMispTypes($type);
|
||||
$intel = array();
|
||||
foreach($mispTypes as $mispType) {
|
||||
|
||||
$conditions['AND']['Attribute.type'] = $mispType;
|
||||
if ($type === 'filehash') {
|
||||
$intel = array_merge($intel, $this->brohids($user, $conditions, $this->getValueField($type, $mispType), $mispType));
|
||||
} else {
|
||||
$intel = array_merge($intel, $this->bronids($user, $conditions,$this->getValueField($type, $mispType)));
|
||||
}
|
||||
$conditions['AND']['Attribute.type'] = $mispType[0];
|
||||
$intel = $this->__bro($intel, $user, $conditions, $mispType[1], $export);
|
||||
}
|
||||
|
||||
return $intel;
|
||||
}
|
||||
|
||||
private function getMispTypes($type)
|
||||
{
|
||||
$mispTypes = array();
|
||||
if ($type !== 'all') {
|
||||
switch ($type) {
|
||||
case 'ip':
|
||||
$mispTypes = array('ip-src', 'ip-dst', 'domain|ip');
|
||||
break;
|
||||
case 'url':
|
||||
$mispTypes = array('url');
|
||||
break;
|
||||
case 'domain':
|
||||
$mispTypes = array('hostname', 'domain', 'domain|ip');
|
||||
break;
|
||||
case 'email':
|
||||
$mispTypes = array('email-src', 'email-dst');
|
||||
break;
|
||||
case 'filename':
|
||||
$mispTypes = array('filename', 'email-attachment', 'filename|md5', 'filename|sha1', 'filename|sha256');
|
||||
break;
|
||||
case 'filehash':
|
||||
$mispTypes = array('md5', 'sha1', 'sha256', 'filename|md5', 'filename|sha1', 'filename|sha256');
|
||||
break;
|
||||
case 'certhash':
|
||||
$mispTypes = array('x509-fingerprint-sha1');
|
||||
break;
|
||||
case 'software':
|
||||
$mispTypes = array('user-agent');
|
||||
break;
|
||||
}
|
||||
return $mispTypes;
|
||||
}
|
||||
return $mispTypes;
|
||||
}
|
||||
|
||||
private function getValueField($type, $mispType){
|
||||
$valueField = "value1";
|
||||
switch($type){
|
||||
case 'ip':
|
||||
if($mispType == 'domain|ip'){
|
||||
$valueField = "value2";
|
||||
}
|
||||
break;
|
||||
case 'filehash':
|
||||
if($mispType == 'filename|md5' || $mispType == 'filename|sha1' || $mispType == 'filename|sha256'){
|
||||
$valueField = "value2";
|
||||
}
|
||||
break;
|
||||
}
|
||||
return $valueField;
|
||||
}
|
||||
|
||||
private function brohids($user, $conditions, $valueField, $mispType)
|
||||
{
|
||||
$intel = array();
|
||||
App::uses('HidsBroExport', 'Export');
|
||||
$export = new HidsBroExport();
|
||||
$attributes = $this->fetchAttributes($user, array(
|
||||
'conditions' => $conditions,
|
||||
'order' => 'Attribute.'.$valueField.' ASC',
|
||||
'fields' => array('Attribute.id', 'Attribute.event_id', 'Attribute.type',
|
||||
'Attribute.'.$valueField." as value"),
|
||||
'contain' => array('Event' => array(
|
||||
'fields' => array(
|
||||
'Event.id', 'Event.published', 'Event.date',
|
||||
'Event.publish_timestamp', 'Event.orgc_id'),
|
||||
),
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
$intel = array_merge($intel, $export->export($attributes, strtoupper($mispType), true));
|
||||
return $intel;
|
||||
}
|
||||
|
||||
private function bronids($user, $conditions, $valueField)
|
||||
{
|
||||
$intel = array();
|
||||
App::uses('NidsBroExport', 'Export');
|
||||
$export = new NidsBroExport();
|
||||
private function __bro($intel, $user, $conditions, $valueField, $export) {
|
||||
$attributes = $this->fetchAttributes($user, array(
|
||||
'conditions' => $conditions, // array of conditions
|
||||
'order' => 'Attribute.'.$valueField.' ASC',
|
||||
'order' => 'Attribute.value' . $valueField . ' ASC',
|
||||
'recursive' => -1, // int
|
||||
'fields' => array('Attribute.id', 'Attribute.event_id', 'Attribute.type', 'Attribute.'.$valueField." as value"),
|
||||
'contain' => array('Event' => array('fields' => array('Event.id', 'Event.threat_level_id'))),
|
||||
'group' => array('Attribute.type', 'Attribute.'.$valueField), // fields to GROUP BY
|
||||
'fields' => array('Attribute.id', 'Attribute.event_id', 'Attribute.type', 'Attribute.value' . $valueField . " as value"),
|
||||
'contain' => array('Event' => array('fields' => array('Event.id', 'Event.threat_level_id', 'Event.orgc_id'))),
|
||||
'group' => array('Attribute.type', 'Attribute.value' . $valueField), // fields to GROUP BY
|
||||
)
|
||||
);
|
||||
$intel = array_merge($intel, $export->export($attributes, $user['nids_sid'], 'bro', true));
|
||||
return $intel;
|
||||
$orgs = $this->Event->Orgc->find('list', array(
|
||||
'fields' => array('Orgc.id', 'Orgc.name')
|
||||
));
|
||||
return $export->export($attributes, $orgs, $valueField, $intel);
|
||||
}
|
||||
|
||||
public function generateCorrelation($jobId = false, $startPercentage = 0) {
|
||||
|
|
|
|||
|
|
@ -98,9 +98,11 @@ class Event extends AppModel {
|
|||
'description' => 'Click this to download all network related attributes that you have access to under the Snort rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a whitelist containing host, domain name and IP numbers to exclude from the NIDS export.',
|
||||
),
|
||||
'bro' => array(
|
||||
'extension' => '.intel.zip',
|
||||
'type' => 'Bro',
|
||||
'description' => 'Click this to download all network related attributes that you have access to under the Bro rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a whitelist containing host, domain name and IP numbers to exclude from the NIDS export.',
|
||||
'extension' => '.intel.zip',
|
||||
'type' => 'Bro',
|
||||
'requiresPublished' => 1,
|
||||
'canHaveAttachments' => false,
|
||||
'description' => 'Click this to download all network related attributes that you have access to under the Bro rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a whitelist containing host, domain name and IP numbers to exclude from the NIDS export.',
|
||||
),
|
||||
'rpz' => array(
|
||||
'extension' => '.txt',
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@
|
|||
</div>
|
||||
<div class="row bottom-buffer">
|
||||
<div class="span3">
|
||||
<?php echo $this->Html->link('Download Bro signatures', array('action' => 'nids', 'bro', 'download'), array('class' => 'btn btn-block full-width')); ?>
|
||||
<?php echo $this->Html->link('Download Bro signatures', array('controller' => 'attributes', 'action' => 'bro', 'download'), array('class' => 'btn btn-block full-width')); ?>
|
||||
</div>
|
||||
<div class="span9">Click these to download all network related attributes that you
|
||||
have access to under the Bro rule format. Only <em>published</em>
|
||||
|
|
|
|||
Loading…
Reference in New Issue