bro export funtionality

pull/1465/head
ppanero 2016-08-23 14:06:06 +02:00
parent 9ae56cd557
commit 131e2f760a
14 changed files with 512 additions and 7 deletions

1
.gitignore vendored
View File

@ -61,3 +61,4 @@
/app/tmp/cached_exports/csv_all/*
/app/tmp/cached_exports/csv_sig/*
*.swp
*.iml

View File

@ -299,6 +299,53 @@ class EventShell extends AppShell
$this->Job->saveField('date_modified', date("y-m-d H:i:s"));
}
public function cachebro()
{
$broHeader = "#fields indicator\tindicator_type\tmeta.source\tmeta.url\tmeta.do_notice\tmeta.if_in\n";
$userId = $this->args[0];
$user = $this->User->getAuthUser($userId);
$id = $this->args[1];
$this->Job->id = $id;
$format = $this->args[2];
$this->Job->saveField('progress', 1);
$types = array('ip', 'email', 'domain', 'filename', 'filehash', 'certhash', 'software', 'url'); //Bro types
$typeCount = count($types);
$dir = new Folder(APP . DS . '/tmp/cached_exports/' . $format, true, 0750);
if ($user['Role']['perm_site_admin']) {
$zipname = DS . 'misp.bro.ADMIN.intel.zip';
} else {
$zipname = DS . 'misp.bro.' . $user['Organisation']['name'] . '.intel.zip';
}
$tmpZipname = DS . "bro_export_tmp.zip";
$zip = new File($dir->pwd() . $tmpZipname);
foreach ($types as $k => $type) {
$final = $this->Attribute->bro($user, $type);
$filename = $type . '.intel';
$file = new File($dir->pwd() . DS . $filename);
$file->write($broHeader);
foreach ($final as $attribute) {
$file->append($attribute . "\n");
}
$file->close();
$execRetval = '';
$execOutput = array();
exec('zip -gj ' . $zip->path . ' ' . $dir->pwd() . '/' . $filename,
$execOutput, $execRetval);
if ($execRetval != 0) { // not EXIT_SUCCESS
throw new Exception('An error has occured while attempting to zip the intel files.');
}
$file->delete(); // delete the original non-zipped-file
$this->Job->saveField('progress', $k / $typeCount * 100);
}
$zip->close();
rename($dir->pwd() . $tmpZipname, $dir->pwd() . $zipname);
$this->Job->saveField('progress', 100);
$this->Job->saveField('message', 'Job done.');
}
public function alertemail() {
$userId = $this->args[0];
$processId = $this->args[1];

View File

@ -1850,6 +1850,36 @@ class AttributesController extends AppController {
$this->render('/Attributes/rpz');
}
public function bro($key='download', $type='all', $tags=false, $eventId=false, $allowNonIDS=false, $from=false, $to=false, $last=false) {
$simpleFalse = array('eventId', 'allowNonIDS', 'tags', 'from', 'to', 'last');
foreach ($simpleFalse as $sF) {
if (!is_array(${$sF}) && (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false')) ${$sF} = false;
}
if ($type !== 'null' || $type !== '0' || $type !== 'false') {
if ($from) $from = $this->Attribute->Event->dateFieldCheck($from);
if ($to) $to = $this->Attribute->Event->dateFieldCheck($to);
if ($last) $last = $this->Attribute->Event->resolveTimeDelta($last);
if ($key != 'download') {
// check if the key is valid -> search for users based on key
$user = $this->checkAuthUser($key);
if (!$user) {
throw new UnauthorizedException('This authentication key is not authorized to be used for exports. Contact your administrator.');
}
} else {
if (!$this->Auth->user('id')) {
throw new UnauthorizedException('You have to be logged in to do that.');
}
}
$this->response->type('txt'); // set the content type
$this->header('Content-Disposition: download; filename="misp.' . $type . '.intel"');
$this->layout = 'text/default';
$attributes = array("#fields indicator\tindicator_type\tmeta.source\tmeta.url\tmeta.do_notice\tmeta.if_in");
$attributes = array_merge($attributes, $this->Attribute->bro($this->Auth->user(), $type, $tags, $eventId, $allowNonIDS, $from, $to, $last));
$this->set('attributes', $attributes);
$this->render('/Attributes/bro');
}
}
public function reportValidationIssuesAttributes($eventId = false) {
// TODO improve performance of this function by eliminating the additional SQL query per attribute
// search for validation problems in the attributes

View File

@ -3141,6 +3141,12 @@ class EventsController extends AppController {
'requiresPublished' => true,
'checkbox' => false,
),
'bro' => array(
'url' => '/events/bro/download/' . $id,
'text' => 'Download Bro rules',
'requiresPublished' => true,
'checkbox' => false,
),
'text' => array(
'url' => '/attributes/text/download/all/false/' . $id,
'text' => 'Export all attribute values as a text file',

View File

@ -0,0 +1,50 @@
<?php
App::uses('HidsExport', 'Export');
/**
* Created by IntelliJ IDEA.
* User: ppanero
* Date: 11/08/16
* Time: 11:04
*/
class HidsBroExport extends HidsExport
{
// below overwrite functions from HidsExport
public function export($items, $type = 'MD5', $continue = false)
{
$orgsName = array();
if (!empty($items)) {
foreach ($items as &$item) {
$ruleFormatReference = Configure::read('MISP.baseurl') . '/events/view/' . $item['Event']['id'];
if (array_key_exists($item['Event']['orgc_id'], $orgsName)) {
$orgName = $orgsName[$item['Event']['orgc_id']];
} else {
$orgModel = ClassRegistry::init('Organisation');
$org = $orgModel->find('first', array(
'fields' => array('Organisation.name'),
'conditions' => array('id' => $item['Event']['orgc_id']),
)
);
$orgName = $org['Organisation']['name'];
$orgsName[$item['Event']['orgc_id']] = $orgName;
}
$orgFormatReference = $orgName;
$ruleFormat = "%s\t%s\t" . $orgFormatReference . "\t" . $ruleFormatReference . "\t%s\t%s";
$attribute = &$item['Attribute'];
$this->rules[] = sprintf($ruleFormat,
$attribute['value'], // hash value
'Intel:FILE_HASH', // type
'T', // meta.do_notice
'-' // meta.if_in
);
}
}
if (!$continue) $this->explain($type);
return $this->rules;
}
}

View File

@ -0,0 +1,144 @@
<?php
App::uses('NidsExport', 'Export');
class NidsBroExport extends NidsExport
{
public function export($items, $startSid, $format = "suricata", $continue = false)
{
// set the specific format
$this->format = "bro";
// call the generic function
return parent::export($items, $startSid, $format, $continue);
}
// below overwrite functions from NidsExport
public function ipDstRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
$this->rules[] = sprintf($ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' :
$attribute['value'], // dst_ip
'Intel:ADDR', // type
'T', // meta.do_notice
'-' // meta.if_in
);
}
public function ipSrcRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
$this->rules[] = sprintf($ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' :
$attribute['value'], // dst_ip
'Intel:ADDR', // type
'T', // meta.do_notice
'-' // meta.if_in
);
}
public function emailSrcRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
$this->rules[] = sprintf($ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' :
$attribute['value'], // dst_ip
'Intel:EMAIL', // type
'T', // meta.do_notice
'-' // meta.if_in
);
}
public function emailDstRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
$this->rules[] = sprintf($ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' :
$attribute['value'], // dst_ip
'Intel:EMAIL', // type
'T', // meta.do_notice
'-' // meta.if_in
);
}
public function emailSubjectRule($ruleFormat, $attribute, &$sid)
{
// Nothing to return, there is no clear mapping to Bro intel
}
public function emailAttachmentRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
$this->rules[] = sprintf($ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' :
$attribute['value'], // dst_ip
'Intel:FILE_NAME', // type
'T', // meta.do_notice
'-' // meta.if_in
);
}
public function hostnameRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
$this->rules[] = sprintf($ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' :
$attribute['value'], // dst_ip
'Intel:DOMAIN', // type
'T', // meta.do_notice
'-' // meta.if_in
);
}
public function domainRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
$this->rules[] = sprintf($ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' :
$attribute['value'], // dst_ip
'Intel:DOMAIN', // type
'T', // meta.do_notice
'-' // meta.if_in
);
}
public function urlRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
$attribute['value'] = preg_replace('#^https?://#', '', $attribute['value']);
$this->rules[] = sprintf($ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' :
$attribute['value'], // dst_ip
'Intel:URL', // type
'T', // meta.do_notice
'-' // meta.if_in
);
}
public function userAgentRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
$this->rules[] = sprintf($ruleFormat,
($overruled) ? '#OVERRULED BY WHITELIST# ' :
$attribute['value'], // dst_ip
'Intel:SOFTWARE', // type
'T', // meta.do_notice
'-' // meta.if_in
);
}
public function snortRule($ruleFormat, $attribute, &$sid, $ruleFormatMsg, $ruleFormatReference)
{
//Nothing to export
}
}

View File

@ -29,6 +29,9 @@ class NidsExport {
$this->Whitelist = ClassRegistry::init('Whitelist');
$this->whitelist = $this->Whitelist->getBlockedValues();
//For bro format organisation
$orgsName = array();
// output a short explanation
if (!$continue) {
$this->explain();
@ -36,10 +39,29 @@ class NidsExport {
// generate the rules
foreach ($items as $item) {
# proto src_ip src_port direction dst_ip dst_port msg rule_content tag sid rev
$ruleFormatMsg = 'msg: "MISP e' . $item['Event']['id'] . ' %s"';
$ruleFormatReference = 'reference:url,' . Configure::read('MISP.baseurl') . '/events/view/' . $item['Event']['id'];
$ruleFormat = '%salert %s %s %s %s %s %s (' . $ruleFormatMsg . '; %s %s classtype:' . $this->classtype . '; sid:%d; rev:%d; priority:' . $item['Event']['threat_level_id'] . '; ' . $ruleFormatReference . ';) ';
if($format != 'bro') {
# proto src_ip src_port direction dst_ip dst_port msg rule_content tag sid rev
$ruleFormatMsg = 'msg: "MISP e' . $item['Event']['id'] . ' %s"';
$ruleFormatReference = 'reference:url,' . Configure::read('MISP.baseurl') . '/events/view/' . $item['Event']['id'];
$ruleFormat = '%salert %s %s %s %s %s %s (' . $ruleFormatMsg . '; %s %s classtype:' . $this->classtype . '; sid:%d; rev:%d; priority:' . $item['Event']['threat_level_id'] . '; ' . $ruleFormatReference . ';) ';
}
else{
if (array_key_exists($item['Event']['orgc_id'], $orgsName)) {
$orgName = $orgsName[$item['Event']['orgc_id']];
} else {
$orgModel = ClassRegistry::init('Organisation');
$org = $orgModel->find('first', array(
'fields' => array('Organisation.name'),
'conditions' => array('id' => $item['Event']['orgc_id']),
)
);
$orgName = $org['Organisation']['name'];
$orgsName[$item['Event']['orgc_id']] = $orgName;
}
$orgFormatReference = $orgName;
$ruleFormatReference = Configure::read('MISP.baseurl') . '/events/view/' . $item['Event']['id'];
$ruleFormat = "%s\t%s\t" . $orgFormatReference . "\t" . $ruleFormatReference . "\t%s\t%s";
}
$sid = $startSid + ($item['Attribute']['id'] * 10); // leave 9 possible rules per attribute type
$sid++;

View File

@ -1412,6 +1412,141 @@ class Attribute extends AppModel {
return $values;
}
function bro($user, $type, $tags = false, $eventId = false, $from = false, $to = false, $last = false){
//restricting to non-private or same org if the user is not a site-admin.
$conditions['AND'] = array('Attribute.to_ids =' => 1, 'Event.published =' => 1);
if ($from) $conditions['AND']['Event.date >='] = $from;
if ($to) $conditions['AND']['Event.date <='] = $to;
if ($last) $conditions['AND']['Event.publish_timestamp >='] = $last;
if ($eventId !== false) {
$conditions['AND'][] = array('Event.id' => $eventId);
}
else if ($tags !== false) {
// If we sent any tags along, load the associated tag names for each attribute
$tag = ClassRegistry::init('Tag');
$args = $this->dissectArgs($tags);
$tagArray = $tag->fetchEventTagIds($args[0], $args[1]);
$temp = array();
foreach ($tagArray[0] as $accepted) {
$temp['OR'][] = array('Event.id' => $accepted);
}
$conditions['AND'][] = $temp;
$temp = array();
foreach ($tagArray[1] as $rejected) {
$temp['AND'][] = array('Event.id !=' => $rejected);
}
$conditions['AND'][] = $temp;
}
$mispTypes = $this->getMispTypes($type);
$intel = array();
foreach($mispTypes as $mispType) {
$conditions['AND']['Attribute.type'] = $mispType;
if ($type === 'filehash') {
$intel = array_merge($intel, $this->brohids($user, $conditions, $this->getValueField($type, $mispType), $mispType));
} else {
$intel = array_merge($intel, $this->bronids($user, $conditions,$this->getValueField($type, $mispType)));
}
}
return $intel;
}
private function getMispTypes($type)
{
$mispTypes = array();
if ($type !== 'all') {
switch ($type) {
case 'ip':
$mispTypes = array('ip-src', 'ip-dst', 'domain|ip');
break;
case 'url':
$mispTypes = array('url');
break;
case 'domain':
$mispTypes = array('hostname', 'domain', 'domain|ip');
break;
case 'email':
$mispTypes = array('email-src', 'email-dst');
break;
case 'filename':
$mispTypes = array('filename', 'email-attachment', 'filename|md5', 'filename|sha1', 'filename|sha256');
break;
case 'filehash':
$mispTypes = array('md5', 'sha1', 'sha256', 'filename|md5', 'filename|sha1', 'filename|sha256');
break;
case 'certhash':
$mispTypes = array('x509-fingerprint-sha1');
break;
case 'software':
$mispTypes = array('user-agent');
break;
}
return $mispTypes;
}
return $mispTypes;
}
private function getValueField($type, $mispType){
$valueField = "value1";
switch($type){
case 'ip':
if($mispType == 'domain|ip'){
$valueField = "value2";
}
break;
case 'filehash':
if($mispType == 'filename|md5' || $mispType == 'filename|sha1' || $mispType == 'filename|sha256'){
$valueField = "value2";
}
break;
}
return $valueField;
}
private function brohids($user, $conditions, $valueField, $mispType)
{
$intel = array();
App::uses('HidsBroExport', 'Export');
$export = new HidsBroExport();
$attributes = $this->fetchAttributes($user, array(
'conditions' => $conditions,
'order' => 'Attribute.'.$valueField.' ASC',
'fields' => array('Attribute.id', 'Attribute.event_id', 'Attribute.type',
'Attribute.'.$valueField." as value"),
'contain' => array('Event' => array(
'fields' => array(
'Event.id', 'Event.published', 'Event.date',
'Event.publish_timestamp', 'Event.orgc_id'),
),
)
)
);
$intel = array_merge($intel, $export->export($attributes, strtoupper($mispType), true));
return $intel;
}
private function bronids($user, $conditions, $valueField)
{
$intel = array();
App::uses('NidsBroExport', 'Export');
$export = new NidsBroExport();
$attributes = $this->fetchAttributes($user, array(
'conditions' => $conditions, // array of conditions
'order' => 'Attribute.'.$valueField.' ASC',
'recursive' => -1, // int
'fields' => array('Attribute.id', 'Attribute.event_id', 'Attribute.type', 'Attribute.'.$valueField." as value"),
'contain' => array('Event' => array('fields' => array('Event.id', 'Event.threat_level_id'))),
'group' => array('Attribute.type', 'Attribute.'.$valueField), // fields to GROUP BY
)
);
$intel = array_merge($intel, $export->export($attributes, $user['nids_sid'], 'bro', true));
return $intel;
}
public function generateCorrelation($jobId = false, $startPercentage = 0) {
$this->Correlation = ClassRegistry::init('Correlation');
$this->Correlation->deleteAll(array('id !=' => 0), false);
@ -1697,12 +1832,12 @@ class Attribute extends AppModel {
'recursive' => -1,
'contain' => array(
'Event' => array(
'fields' => array('id', 'info', 'org_id'),
'fields' => array('id', 'info', 'org_id', 'orgc_id'),
),
),
);
if (isset($options['contain'])) $params['contain'] = array_merge_recursive($params['contain'], $options['contain']);
else $option['contain']['Event']['fields'] = array('id', 'info', 'org_id');
else $option['contain']['Event']['fields'] = array('id', 'info', 'org_id', 'orgc_id');
if (isset($options['fields'])) $params['fields'] = $options['fields'];
if (isset($options['conditions'])) $params['conditions']['AND'][] = $options['conditions'];
if (isset($options['order'])) $params['order'] = $options['order'];

View File

@ -97,6 +97,11 @@ class Event extends AppModel {
'canHaveAttachments' => false,
'description' => 'Click this to download all network related attributes that you have access to under the Snort rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a whitelist containing host, domain name and IP numbers to exclude from the NIDS export.',
),
'bro' => array(
'extension' => '.intel.zip',
'type' => 'Bro',
'description' => 'Click this to download all network related attributes that you have access to under the Bro rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a whitelist containing host, domain name and IP numbers to exclude from the NIDS export.',
),
'rpz' => array(
'extension' => '.txt',
'type' => 'RPZ',

View File

@ -50,6 +50,11 @@ class Job extends AppModel {
$type = 'nids';
$extra2 = isset($user['nids_sid']) ? $user['nids_sid'] : 0;
}
if ($type === 'bro') {
$extra = $type;
$type = 'bro';
$extra2 = isset($user['nids_sid']) ? $user['nids_sid'] : 0;
}
if ($type === 'rpz') $extra = $type;
$this->save($data);
$id = $this->id;

View File

@ -2425,6 +2425,7 @@ class Server extends AppModel {
APP . 'tmp' . DS . 'xml' => 0,
APP . 'tmp' . DS . 'files' => 0,
APP . 'tmp' . DS . 'logs' => 0,
APP . 'tmp' . DS . 'bro' => 0,
);
foreach ($writeableDirs as $path => &$error) {
$dir = new Folder($path);

View File

@ -0,0 +1,5 @@
<?php
foreach ($attributes as $attribute) {
echo $attribute;
echo PHP_EOL;
}

View File

@ -204,6 +204,49 @@ Use semicolons instead (the search will automatically search for colons instead)
echo $baseurl.'/attributes/text/download/all/tag1&amp;&amp;tag2&amp;&amp;!tag3';
?>
</pre>
<h3>Bro IDS export</h3>
<p>An export of all attributes of a specific bro type to a formatted plain text file. By default only published and IDS flagged attributes are exported.</p>
<p>You can configure your tools to automatically download a file one of the Bro types. There is no option to download all attributes in the same file:</p>
<pre>
<?php
$broTypes = array('ip', 'email', 'domain', 'filename', 'filehash', 'certhash', 'software', 'url');
foreach ($broTypes as $broType) {
echo $baseurl.'/attributes/bro/download/'.$broType . "\n";
}
?>
</pre>
<p>To restrict the results by tags, use the usual syntax. Please be aware the colons (:) cannot be used in the tag search. Use semicolons instead (the search will automatically search for colons instead). To get ip values from events tagged tag1 but not tag2 use:</p>
<pre>
<?php
echo $baseurl.'/attributes/bro/download/ip/tag1&&!tag2';
?>
</pre>
<p>As of version 2.3.38, it is possible to restrict the text exports on two additional flags. The first allows the user to restrict based on event ID, whilst the second is a boolean switch allowing non IDS flagged attributes to be exported. Additionally, choosing "all" in the type field will return all eligible attributes. </p>
<pre>
<?php
echo $baseurl.'/attributes/bro/download/[type]/[tags]/[event_id]/[allowNonIDS]/[from]/[to]/[last]';
?>
</pre>
<b>type</b>: The attribute type, any valid Bro attribute type is accepted. The matching between Bro and MISP types is the following:<br />
<pre>
<b>ip</b>:ip-src, ip-dst, domain|ip
<b>domain</b>: hostname, domain, , domain|ip
<b>url</b>: url
<b>filename</b>: filename, filename|md5, filename|sha1, filename|sha256, email-attachment
<b>software</b>: user-agent
<b>filehash</b>: md5, sha1, sha256, filename|md5, filename|sha1, filename|sha256
<b>certhash</b>: x509-fingerprint-sha1
<b>email</b>: email-src, email-dst
</pre>
<b>tags</b>: To include a tag in the results just write its names into this parameter. To exclude a tag prepend it with a '!'.
You can also chain several tag commands together with the '&amp;&amp;' operator. Please be aware the colons (:) cannot be used in the tag search.
Use semicolons instead (the search will automatically search for colons instead). For example, to include tag1 and tag2 but exclude tag3 you would use:<br />
<pre>
<?php
echo $baseurl.'/attributes/bro/download/ip/tag1&amp;&amp;tag2&amp;&amp;!tag3';
?>
</pre>
<p>
<b>event_id</b>: Restrict the results to the given event IDs. <br />
<b>allowNonIDS</b>: Allow attributes to be exported that are not marked as "to_ids".<br />

View File

@ -38,7 +38,18 @@
domain name and IP numbers to exclude from the NIDS export.
</div>
</div>
<div class="row bottom-buffer">
<div class="row bottom-buffer">
<div class="span3">
<?php echo $this->Html->link('Download Bro signatures', array('action' => 'nids', 'bro', 'download'), array('class' => 'btn btn-block full-width')); ?>
</div>
<div class="span9">Click these to download all network related attributes that you
have access to under the Bro rule format. Only <em>published</em>
events and attributes marked as <em>IDS Signature</em> are exported.
Administration is able to maintain a whitelist containing host,
domain name and IP numbers to exclude from the NIDS export.
</div>
</div>
<div class="row bottom-buffer">
<div class="span3">
<?php echo $this->Html->link('Download RPZ Zone File', array('controller' => 'attributes', 'action' => 'rpz', 'download'), array('class' => 'btn btn-block full-width')); ?>
</div>