MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Updates to the STIX export

pull/304/merge
iglocska 2014-08-04 18:07:15 +02:00
parent 525ef65008
commit 83d5e191fc
7 changed files with 182 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/Controller/AppController.php
View File

@ -45,7 +45,7 @@ class AppController extends Controller {
// Used for _isAutomation(), a check that returns true if the controller & action combo matches an action that is a non-xml and non-json automation method
// This is used to allow authentication via headers for methods not covered by _isRest() - as that only checks for JSON and XML formats
public $automationArray = array(
'events' => array('csv', 'nids', 'hids'),
'events' => array('csv', 'nids', 'hids', 'stix'),
'attributes' => array('text', 'downloadAttachment'),
);
@ -75,7 +75,7 @@ class AppController extends Controller {
),
);
public $mispVersion = '2.2.0';
public $mispVersion = '2.3.0';
public function beforeFilter() {
// send users away that are using ancient versions of IE

113
app/Controller/EventsController.php
View File

@ -44,6 +44,7 @@ class EventsController extends AppController {
$this->Auth->allow('text');
$this->Auth->allow('dot');
$this->Auth->allow('restSearch');
$this->Auth->allow('stix');
// TODO Audit, activate logable in a Controller
if (count($this->uses) && $this->{$this->modelClass}->Behaviors->attached('SysLogLogable')) {
@ -2369,76 +2370,64 @@ class EventsController extends AppController {
}
}
public function stix($id, $attachments = false) {
if (!$id) throw new MethodNotAllowedException('No event ID given.');
$event = $this->Event->find('first', array(
'conditions' => array('id' => $id),
'recursive' => -1,
'fields' => array('orgc', 'id'),
));
//
if (!$this->_isSiteAdmin() && !empty($event) && $event['Event']['orgc'] != $this->Auth->user('org')) throw new MethodNotAllowedException('Event not found or you don\'t have permissions to create attributes');
$events = $this->Event->fetchEvent($id, null, $this->Auth->user('org'), $this->_isSiteAdmin());
// If a second argument is passed (and it is either "yes", "true", or 1) base64 encode all of the attachments
if ($attachments == "yes" || $attachments == "true" || $attachments == 1) {
foreach ($events as &$event) {
foreach ($event['Attribute'] as &$attribute) {
if ($this->Event->Attribute->typeIsAttachment($attribute['type'])) {
$encodedFile = $this->Event->Attribute->base64EncodeAttachment($attribute);
$attribute['data'] = $encodedFile;
}
}
public function stix($key, $id = null, $withAttachments = false, $tags = null) {
if ($key != 'download') {
// check if the key is valid -> search for users based on key
$user = $this->checkAuthUser($key);
if (!$user) {
throw new UnauthorizedException('This authentication key is not authorized to be used for exports. Contact your administrator.');
}
$isSiteAdmin = $user['User']['siteAdmin'];
$org = $user['User']['org'];
} else {
if (!$this->Auth->user('id')) {
throw new UnauthorizedException('You have to be logged in to do that.');
}
$isSiteAdmin = $this->_isSiteAdmin();
$org = $this->Auth->user('org');
}
// set null if a null string is passed
if ($id == 'false' || $id == 'null') $id = null;
$numeric = false;
if (is_numeric($id)) $numeric = true;
if ($tags == 'false' || $tags == 'null') $tags = null;
if ($withAttachments == 'false' || 'null') $withAttachments = false;
// set the export type based on the request
if ($this->response->type() === 'application/json') $returnType = 'json';
else {
$returnType = 'xml';
$this->response->type('xml'); // set the content type
$this->layout = 'xml/default';
}
// request handler for POSTed queries. If the request is a post, the parameters (apart from the key) will be ignored and replaced by the terms defined in the posted xml object.
// The correct format for a posted xml is a "request" root element, as shown by the examples below:
// For XML: <request><id>!3&amp;!4</id><tags>OSINT</tags></request>
// This would return all OSINT tagged events except for event #3 and #4
if ($this->request->is('post')) {
if (empty($this->request->data)) {
throw new BadRequestException('Either specify the search terms in the url, or POST an xml (with the root element being "request".');
} else {
$data = $this->request->data;
}
$paramArray = array('id', 'withAttachment', 'tags');
foreach ($paramArray as $p) {
if (isset($data['request'][$p])) ${$p} = $data['request'][$p];
else ${$p} = null;
}
}
// generate a randomised filename for the temporary file that will be passed to the python script
$randomFileName = $this->__generateRandomFileName();
$tempFile = new File (APP . "files" . DS . "scripts" . DS . "tmp" . DS . $randomFileName, true, 0644);
$result = $this->Event->stix($id, $tags, $withAttachments, $this->Auth->user('org'), $this->_isSiteAdmin(), $returnType);
// save the json_encoded event(s) to the temporary file
$result = $tempFile->write(json_encode($events));
$scriptFile = APP . "files" . DS . "scripts" . DS . "misp2stix.py";
$returnType = 'xml';
if ($this->response->type() === 'application/json') $returnType = 'json';
// Execute the python script and point it to the temporary filename
$result = shell_exec('python ' . $scriptFile . ' ' . $randomFileName . ' ' . $returnType);
// The result of the script will be a returned JSON object with 2 variables: success (boolean) and message
// If success = 1 then the temporary output file was successfully written, otherwise an error message is passed along
$result = json_decode($result);
if ($result->success == 1) {
// delete the temporary file containing the json from MISP to the stix script
$tempFile->delete();
$file = new File(APP . "files" . DS . "scripts" . DS . "tmp" . DS . $randomFileName . ".out");
if ($result['success'] == 1) {
// read the output file and pass it to the view
if ($id == null) {
$this->header('Content-Disposition: download; filename="misp.stix.all.' . $returnType . '"');
if (!$numeric) {
$this->header('Content-Disposition: download; filename="misp.stix.event.collection.' . $returnType . '"');
} else {
$this->header('Content-Disposition: download; filename="misp.stix.event' . $id . '.' . $returnType . '"');
}
$this->set('data', $file->read());
// delete the output file
$file->delete();
$this->set('data', $result['data']);
} else {
// delete all of the created temporary files even if the import failed.
$tempFile->delete();
$file = new File(APP . "files" . DS . "scripts" . DS . "tmp" . DS . $randomFileName . ".out");
$file->delete();
throw new Exception(h($result->message));
throw new Exception(h($result['message']));
}
}
private function __generateRandomFileName() {
$length = 12;
$characters = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
$charLen = strlen($characters) - 1;
$fn = '';
for ($p = 0; $p < $length; $p++) {
$fn .= $characters[rand(0, $charLen)];
}
return $fn;
}
}

7
app/Model/Attribute.php
View File

@ -1336,7 +1336,12 @@ class Attribute extends AppModel {
// This method takes a string from an argument with several elements (separated by '&&' and negated by '!') and returns 2 arrays
// array 1 will have all of the non negated terms and array 2 all the negated terms
public function dissectArgs($args) {
$argArray = explode('&&', $args);
if (!$args) return array(null, null);
if (is_array($args)) {
$argArray = $args;
} else {
$argArray = explode('&&', $args);
}
$accept = $reject = $result = array();
$reject = array();
foreach ($argArray as $arg) {

80
app/Model/Event.php
View File

@ -1789,10 +1789,88 @@ class Event extends AppModel {
return $xmlArray;
}
public function checkIfNewer($incomingEvent) {
$localEvent = $this->find('first', array('conditions' => array('uuid' => $incomingEvent['uuid']), 'recursive' => -1));
if (empty($localEvent) || $incomingEvent['timestamp'] > $localEvent['Event']['timestamp']) return true;
return false;
}
public function stix($id, $tags, $attachments, $org, $isSiteAdmin, $returnType) {
$eventIDs = $this->Attribute->dissectArgs($id);
$tagIDs = $this->Attribute->dissectArgs($tags);
$idList = $this->getAccessibleEventIds($eventIDs[0], $eventIDs[1], $tagIDs[0], $tagIDs[1]);
$events = $this->fetchEvent(null, $idList, $org, $isSiteAdmin);
// If a second argument is passed (and it is either "yes", "true", or 1) base64 encode all of the attachments
if ($attachments == "yes" || $attachments == "true" || $attachments == 1) {
foreach ($events as &$event) {
foreach ($event['Attribute'] as &$attribute) {
if ($this->Attribute->typeIsAttachment($attribute['type'])) {
$encodedFile = $this->Attribute->base64EncodeAttachment($attribute);
$attribute['data'] = $encodedFile;
}
}
}
}
// generate a randomised filename for the temporary file that will be passed to the python script
$randomFileName = $this->__generateRandomFileName();
$tempFile = new File (APP . "files" . DS . "scripts" . DS . "tmp" . DS . $randomFileName, true, 0644);
// save the json_encoded event(s) to the temporary file
$result = $tempFile->write(json_encode($events));
$scriptFile = APP . "files" . DS . "scripts" . DS . "misp2stix.py";
// Execute the python script and point it to the temporary filename
$result = shell_exec('python ' . $scriptFile . ' ' . $randomFileName . ' ' . $returnType);
// The result of the script will be a returned JSON object with 2 variables: success (boolean) and message
// If success = 1 then the temporary output file was successfully written, otherwise an error message is passed along
$decoded = json_decode($result);
$result = array();
$result['success'] = $decoded->success;
$result['message'] = $decoded->message;
if ($result['success'] == 1) {
$file = new File(APP . "files" . DS . "scripts" . DS . "tmp" . DS . $randomFileName . ".out");
$result['data'] = $file->read();
}
$tempFile->delete();
$file = new File(APP . "files" . DS . "scripts" . DS . "tmp" . DS . $randomFileName . ".out");
$file->delete();
return $result;
}
public function getAccessibleEventIds($include, $exclude, $includedTags, $excludedTags) {
$conditions = array();
// get all of the event IDs based on include / exclude
if (!empty($include)) $conditions['OR'] = array('id' => $include);
if (!empty($exclude)) $conditions['NOT'] = array('id' => $exclude);
$events = $this->find('all', array(
'recursive' => -1,
'fields' => array('id', 'org', 'orgc', 'distribution'),
'conditions' => $conditions
));
$ids = array();
foreach ($events as $event) {
$ids[] = $event['Event']['id'];
}
// get all of the event IDs based on includedTags / excludedTags
if (!empty($includedTags) || !empty($excludedTags)) {
$eventIDsFromTags = $this->EventTag->getEventIDsFromTags($includedTags, $excludedTags);
// get the intersect of the two
$ids = array_intersect($ids, $eventIDsFromTags);
}
return $ids;
}
private function __generateRandomFileName() {
$length = 12;
$characters = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
$charLen = strlen($characters) - 1;
$fn = '';
for ($p = 0; $p < $length; $p++) {
$fn .= $characters[rand(0, $charLen)];
}
return $fn;
}
}

26
app/Model/EventTag.php
View File

@ -28,4 +28,30 @@ class EventTag extends AppModel {
'className' => 'Tag',
),
);
// take an array of tag names to be included and an array with tagnames to be excluded and find all event IDs that fit the criteria
public function getEventIDsFromTags($includedTags, $excludedTags) {
$conditions = array();
if (!empty($includedTags)) $conditions['OR'] = array('name' => $includedTags);
if (!empty($excludedTags)) $conditions['NOT'] = array('name' => $excludedTags);
$tags = $this->Tag->find('all', array(
'recursive' => -1,
'fields' => array('id', 'name'),
'conditions' => $conditions
));
$tagIDs = array();
foreach ($tags as $tag) {
$tagIDs[] = $tag['Tag']['id'];
}
$eventTags = $this->find('all', array(
'recursive' => -1,
'conditions' => array('tag_id' => $tagIDs)
));
$eventIDs = array();
foreach ($eventTags as $eventTag) {
$eventIDs[] = $eventTag['EventTag']['event_id'];
}
$eventIDs = array_unique($eventIDs);
return $eventIDs;
}
}

4
app/View/Elements/side_menu.ctp
View File

@ -42,8 +42,8 @@
<?php if (isset($event['Event']['published']) && $event['Event']['published']): ?>
<li><a href="/events/downloadOpenIOCEvent/<?php echo $event['Event']['id'];?>">Download as IOC</a></li>
<li><a href="/events/csv/download/<?php echo $event['Event']['id'];?>/1">Download as CSV</a></li>
<li><a href="/events/stix/<?php echo $event['Event']['id'];?>.xml">Download as STIX XML</a></li>
<li><a href="/events/stix/<?php echo $event['Event']['id'];?>.json">Download as STIX JSON</a></li>
<li><a href="/events/stix/download/<?php echo $event['Event']['id'];?>.xml">Download as STIX XML</a></li>
<li><a href="/events/stix/download/<?php echo $event['Event']['id'];?>.json">Download as STIX JSON</a></li>
<?php endif; ?>
<li class="divider"></li>
<li><a href="/events/index">List Events</a></li>

17
app/View/Events/automation.ctp
View File

@ -48,7 +48,22 @@ You can <?php echo $this->Html->link('reset', array('controller' => 'users', 'ac
<pre><?php echo Configure::read('MISP.baseurl');?>/events/hids/sha1/download</pre>
<p>You can also use the tag syntax similar to the XML import. Please be aware the colons (:) cannot be used in the tag search. Use semicolons instead (the search will automatically search for colons instead). For example, to only show sha1 values from events tagged tag1, use:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/hids/sha1/download/tag1</pre>
<h3>STIX export</h3>
<p>You can export MISP events in Mitre's STIX format (to read more about STIX, click <a href="https://stix.mitre.org/">here</a>). The STIX XML export is currently very slow and can lead to timeouts with larger events or collections of events. The JSON return format does not suffer from this issue. Usage:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/stix/download</pre>
<p>Search parameters can be passed to the function via url parameters or by POSTing an xml or json object (depending on the return type). The following parameters can be passed to the STIX export tool: <code>id</code>, <code>withAttachments</code>, <code>tags</code>. Both <code>id</code> and <code>tags</code> can use the <code>&&</code> (and) and <code>!</code> (not) operators to build queries. Using the url parameters, the syntax is as follows:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/stix/download/[id]/[withAttachments]/[tags]</pre>
<h4>Various ways to narrow down the search results of the STIX export</h4>
<p>For example, to retrieve all events tagged "APT1" but excluding events tagged "OSINT" and excluding events #51 and #62 without any attachments:
<pre><?php echo Configure::read('MISP.baseurl');?>/events/stix/download/!51&&!62/false/APT1&&!OSINT</pre>
<p>To export the same events using a POST request use:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/stix/download.json</pre>
<p>Together with this JSON object in the POST message:</p>
<code>{"request": {"id":["!51","!62"],"tags":["APT1","!OSINT"]}}</code><br /><br />
<p>XML is automatically assumed when using the stix export:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/stix/download</pre>
<p>The same search could be accomplished using the following POSTed XML object (note that ampersands need to be escaped, or alternatively separate id and tag elements can be used): </p>
<code>&lt;request&gt;&lt;id&gt;!51&lt;/id&gt;&lt;id&gt;!62&lt;/id&gt;&lt;tags&gt;APT1&lt;/tags&gt;&lt;tags&gt;!OSINT&lt;/tags&gt;&lt;/request&gt;</code>
<h3>Text export</h3>
<p>An automatic export of all attributes of a specific type to a plain text file.</p>
<p>You can configure your tools to automatically download the following files:</p>