MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Several changes in one (xml version, tag filters for exports) 

- xml version now included in the xml exports
- MISP will now check the xml version on all imports related to sync / add MISP XML and try to update the incoming info if it detects an older version

- exports now take tag names as a parameter (affected exports: XML, text, HIDS, NIDS)

- eventtags now correctly get removed when an event is deleted
pull/217/head
iglocska 2014-02-02 18:10:21 +01:00
parent 70c7f650f6
commit fc94376f0e
13 changed files with 259 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
app/Controller/AppController.php
View File

@ -67,7 +67,9 @@ class AppController extends Controller {
//'Actions' => array('actionPath' => 'controllers')) // TODO ACL, 4: tell actionPath
),
);
public $mispVersion = '2.2.0';
public function beforeFilter() {
// send users away that are using ancient versions of IE
@ -147,6 +149,7 @@ class AppController extends Controller {
$proposalCount = $this->_getProposalCount();
$this->set('proposalCount', $proposalCount[0]);
$this->set('proposalEventCount', $proposalCount[1]);
$this->set('mispVersion', $this->mispVersion);
}
public $userRole = null;

30
app/Controller/AttributesController.php
View File

@ -1095,7 +1095,7 @@ class AttributesController extends AppController {
// the last 4 fields accept the following operators:
// && - you can use && between two search values to put a logical OR between them. for value, 1.1.1.1&&2.2.2.2 would find attributes with the value being either of the two.
// ! - you can negate a search term. For example: google.com&&!mail would search for all attributes with value google.com but not ones that include mail. www.google.com would get returned, mail.google.com wouldn't.
public function restSearch($key='download', $value=null, $type=null, $category=null, $org=null) {
public function restSearch($key='download', $value=null, $type=null, $category=null, $org=null, $tags=null) {
if ($key!=null && $key!='download') {
$user = $this->checkAuthUser($key);
} else {
@ -1117,7 +1117,7 @@ class AttributesController extends AppController {
$parameters = array('value', 'type', 'category', 'org');
foreach ($parameters as $k => $param) {
if (isset(${$parameters[$k]})) {
if (isset(${$parameters[$k]}) && ${$parameters[$k]}!=='null') {
$elements = explode('&&', ${$parameters[$k]});
foreach($elements as $v) {
if (substr($v, 0, 1) == '!') {
@ -1162,15 +1162,30 @@ class AttributesController extends AppController {
$subcondition['OR'][] = array('Event.org' => $user['User']['org']);
array_push($conditions['AND'], $subcondition);
}
// If we sent any tags along, load the associated tag names for each attribute
if ($tags !== '') {
$args = $this->Attribute->dissectArgs($tags);
$this->loadModel('Tag');
$tagArray = $this->Tag->fetchEventTagIds($args[0], $args[1]);
$temp = array();
foreach ($tagArray[0] as $accepted) {
$temp['OR'][] = array('Event.id' => $accepted);
}
$conditions['AND'][] = $temp;
$temp = array();
foreach ($tagArray[1] as $rejected) {
$temp['AND'][] = array('Event.id !=' => $rejected);
}
$conditions['AND'][] = $temp;
}
// change the fields here for the attribute export!!!! Don't forget to check for the permissions, since you are not going through fetchevent. Maybe create fetchattribute?
$params = array(
'conditions' => $conditions,
'fields' => array('Attribute.*', 'Event.org', 'Event.distribution'),
'contain' => 'Event'
'contain' => array('Event' => array())
);
$results = $this->Attribute->find('all', $params);
$this->loadModel('Whitelist');
$this->response->type('xml');
@ -1291,7 +1306,7 @@ class AttributesController extends AppController {
$this->__downloadAttachment($this->Attribute->data['Attribute']);
}
public function text($key='download', $type="") {
public function text($key='download', $type="", $tags='') {
if ($key != 'download') {
// check if the key is valid -> search for users based on key
$user = $this->checkAuthUser($key);
@ -1306,8 +1321,7 @@ class AttributesController extends AppController {
$this->response->type('txt'); // set the content type
$this->header('Content-Disposition: download; filename="misp.' . $type . '.txt"');
$this->layout = 'text/default';
$attributes = $this->Attribute->text($this->_checkOrg(), $this->_isSiteAdmin(), $type);
$attributes = $this->Attribute->text($this->_checkOrg(), $this->_isSiteAdmin(), $type, $tags);
$this->loadModel('Whitelist');
$attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true);
$this->set('attributes', $attributes);

31
app/Controller/EventsController.php
View File

@ -473,6 +473,7 @@ class EventsController extends AppController {
is_uploaded_file($this->data['Event']['submittedgfi']['tmp_name'])) {
$this->Session->setFlash(__('You may only upload GFI Sandbox zip files.'));
} else {
if ($this->_isRest()) $this->request->data = $this->updateXMLArray($this->request->data, false);
$add = $this->Event->_add($this->request->data, $this->_isRest(), $this->Auth->user(), '');
if ($add && !is_numeric($add)) {
if ($this->_isRest()) {
@ -583,7 +584,7 @@ class EventsController extends AppController {
}
if (isset($this->data['Event']['submittedxml']) && ($ext != 'xml') && $this->data['Event']['submittedxml']['size'] > 0 &&
is_uploaded_file($this->data['Event']['submittedxml']['tmp_name'])) {
$this->Session->setFlash(__('You may only upload OpenIOC ioc files.'));
$this->Session->setFlash(__('You may only upload MISP XML files.'));
}
if (isset($this->data['Event']['submittedxml'])) $this->_addXMLFile();
@ -738,6 +739,7 @@ class EventsController extends AppController {
if ($this->request->is('post') || $this->request->is('put')) {
if ($this->_isRest()) {
$saveEvent = false;
if ($this->_isRest()) $this->request->data = $this->updateXMLArray($this->request->data, false);
// Workaround for different structure in XML/array than what CakePHP expects
$this->Event->cleanupEventArrayFromXML($this->request->data);
@ -904,14 +906,12 @@ class EventsController extends AppController {
throw new MethodNotAllowedException();
}
}
if ($this->Event->delete()) {
// delete the event from remote servers
//if ('true' == Configure::read('CyDefSIG.sync')) { // TODO test..(!$this->_isRest()) &&
// $this->__deleteEventFromServers($uuid);
//}
$this->Session->setFlash(__('Event deleted'));
// if coming from index, redirect to referer (to have the filter working)
@ -1180,7 +1180,7 @@ class EventsController extends AppController {
return $difference . " " . $periods[$j] . " ago";
}
public function xml($key, $eventid=null, $withAttachment = false) {
public function xml($key, $eventid=null, $withAttachment = false, $tags = '') {
if ($key != 'download') {
// check if the key is valid -> search for users based on key
$user = $this->checkAuthUser($key);
@ -1204,7 +1204,7 @@ class EventsController extends AppController {
$this->header('Content-Disposition: download; filename="misp.export.event' . $eventid . '.xml"');
}
}
$results = $this->__fetchEvent($eventid);
$results = $this->__fetchEvent($eventid, null, null, false, $tags);
if ($withAttachment) {
$this->loadModel('Attribute');
foreach ($results[0]['Attribute'] as &$attribute) {
@ -1228,7 +1228,7 @@ class EventsController extends AppController {
// Grab an event or a list of events for the event view or any of the XML exports. The returned object includes an array of events (or an array that only includes a single event if an ID was given)
// Included with the event are the attached attributes, shadow attributes, related events, related attribute information for the event view and the creating user's email address where appropriate
private function __fetchEvent($eventid = null, $idList = null, $orgFromFetch = null, $isSiteAdmin = false) {
private function __fetchEvent($eventid = null, $idList = null, $orgFromFetch = null, $isSiteAdmin = false, $tags = '') {
// if we come from automation, we may not be logged in - instead we used an auth key in the URL.
if (!empty($orgFromFetch)) {
$org = $orgFromFetch;
@ -1238,11 +1238,11 @@ class EventsController extends AppController {
}
if (!empty($orgFromFetch)) $org = $orgFromFetch;
else $org = $this->_checkOrg();
$results = $this->Event->fetchEvent($eventid, $idList, $org, $isSiteAdmin);
$results = $this->Event->fetchEvent($eventid, $idList, $org, $isSiteAdmin, $tags);
return $results;
}
public function nids($format = 'suricata', $key = '', $id = null, $continue = false) {
public function nids($format = 'suricata', $key = '', $id = null, $continue = false, $tags = '') {
// backwards compatibility, swap key and format
if ($format != 'snort' && $format != 'suricata') {
@ -1268,11 +1268,11 @@ class EventsController extends AppController {
// display the full snort rulebase
$this->loadModel('Attribute');
$rules = $this->Attribute->nids($user['User']['siteAdmin'], $user['User']['org'], $format, $user['User']['nids_sid'], $id, $continue);
$rules = $this->Attribute->nids($user['User']['siteAdmin'], $user['User']['org'], $format, $user['User']['nids_sid'], $id, $continue, $tags);
$this->set('rules', $rules);
}
public function hids($type, $key) {
public function hids($type, $key, $tags = '') {
$this->response->type('txt'); // set the content type
$this->header('Content-Disposition: download; filename="misp.' . $type . '.rules"');
$this->layout = 'text/default';
@ -1291,7 +1291,7 @@ class EventsController extends AppController {
}
$this->loadModel('Attribute');
$rules = $this->Attribute->hids($user['User']['siteAdmin'], $user['User']['org'], $type);
$rules = $this->Attribute->hids($user['User']['siteAdmin'], $user['User']['org'], $type, $tags);
$this->set('rules', $rules);
}
@ -1538,6 +1538,9 @@ class EventsController extends AppController {
if (!isset($xmlArray['response']) || !isset($xmlArray['response']['Event'])) {
throw new Exception('This is not a valid MISP XML file.');
}
$xmlArray = $this->Event->updateXMLArray($xmlArray);
if (isset($xmlArray['response']['Event'][0])) {
foreach ($xmlArray['response']['Event'] as $event) {
$temp['Event'] = $event;
@ -2055,4 +2058,10 @@ class EventsController extends AppController {
$this->Session->setFlash('Tag removed.');
$this->redirect(array('action' => 'view', $id));
}
public function test() {
debug($this->Event->find('all', array(
'contain' => array('EventTag' => array('Tag'))
)));
}
}

88
app/Model/Attribute.php
View File

@ -1133,7 +1133,7 @@ class Attribute extends AppModel {
return $fails;
}
public function hids($isSiteAdmin, $org ,$type) {
public function hids($isSiteAdmin, $org ,$type, $tags = '') {
// check if it's a valid type
if ($type != 'md5' && $type != 'sha1') {
throw new UnauthorizedException('Invalid hash type.');
@ -1149,6 +1149,23 @@ class Attribute extends AppModel {
$conditions['OR'] = $temp;
}
// If we sent any tags along, load the associated tag names for each attribute
if ($tags !== '') {
$tag = ClassRegistry::init('Tag');
$args = $this->dissectArgs($tags);
$tagArray = $tag->fetchEventTagIds($args[0], $args[1]);
$temp = array();
foreach ($tagArray[0] as $accepted) {
$temp['OR'][] = array('Event.id' => $accepted);
}
$conditions['AND'][] = $temp;
$temp = array();
foreach ($tagArray[1] as $rejected) {
$temp['AND'][] = array('Event.id !=' => $rejected);
}
$conditions['AND'][] = $temp;
}
$params = array(
'conditions' => $conditions, //array of conditions
'recursive' => 0, //int
@ -1161,7 +1178,7 @@ class Attribute extends AppModel {
return $rules;
}
public function nids($isSiteAdmin, $org, $format, $sid, $id = null, $continue = false) {
public function nids($isSiteAdmin, $org, $format, $sid, $id = null, $continue = false, $tags = '') {
//restricting to non-private or same org if the user is not a site-admin.
$conditions['AND'] = array('Attribute.to_ids' => 1, "Event.published" => 1);
if (!$isSiteAdmin) {
@ -1175,6 +1192,22 @@ class Attribute extends AppModel {
if ($id) {
array_push($conditions['AND'], array('Event.id' => $id));
}
// If we sent any tags along, load the associated tag names for each attribute
if ($tags !== '') {
$tag = ClassRegistry::init('Tag');
$args = $this->dissectArgs($tags);
$tagArray = $tag->fetchEventTagIds($args[0], $args[1]);
$temp = array();
foreach ($tagArray[0] as $accepted) {
$temp['OR'][] = array('Event.id' => $accepted);
}
$conditions['AND'][] = $temp;
$temp = array();
foreach ($tagArray[1] as $rejected) {
$temp['AND'][] = array('Event.id !=' => $rejected);
}
$conditions['AND'][] = $temp;
}
$params = array(
'conditions' => $conditions, //array of conditions
@ -1199,7 +1232,7 @@ class Attribute extends AppModel {
return $rules;
}
public function text($org, $isSiteAdmin, $type) {
public function text($org, $isSiteAdmin, $type, $tags) {
//restricting to non-private or same org if the user is not a site-admin.
$conditions['AND'] = array('Attribute.type' => $type, 'Attribute.to_ids =' => 1, 'Event.published =' => 1);
if (!$isSiteAdmin) {
@ -1210,15 +1243,36 @@ class Attribute extends AppModel {
$conditions['OR'] = $temp;
}
// If we sent any tags along, load the associated tag names for each attribute
if ($tags !== '') {
$tag = ClassRegistry::init('Tag');
$args = $this->dissectArgs($tags);
$tagArray = $tag->fetchEventTagIds($args[0], $args[1]);
$temp = array();
foreach ($tagArray[0] as $accepted) {
$temp['OR'][] = array('Event.id' => $accepted);
}
$conditions['AND'][] = $temp;
$temp = array();
foreach ($tagArray[1] as $rejected) {
$temp['AND'][] = array('Event.id !=' => $rejected);
}
$conditions['AND'][] = $temp;
}
$params = array(
'conditions' => $conditions, //array of conditions
'recursive' => 0, //int
'fields' => array('Attribute.value'), //array of field names
//'recursive' => 2, //int
//'fields' => array('Attribute.value'), //array of field names
'order' => array('Attribute.value'), //string or array defining order
'group' => array('Attribute.value'), //fields to GROUP BY
'contain' => array('Event.id', 'Event.published'),
);
return $this->find('all', $params);
'contain' => array('Event' => array(
'fields' => array('Event.id', 'Event.published'),
)));
$attributes = $this->find('all', $params);
return $attributes;
}
public function generateCorrelation() {
@ -1258,4 +1312,22 @@ class Attribute extends AppModel {
}
return $result;
}
// This method takes a string from an argument with several elements (separated by '&&' and negated by '!') and returns 2 arrays
// array 1 will have all of the non negated terms and array 2 all the negated terms
public function dissectArgs($args) {
$argArray = explode('&&', $args);
$accept = $reject = $result = array();
$reject = array();
foreach ($argArray as $arg) {
if (substr($arg, 0, 1) == '!') {
$reject[] = substr($arg, 1);
} else {
$accept[] = $arg;
}
}
$result[0] = $accept;
$result[1] = $reject;
return $result;
}
}

93
app/Model/Event.php
View File

@ -27,6 +27,8 @@ class Event extends AppModel {
public $displayField = 'id';
public $virtualFields = array();
public $mispVersion = '2.2.0';
/**
* Description field
@ -311,6 +313,10 @@ class Event extends AppModel {
public function beforeDelete($cascade = true) {
// delete event from the disk
$this->read(); // first read the event from the db
// delete all of the event->tag combinations that involve the deleted event
$this->EventTag->deleteAll(array('event_id' => $this->id));
// FIXME secure this filesystem access/delete by not allowing to change directories or go outside of the directory container.
// only delete the file if it exists
$filepath = APP . "files" . DS . $this->data['Event']['id'];
@ -739,6 +745,7 @@ class Event extends AppModel {
$response = $HttpSocket->get($uri, $data = '', $request);
if ($response->isOk()) {
$xmlArray = Xml::toArray(Xml::build($response->body));
$xmlArray = $this->updateXMLArray($xmlArray);
return $xmlArray['response'];
} else {
// TODO parse the XML response and keep the reason why it failed
@ -831,7 +838,7 @@ class Event extends AppModel {
}
//Once the data about the user is gathered from the appropriate sources, fetchEvent is called from the controller.
public function fetchEvent($eventid = null, $idList = null, $org, $isSiteAdmin, $bkgrProcess = null) {
public function fetchEvent($eventid = null, $idList = null, $org, $isSiteAdmin, $bkgrProcess = null, $tags = '') {
if (isset($eventid)) {
$this->id = $eventid;
if (!$this->exists()) {
@ -857,9 +864,27 @@ class Event extends AppModel {
);
}
if ($idList) {
if ($idList && $tags == '') {
$conditions['AND'][] = array('Event.id' => $idList);
}
// If we sent any tags along, load the associated tag names for each attribute
if ($tags !== '') {
$tag = ClassRegistry::init('Tag');
$args = $this->dissectArgs($tags);
$tagArray = $tag->fetchEventTagIds($args[0], $args[1]);
$temp = array();
if ($idList) $tagArray[0] = array_intersect($tagArray[0], $idList);
foreach ($tagArray[0] as $accepted) {
$temp['OR'][] = array('Event.id' => $accepted);
}
$conditions['AND'][] = $temp;
$temp = array();
foreach ($tagArray[1] as $rejected) {
$temp['AND'][] = array('Event.id !=' => $rejected);
}
$conditions['AND'][] = $temp;
}
// removing this for now, we export the to_ids == 0 attributes too, since there is a to_ids field indicating it in the .xml
// $conditionsAttributes['AND'] = array('Attribute.to_ids =' => 1);
// Same idea for the published. Just adjust the tools to check for this
@ -1619,4 +1644,68 @@ class Event extends AppModel {
}
return $k;
}
// check two version strings. If version 1 is older than 2, return -1, if they are the same return 0, if version 2 is older return 1
public function compareVersions($version1, $version2) {
$version1Array = explode('.', $version1);
$version2Array = explode('.', $version2);
if ($version1Array[0] != $version2Array[0]) {
if ($version1Array[0] > $version2Array[0]) return 1;
else return -1;
}
if ($version1Array[1] != $version2Array[1]) {
if ($version1Array[1] > $version2Array[1]) return 1;
else return -1;
}
if ($version1Array[2] != $version2Array[2]) {
if ($version1Array[2] > $version2Array[2]) return 1;
else return -1;
}
}
// main dispatch method for updating an incoming xmlArray - pass xmlArray to all of the appropriate transformation methods to make all the changes necessary to save the imported event
public function updateXMLArray($xmlArray, $response = true) {
if (isset($xmlArray['xml_version'])) {
$xmlArray['response']['xml_version'] = $xmlArray['xml_version'];
unset($xmlArray['xml_version']);
}
// if a version is set, it must be at least 2.2.0 - check the version and save the result of the comparison
if (isset($xmlArray['response']['xml_version'])) $version = $this->compareVersions($xmlArray['response']['xml_version'], $this->mispVersion);
// if no version is set, set the version to older (-1) manually
else $version = -1;
// same version, proceed normally
if ($version == 0) return $xmlArray;
// The xml is from an instance that is newer than the local instance, let the user know that the admin needs to upgrade before it could be imported
if ($version == 1) throw new Exception('This XML file is from a MISP instance that is newer than the current instance. Please contact your administrator about upgrading this instance.');
// if the xml contains an event or events from an older MISP instance, let's try to upgrade it!
// Let's manually set the version to something below 2.2.0 if there is no version set in the xml
if (!isset($xmlArray['response']['xmlVersion'])) $xmlArray['response']['xmlVersion'] = '2.1.0';
// Upgrade from versions below 2.2.0 will need to replace the risk field with threat level id
if ($this->compareVersions($xmlArray['response']['xmlVersion'], '2.2.0') < 0) {
if ($response) $xmlArray['response'] = $this->__updateXMLArray220($xmlArray['response']);
else $xmlArray = $this->__updateXMLArray220($xmlArray);
}
unset ($xmlArray['response']['xml_version']);
return $xmlArray;
}
// replaces the old risk value with the new threat level id
private function __updateXMLArray220($xmlArray) {
$risk = array('Undefined' => 4, 'Low' => 3, 'Medium' => 2, 'High' => 1);
if (isset($xmlArray['Event'][0])) {
foreach ($xmlArray['Event'] as &$event) {
$event['Event']['threat_level_id'] = $risk[$event['Event']['risk']];
}
} else {
$xmlArray['Event']['threat_level_id'] = $risk[$xmlArray['Event']['risk']];
}
return $xmlArray;
}
}

2
app/Model/EventTag.php
View File

@ -3,6 +3,8 @@ App::uses('AppModel', 'Model');
class EventTag extends AppModel {
public $actsAs = array('Containable');
public $validate = array(
'event_id' => array(
'notEmpty' => array(

36
app/Model/Tag.php
View File

@ -21,13 +21,13 @@ class Tag extends AppModel {
* @var string
*/
public $displayField = 'name';
public $actsAs = array(
'SysLogLogable.SysLogLogable' => array( // TODO Audit, logable
'roleModel' => 'Tag',
'roleKey' => 'tag_id',
'change' => 'full'
),
'Containable'
);
public $validate = array(
@ -62,5 +62,37 @@ class Tag extends AppModel {
if (!preg_match('/^#[0-9a-f]{6}$/i', $fields['colour'])) return false;
return true;
}
// find all of the event Ids that belong to the accepted tags and the rejected tags
public function fetchEventTagIds($accept=array(), $reject=array()) {
$acceptIds = $rejectIds = array();
if (!empty($accept)) {
$acceptIds = $this->findTags($accept);
}
if (!empty($reject)) {
$rejectIds = $this->findTags($reject);
}
return array($acceptIds, $rejectIds);
}
public function findTags($array) {
$ids = array();
foreach ($array as $a) {
$conditions['OR'][] = array('name like' => '%' . $a . '%');
}
$params = array(
'recursive' => 1,
'contain' => 'EventTag',
//'fields' => array('id', 'name'),
'conditions' => $conditions
);
$result = $this->find('all', $params);
foreach ($result as $tag) {
foreach ($tag['EventTag'] as $eventTag) {
$ids[] = $eventTag['event_id'];
}
}
return $ids;
}
}

2
app/View/Events/xml.ctp
View File

@ -40,6 +40,6 @@ foreach ($results as $result) {
$xmlArray['response']['Event'][] = $result['Event'];
}
$xmlArray['response']['xml_version'] = $mispVersion;
$xmlObject = Xml::fromArray($xmlArray, array('format' => 'tags'));
echo $xmlObject->asXML();

1
app/View/Events/xml/add.ctp
View File

@ -33,5 +33,6 @@ if (isset($relatedEvents)) {
// display the XML to the user
$xmlArray['response']['Event'][] = $event['Event'];
$xmlArray['response']['xml_version'] = $mispVersion;
$xmlObject = Xml::fromArray($xmlArray, array('format' => 'tags'));
echo $xmlObject->asXML();

1
app/View/Events/xml/index.ctp
View File

@ -20,6 +20,7 @@ foreach ($events as $key => $event) {
// display the XML to the user
$xmlArray['response']['Event'] = $events;
$xmlArray['response']['xml_version'] = $mispVersion;
$xmlObject = Xml::fromArray($xmlArray, array('format' => 'tags'));
echo $xmlObject->asXML();
?><!--

2
app/View/Events/xml/rest_search.ctp
View File

@ -40,6 +40,6 @@ foreach ($results as $result) {
$xmlArray['response']['Event'][] = $result['Event'];
}
$xmlArray['response']['xml_version'] = $mispVersion;
$xmlObject = Xml::fromArray($xmlArray, array('format' => 'tags'));
echo $xmlObject->asXML();

1
app/View/Events/xml/view.ctp
View File

@ -42,5 +42,6 @@ if (isset($event['Event']['RelatedEvent'])) {
// display the XML to the user
$xmlArray['response']['Event'][] = $event['Event'];
$xmlArray['response']['xml_version'] = $mispVersion;
$xmlObject = Xml::fromArray($xmlArray, array('format' => 'tags'));
echo $xmlObject->asXML();

2
app/View/Events/xml/xml.ctp
View File

@ -40,6 +40,6 @@ foreach ($results as $result) {
$xmlArray['response']['Event'][] = $result['Event'];
}
$xmlArray['response']['xml_version'] = $mispVersion;
$xmlObject = Xml::fromArray($xmlArray, array('format' => 'tags'));
echo $xmlObject->asXML();