Reimplementation of the Add XML feature

- called Add MISP export now
- can be an XML / JSON file
- result browser with explanations of failures

- REST XML/JSON add/edit of events returns errors instead of the partially succeeding event
pull/731/merge
iglocska 2015-11-30 02:28:07 +01:00
parent 0611648989
commit 9eb5680ee8
8 changed files with 180 additions and 156 deletions

View File

@ -15,6 +15,7 @@ class AttributesController extends AppController {
public $paginate = array(
'limit' => 60,
'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events
'conditions' => array('AND' => array('Event.id >' => 0))
);
public $helpers = array('Js' => array('Jquery'));

View File

@ -966,7 +966,7 @@ class EventsController extends AppController {
if (isset($this->request->data['Event']['orgc']) && !$this->userRole['perm_sync']) $this->request->data['Event']['orgc'] = $this->Auth->user('org');
}
$add = $this->Event->_add($this->request->data, $this->_isRest(), $this->Auth->user(), '');
if ($add && !is_numeric($add)) {
if ($add === true && !is_numeric($add)) {
if ($this->_isRest()) {
if ($add === 'blocked') {
throw new ForbiddenException('Event blocked by local blacklist.');
@ -994,9 +994,15 @@ class EventsController extends AppController {
$this->response->send();
throw new NotFoundException('Event already exists, if you would like to edit it, use the url in the location header.');
}
// REST users want to see the failed event
$this->view($this->Event->getId());
$this->render('view');
$add = json_decode($add, true);
App::uses('JSONConverterTool', 'Tools');
$converter = new JSONConverterTool();
$add = $converter->jsonPrinter($add);
$this->set('name', 'Add event failed.');
$this->set('message', 'The event could not be saved.');
$this->set('errors', $add);
$this->set('url', '/events/add');
$this->set('_serialize', array('name', 'message', 'url', 'errors'));
} else {
$this->Session->setFlash(__('The event could not be saved. Please, try again.'), 'default', array(), 'error');
// TODO return error if REST
@ -1063,28 +1069,28 @@ class EventsController extends AppController {
$this->set('published', $this->Event->data['Event']['published']);
}
public function add_xml() {
public function add_misp_export() {
if (!$this->userRole['perm_modify']) {
throw new UnauthorizedException('You do not have permission to do that.');
}
if ($this->request->is('post')) {
if (!empty($this->data)) {
$ext = '';
if (isset($this->data['Event']['submittedxml'])) {
if (isset($this->data['Event']['submittedfile'])) {
App::uses('File', 'Utility');
$file = new File($this->data['Event']['submittedxml']['name']);
$file = new File($this->data['Event']['submittedfile']['name']);
$ext = $file->ext();
}
if (isset($this->data['Event']['submittedxml']) && ($ext != 'xml') && $this->data['Event']['submittedxml']['size'] > 0 &&
if (isset($this->data['Event']['submittedfile']) && ($ext != 'xml' && $ext != 'json') && $this->data['Event']['submittedfile']['size'] > 0 &&
is_uploaded_file($this->data['Event']['submittedxml']['tmp_name'])) {
$this->Session->setFlash(__('You may only upload MISP XML files.'));
$this->Session->setFlash(__('You may only upload MISP XML or MISP JSON files.'));
}
if (isset($this->data['Event']['submittedxml'])) {
if (isset($this->data['Event']['submittedfile'])) {
if (Configure::read('MISP.take_ownership_xml_import')
&& (isset($this->data['Event']['takeownership']) && $this->data['Event']['takeownership'] == 1)) {
$this->_addXMLFile(true);
$results = $this->_addMISPExportFile($ext, true);
} else {
$this->_addXMLFile();
$results = $this->_addMISPExportFile($ext);
}
}
@ -1096,74 +1102,8 @@ class EventsController extends AppController {
$this->Session->setFlash(__('The event has been saved. ' . $existingFlash['message']));
}
}
}
}
/**
* Low level function to add an Event based on an Event $data array
*
* @return bool true if success
*/
public function _add(&$data, $fromXml, $or='', $passAlong = null, $fromPull = false) {
$this->Event->create();
// force check userid and orgname to be from yourself
$auth = $this->Auth;
$data['Event']['user_id'] = $auth->user('id');
$date = new DateTime();
//if ($this->checkAction('perm_sync')) $data['Event']['org'] = Configure::read('MISP.org');
//else $data['Event']['org'] = $auth->user('org');
$data['Event']['org'] = $auth->user('org');
// set these fields if the event is freshly created and not pushed from another instance.
// Moved out of if (!$fromXML), since we might get a restful event without the orgc/timestamp set
if (!isset ($data['Event']['orgc'])) $data['Event']['orgc'] = $data['Event']['org'];
if ($fromXml) {
// Workaround for different structure in XML/array than what CakePHP expects
$this->Event->cleanupEventArrayFromXML($data);
// the event_id field is not set (normal) so make sure no validation errors are thrown
// LATER do this with $this->validator()->remove('event_id');
unset($this->Event->Attribute->validate['event_id']);
unset($this->Event->Attribute->validate['value']['unique']); // otherwise gives bugs because event_id is not set
}
unset ($data['Event']['id']);
if (isset($data['Event']['uuid'])) {
// check if the uuid already exists
$existingEventCount = $this->Event->find('count', array('conditions' => array('Event.uuid' => $data['Event']['uuid'])));
if ($existingEventCount > 0) {
// RESTfull, set responce location header..so client can find right URL to edit
if ($fromPull) return false;
$existingEvent = $this->Event->find('first', array('conditions' => array('Event.uuid' => $data['Event']['uuid'])));
$this->response->header('Location', Configure::read('MISP.baseurl') . '/events/' . $existingEvent['Event']['id']);
$this->response->send();
return false;
}
}
if (isset($data['Attribute'])) {
foreach ($data['Attribute'] as &$attribute) {
unset ($attribute['id']);
}
}
// FIXME chri: validatebut the necessity for all these fields...impact on security !
$fieldList = array(
'Event' => array('org', 'orgc', 'date', 'threat_level_id', 'analysis', 'info', 'user_id', 'published', 'uuid', 'timestamp', 'distribution', 'locked'),
'Attribute' => array('event_id', 'category', 'type', 'value', 'value1', 'value2', 'to_ids', 'uuid', 'revision', 'timestamp', 'distribution')
);
$saveResult = $this->Event->saveAssociated($data, array('validate' => true, 'fieldList' => $fieldList,
'atomic' => true));
// FIXME chri: check if output of $saveResult is what we expect when data not valid, see issue #104
if ($saveResult) {
if (!empty($data['Event']['published']) && 1 == $data['Event']['published']) {
// do the necessary actions to publish the event (email, upload,...)
if ('true' != Configure::read('MISP.disablerestalert')) {
$this->Event->sendAlertEmailRouter($this->Event->getId(), $this->Auth->user(), $this->_isSiteAdmin());
}
$this->Event->publish($this->Event->getId(), $passAlong);
}
return true;
} else {
//throw new MethodNotAllowedException("Validation ERROR: \n".var_export($this->Event->validationErrors, true));
return false;
$this->set('results', $results);
$this->render('add_misp_export_result');
}
}
@ -1329,8 +1269,19 @@ class EventsController extends AppController {
return true;
} else {
$message = 'Error';
$this->set(array('message' => $message,'_serialize' => array('message'))); // $this->Event->validationErrors
$this->render('edit');
if ($this->_isRest()) {
App::uses('JSONConverterTool', 'Tools');
$converter = new JSONConverterTool();
$errors = $converter->jsonPrinter($this->Event->validationErrors);
$this->set('name', 'Add event failed.');
$this->set('message', $message);
$this->set('errors', $errors);
$this->set('url', '/events/add');
$this->set('_serialize', array('name', 'message', 'url', 'errors'));
} else {
$this->set(array('message' => $message,'_serialize' => array('message'))); // $this->Event->validationErrors
$this->render('edit');
}
//throw new MethodNotAllowedException("Validation ERROR: \n".var_export($this->Event->validationErrors, true));
return false;
}
@ -2162,38 +2113,49 @@ class EventsController extends AppController {
}
}
public function _addXMLFile($take_ownership = false) {
if (!empty($this->data) && $this->data['Event']['submittedxml']['size'] > 0 &&
is_uploaded_file($this->data['Event']['submittedxml']['tmp_name'])) {
$xmlData = fread(fopen($this->data['Event']['submittedxml']['tmp_name'], "r"),
$this->data['Event']['submittedxml']['size']);
public function _addMISPExportFile($ext, $take_ownership = false) {
$data = fread(fopen($this->data['Event']['submittedfile']['tmp_name'], "r"), $this->data['Event']['submittedfile']['size']);
if ($ext == 'xml') {
App::uses('Xml', 'Utility');
$xmlArray = Xml::toArray(Xml::build($xmlData));
// In case we receive an event that is not encapsulated in a response. This should never happen (unless it's a copy+paste fail),
// but just in case, let's clean it up anyway.
if (isset($xmlArray['Event'])) {
$xmlArray['response']['Event'] = $xmlArray['Event'];
unset($xmlArray['Event']);
}
if (!isset($xmlArray['response']) || !isset($xmlArray['response']['Event'])) {
throw new Exception('This is not a valid MISP XML file.');
}
$xmlArray = $this->Event->updateXMLArray($xmlArray);
if (isset($xmlArray['response']['Event'][0])) {
foreach ($xmlArray['response']['Event'] as $event) {
$temp['Event'] = $event;
if ($take_ownership) $temp['Event']['orgc'] = $this->Auth->user('org');
$this->Event->_add($temp, true, $this->Auth->user());
$dataArray = Xml::toArray(Xml::build($data));
} else {
$dataArray = json_decode($data, true);
if (isset($dataArray['response'][0])) {
foreach ($dataArray['response'] as $k => &$temp) {
$dataArray['Event'][] = $temp['Event'];
unset ($dataArray['response'][$k]);
}
} else {
$temp['Event'] = $xmlArray['response']['Event'];
if ($take_ownership) $temp['Event']['orgc'] = $this->Auth->user('org');
$this->Event->_add($temp, true, $this->Auth->user());
}
}
// In case we receive an event that is not encapsulated in a response. This should never happen (unless it's a copy+paste fail),
// but just in case, let's clean it up anyway.
if (isset($dataArray['Event'])) {
$dataArray['response']['Event'] = $dataArray['Event'];
unset($dataArray['Event']);
}
if (!isset($dataArray['response']) || !isset($dataArray['response']['Event'])) {
throw new Exception('This is not a valid MISP XML file.');
}
$dataArray = $this->Event->updateXMLArray($dataArray);
$results = array();
if (isset($dataArray['response']['Event'][0])) {
foreach ($dataArray['response']['Event'] as $k => $event) {
$result = array('info' => $event['info']);
if ($take_ownership) $event['orgc'] = $this->Auth->user('org');
$event = array('Event' => $event);
$created_id = 0;
$result['result'] = $this->Event->_add($event, true, $this->Auth->user(), '', null, false, null, $created_id);
$result['id'] = $created_id;
$results[] = $result;
}
} else {
$temp['Event'] = $dataArray['response']['Event'];
if ($take_ownership) $temp['Event']['orgc'] = $this->Auth->user('org');
$created_id = 0;
$result = $this->Event->_add($temp, true, $this->Auth->user(), '', null, false, null, $created_id);
$results = array(0 => array('info' => $temp['Event']['info'], 'result' => $result, 'id' => $created_id));
}
return $results;
}
public function _readGfiXML($data, $id) {

View File

@ -44,4 +44,23 @@ class JSONConverterTool {
$result = array('Event' => $event['Event']);
return json_encode($result);
}
public function jsonPrinter($array, $root = true) {
if (is_array($array)) {
$resultArray = array();
foreach ($array as $k => $element) {
$temp = $this->jsonPrinter($element, false);
if (!is_array($temp)) {
$resultArray[] = '[' . $k .']' . $temp;
} else {
foreach ($temp as &$t) $resultArray[] = '[' . $k . ']' . $t;
}
}
} else $resultArray = ': ' . $array . PHP_EOL;
if ($root) {
$text = '';
foreach ($resultArray as &$r) $text .= $r;
return $text;
} else return $resultArray;
}
}

View File

@ -150,7 +150,7 @@ class Event extends AppModel {
'date' => array(
'date' => array(
'rule' => array('date'),
//'message' => 'Your custom message here',
'message' => 'Expected date format: YYYY-MM-DD',
//'allowEmpty' => false,
'required' => true,
//'last' => false, // Stop validation after this rule
@ -508,19 +508,21 @@ class Event extends AppModel {
}
}
}
// get the new attribute uuids in an array
$newerUuids = array();
foreach ($event['Attribute'] as $attribute) {
$newerUuids[$attribute['id']] = $attribute['uuid'];
$attribute['event_id'] = $remoteId;
}
// get the already existing attributes and delete the ones that are not there
foreach ($xml->Event->Attribute as $attribute) {
foreach ($attribute as $key => $value) {
if ($key == 'uuid') {
if (!in_array((string)$value, $newerUuids)) {
$anAttr = ClassRegistry::init('Attribute');
$anAttr->deleteAttributeFromServer((string)$value, $server, $HttpSocket);
if ($remoteId) {
// get the new attribute uuids in an array
$newerUuids = array();
foreach ($event['Attribute'] as $attribute) {
$newerUuids[$attribute['id']] = $attribute['uuid'];
$attribute['event_id'] = $remoteId;
}
// get the already existing attributes and delete the ones that are not there
foreach ($xml->Event->Attribute as $attribute) {
foreach ($attribute as $key => $value) {
if ($key == 'uuid') {
if (!in_array((string)$value, $newerUuids)) {
$anAttr = ClassRegistry::init('Attribute');
$anAttr->deleteAttributeFromServer((string)$value, $server, $HttpSocket);
}
}
}
}
@ -1293,7 +1295,7 @@ class Event extends AppModel {
*
* @return bool true if success
*/
public function _add(&$data, $fromXml, $user, $org='', $passAlong = null, $fromPull = false, $jobId = null) {
public function _add(&$data, $fromXml, $user, $org='', $passAlong = null, $fromPull = false, $jobId = null, &$created_id = 0) {
if ($jobId) {
App::import('Component','Auth');
}
@ -1327,8 +1329,7 @@ class Event extends AppModel {
$data = $this->cleanupEventArrayFromXML($data);
// the event_id field is not set (normal) so make sure no validation errors are thrown
// LATER do this with $this->validator()->remove('event_id');
unset($this->Attribute->validate['event_id']);
unset($this->Attribute->validate['value']['unique']); // otherwise gives bugs because event_id is not set
unset($this->Attribute->validate['event_id']); // otherwise gives bugs because event_id is not set
}
unset ($data['Event']['id']);
if (isset($data['Event']['uuid'])) {
@ -1338,6 +1339,7 @@ class Event extends AppModel {
// RESTfull, set responce location header..so client can find right URL to edit
if ($fromPull) return false;
$existingEvent = $this->find('first', array('conditions' => array('Event.uuid' => $data['Event']['uuid'])));
if ($fromXml) $created_id = $existingEvent['Event']['id'];
return $existingEvent['Event']['id'];
}
}
@ -1355,6 +1357,7 @@ class Event extends AppModel {
'atomic' => true));
// FIXME chri: check if output of $saveResult is what we expect when data not valid, see issue #104
if ($saveResult) {
if ($fromXml) $created_id = $this->id;
if (!empty($data['Event']['published']) && 1 == $data['Event']['published']) {
// do the necessary actions to publish the event (email, upload,...)
if ('true' != Configure::read('MISP.disablerestalert')) {
@ -1365,7 +1368,7 @@ class Event extends AppModel {
return true;
} else {
//throw new MethodNotAllowedException("Validation ERROR: \n".var_export($this->Event->validationErrors, true));
return false;
return json_encode($this->validationErrors);
}
}
@ -1683,34 +1686,27 @@ class Event extends AppModel {
}
if (!$response) {
$temp = $xmlArray;
$xmlArray = array();
$xmlArray['response'] = $temp;
$xmlArray = array('response' => $xmlArray);
}
// if a version is set, it must be at least 2.2.0 - check the version and save the result of the comparison
if (isset($xmlArray['response']['xml_version'])) $version = $this->compareVersions($xmlArray['response']['xml_version'], $this->mispVersion);
// if no version is set, set the version to older (-1) manually
else $version = -1;
// same version, proceed normally
if ($version == 0) {
unset ($xmlArray['response']['xml_version']);
if ($response) return $xmlArray;
else return $xmlArray['response'];
}
// The xml is from an instance that is newer than the local instance, let the user know that the admin needs to upgrade before it could be imported
if ($version == 1) throw new Exception('This XML file is from a MISP instance that is newer than the current instance. Please contact your administrator about upgrading this instance.');
// if the xml contains an event or events from an older MISP instance, let's try to upgrade it!
// Let's manually set the version to something below 2.2.0 if there is no version set in the xml
if (!isset($xmlArray['response']['xml_version'])) $xmlArray['response']['xml_version'] = '2.1.0';
// Upgrade from versions below 2.2.0 will need to replace the risk field with threat level id
if ($this->compareVersions($xmlArray['response']['xml_version'], '2.2.0') < 0) {
if ($response) $xmlArray['response'] = $this->__updateXMLArray220($xmlArray['response']);
else $xmlArray = $this->__updateXMLArray220($xmlArray);
}
if ($version != 0) {
// The xml is from an instance that is newer than the local instance, let the user know that the admin needs to upgrade before it could be imported
if ($version == 1) throw new Exception('This XML file is from a MISP instance that is newer than the current instance. Please contact your administrator about upgrading this instance.');
// if the xml contains an event or events from an older MISP instance, let's try to upgrade it!
// Let's manually set the version to something below 2.2.0 if there is no version set in the xml
if (!isset($xmlArray['response']['xml_version'])) $xmlArray['response']['xml_version'] = '2.1.0';
// Upgrade from versions below 2.2.0 will need to replace the risk field with threat level id
if ($this->compareVersions($xmlArray['response']['xml_version'], '2.2.0') < 0) {
if ($response) $xmlArray['response'] = $this->__updateXMLArray220($xmlArray['response']);
else $xmlArray = $this->__updateXMLArray220($xmlArray);
}
}
unset ($xmlArray['response']['xml_version']);
if ($response) return $xmlArray;
else return $xmlArray['response'];

View File

@ -11,7 +11,7 @@
<?php if ($me != false ):?>
<div class="nav-collapse collapse">
<ul class="nav">
<li><a href="<?php echo $baseurl;?>" style="color:white">Home</a></li>
<li><a href="<?php echo !empty($baseurl) ? $baseurl : '/';?>" style="color:white">Home</a></li>
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" href="#">
Event Actions

View File

@ -55,7 +55,7 @@
<li <?php if ($menuItem === 'index') echo 'class="active"';?>><a href="<?php echo $baseurl;?>/events/index">List Events</a></li>
<?php if ($isAclAdd): ?>
<li <?php if ($menuItem === 'add') echo 'class="active"';?>><a href="<?php echo $baseurl;?>/events/add">Add Event</a></li>
<li <?php if ($menuItem === 'addXML') echo 'class="active"';?>><a href="<?php echo $baseurl;?>/events/add_xml">Add MISP XML</a></li>
<li <?php if ($menuItem === 'addMISPExport') echo 'class="active"';?>><a href="<?php echo $baseurl;?>/events/add_misp_export">Import from MISP Export</a></li>
<?php endif; ?>
<li class="divider"></li>
<li <?php if ($menuItem === 'listAttributes') echo 'class="active"';?>><a href="<?php echo $baseurl;?>/attributes/index">List Attributes</a></li>

View File

@ -1,10 +1,10 @@
<div class="events form">
<?php echo $this->Form->create('Event', array('type' => 'file'));?>
<fieldset>
<legend><?php echo __('Import from MISP XML'); ?></legend>
<legend><?php echo __('Import from MISP Export File'); ?></legend>
<?php
echo $this->Form->input('Event.submittedxml', array(
'label' => '<b>MISP XML</b>',
echo $this->Form->input('Event.submittedfile', array(
'label' => '<b>MISP XML or JSON file</b>',
'type' => 'file',
));
if (Configure::read('MISP.take_ownership_xml_import')):
@ -25,5 +25,5 @@
?>
</div>
<?php
echo $this->element('side_menu', array('menuList' => 'event-collection', 'menuItem' => 'addXML'));
echo $this->element('side_menu', array('menuList' => 'event-collection', 'menuItem' => 'addMISPExport'));
?>

View File

@ -0,0 +1,46 @@
<div class="eventAddXML index">
<h2>Add From MISP Export Result</h2>
<table class="table table-striped table-hover table-condensed">
<tr>
<th>Event info</th>
<th>Result</th>
<th>Details</th>
</tr>
<?php
App::uses('JSONConverterTool', 'Tools');
$converter = new JSONConverterTool();
foreach ($results as &$result):
$status = 'Failed';
$text = $result['result'];
$colour = 'red';
if ($result['result'] === true) {
$colour = 'green';
$status = 'OK';
$text = 'Event created.';
} else if (is_numeric($result['result'])) {
$text = 'Event with this UUID already exists.';
} else {
$temp = json_decode($text, true);
$text = $converter->jsonPrinter($temp);
}
?>
<tr>
<td class="short"><?php echo h($result['info']); ?>&nbsp;</td>
<td class="short" style="color:<?php echo $colour; ?>"><?php echo h($status); ?>&nbsp;</td>
<td class="short">
<?php
echo nl2br(h($text));
if (0 !== ($result['id'])) echo ' <a href="' . $baseurl . '/events/view/' . h($result['id']) . '">Event ' . h($result['id']) . '</a>';
?>
&nbsp;
</td>
</tr>
<?php
endforeach;
?>
</table>
</div>
<?php
echo $this->element('side_menu', array('menuList' => 'admin', 'menuItem' => 'eventBlacklists'));
?>