chg: [cleanup] Removed the deprecated GFI sandbox import

- Burn the heretic. Kill the mutant. Purge the unclean.
pull/3470/head
iglocska 2018-07-06 10:57:44 +02:00
parent 43698812e6
commit 8d567782d9
6 changed files with 74 additions and 369 deletions

View File

@ -1394,103 +1394,89 @@ class EventsController extends AppController {
$this->request->data['Event']['user_id'] = $this->Auth->user('id');
}
if (!empty($this->data)) {
$ext = '';
if (isset($this->data['Event']['submittedgfi'])) {
App::uses('File', 'Utility');
$file = new File($this->data['Event']['submittedgfi']['name']);
$ext = $file->ext();
if (!isset($this->request->data['Event']['distribution'])) {
$this->request->data['Event']['distribution'] = Configure::read('MISP.default_event_distribution') ? Configure::read('MISP.default_event_distribution') : 0;
}
if (isset($this->data['Event']['submittedgfi']) && ($ext != 'zip') && $this->data['Event']['submittedgfi']['size'] > 0 &&
is_uploaded_file($this->data['Event']['submittedgfi']['tmp_name'])) {
$this->Flash->error(__('You may only upload GFI Sandbox zip files.'));
} else {
if (!isset($this->request->data['Event']['distribution'])) {
$this->request->data['Event']['distribution'] = Configure::read('MISP.default_event_distribution') ? Configure::read('MISP.default_event_distribution') : 0;
}
if (!isset($this->request->data['Event']['analysis'])) {
$this->request->data['Event']['analysis'] = 0;
}
if (!isset($this->request->data['Event']['threat_level_id'])) {
$this->request->data['Event']['threat_level_id'] = Configure::read('MISP.default_event_threat_level') ? Configure::read('MISP.default_event_threat_level') : 4;
}
if (!isset($this->request->data['Event']['date'])) {
$this->request->data['Event']['date'] = date('Y-m-d');
}
// If the distribution is set to sharing group, check if the id provided is really visible to the user, if not throw an error.
if ($this->request->data['Event']['distribution'] == 4) {
if ($this->userRole['perm_sync'] && $this->_isRest()) {
if (isset($this->request->data['Event']['SharingGroup'])) {
if (!isset($this->request->data['Event']['SharingGroup']['uuid'])) {
if ($this->Event->SharingGroup->checkIfExists($this->request->data['Event']['SharingGroup']['uuid']) &&
$this->Event->SharingGroup->checkIfAuthorised($this->Auth->user(), $this->request->data['Event']['SharingGroup']['uuid'])) {
throw new MethodNotAllowedException('Invalid Sharing Group or not authorised (Sync user is not contained in the Sharing group).');
}
if (!isset($this->request->data['Event']['analysis'])) {
$this->request->data['Event']['analysis'] = 0;
}
if (!isset($this->request->data['Event']['threat_level_id'])) {
$this->request->data['Event']['threat_level_id'] = Configure::read('MISP.default_event_threat_level') ? Configure::read('MISP.default_event_threat_level') : 4;
}
if (!isset($this->request->data['Event']['date'])) {
$this->request->data['Event']['date'] = date('Y-m-d');
}
// If the distribution is set to sharing group, check if the id provided is really visible to the user, if not throw an error.
if ($this->request->data['Event']['distribution'] == 4) {
if ($this->userRole['perm_sync'] && $this->_isRest()) {
if (isset($this->request->data['Event']['SharingGroup'])) {
if (!isset($this->request->data['Event']['SharingGroup']['uuid'])) {
if ($this->Event->SharingGroup->checkIfExists($this->request->data['Event']['SharingGroup']['uuid']) &&
$this->Event->SharingGroup->checkIfAuthorised($this->Auth->user(), $this->request->data['Event']['SharingGroup']['uuid'])) {
throw new MethodNotAllowedException('Invalid Sharing Group or not authorised (Sync user is not contained in the Sharing group).');
}
} else if (!isset($sgs[$this->request->data['Event']['sharing_group_id']])) {
throw new MethodNotAllowedException('Invalid Sharing Group or not authorised.');
}
} else {
if (!isset($sgs[$this->request->data['Event']['sharing_group_id']])) throw new MethodNotAllowedException('Invalid Sharing Group or not authorised.');
} else if (!isset($sgs[$this->request->data['Event']['sharing_group_id']])) {
throw new MethodNotAllowedException('Invalid Sharing Group or not authorised.');
}
} else {
// If the distribution is set to something "traditional", set the SG id to 0.
$this->request->data['Event']['sharing_group_id'] = 0;
if (!isset($sgs[$this->request->data['Event']['sharing_group_id']])) throw new MethodNotAllowedException('Invalid Sharing Group or not authorised.');
}
// If we are not sync users / site admins, we only allow events to be created for our own org
// Set the orgc ID as our own orgc ID and unset both the 2.4 and 2.3 style creator orgs
if ($this->_isRest() && !$this->userRole['perm_sync']) {
$this->request->data['Event']['orgc_id'] = $this->Auth->user('org_id');
if (isset($this->request->data['Event']['Orgc'])) {
unset($this->request->data['Event']['Orgc']);
}
if (isset($this->request->data['Event']['orgc'])) {
unset($this->request->data['Event']['orgc']);
}
} else {
// If the distribution is set to something "traditional", set the SG id to 0.
$this->request->data['Event']['sharing_group_id'] = 0;
}
// If we are not sync users / site admins, we only allow events to be created for our own org
// Set the orgc ID as our own orgc ID and unset both the 2.4 and 2.3 style creator orgs
if ($this->_isRest() && !$this->userRole['perm_sync']) {
$this->request->data['Event']['orgc_id'] = $this->Auth->user('org_id');
if (isset($this->request->data['Event']['Orgc'])) {
unset($this->request->data['Event']['Orgc']);
}
$validationErrors = array();
$created_id = 0;
$add = $this->Event->_add($this->request->data, $this->_isRest(), $this->Auth->user(), '', null, false, null, $created_id, $validationErrors);
if ($add === true && !is_numeric($add)) {
if ($this->_isRest()) {
if ($add === 'blocked') {
throw new ForbiddenException('Event blocked by local blacklist.');
}
// REST users want to see the newly created event
$results = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $created_id));
$event = $results[0];
if (!empty($validationErrors)) {
$event['errors'] = $validationErrors;
}
$this->set('event', $event);
$this->render('view');
return true;
} else {
// TODO now save uploaded attributes using $this->Event->getID() ..
if (isset($this->data['Event']['submittedgfi'])) $this->_addGfiZip($this->Event->getID());
// redirect to the view of the newly created event
$this->Flash->success(__('The event has been saved'));
$this->redirect(array('action' => 'view', $this->Event->getID()));
if (isset($this->request->data['Event']['orgc'])) {
unset($this->request->data['Event']['orgc']);
}
}
$validationErrors = array();
$created_id = 0;
$add = $this->Event->_add($this->request->data, $this->_isRest(), $this->Auth->user(), '', null, false, null, $created_id, $validationErrors);
if ($add === true && !is_numeric($add)) {
if ($this->_isRest()) {
if ($add === 'blocked') {
throw new ForbiddenException('Event blocked by local blacklist.');
}
// REST users want to see the newly created event
$results = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $created_id));
$event = $results[0];
if (!empty($validationErrors)) {
$event['errors'] = $validationErrors;
}
$this->set('event', $event);
$this->render('view');
return true;
} else {
if ($this->_isRest()) { // TODO return error if REST
if (is_numeric($add)) {
$this->response->header('Location', Configure::read('MISP.baseurl') . '/events/' . $add);
$this->response->send();
throw new NotFoundException('Event already exists, if you would like to edit it, use the url in the location header.');
}
$this->set('name', 'Add event failed.');
$this->set('message', 'The event could not be saved.');
$this->set('errors', $validationErrors);
$this->set('url', '/events/add');
$this->set('_serialize', array('name', 'message', 'url', 'errors'));
return false;
// redirect to the view of the newly created event
$this->Flash->success(__('The event has been saved'));
$this->redirect(array('action' => 'view', $this->Event->getID()));
}
} else {
if ($this->_isRest()) { // TODO return error if REST
if (is_numeric($add)) {
$this->response->header('Location', Configure::read('MISP.baseurl') . '/events/' . $add);
$this->response->send();
throw new NotFoundException('Event already exists, if you would like to edit it, use the url in the location header.');
}
$this->set('name', 'Add event failed.');
$this->set('message', 'The event could not be saved.');
$this->set('errors', $validationErrors);
$this->set('url', '/events/add');
$this->set('_serialize', array('name', 'message', 'url', 'errors'));
return false;
} else {
if ($add === 'blocked') {
$this->Flash->error('A blacklist entry is blocking you from creating any events. Please contact the administration team of this instance' . (Configure::read('MISP.contact') ? ' at ' . Configure::read('MISP.contact') : '') . '.');
} else {
if ($add === 'blocked') {
$this->Flash->error('A blacklist entry is blocking you from creating any events. Please contact the administration team of this instance' . (Configure::read('MISP.contact') ? ' at ' . Configure::read('MISP.contact') : '') . '.');
} else {
$this->Flash->error(__('The event could not be saved. Please, try again.'), 'default', array(), 'error');
}
$this->Flash->error(__('The event could not be saved. Please, try again.'), 'default', array(), 'error');
}
}
}
@ -2601,47 +2587,6 @@ class EventsController extends AppController {
return $this->RestResponse->viewData($final, 'csv', false, true, $filename);
}
public function _addGfiZip($id) {
if (!empty($this->data) && $this->data['Event']['submittedgfi']['size'] > 0 &&
is_uploaded_file($this->data['Event']['submittedgfi']['tmp_name'])) {
App::uses('FileAccessTool', 'Tools');
$fileAccessTool = new FileAccessTool();
$zipData = $fileAccessTool->readFromFile($this->data['Event']['submittedgfi']['tmp_name'], $this->data['Event']['submittedgfi']['size']);
// write
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$this->loadModel('Server');
$attachments_dir = $this->Server->getDefaultAttachments_dir();
}
$rootDir = $attachments_dir . DS . "GFI" . DS . $id . DS;
App::uses('Folder', 'Utility');
$dir = new Folder($rootDir, true);
if (!$this->Event->checkFilename($this->data['Event']['submittedgfi']['name'])) {
throw new Exception ('Filename not allowed.');
}
$zipFile = new File($rootDir . $this->data['Event']['submittedgfi']['name']);
$result = $zipFile->write($zipData);
if (!$result) $this->Flash->error(__('Problem with writing the zip file. Please report to administrator.'));
// extract zip
$execRetval = '';
$execOutput = array();
exec("unzip " . $zipFile->path . ' -d ' . $rootDir, $execOutput, $execRetval);
if ($execRetval != 0) { // not EXIT_SUCCESS
throw new Exception('An error has occured while attempting to unzip the GFI sandbox .zip file. We apologise for the inconvenience.');
}
// open the xml
$xmlFileName = 'analysis.xml';
$xmlFilePath = $rootDir . DS . 'Analysis' . DS . $xmlFileName;
$xmlFileData = $fileAccessTool->readFromFile($xmlFilePath);
// read XML
$this->_readGfiXML($xmlFileData, $id);
}
}
public function _addIOCFile($id) {
if (!empty($this->data) && $this->data['Event']['submittedioc']['size'] > 0 &&
is_uploaded_file($this->data['Event']['submittedioc']['tmp_name'])) {
@ -2794,176 +2739,6 @@ class EventsController extends AppController {
return $results;
}
public function _readGfiXML($data, $id) {
$this->loadModel('Attribute');
$this->Event->recursive = -1;
$this->Event->read(array('id', 'uuid', 'distribution'), $id);
// import XML class
App::uses('Xml', 'Utility');
// now parse it
try {
$parsedXml = Xml::build($data, array('return' => 'simplexml'));
} catch (Exception $e) {
$this->Flash->error('Invalid GFI archive.');
$this->redirect(array('controller' => 'events', 'action' => 'view', $id));
}
// xpath..
if (Configure::read('MISP.default_attribute_distribution') != null) {
if (Configure::read('MISP.default_attribute_distribution') === 'event') {
$dist = $this->Event->data['Event']['distribution'];
} else {
$dist = '';
$dist .= Configure::read('MISP.default_attribute_distribution');
}
} else {
throw new Exception('Couldn\'t read "MISP.default_attribute_distribution".');
}
// Payload delivery -- malware-sample
$realFileName = '';
$results = $parsedXml->xpath('/analysis');
foreach ($results as $result) {
foreach ($result[0]->attributes() as $key => $val) {
if ((string)$key == 'filename') $realFileName = (string)$val;
}
}
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$this->loadModel('Server');
$attachments_dir = $this->Server->getDefaultAttachments_dir();
}
$rootDir = $attachments_dir . DS . $id . DS;
$malware = $rootDir . DS . 'sample';
$this->Event->Attribute->uploadAttachment($malware, $realFileName, true, $id, null, '', $this->Event->data['Event']['uuid'] . '-sample', $dist, true);
// Network activity -- .pcap
$realFileName = 'analysis.pcap';
$rootDir = $attachments_dir . DS . $id . DS;
$malware = $rootDir . DS . 'Analysis' . DS . 'analysis.pcap';
$this->Event->Attribute->uploadAttachment($malware, $realFileName, false, $id, 'Network activity', '', $this->Event->data['Event']['uuid'] . '-analysis.pcap', $dist, true);
// Artifacts dropped -- filename|md5
$files = array();
// TODO what about stored_modified_file ??
$results = $parsedXml->xpath('/analysis/processes/process/stored_files/stored_created_file');
foreach ($results as $result) {
$arrayItemKey = '';
$arrayItemValue = '';
$arrayItemSize = 0;
foreach ($result[0]->attributes() as $key => $val) {
if ($key == 'filename') $arrayItemKey = (string)$val;
if ($key == 'md5') $arrayItemValue = (string)$val;
if ($key == 'filesize') $arrayItemSize = $val;
}
if ($arrayItemSize > 0) {
$files[] = array('key' => $arrayItemKey, 'val' => $arrayItemValue);
}
}
// write content..
$actualFileNameArray = array();
foreach ($files as $file) {
$keyName = $file['key'];
// the actual files..
// seek $val in dirs and add..
$ext = substr($file['key'], strrpos($file['key'], '.'));
$actualFileName = $file['val'] . $ext;
$actualFileNameBase = str_replace('\\', '/', $file['key']);
$actualFileNameArray[] = basename($actualFileNameBase);
$tempExplode = explode('\\', $file['key']);
$realFileName = end($tempExplode);
// have the filename, now look at parents parent for the process number
$express = "/analysis/processes/process/stored_files/stored_created_file[@md5='" . $file['val'] . "']/../..";
$results = $parsedXml->xpath($express);
foreach ($results as $result) {
foreach ($result[0]->attributes() as $key => $val) {
if ((string)$key == 'index') $index = (string)$val;
}
}
if (!isset($index) || !is_numeric($index)) {
throw new Exception('The GFI sandbox xml file seems to be malformed, at least one process with stored_files hasn\'t got a valid numeric index attribute.');
}
$actualFile = $rootDir . DS . 'Analysis' . DS . 'proc_' . $index . DS . 'modified_files' . DS . $actualFileName;
$extraPath = 'Analysis' . DS . 'proc_' . $index . DS . 'modified_files' . DS;
$file = new File($actualFile);
if ($file->exists()) { // TODO put in array for test later
$this->Event->Attribute->uploadAttachment($actualFile, $realFileName, true, $id, null, $extraPath, $keyName, $dist, true); // TODO was false
} else {
}
}
// Network activity -- ip-dst
$ips = array();
$hostnames = array();
$results = $parsedXml->xpath('/analysis/processes/process/networkpacket_section/connect_to_computer');
foreach ($results as $result) {
foreach ($result[0]->attributes() as $key => $val) {
if ($key == 'remote_ip') $ips[] = (string)$val;
if ($key == 'remote_hostname') $hostnames[] = (string)$val;
}
}
// write content..
// ip-s
foreach ($ips as $ip) {
// add attribute..
$this->Attribute->create();
$this->Attribute->save(array(
'event_id' => $id,
'category' => 'Network activity',
'type' => 'ip-dst',
'value' => $ip,
'to_ids' => false,
'distribution' => $dist,
'comment' => 'GFI import',
));
}
foreach ($hostnames as $hostname) {
// add attribute..
$this->Attribute->create();
$this->Attribute->save(array(
'event_id' => $id,
'category' => 'Network activity',
'type' => 'hostname',
'value' => $hostname,
'to_ids' => false,
'distribution' => $dist,
'comment' => 'GFI import',
));
}
// Persistence mechanism -- regkey|value
$regs = array();
$results = $parsedXml->xpath('/analysis/processes/process/registry_section/set_value');
foreach ($results as $result) {
$arrayItemKey = '';
$arrayItemValue = '';
foreach ($result[0]->attributes() as $key => $val) {
if ($key == 'key_name') $arrayItemKey = (string)$val;
if ($key == 'data') $arrayItemValue = (string)$val;
}
$regs[$arrayItemKey] = str_replace('(UNICODE_0x00000000)', '', $arrayItemValue);
}
// write content..
foreach ($regs as $key => $val) {
// add attribute..
$this->Attribute->create();
if ($this->__strposarray($val,$actualFileNameArray)) {
$this->Attribute->save(array(
'event_id' => $id,
'comment' => 'GFI import',
'category' => 'Persistence mechanism', // 'Persistence mechanism'
'type' => 'regkey|value',
'value' => $key . '|' . $val,
'distribution' => $dist,
'to_ids' => false
));
}
}
}
private function __strposarray($string, $array) {
$toReturn = false;
foreach ($array as $item) {

View File

@ -1453,67 +1453,6 @@ class Attribute extends AppModel {
}
}
public function uploadAttachment($fileP, $realFileName, $malware, $eventId = null, $category = null, $extraPath = '', $fullFileName = '', $dist, $fromGFI = false) {
// Check if there were problems with the file upload
// only keep the last part of the filename, this should prevent directory attacks
$filename = basename($fileP);
$tmpfile = new File($fileP);
// save the file-info in the database
$this->create();
$this->data['Attribute']['event_id'] = $eventId;
$this->data['Attribute']['distribution'] = $dist;
if ($malware) {
$md5 = !$tmpfile->size() ? md5_file($fileP) : $tmpfile->md5();
$this->data['Attribute']['category'] = $category ? $category : "Payload delivery";
$this->data['Attribute']['type'] = "malware-sample";
$this->data['Attribute']['value'] = $fullFileName ? $fullFileName . '|' . $md5 : $filename . '|' . $md5; // TODO gives problems with bigger files
$this->data['Attribute']['to_ids'] = 1; // LATER let user choose whether to send this to an IDS
if ($fromGFI) $this->data['Attribute']['comment'] = 'GFI import';
} else {
$this->data['Attribute']['category'] = $category ? $category : "Artifacts dropped";
$this->data['Attribute']['type'] = "attachment";
$this->data['Attribute']['value'] = $fullFileName ? $fullFileName : $realFileName;
$this->data['Attribute']['to_ids'] = 0;
if ($fromGFI) $this->data['Attribute']['comment'] = 'GFI import';
}
if (!$this->save($this->data)) {
// TODO: error handling
}
// no errors in file upload, entry already in db, now move the file where needed and zip it if required.
// no sanitization is required on the filename, path or type as we save
// create directory structure
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$rootDir = $attachments_dir . DS . $eventId;
$dir = new Folder($rootDir, true);
// move the file to the correct location
$destpath = $rootDir . DS . $this->getID(); // id of the new attribute in the database
$file = new File($destpath);
$zipfile = new File($destpath . '.zip');
$fileInZip = new File($rootDir . DS . $extraPath . $filename);
// zip and password protect the malware files
if ($malware) {
$execRetval = '';
$execOutput = array();
exec('zip -j -P infected ' . escapeshellarg($zipfile->path) . ' ' . escapeshellarg($fileInZip->path), $execOutput, $execRetval);
if ($execRetval != 0) { // not EXIT_SUCCESS
throw new Exception('An error has occured while attempting to zip the malware file.');
}
$fileInZip->delete(); // delete the original non-zipped-file
rename($zipfile->path, $file->path); // rename the .zip to .nothing
} else {
$fileAttach = new File($fileP);
rename($fileAttach->path, $file->path);
}
}
public function __beforeSaveCorrelation($a) {
// (update-only) clean up the relation of the old value: remove the existing relations related to that attribute, we DO have a reference, the id
// ==> DELETE FROM correlations WHERE 1_attribute_id = $a_id OR attribute_id = $a_id; */

View File

@ -24,7 +24,6 @@ class Event extends AppModel {
public $fieldDescriptions = array(
'threat_level_id' => array('desc' => 'Risk levels: *low* means mass-malware, *medium* means APT malware, *high* means sophisticated APT malware or 0-day attack', 'formdesc' => 'Risk levels: low: mass-malware medium: APT malware high: sophisticated APT malware or 0-day attack'),
'classification' => array('desc' => 'Set the Traffic Light Protocol classification. <ol><li><em>TLP:AMBER</em>- Share only within the organization on a need-to-know basis</li><li><em>TLP:GREEN:NeedToKnow</em>- Share within your constituency on the need-to-know basis.</li><li><em>TLP:GREEN</em>- Share within your constituency.</li></ol>'),
'submittedgfi' => array('desc' => 'GFI sandbox: export upload', 'formdesc' => 'GFI sandbox: export upload'),
'submittedioc' => array('desc' => '', 'formdesc' => ''),
'analysis' => array('desc' => 'Analysis Levels: *Initial* means the event has just been created, *Ongoing* means that the event is being populated, *Complete* means that the event\'s creation is complete', 'formdesc' => 'Analysis levels: Initial: event has been started Ongoing: event population is in progress Complete: event creation has finished'),
'distribution' => array('desc' => 'Describes who will have access to the event.')

View File

@ -67,13 +67,6 @@
));
?>
<div id="extended_event_preview" style="width:446px;"></div>
<?php
echo $this->Form->input('Event.submittedgfi', array(
'label' => '<b>GFI sandbox</b>',
'type' => 'file',
'div' => 'clear'
));
?>
</fieldset>
<?php
echo $this->Form->button(__('Add'), array('class' => 'btn btn-primary'));

View File

@ -62,7 +62,7 @@
<?php echo __('Administrators can add, edit or delete blacklisted items by using the appropriate functions in the list\'s action menu and the menu on the left.');?><br />
<hr />
<h2><a id="regexp"></a><?php echo __('Import Regexp');?></h2>
<?php echo __('The system allows administrators to set up rules for regular expressions that will automatically alter newly entered or imported events (from GFI Sandbox).');?><br />
<?php echo __('The system allows administrators to set up rules for regular expressions that will automatically alter newly entered or imported events.');?><br />
<h3><?php echo __('The purpose of Import Regexp entries');?></h3>
<?php echo __('They can be used for several things, such as unifying the capitalisation of file paths for more accurate event correlation or to automatically censor the usernames and use system path variable names (changing C:\Users\UserName\Appdata\Roaming\file.exe to %APPDATA%\file.exe).<br />
The second use is blocking, if a regular expression is entered with a blank replacement, any event info or attribute value containing the expression will not be added. Please make sure the entered regexp expression follows the preg_replace pattern rules as described <a href="http://php.net/manual/en/function.preg-replace.php">here</a>.');?><br />

View File

@ -74,7 +74,6 @@ and attachments and finally publishing it.');?><br /><br />
<li><i><?php echo __('Completed');?>:</i> <?php echo __('The analysis is complete');?></li>
</ul></li>
<li><b><?php echo __('Event Description');?>:</b> <?php echo __('The info field, where the malware/incident can get a brief description starting with the internal reference. This field should be as brief and concise as possible, the more detailed description happens through attributes in the next stage of the event\'s creation. Keep in mind that the system will automatically replace detected text strings that match a regular expression entry set up by your server\'s administrator(s).');?></li>
<li><b><?php echo __('GFI Sandbox');?>:</b> <?php echo __('It is possible to upload the exported .zip file from GFI sandbox with the help of this tool. These will be dissected by the MISP and a list of attributes and attachments will automatically be generated from the .zip file. Whilst this does most of the work needed to be done in the second step of the event\'s creation, it is important to manually look over all the data that is being entered.');?></li>
</ul>
<hr />
<a id="create_attribute"></a><h3><?php echo __('Add attributes to the event');?>:</h3>