new: Rework of the attachment uploader

- add attachments and upload_sample now share code
- allow the same features via upload_sample (object creation / use of advanced add attachments)
- new flag: advanced

- example:

  POST to mymisp/events/upload_sample
  BODY:
{"request":{"files": [{"filename": "bla.exe", "data": "U3RhckNyYWZ0IElJIGZvcmV2ZXI="}], "distribution": 1, "advanced":1, "info":"bla"}}

- this commit was brought to you by CEF and

MMMH$= -  .,   ,,.          %H++  ,= %%$$$$X+ ;=== .=  :+HHHMMMHMMM####MMH@@@@@@HHH$=      HHH@HHHHH+XXX$$$$$$$$XXXXXXX+
MMH = -.  . ,-,,-,.         :H@H  =;;++$HH+XX$%+X%+$++=:=.XH@@@HMMMMMMMMH@@@@@@@HHX$   ,X@@@@@@@HHHHHHHHHHXXXXXXXXXXXXXX
  . ---,  -    ,,,            +@ .. ;++$HH+HHH++$+++HH+++, .+%HHMHHHHHHHHH+%%%++++$+   +++HHHHHHH+++++++++HHHHHHHHHHHHHH
- -- ,,,  --,. -                 , ,; +$XHH@@@@HHH@@@HHHH+$+$X+HH+$$+ ;  ;=  .    %   +  ,+$X+++XXXXXXXXXXXXX++HH+++++++
---==,,--,-,-., :     .          -,,:/ $XHH@HMMMMMMMMMM@HHX$H@MHHHHX+H%%$%+H/:.%. $. @,,,. $$XXXXXXXXXXXXXXXXXXXXXXXXXX+
  =  - --,,   , --   ..             =/ +$+H@@HMMMMMMMMH+H+++HHHHHHHH@+++++H+X++X+$$  = ,,, - $$XXXXX$$$$X$$$$$$$$$$$$$$X
====== --,,,, ,= =              ,==== ++$$+HHMMM####MH+$$+++HH@+HH@MHMMH@@H@@@HH+$+    ,,, ,. $$+$++$$$$$$$$$$++$$$$$$$X
 :==-===-,. ,., ==   .           :;; +++%$+H@HMMMMMMM%$%$$$+H@@+HH@MMMMMM@@@@HHH++H. .,,-,,--=/+$$%%%%%%%%$+%%$$$$$XXXXX
,  =  ==- -  .  ==             . =; ++++%++HHHHHHHHHH++%$$X+@@H+HHHMMMMMMHH@@@+X+    , ,,,,-  , ,$$$$$$$+++++$$$$XXXXX$$
,,-       ,    --=    ..       . ;/ ++++%$X+HHHHHHH  ++$++X+HH+X+H@HMMHHHHHHHH+.       ,,  ,,  , .    +$$$$+%+$$$$$$$$$$
,-----=-=--,   ,==             ..;/ +% +%$XX+HH++HH+/+$%++H@@HHXHHH@@@@@@@@HXX  .   .,,,.  ,,,,     ,-=$$$$$$$$$$$$$$$$$
 - ,- --  -,   ,-=     .         =/++%++%+++++XXXXX$$+.  +HHH@+$XHHHHHHHHH++$        -,,,  ,,      ,,,.   ,+$$$$$$$$$$$$
 ---,-----, .   ==               =/+%+++%++$$+++$X$$$$++,$$+++XXHHHHHHHH+X$+%       ,-,-,        ,,    .  .  ,+$$+++++++
== --, -- =--, ,,=          .    ./++$$++$+X$+/++$$XXXX$$$$XXXXXXH+HH+H+X$%%/     .,,,,,,    ..  ..    ,. ,,,-=+%+++ /++
+   -- -  -,,-  .,    .  . .      = +$$++++HH+.  ,+$$+++++++$XX$X$XHHH+X$$+      ..--,-    .. .        .    ,-, = ======
MH - ---- --,,,    .       .. ,      %++$$X++++ +%++++++++%++$$$$$+H++X$$+        --,    .         .   .        =  .====
MM=,-, ---,,,,,    . .     ...,,,   =/++%$$XXXX+/+++@@H@HX$+%$$+HHHHH$$$+:       ,--    .     ,. ..       .. ==::;=-:;;;
MM+ ,----,,,,              , .. ,.      +++X+HH+++++%++$++++$$+HHH+++$$          ,-          ,   .       .   : ;/ +%+.
MMH ,-,-,, ,,.        .    -,     =     = +$+H@HH++++$$X$$+++HHH+++$                       ,    ..       ,  +++++++%%+%+
MM@,--,-,,,,,. .     ,,     .    ,-,    .=+$XHHHXXHHHHHHHH@@@@HX$%+:          ,, .      ..,,  .....    ...%%%%++%%%%%%%%
M@@== ,,,  ,                               ++++XX++HHHHHH++HHH+,              ,         ,  .  ....     . +$+%%%%%%+%%%%%
H@H+=,,,  ..                                  ,,+%$+H@HHHXX++,               ,         ,,  .  ...   . ,$$$$$%%%%%+%+%%%%
@H+,-,,.....       .                          .,.;; ++$$X+%+:-              ,  .     .,,,  .  ...   . XXX$$$%%%%%%+%%%%%
+++ -, . ...                             .  .======== === ,                          ,, . .  ..   . -,XXX$X$+$+%%%%%%%%%
$+     .                                ===:; ++++ ++++-,.  ,                       ,-,          .  $X+XX+XXX$$+%++%%%%%
++: ,. .                         ,-,,-==:; %%%%%+%$$%$$X$$$+%+:==        .        . ,,           ..+X$XXXXXX$$$+%%$$%%%%
=:                              ,,,  ==   ++++++$+$$%+++$$$++$+ . ==     .        .,,,             +$$$$$$$$$$$$$$+$%%%+
 ,                          ,---, =:;/++$$XX$$$$$$X+H@H@HHH$%%%$X$++;===== .      .,            .. +%%+$++$%$$$$$$%%++%+
                               ===; +++$$$$+ +%+++%+HH@@@@HH+++ ++%+$+,  ===      ..             ,=;   +++++++++..   :;;
                      .   =:;   /++%$$++,  ,++HHMMHH@@@@HHHH@HH++++++ ,+$$+ .     ..                :=;;:;;;;;==========
                  .,,-==;;;+%  %%+$$$$ /+++@@@@@@@@@@HH@M@MH@@@HHHHH$$% /%$XXX$X  .                -=====::::=========::
                .    =;  ++++++$+++  , +%H@@@HHH@HH++HHH@MHHH@HHHHHH++++ , +%%+$                    ,, -       --- ==:=:
               ====;    ++++$$+%  ++H@HHHHHHH+X++X++@@@HHH@MMMMHHHHHH@HHHH+++++.                        ,,,,-,--- =:==;;
     .,., ==;// / ++++%+%+%+++$$+@H@@@@H@HHH+XXX$%+HHHH@@HH@HMMMMMMMMMMMMMMH@+%;                       ...,,,,,--==;;;/;
 .  ...=    .,+%$++%+$XXX$++%+++H@@@@HHH@HHH+++.   ++++H+HHHHHHHMMMMMMMMMMMM@++:                            ,,, ===;;;;;
==: .  ++++++++HH%H+++X++HH+H@HHHH@HHHHHHH+++++%++%%+%%++ . ,   = ++$H@@HMHMMH%=                                .  ..,,=
+++%$XXHHHHHH@H@@@@@H@HH@MMM@@HH@HH+HXH@HH%%+HH+XX$$$+++/;:=== ,,,,,, = ::; % :,                                   ...,,
%+++HHH@HHH@@HMHHHH@HHHMHMHHHHHH+XH+HHH++++HHHH@HHHHH++%+ -,  = ,=== ,,  ,,, .
H@HHHH#M#M#MHHHM#MMMMMMMHHHH@H@H++@H$+++HHM#MMMMHMMH@@HHHHHH%+++++%%%+++    ,  .
%%%%%%%%%%%%%%++++%%++   ..   ...  ..  .                                   +++%+++++++%++++%+++++++++%+%++%+%%++%++++++%
pull/2509/head
iglocska 2017-09-25 12:22:19 +02:00
parent 217a047564
commit 3f76fd6ea7
4 changed files with 122 additions and 82 deletions

View File

@ -361,16 +361,18 @@ class AttributesController extends AppController {
if ($this->request->data['Attribute']['malware']) {
if ($this->request->data['Attribute']['advanced']) {
$result = $this->Attribute->advancedAddMalwareSample($tmpfile);
$result = $this->Attribute->advancedAddMalwareSample(
$eventId,
$this->request->data['Attribute'],
$filename,
$tmpfile
);
if ($result) $success++;
else $fails[] = $filename;
} else {
$result = $this->Attribute->simpleAddMalwareSample(
$eventId,
$this->request->data['Attribute']['category'],
$this->request->data['Attribute']['distribution'],
$this->request->data['Attribute']['distribution'] == 4 ? $this->request->data['Attribute']['sharing_group_id'] : 0,
$this->request->data['Attribute']['comment'],
$this->request->data['Attribute'],
$filename,
$tmpfile
);
@ -379,13 +381,6 @@ class AttributesController extends AppController {
}
if (!empty($result)) {
foreach ($result['Object'] as $object) {
$object['distribution'] = $this->request->data['Attribute']['distribution'];
$object['sharing_group_id'] = isset($this->request->data['Attribute']['distribution']) ? $this->request->data['Attribute']['distribution'] : 0;
if (!empty($object['Attribute'])) {
foreach ($object['Attribute'] as $k => $attribute) {
if ($attribute['value'] == $tmpfile->name) $object['Attribute'][$k]['value'] = $value['name'];
}
}
$this->loadModel('MispObject');
$this->MispObject->captureObject(array('Object' => $object), $eventId, $this->Auth->user());
}
@ -1843,7 +1838,7 @@ class AttributesController extends AppController {
if (isset($results['response']['Attribute'][$k]['AttributeTag'])) {
foreach ($results['response']['Attribute'][$k]['AttributeTag'] as $tk => $tag) {
$results['response']['Attribute'][$k]['Attribute']['Tag'][$tk] = $tag;
}
}
$results['response']['Attribute'][$k] = $results['response']['Attribute'][$k]['Attribute'];

View File

@ -3662,7 +3662,7 @@ class EventsController extends AppController {
// API for pushing samples to MISP
// Either send it to an existing event, or let MISP create a new one automatically
public function upload_sample($event_id = null) {
public function upload_sample($event_id = null, $advanced = false) {
$this->loadModel('Log');
$hashes = array('md5' => 'malware-sample', 'sha1' => 'filename|sha1', 'sha256' => 'filename|sha256');
$categoryDefinitions = $this->Event->Attribute->categoryDefinitions;
@ -3697,13 +3697,15 @@ class EventsController extends AppController {
foreach ($parameter_options as $k => $v) {
if (isset($data[$k])) {
if (isset($v['valid_options']) && !in_array($data[$k], $v['valid_options'])) {
$data[$k] = $v['default'];
$data['settings'][$k] = $v['default'];
} else {
$data['settings'][$k] = $data[$k];
}
unset($data[$k]);
} else {
$data[$k] = $v['default'];
$data['settings'][$k] = $v['default'];
}
}
if (isset($data['files'])) {
foreach ($data['files'] as $k => $file) {
if (!isset($file['filename']) || !isset($file['data'])) {
@ -3717,16 +3719,17 @@ class EventsController extends AppController {
if (empty($data['files'])) {
throw new BadRequestException('No samples received, or samples not in the correct format. Please refer to the API documentation on the automation page.');
}
if (isset($event_id)) $data['event_id'] = $event_id;
if (isset($data['event_id'])) {
$this->Event->id = $data['event_id'];
if (isset($event_id)) $data['settings']['event_id'] = $event_id;
if (isset($data['settings']['event_id'])) {
$this->Event->id = $data['settings']['event_id'];
if (!$this->Event->exists()) throw new NotFoundException('Event not found');
}
if (isset($data['advanced'])) $advanced = $data['advanced'];
// check if the user has permission to create attributes for an event, if the event ID has been passed
// If not, create an event
if (isset($data['event_id']) && !empty($data['event_id']) && is_numeric($data['event_id'])) {
$conditions = array('Event.id' => $data['event_id']);
if (isset($data['settings']['event_id']) && !empty($data['settings']['event_id']) && is_numeric($data['settings']['event_id'])) {
$conditions = array('Event.id' => $data['settings']['event_id']);
if (!$this->_isSiteAdmin()) {
$conditions[] = array('Event.orgc_id' => $this->Auth->user('org_id'));
if (!$this->userRole['perm_modify_org']) {
@ -3739,19 +3742,19 @@ class EventsController extends AppController {
'fields' => array('id'),
));
if (empty($event)) throw new NotFoundException('Event not found.');
$this->Event->id = $data['event_id'];
$this->Event->id = $data['settings']['event_id'];
$date = new DateTime();
$this->Event->saveField('timestamp', $date->getTimestamp());
$this->Event->saveField('published', 0);
} else {
$this->Event->create();
if ($data['distribution'] == 5) throw new BadRequestException('Distribution level 5 is not supported when uploading a sample without passing an event ID. Distribution level 5 is meant to take on the distribution level of an existing event.');
if ($data['settings']['distribution'] == 5) throw new BadRequestException('Distribution level 5 is not supported when uploading a sample without passing an event ID. Distribution level 5 is meant to take on the distribution level of an existing event.');
$result = $this->Event->save(
array(
'info' => $data['info'],
'analysis' => $data['analysis'],
'threat_level_id' => $data['threat_level_id'],
'distribution' => $data['distribution'],
'info' => $data['settings']['info'],
'analysis' => $data['settings']['analysis'],
'threat_level_id' => $data['settings']['threat_level_id'],
'distribution' => $data['settings']['distribution'],
'date' => date('Y-m-d'),
'orgc_id' => $this->Auth->user('org_id'),
'org_id' => $this->Auth->user('org_id'),
@ -3767,64 +3770,72 @@ class EventsController extends AppController {
'action' => 'upload_sample',
'user_id' => $this->Auth->user('id'),
'title' => 'Error: Failed to create event using the upload sample functionality',
'change' => 'There was an issue creating an event (' . $data['info'] . '). The validation errors were: ' . json_encode($this->Event->validationErrors),
'change' => 'There was an issue creating an event (' . $data['settings']['info'] . '). The validation errors were: ' . json_encode($this->Event->validationErrors),
));
throw new BadRequestException('The creation of a new event with the supplied information has failed.');
}
$data['event_id'] = $this->Event->id;
$data['settings']['event_id'] = $this->Event->id;
$event_id = $this->Event->id;
}
if (!isset($data['to_ids']) || !in_array($data['to_ids'], array('0', '1', 0, 1))) $data['to_ids'] = 1;
if (!isset($data['settings']['to_ids']) || !in_array($data['settings']['to_ids'], array('0', '1', 0, 1))) $data['settings']['to_ids'] = 1;
$successCount = 0;
$errors = array();
App::uses('FileAccessTool', 'Tools');
$fileAccessTool = new FileAccessTool();
foreach ($data['files'] as $file) {
$temp = $this->Event->Attribute->handleMaliciousBase64($data['event_id'], $file['filename'], $file['data'], array_keys($hashes));
if ($temp['success']) {
foreach ($hashes as $hash => $typeName) {
if ($temp[$hash] == false) continue;
$file[$hash] = $temp[$hash];
$file['data'] = $temp['data'];
$this->Event->Attribute->create();
$attribute = array(
'value' => $file['filename'] . '|' . $file[$hash],
'distribution' => $data['distribution'],
'category' => $data['category'],
'type' => $typeName,
'event_id' => $data['event_id'],
'to_ids' => $data['to_ids'],
'comment' => $data['comment']
);
if ($hash == 'md5') $attribute['data'] = $file['data'];
$result = $this->Event->Attribute->save($attribute);
if (!$result) {
$this->Log->save(array(
'org' => $this->Auth->user('Organisation')['name'],
'model' => 'Event',
'model_id' => $data['event_id'],
'email' => $this->Auth->user('email'),
'action' => 'upload_sample',
'user_id' => $this->Auth->user('id'),
'title' => 'Error: Failed to create attribute using the upload sample functionality',
'change' => 'There was an issue creating an attribute (' . $typeName . ': ' . $file['filename'] . '|' . $file[$hash] . '). ' . 'The validation errors were: ' . json_encode($this->Event->Attribute->validationErrors),
));
if ($typeName == 'malware-sample') {
$errors[] = array('filename' => $file['filename'], 'hash' => $file[$hash], 'error' => $this->Event->Attribute->validationErrors);
$tmpdir = Configure::read('MISP.tmpdir') ? Configure::read('MISP.tmpdir') : '/var/www/MISP/app/tmp';
$tmpfile = $fileAccessTool->createTempFile($tmpdir, $prefix = 'MISP_upload');
$fileAccessTool->writeToFile($tmpfile, base64_decode($file['data']));
$tmpfile = new File($tmpfile);
if ($advanced) {
$result = $this->Event->Attribute->advancedAddMalwareSample(
$event_id,
$data['settings'],
$file['filename'],
$tmpfile
);
if ($result) $successCount++;
else $errors[] = $file['filename'];
} else {
$result = $this->Event->Attribute->simpleAddMalwareSample(
$event_id,
$data['settings'],
$file['filename'],
$tmpfile
);
if ($result) $successCount++;
else $errors[] = $file['filename'];
}
if (!empty($result)) {
foreach ($result['Object'] as $object) {
$object['distribution'] = $data['settings']['distribution'];
$object['sharing_group_id'] = isset($data['settings']['distribution']) ? $data['settings']['distribution'] : 0;
if (!empty($object['Attribute'])) {
foreach ($object['Attribute'] as $k => $attribute) {
if ($attribute['value'] == $tmpfile->name) {
$object['Attribute'][$k]['value'] = $file['filename'];
}
}
} else if ($typeName == 'malware-sample') {
$successCount++;
}
$this->loadModel('MispObject');
$this->MispObject->captureObject(array('Object' => $object), $event_id, $this->Auth->user());
}
if (!empty($result['ObjectReference'])) {
foreach ($result['ObjectReference'] as $reference) {
$this->MispObject->ObjectReference->smartSave($reference, $event_id);
}
}
} else {
$errors[] = array('filename' => $file['filename'], 'hash' => $file['hash'], 'error' => 'Failed to encrypt and compress the file.');
}
$fileAccessTool->deleteFile($tmpfile->path);
}
if (!empty($errors)) {
$this->set('errors', $errors);
if ($successCount > 0) {
$this->set('name', 'Partial success');
$this->set('message', 'Successfuly saved ' . $successCount . ' sample(s), but some samples could not be saved.');
$this->set('url', '/events/view/' . $data['event_id']);
$this->set('id', $data['event_id']);
$this->set('url', '/events/view/' . $data['settings']['event_id']);
$this->set('id', $data['settings']['event_id']);
$this->set('_serialize', array('name', 'message', 'url', 'id', 'errors'));
} else {
$this->set('name', 'Failed');
@ -3834,11 +3845,11 @@ class EventsController extends AppController {
} else {
$this->set('name', 'Success');
$this->set('message', 'Success, saved all attributes.');
$this->set('url', '/events/view/' . $data['event_id']);
$this->set('id', $data['event_id']);
$this->set('url', '/events/view/' . $data['settings']['event_id']);
$this->set('id', $data['settings']['event_id']);
$this->set('_serialize', array('name', 'message', 'url', 'id'));
}
$this->view($data['event_id']);
$this->view($data['settings']['event_id']);
$this->render('view');
}

View File

@ -38,4 +38,9 @@ class FileAccessTool {
throw new MethodNotAllowedException($this->__fileErrorMsgPrefix . $errorMsgPart . '".');
}
}
public function deleteFile($file) {
unlink($file);
return true;
}
}

View File

@ -2690,7 +2690,7 @@ class Attribute extends AppModel {
return array('sgs' => $sgs, 'levels' => $distributionLevels, 'initial' => $initialDistribution);
}
public function simpleAddMalwareSample($event_id, $category, $distribution, $sharing_group_id, $comment, $filename, $tmpfile) {
public function simpleAddMalwareSample($event_id, $attribute_settings, $filename, $tmpfile) {
$attributes = array(
'malware-sample' => array('type' => 'malware-sample', 'data' => 1, 'category' => '', 'to_ids' => 1, 'disable_correlation' => 0, 'object_relation' => 'malware-sample'),
'filename' => array('type' => 'filename', 'category' => '', 'to_ids' => 0, 'disable_correlation' => 0, 'object_relation' => 'filename'),
@ -2702,12 +2702,21 @@ class Attribute extends AppModel {
$hashes = array('md5', 'sha1', 'sha256');
$this->Object = ClassRegistry::init('Object');
$this->ObjectTemplate = ClassRegistry::init('ObjectTemplate');
$object_template = $this->ObjectTemplate->find('first', array(
'conditions' => array(
'ObjectTemplate.uuid' => '688c46fb-5edb-40a3-8273-1af7923e2215'
),
'recursive' => -1
$current = $this->ObjectTemplate->find('first', array(
'fields' => array('MAX(version) AS version', 'uuid'),
'conditions' => array('uuid' => '688c46fb-5edb-40a3-8273-1af7923e2215'),
'recursive' => -1,
'group' => array('uuid')
));
if (!empty($current)) {
$object_template = $this->ObjectTemplate->find('first', array(
'conditions' => array(
'ObjectTemplate.uuid' => '688c46fb-5edb-40a3-8273-1af7923e2215',
'ObjectTemplate.version' => $current[0]['version']
),
'recursive' => -1
));
}
if (empty($object_template)) {
$object_template = array(
'ObjectTemplate' => array(
@ -2720,21 +2729,21 @@ class Attribute extends AppModel {
);
}
$object = array(
'distribution' => $distribution,
'sharing_group_id' => $sharing_group_id,
'distribution' => $attribute_settings['distribution'],
'sharing_group_id' => isset($attribute_settings['sharing_group_id']) ? $attribute_settings['sharing_group_id'] : 0,
'meta-category' => $object_template['ObjectTemplate']['meta-category'],
'name' => $object_template['ObjectTemplate']['name'],
'template_version' => $object_template['ObjectTemplate']['version'],
'description' => $object_template['ObjectTemplate']['description'],
'template_uuid' => $object_template['ObjectTemplate']['uuid'],
'event_id' => $event_id,
'comment' => $comment
'comment' => !empty($attribute_settings['comment']) ? $attribute_settings['comment'] : ''
);
$result = $this->Event->Attribute->handleMaliciousBase64($event_id, $filename, base64_encode($tmpfile->read()), $hashes);
foreach ($attributes as $k => $v) {
$attribute = array(
'distribution' => 5,
'category' => empty($v['category']) ? $category : $v['category'],
'category' => empty($v['category']) ? $attribute_settings['category'] : $v['category'],
'type' => $v['type'],
'to_ids' => $v['to_ids'],
'disable_correlation' => $v['disable_correlation'],
@ -2759,7 +2768,7 @@ class Attribute extends AppModel {
return array('Object' => array($object));
}
public function advancedAddMalwareSample($tmpfile) {
public function advancedAddMalwareSample($event_id, $attribute_settings, $filename, $tmpfile) {
$execRetval = '';
$execOutput = array();
$result = shell_exec('python ' . APP . 'files/scripts/generate_file_objects.py -p ' . $tmpfile->path);
@ -2773,6 +2782,26 @@ class Attribute extends AppModel {
$result['ObjectReference'] = $result['references'];
unset($result['references']);
}
foreach ($result['Object'] as $k => $object) {
$result['Object'][$k]['distribution'] = $attribute_settings['distribution'];
$result['Object'][$k]['sharing_group_id'] = isset($attribute_settings['distribution']) ? $attribute_settings['distribution'] : 0;
if (!empty($result['Object'][$k]['Attribute'])) {
foreach ($result['Object'][$k]['Attribute'] as $k2 => $attribute) {
if ($attribute['value'] == $tmpfile->name) {
$result['Object'][$k]['Attribute'][$k2]['value'] = $filename;
}
if (!empty($attribute['encrypt'])) {
if (!empty($attribute['encrypt']) && $attribute['encrypt']) {
$encrypted = $this->handleMaliciousBase64($event_id, $filename, $attribute['data'], array('md5'));
$result['Object'][$k]['Attribute'][$k2]['data'] = $encrypted['data'];
$result['Object'][$k]['Attribute'][$k2]['value'] = $filename . '|' . $encrypted['md5'];
}
}
}
}
}
} else {
$result = $this->simpleAddMalwareSample($event_id, $attribute_settings, $filename, $tmpfile);
}
return $result;
}