MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

new: Further work on the objects 

- uuids of both sides saved in references
- attachment adding fixed
pull/2489/head
iglocska 2017-09-01 17:42:54 +02:00
parent 5552d7c030
commit c09741ebfc
7 changed files with 75 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
INSTALL/MYSQL.sql
View File

@ -464,14 +464,18 @@ CREATE TABLE IF NOT EXISTS object_references (
`timestamp` int(11) NOT NULL DEFAULT 0,
`object_id` int(11) NOT NULL,
`event_id` int(11) NOT NULL,
`source_uuid` varchar(40) COLLATE utf8_bin DEFAULT NULL,
`destination_uuid` varchar(40) COLLATE utf8_bin DEFAULT NULL,
`referenced_id` int(11) NOT NULL,
`referenced_type` int(11) NOT NULL DEFAULT 0,
`relationship_type` varchar(255) CHARACTER SET utf8 COLLATE utf8_unicode_ci,
`comment` text COLLATE utf8_bin NOT NULL,
`deleted` TINYINT NOT NULL DEFAULT 0,
PRIMARY KEY (id),
INDEX `uuid` (`uuid`),
INDEX `source_uuid` (`source_uuid`),
INDEX `destination_uuid` (`destination_uuid`),
INDEX `timestamp` (`timestamp`),
INDEX `object_id` (`object_id`),
INDEX `referenced_id` (`referenced_id`),
INDEX `relationship_type` (`relationship_type`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

18
app/Controller/AttributesController.php
View File

@ -361,16 +361,7 @@ class AttributesController extends AppController {
if ($this->request->data['Attribute']['malware']) {
if ($this->request->data['Attribute']['advanced']) {
$execRetval = '';
$execOutput = array();
$result = shell_exec('python ' . APP . 'files/scripts/generate_file_objects.py -p ' . $tmpfile->path);
if (!empty($result)) {
$result = json_decode($result, true);
if (isset($result['objects'])) {
$result['Object'] = $result['objects'];
unset($result['objects']);
}
}
$result = $this->Attribute->advancedAddMalwareSample($tmpfile);
} else {
$result = $this->Attribute->simpleAddMalwareSample(
$eventId,
@ -387,7 +378,11 @@ class AttributesController extends AppController {
foreach ($object['Attribute'] as $k => $attribute) {
if ($attribute['value'] == $tmpfile->name) $object['Attribute'][$k]['value'] = $value['name'];
}
$result = $this->Attribute->Object->captureObject($eventId, array('Object' => $object), $this->Auth->user());
$this->loadModel('MispObject');
$result = $this->MispObject->captureObject($eventId, array('Object' => $object), $this->Auth->user());
}
foreach ($result['ObjectReference'] as $reference) {
$result = $this->MispObject->ObjectReference->smartSave($reference, $eventId);
}
}
} else {
@ -410,7 +405,6 @@ class AttributesController extends AppController {
else $success++;
}
}
throw new Exception();
$message = 'The attachment(s) have been uploaded.';
if (!empty($partialFails)) $message .= ' Some of the attributes however could not be created.';
if (!empty($fails)) $message = 'Some of the attachments failed to upload. The failed files were: ' . implode(', ', $fails) . ' - This can be caused by the attachments already existing in the event.';

11
app/Controller/ObjectReferencesController.php
View File

@ -54,6 +54,7 @@ class ObjectReferencesController extends AppController {
));
if (!empty($target_object)) {
$referenced_id = $target_object['Object']['id'];
$target_uuid = $target_attribute['Object']['uuid'];
if ($target_object['Object']['event_id'] != $object['Event']['id']) {
throw new NotFoundException('Invalid target. Target has to be within the same event.');
}
@ -70,6 +71,7 @@ class ObjectReferencesController extends AppController {
throw new NotFoundException('Invalid target. Target has to be within the same event.');
}
$referenced_id = $target_attribute['Attribute']['id'];
$destination_uuid = $target_attribute['Attribute']['uuid'];
$referenced_type = 0;
}
$relationship_type = empty($this->request->data['ObjectReference']['relationship_type']) ? '' : $this->request->data['ObjectReference']['relationship_type'];
@ -79,14 +81,15 @@ class ObjectReferencesController extends AppController {
$data = array(
'referenced_type' => $referenced_type,
'referenced_id' => $referenced_id,
'uuid' => $this->request->data['ObjectReference']['uuid'],
'destination_uuid' => $destination_uuid,
'relationship_type' => $relationship_type,
'comment' => !empty($this->request->data['ObjectReference']['comment']) ? $this->request->data['ObjectReference']['comment'] : '',
'event_id' => $object['Event']['id'],
'object_id' => $objectId
'source_uuid' => $object['Object']['uuid'],
'object_id' => $objectId,
'referenced_type' => $referenced_type,
'uuid' => CakeText::uuid()
);
$data['referenced_type'] = $referenced_type;
$data['uuid'] = $this->request->data['ObjectReference']['uuid'];
$this->ObjectReference->create();
$result = $this->ObjectReference->save(array('ObjectReference' => $data));
if ($result) {

12
app/Model/AppModel.php
View File

@ -741,16 +741,20 @@ class AppModel extends Model {
`timestamp` int(11) NOT NULL DEFAULT 0,
`object_id` int(11) NOT NULL,
`event_id` int(11) NOT NULL,
`source_uuid` varchar(40) COLLATE utf8_bin DEFAULT NULL,
`destination_uuid` varchar(40) COLLATE utf8_bin DEFAULT NULL,
`referenced_id` int(11) NOT NULL,
`referenced_type` int(11) NOT NULL DEFAULT 0,
`relationship_type` varchar(255) CHARACTER SET utf8 COLLATE utf8_unicode_ci,
`comment` text COLLATE utf8_bin NOT NULL,
`deleted` TINYINT(1) NOT NULL DEFAULT 0,
PRIMARY KEY (id),
INDEX `uuid` (`uuid`),
INDEX `timestamp` (`timestamp`),
INDEX `referenced_uuid` (`referenced_uuid`),
INDEX `relationship_type` (`relationship_type`)
INDEX `source_uuid` (`source_uuid`),
INDEX `destination_uuid` (`destination_uuid`),
INDEX `timestamp` (`timestamp`),
INDEX `object_id` (`object_id`),
INDEX `referenced_id` (`referenced_id`),
INDEX `relationship_type` (`relationship_type`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;";
$sqlArray[] = "CREATE TABLE IF NOT EXISTS object_relationships (

41
app/Model/Attribute.php
View File

@ -2669,12 +2669,12 @@ class Attribute extends AppModel {
public function simpleAddMalwareSample($event_id, $category, $distribution, $sharing_group_id, $comment, $filename, $tmpfile) {
$attributes = array(
'malware-sample' => array('type' => 'malware-sample', 'data' => 1, 'category' => '', 'to_ids' => 1, 'disable_correlation' => 0),
'filename' => array('type' => 'filename', 'category' => '', 'to_ids' => 0, 'disable_correlation' => 0),
'md5' => array('type' => 'md5', 'category' => '', 'to_ids' => 1, 'disable_correlation' => 0),
'sha1' => array('type' => 'sha1', 'category' => '', 'to_ids' => 1, 'disable_correlation' => 0),
'sha256' => array('type' => 'sha256', 'category' => '', 'to_ids' => 1, 'disable_correlation' => 0),
'size-in-bytes' => array('type' => 'size-in-bytes', 'category' => '', 'to_ids' => 0, 'disable_correlation' => 1)
'malware-sample' => array('type' => 'malware-sample', 'data' => 1, 'category' => '', 'to_ids' => 1, 'disable_correlation' => 0, 'object_relation' => 'malware-sample'),
'filename' => array('type' => 'filename', 'category' => '', 'to_ids' => 0, 'disable_correlation' => 0, 'object_relation' => 'filename'),
'md5' => array('type' => 'md5', 'category' => '', 'to_ids' => 1, 'disable_correlation' => 0, 'object_relation' => 'md5'),
'sha1' => array('type' => 'sha1', 'category' => '', 'to_ids' => 1, 'disable_correlation' => 0, 'object_relation' => 'sha1'),
'sha256' => array('type' => 'sha256', 'category' => '', 'to_ids' => 1, 'disable_correlation' => 0, 'object_relation' => 'sha256'),
'size-in-bytes' => array('type' => 'size-in-bytes', 'category' => 'Other', 'to_ids' => 0, 'disable_correlation' => 1, 'object_relation' => 'filesize')
);
$hashes = array('md5', 'sha1', 'sha256');
$this->Object = ClassRegistry::init('Object');
@ -2707,16 +2707,17 @@ class Attribute extends AppModel {
'event_id' => $event_id,
'comment' => $comment
);
$result = $this->Event->Attribute->handleMaliciousBase64($event_id, $filename, base64_encode($tmpfile->read()), $hashes);
foreach ($attributes as $k => $v) {
$result = $this->Event->Attribute->handleMaliciousBase64($event_id, $filename, base64_encode($tmpfile->read()), $hashes);
$attribute = array(
'distribution' => 5,
'category' => $category,
'category' => empty($v['category']) ? $category : $v['category'],
'type' => $v['type'],
'to_ids' => $v['to_ids'],
'disable_correlation' => $v['disable_correlation'],
'object_id' => $this->Object->id,
'event_id' => $event_id
'event_id' => $event_id,
'object_relation' => $v['object_relation']
);
if (isset($v['data'])) {
$attribute['data'] = $result['data'];
@ -2724,7 +2725,7 @@ class Attribute extends AppModel {
if ($k == 'malware-sample') {
$attribute['value'] = $filename . '|' . $result['md5'];
} else if ($k == 'size-in-bytes') {
$attribute['value'] = 0;
$attribute['value'] = $tmpfile->size();
} else if ($k == 'filename') {
$attribute['value'] = $filename;
} else {
@ -2732,11 +2733,25 @@ class Attribute extends AppModel {
}
$object['Attribute'][] = $attribute;
}
return array('Object' => $object);
return array('Object' => array($object));
}
public function advancedAddMalwareSample() {
public function advancedAddMalwareSample($tmpfile) {
$execRetval = '';
$execOutput = array();
$result = shell_exec('python ' . APP . 'files/scripts/generate_file_objects.py -p ' . $tmpfile->path);
if (!empty($result)) {
$result = json_decode($result, true);
if (isset($result['objects'])) {
$result['Object'] = $result['objects'];
unset($result['objects']);
}
if (isset($result['references'])) {
$result['ObjectReference'] = $result['references'];
unset($result['references']);
}
}
return $result;
}
}

19
app/Model/ObjectReference.php
View File

@ -65,7 +65,8 @@ class ObjectReference extends AppModel {
}
public function smartSave($objectReference, $eventId) {
$sides = array('source', 'referenced');
$sides = array('source', 'destination');
$data = array();
foreach ($sides as $side) {
$data[$side] = $this->Object->find('first', array(
'conditions' => array(
@ -75,7 +76,7 @@ class ObjectReference extends AppModel {
'recursive' => -1,
'fields' => array('Object.id')
));
if (empty($data[$side]) && $side == 'referenced') {
if (empty($data[$side]) && $side == 'destination') {
$data[$side] = $this->Attribute->find('first', array(
'conditions' => array(
'Attribute.uuid' => $objectReference[$side . '_uuid'],
@ -84,11 +85,11 @@ class ObjectReference extends AppModel {
'recursive' => -1,
'fields' => array('Attribute.id')
));
$referenced_id = $data[$side]['Attribute']['id'];
$referenced_type = 0;
} else if (!empty($data[$side]) && $side == 'referenced') {
$referenced_id = $data[$side]['Object']['id'];
$referenced_type = 1;
$destination_id = $data[$side]['Attribute']['id'];
$destination_type = 0;
} else if (!empty($data[$side]) && $side == 'destination') {
$destination_id = $data[$side]['Object']['id'];
$destination_type = 1;
} else if (!empty($data[$side]) && $side = 'source') {
$object_id = $data[$side]['Object']['id'];
} else {
@ -96,8 +97,8 @@ class ObjectReference extends AppModel {
}
}
$this->create();
$objectReference['referenced_type'] = $referenced_type;
$objectReference['referenced_id'] = $referenced_id;
$objectReference['destination_type'] = $destination_type;
$objectReference['destination_id'] = $destination_id;
$objectReference['object_id'] = $object_id;
$result = $this->save(array('ObjectReference' => $ojectReference));
if (!$result) {

21
app/files/scripts/generate_file_objects.py
View File

@ -10,6 +10,7 @@ try:
except ImportError:
pass
def check():
missing_dependencies = {'pydeep': False, 'lief': False, 'magic': False, 'pymisp': False}
try:
@ -20,10 +21,10 @@ def check():
import pydeep # noqa
except ImportError:
missing_dependencies['pydeep'] = 'Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git'
#try:
import lief # noqa
#except ImportError:
# missing_dependencies['lief'] = 'Please install lief, documentation here: https://github.com/lief-project/LIEF'
try:
import lief # noqa
except ImportError:
missing_dependencies['lief'] = 'Please install lief, documentation here: https://github.com/lief-project/LIEF'
try:
import magic # noqa
except ImportError:
@ -38,18 +39,18 @@ def make_objects(path):
if seos:
for s in seos:
to_return['objects'].append(s)
if s.references:
to_return['references'] += s.references
if s.ObjectReference:
to_return['references'] += s.ObjectReference
if peo:
to_return['objects'].append(peo)
if peo.references:
to_return['references'] += peo.references
if peo.ObjectReference:
to_return['references'] += peo.ObjectReference
if fo:
to_return['objects'].append(fo)
if fo.references:
to_return['references'] += fo.references
if fo.ObjectReference:
to_return['references'] += fo.ObjectReference
return json.dumps(to_return, cls=MISPEncode)
if __name__ == '__main__':