MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch '2.4' into py-virtualenv

pull/3762/head
Steve Clement 2018-10-02 20:55:46 +08:00
parent 602fb17335 056e0ae688
commit a699c5fcd4
11 changed files with 247 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/Controller/AttributesController.php
View File

@ -2184,6 +2184,7 @@ class AttributesController extends AppController
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
$final = fread($tmpfile, fstat($tmpfile)['size']);
fclose($tmpfile);
$responseType = $validFormats[$returnFormat][0];
return $this->RestResponse->viewData($final, $responseType, false, true);
}

9
app/Controller/Component/RestResponseComponent.php
View File

@ -379,6 +379,15 @@ class RestResponseComponent extends Component
return $this->__sendResponse($data, 200, $format, $raw, $download);
}
public function sendFile($path, $format = false, $download = false, $name = 'download') {
$cakeResponse = new CakeResponse(array(
'status' => 200,
'type' => $format
));
$cakeResponse->file($path, array('name' => $name, 'download' => true));
return $cakeResponse;
}
public function throwException($code, $message, $url = '', $format = false, $raw = false)
{
$message = array(

116
app/Controller/EventsController.php
View File

@ -996,6 +996,7 @@ class EventsController extends AppController
}
$conditions['includeFeedCorrelations'] = true;
$conditions['includeAllTags'] = true;
$conditions['includeGranularCorrelations'] = 1;
$results = $this->Event->fetchEvent($this->Auth->user(), $conditions);
if (empty($results)) {
throw new NotFoundException(__('Invalid event'));
@ -1382,6 +1383,9 @@ class EventsController extends AppController
$this->set('extended', 0);
}
$conditions['includeFeedCorrelations'] = true;
if (!$this->_isRest()) {
$conditions['includeGranularCorrelations'] = 1;
}
$results = $this->Event->fetchEvent($this->Auth->user(), $conditions);
if (empty($results)) {
throw new NotFoundException(__('Invalid event'));
@ -1810,7 +1814,7 @@ class EventsController extends AppController
}
$this->Event->insertLock($this->Auth->user(), $target_id);
if ($this->request->is('post')) {
$source_id = $this->request->data['Event']['source_id'];
$source_id = trim($this->request->data['Event']['source_id']);
$to_ids = $this->request->data['Event']['to_ids'];
if (!is_numeric($source_id)) {
$this->Flash->error(__('Invalid event ID entered.'));
@ -3002,6 +3006,43 @@ class EventsController extends AppController
return $this->response;
}
/*
* Receive a list of eventids in the id=>count format
* Chunk them by the attribute count to fit the memory limits
*
*/
private function __clusterEventIds($exportTool, $eventIds) {
$memory_in_mb = $this->Event->Attribute->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
$memory_scaling_factor = isset($exportTool->memory_scaling_factor) ? $exportTool->memory_scaling_factor : 100;
$limit = $memory_in_mb * $memory_scaling_factor;
$eventIdList = array();
$continue = true;
$i = 0;
$current_chunk_size = 0;
while (!empty($eventIds)) {
foreach ($eventIds as $id => $count) {
if ($current_chunk_size == 0 && $count > $limit) {
$eventIdList[$i][] = $id;
$current_chunk_size = $count;
unset($eventIds[$id]);
$i++;
break;
} else {
if (($current_chunk_size + $count) > $limit) {
$i++;
$current_chunk_size = 0;
break;
} else {
$current_chunk_size += $count;
$eventIdList[$i][] = $id;
unset($eventIds[$id]);
}
}
}
}
return $eventIdList;
}
// Use the REST interface to search for attributes or events. Usage:
// MISP-base-url/events/restSearch/[api-key]/[value]/[type]/[category]/[orgc]
// value, type, category, orgc are optional
@ -3019,13 +3060,13 @@ class EventsController extends AppController
'ordered_url_params' => compact($paramArray)
);
$validFormats = array(
'openioc' => array('xml', 'OpeniocExport'),
'json' => array('json', 'JsonExport'),
'xml' => array('xml', 'XmlExport'),
'suricata' => array('txt', 'NidsSuricataExport'),
'snort' => array('txt', 'NidsSnortExport'),
'rpz' => array('rpz', 'RPZExport'),
'text' => array('text', 'TextExport')
'openioc' => array('xml', 'OpeniocExport', 'ioc'),
'json' => array('json', 'JsonExport', 'json'),
'xml' => array('xml', 'XmlExport', 'xml'),
'suricata' => array('txt', 'NidsSuricataExport', 'rules'),
'snort' => array('txt', 'NidsSnortExport', 'rules'),
'rpz' => array('rpz', 'RPZExport', 'rpz'),
'text' => array('text', 'TextExport', 'txt')
);
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
@ -3058,7 +3099,24 @@ class EventsController extends AppController
$filters['published'] = 1;
}
}
if (isset($filters['ignore'])) {
$filters['to_ids'] = array(0, 1);
$filters['published'] = array(0, 1);
}
if (isset($filters['searchall'])) {
$filters['tags'] = $filters['searchall'];
$filters['eventinfo'] = $filters['searchall'];
$filters['value'] = $filters['searchall'];
$filters['comment'] = $filters['searchall'];
}
if (!empty($filters['quickfilter']) && !empty($filters['value'])) {
$filters['tags'] = $filters['value'];
$filters['eventinfo'] = $filters['value'];
$filters['comment'] = $filters['value'];
}
$filters['include_attribute_count'] = 1;
$eventid = $this->Event->filterEventIds($user, $filters);
$eventids_chunked = $this->__clusterEventIds($exportTool, $eventid);
if (!empty($exportTool->additional_params)) {
$filters = array_merge($filters, $exportTool->additional_params);
}
@ -3077,14 +3135,15 @@ class EventsController extends AppController
$filters['published'] = 1;
}
}
$final = $exportTool->header($exportToolParams);
$tmpfile = tmpfile();
fwrite($tmpfile, $exportTool->header($exportToolParams));
$eventCount = count($eventid);
$i = 0;
if (!empty($filters['withAttachments'])) {
$filters['includeAttachments'] = 1;
}
foreach ($eventid as $k => $currentEventId) {
$filters['eventid'] = $currentEventId;
foreach ($eventids_chunked as $chunk_index => $chunk) {
$filters['eventid'] = $chunk;
if (!empty($filters['tags']['NOT'])) {
$filters['blockedAttributeTags'] = $filters['tags']['NOT'];
}
@ -3093,20 +3152,25 @@ class EventsController extends AppController
$filters,
true
);
if (!empty($result)) {
$this->loadModel('Whitelist');
$result = $this->Whitelist->removeWhitelistedFromArray($result, false);
$temp = $exportTool->handler($result[0], $exportToolParams);
if ($temp !== '') {
if ($k !== 0) {
$final .= $exportTool->separator($exportToolParams);
if (!empty($result)) {
foreach ($result as $event) {
$this->loadModel('Whitelist');
$result = $this->Whitelist->removeWhitelistedFromArray($result, false);
$temp = $exportTool->handler($event, $exportToolParams);
if ($temp !== '') {
if ($i !== 0) {
$temp = $exportTool->separator($exportToolParams) . $temp;
}
fwrite($tmpfile, $temp);
$i++;
}
$final .= $temp;
}
$i++;
}
}
$final .= $exportTool->footer($exportToolParams);
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
$final = fread($tmpfile, fstat($tmpfile)['size']);
fclose($tmpfile);
$responseType = $validFormats[$returnFormat][0];
return $this->RestResponse->viewData($final, $responseType, false, true);
}
@ -4369,7 +4433,10 @@ class EventsController extends AppController
public function viewGraph($id)
{
$event = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $id));
$event = $this->Event->fetchEvent($this->Auth->user(), array(
'eventid' => $id,
'includeGranularCorrelations' => 1
));
if (empty($event)) {
throw new MethodNotAllowedException(__('Invalid Event.'));
}
@ -4379,10 +4446,11 @@ class EventsController extends AppController
$this->set('id', $id);
}
public function viewEventGraph()
{
$event = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $id));
$event = $this->Event->fetchEvent($this->Auth->user(), array(
'eventid' => $id
));
if (empty($event)) {
throw new MethodNotAllowedException(__('Invalid Event.'));
}

2
app/Lib/Tools/CorrelationGraphTool.php
View File

@ -26,7 +26,7 @@
private function __expandEvent($id)
{
$event = $this->__eventModel->fetchEvent($this->__user, array('eventid' => $id, 'flatten' => 0, 'includeTagRelations' => 1, 'includeGalaxy' => 1));
$event = $this->__eventModel->fetchEvent($this->__user, array('eventid' => $id, 'flatten' => 0, 'includeTagRelations' => 1, 'includeGalaxy' => 1, 'includeGranularCorrelations' => 1));
if (empty($event)) {
return $this->__json;
}

11
app/Model/Attribute.php
View File

@ -3662,17 +3662,6 @@ class Attribute extends AppModel
$params['to_ids'] = array(0, 1);
$params['published'] = array(0, 1);
}
if (isset($params['searchall'])) {
$params['tags'] = $params['searchall'];
$params['eventinfo'] = $params['searchall'];
$params['value'] = $params['searchall'];
$params['comment'] = $params['searchall'];
}
if (!empty($params['quickfilter']) && !empty($params['value'])) {
$params['tags'] = $params['value'];
$params['eventinfo'] = $params['value'];
$params['comment'] = $params['value'];
}
$simple_params = array(
'Attribute' => array(
'value' => array('function' => 'set_filter_value'),

115
app/Model/Event.php
View File

@ -705,7 +705,7 @@ class Event extends AppModel
$relatedEvents = $this->find(
'all',
array('conditions' => $conditions,
'recursive' => 0,
'recursive' => -1,
'order' => 'Event.date DESC',
'fields' => $fields,
'contain' => array(
@ -1297,21 +1297,6 @@ class Event extends AppModel
public function filterEventIds($user, &$params = array())
{
$conditions = $this->createEventConditions($user);
if (isset($params['ignore'])) {
$params['to_ids'] = array(0, 1);
$params['published'] = array(0, 1);
}
if (isset($params['searchall'])) {
$params['tags'] = $params['searchall'];
$params['eventinfo'] = $params['searchall'];
$params['value'] = $params['searchall'];
$params['comment'] = $params['searchall'];
}
if (!empty($params['quickfilter']) && !empty($params['value'])) {
$params['tags'] = $params['value'];
$params['eventinfo'] = $params['value'];
$params['comment'] = $params['value'];
}
$simple_params = array(
'Event' => array(
'eventid' => array('function' => 'set_filter_eventid', 'pop' => true),
@ -1370,11 +1355,22 @@ class Event extends AppModel
}
}
}
$results = array_values($this->find('list', array(
'conditions' => $conditions,
$fields = array('Event.id');
if (!empty($params['include_attribute_count'])) {
$fields[] = 'Event.attribute_count';
}
$find_params = array(
'conditions' => $conditions,
'recursive' => -1,
'fields' => array('Event.id')
)));
'fields' => $fields
);
if (isset($params['limit'])) {
$find_params['limit'] = $params['limit'];
if (isset($params['page'])) {
$find_params['page'] = $params['page'];
}
}
$results = $this->find('list', $find_params);
return $results;
}
@ -1674,16 +1670,14 @@ class Event extends AppModel
'ThreatLevel' => array(
'fields' => array('ThreatLevel.name')
),
'Org' => array('fields' => $fieldsOrg),
'Orgc' => array('fields' => $fieldsOrg),
'Attribute' => array(
'fields' => $fieldsAtt,
'conditions' => $conditionsAttributes,
'order' => false,
'AttributeTag' => array(
'Tag' => array('conditions' => $tagConditions, 'order' => false),
'order' => false
),
'AttributeTag' => array(
'Tag' => array('conditions' => $tagConditions, 'order' => false),
'order' => false
),
'order' => false
),
'Object' => array(
'fields' => $fieldsObj,
@ -1700,10 +1694,10 @@ class Event extends AppModel
'Org' => array('fields' => $fieldsOrg),
'order' => false
),
'EventTag' => array(
'Tag' => array('conditions' => $tagConditions, 'order' => false),
'order' => false
)
'EventTag' => array(
'Tag' => array('conditions' => $tagConditions, 'order' => false),
'order' => false
)
)
);
if ($flatten) {
@ -1720,7 +1714,6 @@ class Event extends AppModel
if (empty($results)) {
return array();
}
// Do some refactoring with the event
$this->Sighting = ClassRegistry::init('Sighting');
$userEmails = array();
@ -1730,29 +1723,8 @@ class Event extends AppModel
'Object' => array('name', 'meta-category')
);
foreach ($results as $eventKey => &$event) {
if (!empty($event['Object'])) {
foreach ($event['Object'] as $k => $object) {
if (!empty($object['ObjectReference'])) {
foreach ($object['ObjectReference'] as $k2 => $reference) {
$type = array('Attribute', 'Object')[$reference['referenced_type']];
$temp = $this->{$type}->find('first', array(
'recursive' => -1,
'fields' => array_merge($fields['common'], $fields[array('Attribute', 'Object')[$reference['referenced_type']]]),
'conditions' => array('id' => $reference['referenced_id'])
));
if (!empty($temp)) {
if (!$isSiteAdmin && $user['org_id'] != $event['Event']['orgc_id']) {
if ($temp[$type]['distribution'] == 0 || ($temp[$type]['distribution'] == 4 && !in_array($temp[$type]['sharing_group_id'], $sgsids))) {
unset($object['ObjectReference'][$k2]);
continue;
}
}
$event['Object'][$k]['ObjectReference'][$k2][$type] = $temp[$type];
}
}
}
}
}
$this->__attachReferences($user, $event, $sgids, $fields);
$event = $this->Orgc->attachOrgsToEvent($event, $fieldsOrg);
if (!$options['sgReferenceOnly'] && $event['Event']['sharing_group_id']) {
$event['SharingGroup'] = $sharingGroupData[$event['Event']['sharing_group_id']]['SharingGroup'];
}
@ -1767,8 +1739,10 @@ class Event extends AppModel
// Let's find all the related events and attach it to the event itself
$results[$eventKey]['RelatedEvent'] = $this->getRelatedEvents($user, $event['Event']['id'], $sgids);
// Let's also find all the relations for the attributes - this won't be in the xml export though
$results[$eventKey]['RelatedAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], $sgids);
$results[$eventKey]['RelatedShadowAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], $sgids, true);
if (!empty($options['includeGranularCorrelations'])) {
$results[$eventKey]['RelatedAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], $sgids);
$results[$eventKey]['RelatedShadowAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], $sgids, true);
}
if (isset($event['ShadowAttribute']) && !empty($event['ShadowAttribute']) && isset($options['includeAttachments']) && $options['includeAttachments']) {
foreach ($event['ShadowAttribute'] as $k => $sa) {
if ($this->ShadowAttribute->typeIsAttachment($sa['type'])) {
@ -5270,4 +5244,31 @@ class Event extends AppModel
return ($this->processFreeTextData($user, $attributes, $id, $default_comment = '', $force = false, $adhereToWarninglists = false));
}
}
private function __attachReferences($user, &$event, $sgids, $fields)
{
if (!empty($event['Object'])) {
foreach ($event['Object'] as $k => $object) {
if (!empty($object['ObjectReference'])) {
foreach ($object['ObjectReference'] as $k2 => $reference) {
$type = array('Attribute', 'Object')[$reference['referenced_type']];
$temp = $this->{$type}->find('first', array(
'recursive' => -1,
'fields' => array_merge($fields['common'], $fields[array('Attribute', 'Object')[$reference['referenced_type']]]),
'conditions' => array('id' => $reference['referenced_id'])
));
if (!empty($temp)) {
if (!$user['Role']['perm_site_admin'] && $user['org_id'] != $event['Event']['orgc_id']) {
if ($temp[$type]['distribution'] == 0 || ($temp[$type]['distribution'] == 4 && !in_array($temp[$type]['sharing_group_id'], $sgsids))) {
unset($object['ObjectReference'][$k2]);
continue;
}
}
$event['Object'][$k]['ObjectReference'][$k2][$type] = $temp[$type];
}
}
}
}
}
}
}

5
app/Model/GalaxyCluster.php
View File

@ -24,6 +24,7 @@ class GalaxyCluster extends AppModel
)
);
private $__clusterCache = array();
public $hasMany = array(
'GalaxyElement' => array('dependent' => true),
@ -144,6 +145,9 @@ class GalaxyCluster extends AppModel
if (is_numeric($name)) {
$conditions = array('GalaxyCluster.id' => $name);
}
if (isset($this->__clusterCache[$name])) {
return $this->__clusterCache[$name];
}
$objects = array('Galaxy', 'GalaxyElement');
$cluster = $this->find('first', array(
'conditions' => $conditions,
@ -179,6 +183,7 @@ class GalaxyCluster extends AppModel
}
$cluster['GalaxyCluster']['meta'] = $elements;
}
$this->__clusterCache[$name] = $cluster;
return $cluster;
}

29
app/Model/Organisation.php
View File

@ -16,6 +16,8 @@ class Organisation extends AppModel
),
);
private $__orgCache = array();
public $validate = array(
'name' => array(
'unique' => array(
@ -372,6 +374,31 @@ class Organisation extends AppModel
'conditions' => $conditions,
'recursive' => -1
));
return (empty($org)) ? false : $org;
return (empty($org)) ? false : $org[$this->alias];
}
public function attachOrgsToEvent($event, $fields)
{
if (empty($this->__orgCache[$event['Event']['orgc_id']])) {
$temp = $this->find('first', array(
'conditions' => array('id' => $event['Event']['orgc_id']),
'recursive' => -1,
'fields' => $fields
));
if (!empty($temp)) $temp = $temp[$this->alias];
$this->__orgCache[$event['Event']['orgc_id']] = $temp;
}
$event['Orgc'] = $this->__orgCache[$event['Event']['orgc_id']];
if (empty($this->__orgCache[$event['Event']['org_id']])) {
$temp = $this->find('first', array(
'conditions' => array('id' => $event['Event']['org_id']),
'recursive' => -1,
'fields' => $fields
));
if (!empty($temp)) $temp = $temp[$this->alias];
$this->__orgCache[$event['Event']['org_id']] = $temp;
}
$event['Org'] = $this->__orgCache[$event['Event']['org_id']];
return $event;
}
}

27
app/Model/SharingGroup.php
View File

@ -53,6 +53,8 @@ class SharingGroup extends AppModel
)
);
private $__sgoCache = array();
public function beforeValidate($options = array())
{
@ -163,10 +165,7 @@ class SharingGroup extends AppModel
array(
'fields' => array('SharingGroup.*'),
'contain' => array(
'Organisation' => array('fields' => $fieldsOrg),
'SharingGroupOrg' => array(
'Organisation' => array('fields' => $fieldsOrg),
),
'SharingGroupOrg',
'SharingGroupServer' => array(
'Server' => array('fields' => $fieldsServer),
)
@ -179,6 +178,26 @@ class SharingGroup extends AppModel
'fields' => $fieldsSharingGroup[$permissionTree]['fields'],
'order' => 'SharingGroup.name ASC'
));
foreach ($sgs as &$sg) {
if (!isset($this->__sgoCache[$sg['SharingGroup']['org_id']])) {
$this->__sgoCache[$sg['SharingGroup']['org_id']] = $this->Organisation->find('first', array(
'recursive' => -1,
'fields' => $fieldsOrg,
'conditions' => array('id' => $sg['SharingGroup']['org_id'])
));
}
$sg['Organisation'] = $this->__sgoCache[$sg['SharingGroup']['org_id']];
foreach ($sg['SharingGroupOrg'] as &$sgo) {
if (!isset($this->__sgoCache[$sgo['org_id']])) {
$this->__sgoCache[$sgo['org_id']] = $this->Organisation->find('first', array(
'recursive' => -1,
'fields' => $fieldsOrg,
'conditions' => array('id' => $sgo['org_id'])
));
}
$sgo['Organisation'] = $this->__sgoCache[$sgo['org_id']];
}
}
return $sgs;
} elseif ($scope == 'name') {
$sgs = $this->find('list', array(

52
app/Model/User.php
View File

@ -273,31 +273,33 @@ class User extends AppModel
{
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_user_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
$user = $this->data;
if (!isset($user['User'])) {
$user['User'] = $user;
}
$action = $created ? 'edit' : 'add';
if (isset($user['User']['action'])) {
$action = $user['User']['action'];
}
if (isset($user['User']['id'])) {
$user = $this->find('first', array(
'recursive' => -1,
'conditons' => array('User.id' => $user['User']['id']),
'fields' => array('id', 'email', 'last_login', 'org_id', 'termsaccepted', 'autoalert', 'newsread', 'disabled'),
'contain' => array(
'Organisation' => array(
'fields' => array('Organisation.id', 'Organisation.name', 'Organisation.description', 'Organisation.uuid', 'Organisation.nationality', 'Organisation.sector', 'Organisation.type', 'Organisation.local')
)
)
));
}
if (isset($user['User']['password'])) {
unset($user['User']['password']);
unset($user['User']['confirm_password']);
}
$pubSubTool->modified($user, 'user', $action);
if (!empty($this->data)) {
$user = $this->data;
if (!isset($user['User'])) {
$user['User'] = $user;
}
$action = $created ? 'edit' : 'add';
if (isset($user['User']['action'])) {
$action = $user['User']['action'];
}
if (isset($user['User']['id'])) {
$user = $this->find('first', array(
'recursive' => -1,
'conditions' => array('User.id' => $user['User']['id']),
'fields' => array('id', 'email', 'last_login', 'org_id', 'termsaccepted', 'autoalert', 'newsread', 'disabled'),
'contain' => array(
'Organisation' => array(
'fields' => array('Organisation.id', 'Organisation.name', 'Organisation.description', 'Organisation.uuid', 'Organisation.nationality', 'Organisation.sector', 'Organisation.type', 'Organisation.local')
)
)
));
}
if (isset($user['User']['password'])) {
unset($user['User']['password']);
unset($user['User']['confirm_password']);
}
$pubSubTool->modified($user, 'user', $action);
}
}
return true;
}

6
app/webroot/js/event-distribution-graph.js
View File

@ -30,12 +30,12 @@ function generate_additional_info(info) {
var to_ret = "\n\nInvolved:\n";
var sel = document.createElement('select');
sel.classList.add('distributionInfo');
for (var i in info) {
info.forEach(function(i) {
var opt = document.createElement('option');
opt.val = i;
opt.innerHTML = i;
sel.appendChild(opt);
}
});
return to_ret += sel.outerHTML;
}
}
@ -87,7 +87,7 @@ function get_minimum_distribution(array, event_dist) {
var connected = array[2];
var all = array[3];
var sharing = array[4];
if (connected != 0 && 3 < event_distribution) {
if (connected != 0 && 3 == event_distribution) {
return 2;
} else if (community != 0 && 1 < event_distribution) {
return 1;