Merge branch '2.4' of github.com:MISP/MISP into 2.4

pull/2628/head
Alexandre Dulaunoy 2017-11-04 10:55:52 +01:00
commit b720b4bc16
No known key found for this signature in database
GPG Key ID: 09E2CD4944E6CBCD
28 changed files with 481 additions and 109 deletions

View File

@ -42,7 +42,7 @@ sudo a2dissite 000-default
sudo a2ensite default-ssl
# Install PHP and dependencies
sudo apt-get install -y libapache2-mod-php7.0 php7.0 php7.0-cli php7.0-dev php7.0-json php7.0-xml php7.0-mysql php7.0-readline php7.0-redis php7.0-mbstring
sudo apt-get install -y libapache2-mod-php7.0 php7.0 php7.0-cli php7.0-dev php7.0-json php7.0-xml php7.0-mysql php7.0-readline php-redis php7.0-mbstring php-pear
sudo pear install Crypt_GPG
sudo a2enmod php7.0
@ -192,6 +192,12 @@ sudo openssl req -newkey rsa:4096 -days 365 -nodes -x509 \
sudo a2dissite default-ssl
sudo a2ensite misp-ssl
# Recommended: Change some PHP settings in /etc/php/7.0/apache2/php.ini
# max_execution_time = 300
# memory_limit = 512M
# upload_max_filesize = 50M
# post_max_size = 50M
# Restart apache
sudo systemctl restart apache2
@ -233,6 +239,7 @@ sudo -u www-data vim /var/www/MISP/app/Config/database.php
# The admin user account will be generated on the first login, make sure that the salt is changed before you create that user
# If you forget to do this step, and you are still dealing with a fresh installation, just alter the salt,
# delete the user from mysql and log in again using the default admin credentials (admin@admin.test / admin)
# e.g. https://pythontips.com/2013/07/28/generating-a-random-string/
# Change base url in config.php
sudo -u www-data vim /var/www/MISP/app/Config/config.php
@ -268,6 +275,10 @@ sudo -u www-data bash /var/www/MISP/app/Console/worker/start.sh
# Don't forget to change the email, password and authentication key after installation.
# Start the workers
/var/www/MISP/app/Console/worker/start.sh
# Once done, have a look at the diagnostics
# If any of the directories that MISP uses to store files is not writeable to the apache user, change the permissions
@ -329,3 +340,5 @@ sudo ldconfig
## install pyzmq
sudo pip install pyzmq

View File

@ -18,4 +18,6 @@
ErrorLog /var/log/apache2/misp.local_error.log
CustomLog /var/log/apache2/misp.local_access.log combined
ServerSignature Off
Header set X-Content-Type-Options nosniff
Header set X-Frame-Options DENY
</VirtualHost>

View File

@ -17,4 +17,6 @@
ErrorLog /var/log/apache2/misp.local_error.log
CustomLog /var/log/apache2/misp.local_access.log combined
ServerSignature Off
Header set X-Content-Type-Options nosniff
Header set X-Frame-Options DENY
</VirtualHost>

View File

@ -19,4 +19,6 @@
ErrorLog /var/log/httpd/misp.local_error.log
CustomLog /var/log/httpd/misp.local_access.log combined
ServerSignature Off
Header set X-Content-Type-Options nosniff
Header set X-Frame-Options DENY
</VirtualHost>

View File

@ -20,5 +20,7 @@
LogLevel warn
ErrorLog /var/log/httpd/misp.local_error.log
CustomLog /var/log/httpd/misp.local_access.log combined
ServerSignature Off
ServerSignature Off
Header set X-Content-Type-Options nosniff
Header set X-Frame-Options DENY
</VirtualHost>

View File

@ -13,4 +13,6 @@
ErrorLog /var/log/apache2/misp.local_error.log
CustomLog /var/log/apache2/misp.local_access.log combined
ServerSignature Off
Header set X-Content-Type-Options nosniff
Header set X-Frame-Options DENY
</VirtualHost>

View File

@ -1 +1 @@
{"major":2, "minor":4, "hotfix":81}
{"major":2, "minor":4, "hotfix":82}

View File

@ -194,9 +194,7 @@ class AttributesController extends AppController {
}
}
if (!empty($successes)) {
$this->Event->set('timestamp', $date->getTimestamp());
$this->Event->set('published', 0);
$this->Event->save($this->Event->data, array('fieldList' => array('published', 'timestamp', 'info')));
$this->Event->unpublishEvent($eventId);
}
if ($this->_isRest()) {
if (!empty($successes)) {
@ -738,9 +736,6 @@ class AttributesController extends AppController {
$this->Session->setFlash(__('The attribute has been saved'));
// remove the published flag from the event
$this->Event->unpublishEvent($eventId);
$event['Event']['timestamp'] = $date->getTimestamp();
$event['Event']['published'] = 0;
$this->Event->save($event);
if (!empty($this->Attribute->data['Attribute']['object_id'])) {
$object = $this->Attribute->Object->find('first', array(
'recursive' => -1,
@ -1093,9 +1088,7 @@ class AttributesController extends AppController {
$this->ShadowAttribute->deleteAll(array('ShadowAttribute.old_id' => $id), false);
// remove the published flag from the event
$result['Event']['timestamp'] = $date->getTimestamp();
$result['Event']['published'] = 0;
$this->Attribute->Event->save($result, array('fieldList' => array('published', 'timestamp', 'info')));
$this->Attribute->Event->unpublishEvent($result['Event']['id']);
return true;
} else {
return false;
@ -2199,7 +2192,8 @@ class AttributesController extends AppController {
'Event' => array(
'fields' => array('distribution', 'id', 'org_id'),
)
)
),
'flatten' => 1
);
$attribute = $this->Attribute->fetchAttributes($this->Auth->user(), $params);
if (empty($attribute)) throw new NotFoundException(__('Invalid attribute'));
@ -2235,6 +2229,7 @@ class AttributesController extends AppController {
$params = array(
'conditions' => array('Attribute.id' => $id),
'fields' => $fields,
'flatten' => 1,
'contain' => array(
'Event' => array(
'fields' => array('distribution', 'id', 'user_id', 'orgc_id'),

View File

@ -2251,12 +2251,32 @@ class EventsController extends AppController {
}
}
}
$requested_attributes = array('uuid', 'event_id', 'category', 'type',
'value', 'comment', 'to_ids', 'timestamp');
$requested_obj_attributes = array('uuid', 'name', 'meta-category');
if (isset($this->params['url']['attributes'])) {
$requested_attributes = explode(',', $this->params['url']['attributes']);
}
if (isset($this->params['url']['obj_attributes'])) {
$requested_obj_attributes = explode(',', $this->params['url']['obj_attributes']);
}
if (isset($events)) {
foreach ($events as $eventid) {
$attributes = $this->Event->csv($user, $eventid, $ignore, $list, false, $category, $type, $includeContext, $enforceWarninglist);
$attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true);
foreach ($attributes as $attribute) {
$line = $attribute['Attribute']['uuid'] . ',' . $attribute['Attribute']['event_id'] . ',' . $attribute['Attribute']['category'] . ',' . $attribute['Attribute']['type'] . ',' . $attribute['Attribute']['value'] . ',' . $attribute['Attribute']['comment'] . ',' . intval($attribute['Attribute']['to_ids']) . ',' . $attribute['Attribute']['timestamp'] . ',' . $attribute['Object']['uuid'] . ',' . $attribute['Object']['name'] . ',' . $attribute['Object']['meta-category'];
$line1 = '';
$line2 = '';
foreach ($requested_attributes as $requested_attribute) {
$line1 .= $attribute['Attribute'][$requested_attribute] . ',';
}
$line1 = rtrim($line1, ",");
foreach ($requested_obj_attributes as $requested_obj_attribute) {
$line2 .= $attribute['Object'][$requested_obj_attribute] . ',';
}
$line2 = rtrim($line2, ",");
$line = $line1 . ',' . $line2;
$line = rtrim($line, ",");
if ($includeContext) {
foreach ($this->Event->csv_event_context_fields_to_fetch as $header => $field) {
if ($field['object']) $line .= ',' . $attribute['Event'][$field['object']][$field['var']];
@ -2276,11 +2296,18 @@ class EventsController extends AppController {
$filename = "misp.event_" . $exportType . ".csv";
}
$this->layout = 'text/default';
$headers = array('uuid', 'event_id', 'category', 'type', 'value', 'comment', 'to_ids', 'date', 'object_uuid', 'object_name', 'object_meta_category');
if (!empty($requested_obj_attributes)) {
array_walk($requested_obj_attributes, function(&$value, $key) { $value = 'object-'.$value; } );
}
$headers = array_merge($requested_attributes, $requested_obj_attributes);
if ($includeContext) $headers = array_merge($headers, array_keys($this->Event->csv_event_context_fields_to_fetch));
foreach ($headers as $k => $v) {
$headers[$k] = str_replace('-', '_', $v);
if ($v == 'timestamp') $headers[$k] = 'date';
}
$headers = implode(',', $headers);
$final = array_merge(array($headers), $final);
$final = implode (PHP_EOL, $final);
$final = implode(PHP_EOL, $final);
$final .= PHP_EOL;
return $this->RestResponse->viewData($final, 'csv', false, true, $filename);
}
@ -3806,8 +3833,10 @@ class EventsController extends AppController {
$categories[] = $k;
}
}
$default_distribution = !empty(Configure::read('MISP.default_attribute_distribution')) ? Configure::read('MISP.default_attribute_distribution') : 5;
if ($default_distribution == 'event') $default_distribution = 5;
$parameter_options = array(
'distribution' => array('valid_options' => array(0, 1, 2, 3, 5), 'default' => 0),
'distribution' => array('valid_options' => array(0, 1, 2, 3, 5), 'default' => $default_distribution),
'threat_level_id' => array('valid_options' => array(1, 2, 3, 4), 'default' => 4),
'analysis' => array('valid_options' => array(0, 1, 2), 'default' => 0),
'info' => array('default' => 'Malware samples uploaded on ' . date('Y-m-d')),
@ -3827,7 +3856,6 @@ class EventsController extends AppController {
}
if (isset($data['request'])) $data = $data['request'];
foreach ($parameter_options as $k => $v) {
if (isset($data[$k])) {
if (isset($v['valid_options']) && !in_array($data[$k], $v['valid_options'])) {
@ -3943,13 +3971,15 @@ class EventsController extends AppController {
}
if (!empty($result)) {
foreach ($result['Object'] as $object) {
$object['distribution'] = $data['settings']['distribution'];
$object['sharing_group_id'] = isset($data['settings']['distribution']) ? $data['settings']['distribution'] : 0;
if (isset($data['settings']['distribution'])) $object['distribution'] = $data['settings']['distribution'];
$object['sharing_group_id'] = isset($data['settings']['sharing_group_id']) ? $data['settings']['sharing_group_id'] : 0;
if (!empty($object['Attribute'])) {
foreach ($object['Attribute'] as $k => $attribute) {
if ($attribute['value'] == $tmpfile->name) {
$object['Attribute'][$k]['value'] = $file['filename'];
}
if (isset($data['settings']['distribution'])) $object['Attribute'][$k]['distribution'] = $data['settings']['distribution'];
$object['Attribute'][$k]['sharing_group_id'] = isset($data['settings']['sharing_group_id']) ? $data['settings']['sharing_group_id'] : 0;
}
}
$this->loadModel('MispObject');
@ -4174,7 +4204,8 @@ class EventsController extends AppController {
if ($this->request->is('post')) {
$fail = false;
$modulePayload = array(
'module' => $module['name']
'module' => $module['name'],
'event_id' => $eventId
);
if (isset($module['meta']['config'])) {
foreach ($module['meta']['config'] as $conf) {

View File

@ -510,9 +510,7 @@ class ObjectsController extends AppController {
false
);
}
$object['Event']['timestamp'] = $date->getTimestamp();
$object['Event']['published'] = 0;
$this->MispObject->Event->save($object, array('fieldList' => array('published', 'timestamp', 'info')));
$this->MispObject->Event->unpublishEvent($object['Event']['id']);
$object_refs = $this->MispObject->ObjectReference->find('all', array(
'conditions' => array(
'ObjectReference.referenced_type' => 1,

View File

@ -156,7 +156,6 @@ class RolesController extends AppController {
}
public function index() {
if (!$this->_isSiteAdmin()) $this->redirect(array('controller' => 'roles', 'action' => 'index', 'admin' => false));
$this->recursive = 0;
if ($this->_isRest()) {
$roles = $this->Role->find('all', array(

View File

@ -102,25 +102,18 @@ class ShadowAttributesController extends AppController {
$this->Event->recursive = -1;
// Unpublish the event, accepting a proposal is modifying the event after all. Also, reset the lock.
$event = $this->Event->read(null, $activeAttribute['Attribute']['event_id']);
$fieldList = array('proposal_email_lock', 'id', 'info', 'published', 'timestamp');
$event['Event']['timestamp'] = $date->getTimestamp();
$event['Event']['proposal_email_lock'] = 0;
$event['Event']['published'] = 0;
if ($this->Event->save($event, array('fieldList' => $fieldList))) {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org_id' => $this->Auth->user('org_id'),
'model' => 'ShadowAttribute',
'model_id' => $id,
'email' => $this->Auth->user('email'),
'action' => 'accept',
'title' => 'Proposal (' . $shadow['id'] . ') of ' . $shadow['org_id'] . ' to Attribute (' . $shadow['old_id'] . ') of Event (' . $shadow['event_id'] . ') accepted - ' . $shadow['category'] . '/' . $shadow['type'] . ' ' . $shadow['value'],
));
return array('saved' => true, 'success' => 'Proposed change accepted.');
} else {
return array('false' => true, 'errors' => 'Could not accept proposal.');
}
$this->Event->unpublishEvent($activeAttribute['Attribute']['event_id'], true);
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org_id' => $this->Auth->user('org_id'),
'model' => 'ShadowAttribute',
'model_id' => $id,
'email' => $this->Auth->user('email'),
'action' => 'accept',
'title' => 'Proposal (' . $shadow['id'] . ') of ' . $shadow['org_id'] . ' to Attribute (' . $shadow['old_id'] . ') of Event (' . $shadow['event_id'] . ') accepted - ' . $shadow['category'] . '/' . $shadow['type'] . ' ' . $shadow['value'],
));
return array('saved' => true, 'success' => 'Proposed change accepted.');
} else {
// If the old_id is set to 0, then we're dealing with a brand new proposed attribute
// The idea is to load the event that the new attribute will be attached to, create an attribute to it and set the distribution equal to that of the event
@ -147,29 +140,24 @@ class ShadowAttributesController extends AppController {
$this->Attribute->save($attribute);
$this->ShadowAttribute->setDeleted($toDeleteId);
$fieldList = array('proposal_email_lock', 'id', 'info', 'published');
if ($this->Auth->user('org_id') == $event['Event']['orgc_id']) {
$this->Event->unpublishEvent($activeAttribute['Attribute']['event_id'], true);
$event['Event']['proposal_email_lock'] = 0;
}
$event['Event']['published'] = 0;
$date = new DateTime();
$event['Event']['timestamp'] = $date->getTimestamp();
if ($this->Event->save($event, array('fieldList' => $fieldList))) {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org_id' => $this->Auth->user('org_id'),
'model' => 'ShadowAttribute',
'model_id' => $id,
'email' => $this->Auth->user('email'),
'action' => 'accept',
'title' => 'Proposal (' . $shadowForLog['id'] . ') of ' . $shadowForLog['org_id'] . ' to Event(' . $shadowForLog['event_id'] . ') accepted',
'change' => null,
));
return array('saved' => true, 'success' => 'Proposal accepted.');
} else {
return array('false' => true, 'errors' => 'Could not accept proposal.');
$this->Event->unpublishEvent($activeAttribute['Attribute']['event_id']);
}
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org_id' => $this->Auth->user('org_id'),
'model' => 'ShadowAttribute',
'model_id' => $id,
'email' => $this->Auth->user('email'),
'action' => 'accept',
'title' => 'Proposal (' . $shadowForLog['id'] . ') of ' . $shadowForLog['org_id'] . ' to Event(' . $shadowForLog['event_id'] . ') accepted',
'change' => null,
));
return array('saved' => true, 'success' => 'Proposal accepted.');
}
}

View File

@ -936,7 +936,7 @@ class UsersController extends AppController {
}
public function histogram($selected = null) {
if (!$this->request->is('ajax') && !$this->_isRest()) throw new MethodNotAllowedException('This function can only be accessed via AJAX or the API.');
//if (!$this->request->is('ajax') && !$this->_isRest()) throw new MethodNotAllowedException('This function can only be accessed via AJAX or the API.');
if ($selected == '[]') $selected = null;
$selectedTypes = array();
if ($selected) $selectedTypes = json_decode($selected);
@ -953,6 +953,23 @@ class UsersController extends AppController {
// What org posted what type of attribute
$this->loadModel('Attribute');
$conditions = array();
foreach ($temp as $t) {
debug($t);
$list = $this->Attribute->find('list', array(
'recursive' => -1,
'contain' => array('Event'),
'fields' => array('Attribute.type', 'COUNT(Attribute.id) AS num_types'),
'conditions' => array(
'Event.orgc_id' => $t['Event']['orgc_id']
),
'group' => array('Attribute.type')
));
debug($list);
}
throw new Exception();
/*
*/
if ($selected) $conditions[] = array('Attribute.type' => $selectedTypes, 'Attribute.deleted' => 0);
$fields = array('Event.orgc_id', 'Attribute.type', 'COUNT(Attribute.type) AS num_types');
$params = array('recursive' => 0,

View File

@ -50,6 +50,16 @@ class JSONConverterTool {
if (isset($event['Event']['Attribute'])) {
$event['Event']['Attribute'] = $this->__cleanAttributes($event['Event']['Attribute']);
if (!empty($event['Sighting'])) {
foreach ($event['Event']['Attribute'] as $ak => $attribute) {
foreach ($event['Sighting'] as $as => $sighting) {
if ($attribute['id'] == $sighting['attribute_id']) {
$event['Event']['Attribute'][$ak]['Sighting'][] = $sighting;
}
}
}
unset($event['Sighting']);
}
}
if (isset($event['Event']['Object'])) {
$event['Event']['Object'] = $this->__cleanObjects($event['Event']['Object']);

View File

@ -88,6 +88,21 @@ class PubSubTool {
return $this->__pushToRedis(':data:misp_json', $json);
}
public function event_save($event, $action) {
if (!empty($action)) $event['action'] = $action;
return $this->__pushToRedis(':data:misp_json_event', json_encode($event, JSON_PRETTY_PRINT));
}
public function object_save($object, $action) {
if (!empty($action)) $object['action'] = $action;
return $this->__pushToRedis(':data:misp_json_object', json_encode($object, JSON_PRETTY_PRINT));
}
public function object_reference_save($object_reference, $action) {
if (!empty($action)) $object_reference['action'] = $action;
return $this->__pushToRedis(':data:misp_json_object_reference', json_encode($object_reference, JSON_PRETTY_PRINT));
}
public function publishConversation($message) {
return $this->__pushToRedis(':data:misp_json_conversation', json_encode($message, JSON_PRETTY_PRINT));
}
@ -99,15 +114,18 @@ class PubSubTool {
return true;
}
public function attribute_save($attribute) {
public function attribute_save($attribute, $action = false) {
if (!empty($action)) $attribute['action'] = $action;
return $this->__pushToRedis(':data:misp_json_attribute', json_encode($attribute, JSON_PRETTY_PRINT));
}
public function sighting_save($sighting) {
public function sighting_save($sighting, $action = false) {
if (!empty($action)) $sighting['action'] = $action;
return $this->__pushToRedis(':data:misp_json_sighting', json_encode($sighting, JSON_PRETTY_PRINT));
}
public function modified($data, $type) {
public function modified($data, $type, $action = false) {
if (!empty($action)) $data['action'] = $action;
return $this->__pushToRedis(':data:misp_json_' . $type, json_encode($data, JSON_PRETTY_PRINT));
}

View File

@ -1079,8 +1079,6 @@ class AppModel extends Model {
foreach ($hotfixes as $hotfix => $requiresLogout) $updates[$major . '.' . $minor . '.' . $hotfix] = $requiresLogout;
}
}
} else {
// we'll fill this out when 3.0 comes around
}
}
return $updates;

View File

@ -559,7 +559,20 @@ class Attribute extends AppModel {
}
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_attribute_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->attribute_save($this->data);
$attribute = $this->fetchAttribute($this->id);
if (!empty($attribute)) {
$user = array(
'org_id' => -1,
'Role' => array(
'perm_site_admin' => 1
)
);
$attribute['Attribute']['Sighting'] = $this->Sighting->attachToEvent($attribute, $user, $this->id);
if (empty($attribute['Object']['id'])) unset($attribute['Object']);
$action = $created ? 'add' : 'edit';
if (!empty($this->data['Attribute']['deleted'])) $action = 'soft-delete';
$pubSubTool->attribute_save($attribute, $action);
}
}
if (Configure::read('MISP.enable_advanced_correlations') && in_array($this->data['Attribute']['type'], array('ip-src', 'ip-dst', 'domain-ip')) && strpos($this->data['Attribute']['value'], '/')) {
$this->setCIDRList();
@ -592,6 +605,12 @@ class Attribute extends AppModel {
}
// update correlation..
$this->__beforeDeleteCorrelation($this->data['Attribute']['id']);
if (!empty($this->data['Attribute']['id'])) {
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_attribute_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->attribute_save($this->data, 'delete');
}
}
}
public function afterDelete() {
@ -2258,6 +2277,39 @@ class Attribute extends AppModel {
return $this->find('list', $params);
}
/*
* Unlike the other fetchers, this one foregoes any ACL checks.
* the objective is simple: Fetch the given attribute with all related objects needed for the ZMQ output,
* standardising on this function for fetching the attribute to be passed to Attribute->save()
*/
public function fetchAttribute($id) {
$attribute = $this->find('first', array(
'recursive' => -1,
'conditions' => array('Attribute.id' => $id),
'contain' => array(
'Event' => array(
'Orgc' => array(
'fields' => array('Orgc.id', 'Orgc.uuid', 'Orgc.name')
),
'fields' => array('Event.id', 'Event.date', 'Event.info', 'Event.uuid', 'Event.published', 'Event.analysis', 'Event.threat_level_id', 'Event.org_id', 'Event.orgc_id', 'Event.distribution', 'Event.sharing_group_id')
),
'AttributeTag' => array(
'Tag' => array('fields' => array('Tag.id', 'Tag.name', 'Tag.colour', 'Tag.exportable'))
),
'Object'
)
));
if (!empty($attribute)) {
if (!empty($attribute['AttributeTag'])) {
foreach ($attribute['AttributeTag'] as $at) {
if ($at['Tag']['exportable']) $attribute['Attribute']['Tag'][] = $at['Tag'];
}
}
unset($attribute['AttributeTag']);
}
return $attribute;
}
public function fetchAttributesSimple($user, $options = array()) {
$params = array(
'conditions' => $this->buildConditions($user),
@ -2701,7 +2753,7 @@ class Attribute extends AppModel {
'md5' => array('type' => 'md5', 'category' => '', 'to_ids' => 1, 'disable_correlation' => 0, 'object_relation' => 'md5'),
'sha1' => array('type' => 'sha1', 'category' => '', 'to_ids' => 1, 'disable_correlation' => 0, 'object_relation' => 'sha1'),
'sha256' => array('type' => 'sha256', 'category' => '', 'to_ids' => 1, 'disable_correlation' => 0, 'object_relation' => 'sha256'),
'size-in-bytes' => array('type' => 'size-in-bytes', 'category' => 'Other', 'to_ids' => 0, 'disable_correlation' => 1, 'object_relation' => 'filesize')
'size-in-bytes' => array('type' => 'size-in-bytes', 'category' => 'Other', 'to_ids' => 0, 'disable_correlation' => 1, 'object_relation' => 'size-in-bytes')
);
$hashes = array('md5', 'sha1', 'sha256');
$this->Object = ClassRegistry::init('Object');
@ -2864,6 +2916,11 @@ class Attribute extends AppModel {
$this->AttributeTag->save($at);
}
}
if (!empty($attribute['Sighting'])) {
foreach ($attribute['Sighting'] as $k => $sighting) {
$this->Sighting->captureSighting($sighting, $this->id, $eventId, $user);
}
}
}
return $attribute;
}

View File

@ -345,6 +345,12 @@ class Event extends AppModel {
$this->EventBlacklist->create();
$orgc = $this->Orgc->find('first', array('conditions' => array('Orgc.id' => $this->data['Event']['orgc_id']), 'recursive' => -1, 'fields' => array('Orgc.name')));
$this->EventBlacklist->save(array('event_uuid' => $this->data['Event']['uuid'], 'event_info' => $this->data['Event']['info'], 'event_orgc' => $orgc['Orgc']['name']));
if (!empty($this->data['Event']['id'])) {
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_attribute_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->event_save(array('Event' => $this->data['Event']), 'delete');
}
}
}
// delete all of the event->tag combinations that involve the deleted event
@ -426,6 +432,11 @@ class Event extends AppModel {
$this->Correlation->updateAll(array('Correlation.info' => $db->value($this->data['Event']['info'])), array('Correlation.event_id' => $db->value($this->data['Event']['id'])));
}
}
if (empty($this->data['Event']['unpublishAction']) && empty($this->data['Event']['skip_zmq']) && Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
$event = $this->quickFetchEvent($this->data['Event']['id']);
if (!empty($event)) $pubSubTool->event_save($event, $created ? 'add' : 'edit');
}
}
public function isOwnedByOrg($eventid, $org) {
@ -907,6 +918,7 @@ class Event extends AppModel {
}
$serverModel = ClassRegistry::init('Server');
$server = $serverModel->eventFilterPushableServers($event, array($server));
if (empty($server)) return 403;
$server = $server[0];
if ($this->checkDistributionForPush($event, $server, $context = 'Event')) {
@ -1336,6 +1348,27 @@ class Event extends AppModel {
return $results;
}
/*
* Unlike the other fetchers, this one foregoes any ACL checks.
* the objective is simple: Fetch the given event with all related objects needed for the ZMQ output,
* standardising on this function for fetching the event to be passed to the pubsub handler
*/
public function quickFetchEvent($id) {
$event = $this->find('first', array(
'recursive' => -1,
'conditions' => array('Event.id' => $id),
'contain' => array(
'Orgc' => array(
'fields' => array('Orgc.id', 'Orgc.uuid', 'Orgc.name')
),
'EventTag' => array(
'Tag' => array('fields' => array('Tag.id', 'Tag.name', 'Tag.colour', 'Tag.exportable'))
)
)
));
return $event;
}
//Once the data about the user is gathered from the appropriate sources, fetchEvent is called from the controller or background process.
// Possible options:
// eventid: single event ID
@ -1738,6 +1771,11 @@ class Event extends AppModel {
return $results;
}
private function __escapeCSVField(&$field) {
$field = str_replace(array('"'), '""', $field);
$field = '"' . $field . '"';
}
public function csv($user, $eventid=false, $ignore=false, $attributeIDList = array(), $tags = false, $category = false, $type = false, $includeContext = false, $from = false, $to = false, $last = false, $enforceWarninglist = false) {
$this->recursive = -1;
$conditions = array();
@ -1807,31 +1845,31 @@ class Event extends AppModel {
$attributes = $this->Attribute->fetchAttributes($user, $params);
if (empty($attributes)) return array();
foreach ($attributes as &$attribute) {
$attribute['Attribute']['value'] = str_replace(array('"'), '""', $attribute['Attribute']['value']);
$attribute['Attribute']['value'] = '"' . $attribute['Attribute']['value'] . '"';
$attribute['Attribute']['comment'] = str_replace(array('"'), '""', $attribute['Attribute']['comment']);
$attribute['Attribute']['comment'] = '"' . $attribute['Attribute']['comment'] . '"';
$this->__escapeCSVField($attribute['Attribute']['value']);
$this->__escapeCSVField($attribute['Attribute']['comment']);
$attribute['Attribute']['timestamp'] = date('Ymd', $attribute['Attribute']['timestamp']);
if (empty($attribute['Object'])) {
$attribute['Object']['uuid'] = '""';
$attribute['Object']['name'] = '';
$attribute['Object']['meta-category'] = '';
}
$attribute['Object']['name'] = str_replace(array('"'), '""', $attribute['Object']['name']);
$attribute['Object']['name'] = '"' . $attribute['Object']['name'] . '"';
$attribute['Object']['meta-category'] = str_replace(array('"'), '""', $attribute['Object']['meta-category']);
$attribute['Object']['meta-category'] = '"' . $attribute['Object']['meta-category'] . '"';
$this->__escapeCSVField($attribute['Object']['name']);
$this->__escapeCSVField($attribute['Object']['meta-category']);
if ($includeContext) {
$attribute['Event']['info'] = str_replace(array('"'), '""', $attribute['Event']['info']);
$attribute['Event']['info'] = '"' . $attribute['Event']['info'] . '"';
$this->__escapeCSVField($attribute['Event']['info']);
$this->__escapeCSVField($attribute['Org']['name']);
$this->__escapeCSVField($attribute['Orgc']['name']);
$attribute['Event']['Tag']['name'] = '';
if (!empty($attribute['Event']['EventTag'])) {
$tags = array();
foreach ($attribute['Event']['EventTag'] as $eventTag) {
if (!empty($attribute['Event']['Tag']['name'])) $attribute['Event']['Tag']['name'] .= ',';
$attribute['Event']['Tag']['name'] .= str_replace(array('"'), '""', $eventTag['Tag']['name']);
if (!empty($eventTag['Tag']['name'])) {
$tags[] = $eventTag['Tag']['name'];
}
}
$attribute['Event']['Tag']['name'] = implode(',', $tags);
}
if (!empty($attribute['Event']['Tag']['name'])) $attribute['Event']['Tag']['name'] = '"' . $attribute['Event']['Tag']['name'] . '"';
$this->__escapeCSVField($attribute['Event']['Tag']['name']);
}
}
return $attributes;
@ -2767,6 +2805,7 @@ class Event extends AppModel {
$fieldList = array('published', 'id', 'info', 'publish_timestamp');
$event['Event']['published'] = 1;
$event['Event']['publish_timestamp'] = time();
$event['Event']['skip_zmq'] = 1;
$this->save($event, array('fieldList' => $fieldList));
}
if (Configure::read('Plugin.ZeroMQ_enable')) {
@ -2775,7 +2814,7 @@ class Event extends AppModel {
if (!empty($hostOrg)) {
$user = array('org_id' => $hostOrg['Org']['id'], 'Role' => array('perm_sync' => 0, 'perm_audit' => 0, 'perm_site_admin' => 0), 'Organisation' => $hostOrg['Org']);
$fullEvent = $this->fetchEvent($user, array('eventid' => $id));
if (!empty($fullEvent)) $pubSubTool->publishEvent($fullEvent[0]);
if (!empty($fullEvent)) $pubSubTool->publishEvent($fullEvent[0], 'publish');
}
}
$uploaded = $this->uploadEventToServersRouter($id, $passAlong);
@ -3791,7 +3830,9 @@ class Event extends AppModel {
));
$sharingGroupData = array();
foreach ($sharingGroupDataTemp as $k => $v) {
$sharingGroupData[$v['SharingGroup']['id']] = $v;
$v['SharingGroup']['SharingGroupOrg'] = $v['SharingGroupOrg'];
$v['SharingGroup']['SharingGroupServer'] = $v['SharingGroupServer'];
$sharingGroupData[$v['SharingGroup']['id']] = array('SharingGroup' => $v['SharingGroup']);
}
if ($useCache) $this->__assetCache['sharingGroupData'] = $sharingGroupData;
return $sharingGroupData;
@ -3839,7 +3880,7 @@ class Event extends AppModel {
$this->__assetCache = array();
}
public function unpublishEvent($id) {
public function unpublishEvent($id, $proposalLock = false) {
$event = $this->find('first', array(
'recursive' => -1,
'conditions' => array('Event.id' => $id)
@ -3848,6 +3889,8 @@ class Event extends AppModel {
$event['Event']['published'] = 0;
$date = new DateTime();
$event['Event']['timestamp'] = $date->getTimestamp();
if ($proposalLock) $event['Event']['proposal_email_lock'] = 0;
$event['Event']['unpublishAction'] = true;
return $this->save($event);
}
}

View File

@ -74,6 +74,35 @@ class MispObject extends AppModel {
return true;
}
public function afterSave($created, $options = array()) {
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_attribute_notifications_enable')) {
if (empty($this->data['Object']['skip_zmq'])) {
$pubSubTool = $this->getPubSubTool();
$object = $this->find('first', array(
'conditions' => array('Object.id' => $this->id),
'recursive' => -1
));
$action = $created ? 'add' : 'edit';
if (!empty($this->data['Object']['deleted'])) $action = 'soft-delete';
$pubSubTool->object_save($object, $action);
}
}
return true;
}
public function beforeDelete($cascade = true) {
if (!empty($this->data['Object']['id'])) {
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_object_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
$object = $this->find('first', array(
'recursive' => -1,
'conditions' => array('Object.id' => $this->data['Object']['id'])
));
$pubSubTool->object_save($object, 'delete');
}
}
}
public function afterDelete() {
if (!empty($this->data[$this->alias]['id'])) {
$this->ObjectReference->deleteAll(
@ -550,6 +579,7 @@ class MispObject extends AppModel {
'conditions' => array('Object.id' => $id)
));
$object['Object']['timestamp'] = $date->getTimestamp();
$object['Object']['skip_zmq'] = 1;
$result = $this->save($object);
return $result;
}

View File

@ -48,6 +48,20 @@ class ObjectReference extends AppModel {
return true;
}
public function afterSave($created, $options = array()) {
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_object_reference_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
$object_reference = $this->find('first', array(
'conditions' => array('ObjectReference.id' => $this->id),
'recursive' => -1
));
$action = $created ? 'add' : 'edit';
if (!empty($this->data['ObjectReference']['deleted'])) $action = 'soft-delete';
$pubSubTool->object_reference_save($object_reference, $action);
}
return true;
}
public function updateTimestamps($id, $objectReference = false) {
if (!$objectReference) {
$objectReference = $this->find('first', array(

View File

@ -1188,6 +1188,30 @@ class Server extends AppModel {
'type' => 'string',
'afterHook' => 'zmqAfterHook',
),
'ZeroMQ_event_notifications_enable' => array(
'level' => 2,
'description' => 'Enables or disables the publishing of any event creations/edits/deletions.',
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean'
),
'ZeroMQ_object_notifications_enable' => array(
'level' => 2,
'description' => 'Enables or disables the publishing of any object creations/edits/deletions.',
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean'
),
'ZeroMQ_object_reference_notifications_enable' => array(
'level' => 2,
'description' => 'Enables or disables the publishing of any object reference creations/deletions.',
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean'
),
'ZeroMQ_attribute_notifications_enable' => array(
'level' => 2,
'description' => 'Enables or disables the publishing of any attribute creations/edits/soft deletions.',

View File

@ -488,6 +488,7 @@ class ShadowAttribute extends AppModel {
$event['Event']['proposal_email_lock'] = 0;
}
$fieldList = array('proposal_email_lock', 'id', 'info');
$event['Event']['skip_zmq'] = 1;
$this->Event->save($event, array('fieldList' => $fieldList));
}

View File

@ -51,14 +51,109 @@ class Sighting extends AppModel {
}
public function afterSave($created, $options = array()) {
parent::afterSave($created, $options = array());
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_sighting_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->sighting_save($this->data);
$user = array(
'org_id' => -1,
'Role' => array(
'perm_site_admin' => 1
)
);
$sighting = $this->getSighting($this->id, $user);
$pubSubTool->sighting_save($sighting, 'add');
}
return true;
}
public function attachToEvent($event, $user, $attribute_id = false, $extraConditions = false) {
public function beforeDelete($cascade = true) {
parent::beforeDelete();
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_sighting_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
$user = array(
'org_id' => -1,
'Role' => array(
'perm_site_admin' => 1
)
);
$sighting = $this->getSighting($this->id, $user);
$pubSubTool->sighting_save($sighting, 'delete');
}
}
public function captureSighting($sighting, $attribute_id, $event_id, $user) {
$org_id = 0;
if (!empty($sighting['Organisation'])) {
$org_id = $this->Organisation->captureOrg($sighting['Organisation'], $user);
}
if (isset($sighting['id'])) {
unset($sighting['id']);
}
$sighting['org_id'] = $org_id;
$sighting['event_id'] = $event_id;
$sighting['attribute_id'] = $attribute_id;
return $this->save($sighting);
}
public function getSighting($id, $user) {
$sighting = $this->find('first', array(
'recursive' => -1,
'contain' => array(
'Attribute' => array(
'fields' => array('Attribute.value', 'Attribute.id')
),
'Event' => array(
'fields' => array('Event.id', 'Event.uuid', 'Event.orgc_id', 'Event.org_id')
)
),
'conditions' => array('Sighting.id' => $id)
));
if (empty($sighting)) return array();
if ($user['Role']['perm_site_admin'] || $event['Event']['org_id'] == $user['org_id']) $ownEvent = true;
if (!$ownEvent) {
// if sighting policy == 0 then return false if the sighting doesn't belong to the user
if (!Configure::read('Plugin.Sightings_policy') || Configure::read('Plugin.Sightings_policy') == 0) {
if ($sighting['Sighting']['org_id'] != $user['org_id']) return array();
}
// if sighting policy == 1, the user can only see the sighting if they've sighted something in the event once
if (Configure::read('Plugin.Sightings_policy') == 1) {
$temp = $this->find('first',
array(
'recursive' => -1,
'conditions' => array(
'Sighting.event_id' => $sighting['Sighting']['event_id'],
'Sighting.org_id' => $user['org_id']
)
)
);
if (empty($temp)) return array();
}
}
$anonymise = Configure::read('Plugin.Sightings_anonymise');
if ($anonymise) {
if ($sighting['Sighting']['org_id'] != $user['org_id']) {
unset($sighting['Sighting']['org_id']);
unset($sighting['Organisation']);
}
}
// rearrange it to match the event format of fetchevent
if (isset($sighting['Organisation'])) {
$sighting['Sighting']['Organisation'] = $sighting['Organisation'];
unset($sighting['Organisation']);
}
$sighting['Sighting']['value'] = $sighting['Attribute']['value'];
return array('Sighting' => $sighting['Sighting']);
}
public function attachToEvent($event, $user = array(), $attribute_id = false, $extraConditions = false) {
if (empty($user)) {
$user = array(
'org_id' => -1,
'Role' => array(
'perm_site_admin' => 0
)
);
}
$ownEvent = false;
if ($user['Role']['perm_site_admin'] || $event['Event']['org_id'] == $user['org_id']) $ownEvent = true;
$conditions = array('Sighting.event_id' => $event['Event']['id']);
@ -93,7 +188,10 @@ class Sighting extends AppModel {
$anonymise = Configure::read('Plugin.Sightings_anonymise');
foreach ($sightings as $k => $sighting) {
if ($anonymise && !$user['Role']['perm_site_admin']) {
if (
$sighting['Sighting']['org_id'] == 0 && !empty($sighting['Organisation']) ||
$anonymise
) {
if ($sighting['Sighting']['org_id'] != $user['org_id']) {
unset($sightings[$k]['Sighting']['org_id']);
unset($sightings[$k]['Organisation']);
@ -127,7 +225,7 @@ class Sighting extends AppModel {
if (!in_array($type, array(0, 1, 2))) {
return 'Invalid type, please change it before you POST 1000000 sightings.';
}
$attributes = $this->Attribute->fetchAttributes($user, array('conditions' => $conditions));
$attributes = $this->Attribute->fetchAttributes($user, array('conditions' => $conditions, 'flatten' => 1));
if (empty($attributes)) return 'No valid attributes found that match the criteria.';
$sightingsAdded = 0;
foreach ($attributes as $attribute) {

View File

@ -119,7 +119,7 @@ def setDates(incident, date, published):
def resolveObjects(incident, ttps, objects, eventTags, org):
for obj in objects:
tmp_incident = Incident()
resolveAttributes(tmp_incident, ttps, obj["Attribute"], eventTags)
resolveAttributes(tmp_incident, ttps, obj["Attribute"], eventTags, org)
indicator = Indicator(timestamp=getDateFromTimestamp(int(obj["timestamp"])))
indicator.id_= namespace[1] + ":MispObject-" + obj["uuid"]
setProd(indicator, org)

View File

@ -74,8 +74,9 @@ def main(args):
command = r.lpop(namespace + ":command")
if command is not None:
handleCommand(command)
topics = ["misp_json", "misp_json_attribute", "misp_json_sighting",
"misp_json_organisation", "misp_json_user", "misp_json_conversation"]
topics = ["misp_json", "misp_json_event", "misp_json_attribute", "misp_json_sighting",
"misp_json_organisation", "misp_json_user", "misp_json_conversation",
"misp_json_object", "misp_json_object_reference"]
message_received = False
for topic in topics:
data = r.lpop(namespace + ":data:" + topic)

View File

@ -17,6 +17,7 @@
import sys, json, os, datetime, re
import pymisp
# import stix2
from stix2 import *
namespace = ['https://github.com/MISP/MISP', 'MISP']
@ -169,11 +170,13 @@ def saveFile(args, pathname, package):
def getDateFromTimestamp(timestamp):
return datetime.datetime.utcfromtimestamp(timestamp).isoformat() + "+00:00"
def setIdentity(event):
def setIdentity(event, SDOs):
org = event.Orgc
identity = Identity(type="identity", id="identity--{}".format(org["uuid"]),
identity_id = 'identity--{}'.format(org['uuid'])
identity = Identity(type="identity", id=identity_id,
name=org["name"], identity_class="organization")
return identity
SDOs.append(identity)
return identity_id
def readAttributes(event, identity, object_refs, external_refs):
attributes = []
@ -199,7 +202,7 @@ def readAttributes(event, identity, object_refs, external_refs):
else:
addObservedData(object_refs, attributes, attribute, identity)
else:
continue
addCustomObject(object_refs, attributes, attribute, identity)
if event.Galaxy:
galaxies = event.Galaxy
for galaxy in galaxies:
@ -216,6 +219,13 @@ def readAttributes(event, identity, object_refs, external_refs):
addThreatActor(object_refs, attributes, galaxy, identity)
elif galaxyType in ['rat', 'exploit-kit'] or 'tool' in galaxyType:
addTool(object_refs, attributes, galaxy, identity)
if event.Object:
for obj in event.Object:
obj_id = obj.uuid
obj_timestamp = obj.timestamp
obj_attributes = obj.Attribute
for obj_attr in obj_attributes:
print(obj_attr.type, obj_attr.value)
return attributes
def handleLink(attribute, external_refs):
@ -277,13 +287,27 @@ def addCustomObject(object_refs, attributes, attribute, identity):
customObject_type = 'x-misp-object'.format(attribute.type)
value = attribute.value
labels = 'misp:to_ids=\"{}\"'.format(attribute.to_ids)
customObject_args = {'type': customObject_type, 'id': customObject_id, 'timestamp': timestamp,
'to_ids': labels, 'value': value, 'created_by_ref': identity, 'labels': labels}
# customObject_args = {'type': customObject_type, 'id': customObject_id, 'timestamp': timestamp,
# 'to_ids': labels, 'value': value, 'created_by_ref': identity, 'labels': labels}
customObject_args = {'id': customObject_id, 'x_misp_timestamp': timestamp, 'x_misp_to_ids': labels,
'x_misp_value': value, 'created_by_ref': identity}
if attribute.comment:
customObject_args['comment'] = attribute.comment
# At the moment, we skip it
# attributes.append(customObject_args)
# object_refs.append(customObject_id)
customObject_args['x_misp_comment'] = attribute.comment
@CustomObject(customObject_type, [('id', properties.StringProperty(required=True)),
('x_misp_timestamp', properties.StringProperty(required=True)),
('x_misp_to_ids', properties.StringProperty(required=True)),
('x_misp_value', properties.StringProperty(required=True)),
('created_by_ref', properties.StringProperty(required=True)),
('x_misp_comment', properties.StringProperty()),
])
class Custom(object):
def __init__(self, **kwargs):
return
custom = Custom(**customObject_args)
# print(custom)
# custom = CustomObject(**customObject_args)
attributes.append(custom)
object_refs.append(customObject_id)
def addIdentity(object_refs, attributes, attribute, identity, identityClass):
identity_id = "identity--{}".format(attribute.uuid)
@ -293,7 +317,7 @@ def addIdentity(object_refs, attributes, attribute, identity, identityClass):
identity_args['description'] = attribute.comment
identityObject = Identity(**identity_args)
attributes.append(identityObject)
object_refs.append(identityObject)
object_refs.append(identity_id)
def addIntrusionSet(object_refs, attributes, galaxy, identity):
cluster = galaxy['GalaxyCluster'][0]
@ -505,7 +529,7 @@ def eventReport(event, identity, object_refs, external_refs):
for tag in tags:
labels.append(tag['name'])
args_report = {'type': "report", 'id': "report--{}".format(event.uuid), 'created_by_ref': identity["id"],
args_report = {'type': "report", 'id': "report--{}".format(event.uuid), 'created_by_ref': identity,
'name': name, 'published': timestamp}
if labels:
@ -526,6 +550,9 @@ def generateEventPackage(event, SDOs):
return bundle
def main(args):
# for i in dir(stix2):
# print(i)
# sys.exit(0)
pathname = os.path.dirname(sys.argv[0])
if len(sys.argv) > 3:
namespace[0] = sys.argv[3]
@ -537,8 +564,7 @@ def main(args):
SDOs = []
object_refs = []
external_refs = []
identity = setIdentity(misp)
SDOs.append(identity)
identity = setIdentity(misp, SDOs)
attributes = readAttributes(misp, identity, object_refs, external_refs)
buildRelationships(attributes, object_refs)
report = eventReport(misp, identity, object_refs, external_refs)

View File

@ -29,6 +29,7 @@ line.link {
padding: 0px;
margin: 0px;
width: 250px;
max-height: 300px;
}
.menu li {
color: white;

View File

@ -21,7 +21,7 @@ parser = argparse.ArgumentParser(description='Generic ZMQ client to gather event
parser.add_argument("-s","--stats", default=False, action='store_true', help='print regular statistics on stderr')
parser.add_argument("-p","--port", default="50000", help='set TCP port of the MISP ZMQ (default: 50000)')
parser.add_argument("-r","--host", default="127.0.0.1", help='set host of the MISP ZMQ (default: 127.0.0.1)')
parser.add_argument("-o","--only", action="append", default=None, help="set filter (misp_json, misp_json_attribute or misp_json_sighting) to limit the output a specific type (default: no filter)")
parser.add_argument("-o","--only", action="append", default=None, help="set filter (misp_json, misp_json_event, misp_json_attribute or misp_json_sighting) to limit the output a specific type (default: no filter)")
parser.add_argument("-t","--sleep", default=2, help='sleep time (default: 2)')
args = parser.parse_args()