diff --git a/INSTALL/INSTALL.txt b/INSTALL/INSTALL.txt index 0994a8bb8..98fecbf1c 100755 --- a/INSTALL/INSTALL.txt +++ b/INSTALL/INSTALL.txt @@ -72,10 +72,11 @@ php composer.phar install # CakeResque normally uses phpredis to connect to redis, but it has a (buggy) fallback connector through Redisent. It is highly advised to install phpredis pecl install redis +apt get install php5-redis # After installing it, enable it in your php.ini file vim /etc/php5/apache2/php.ini # add the following line: -extension=redis.so +extension=redis.so # To use the scheduler worker for scheduled tasks, do the following: cp -fa /var/www/MISP/INSTALL/setup/config.php /var/www/MISP/app/Plugin/CakeResque/Config/config.php @@ -118,6 +119,7 @@ cp /var/www/MISP/INSTALL/apache.misp /etc/apache2/sites-available/misp.conf # For more information, visit http://httpd.apache.org/docs/2.4/upgrading.html a2dissite default +# default can be called 000-default based on distribution, in which case run a2dissite 000-default a2ensite misp # Enable modules diff --git a/INSTALL/MYSQL.sql b/INSTALL/MYSQL.sql index 1f3af245e..ab756b9f7 100644 --- a/INSTALL/MYSQL.sql +++ b/INSTALL/MYSQL.sql @@ -665,4 +665,7 @@ INSERT INTO `template_element_texts` (`id`, `name`, `template_element_id`, `text (11, 'Persistence mechanism', 41, 'The following fields allow you to describe the persistence mechanism used by the malware'), (12, 'Indicators', 45, 'Just paste your list of indicators based on type into the appropriate field. All of the fields are optional, so inputting a list of IP addresses into the Network indicator field for example is sufficient to complete this template.'); - +INSERT INTO `tasks` (`id`, `type`, `timer`, `scheduled_time`, `job_id`, `description`, `next_execution_time`, `message`) VALUES +(1, 'cache_exports', 0, '12:00', 0, 'Generates export caches for every export type and for every organisation. This process is heavy, schedule so it might be a good idea to schedule this outside of working hours and before your daily automatic imports on connected services are scheduled.', 1391601600, 'Not scheduled yet.'), +(2, 'pull_all', 0, '12:00', 0, 'Initiates a full pull for all eligible instances.', 1391601600, 'Not scheduled yet.'), +(3, 'push_all', 0, '12:00', 0, 'Initiates a full push for all eligible instances.', 1391601600, 'Not scheduled yet.'); diff --git a/VERSION.json b/VERSION.json index 0c9d4a63f..661ef05e9 100644 --- a/VERSION.json +++ b/VERSION.json @@ -1 +1 @@ -{"major":2, "minor":3, "hotfix":27} \ No newline at end of file +{"major":2, "minor":3, "hotfix":40} diff --git a/app/Console/Command/AdminShell.php b/app/Console/Command/AdminShell.php index 7f59c255b..9a145f092 100644 --- a/app/Console/Command/AdminShell.php +++ b/app/Console/Command/AdminShell.php @@ -5,18 +5,9 @@ class AdminShell extends AppShell public $uses = array('Event'); public function jobGenerateCorrelation() { + $jobId = $this->args[0]; $this->loadModel('Job'); - $this->Job->create(); - $data = array( - 'worker' => 'default', - 'job_type' => 'generate correlation', - 'job_input' => 'All attributes', - 'status' => 0, - 'retries' => 0, - 'message' => 'Job created.', - ); - $this->Job->save($data); - $jobID = $this->Job->id; + $this->Job->id = $jobId; $this->loadModel('Correlation'); $this->Correlation->deleteAll(array('id !=' => ''), false); $this->loadModel('Attribute'); @@ -26,7 +17,9 @@ class AdminShell extends AppShell // for all attributes.. $total = count($attributes); foreach ($attributes as $k => $attribute) { - $this->Job->saveField('progress', $k/$total*100); + if ($k > 0 && $k % 1000 == 0) { + $this->Job->saveField('progress', $k/$total*100); + } $this->Attribute->__afterSaveCorrelation($attribute['Attribute']); } $this->Job->saveField('progress', 100); diff --git a/app/Console/Command/EventShell.php b/app/Console/Command/EventShell.php index 7a8f16c47..ae031324a 100755 --- a/app/Console/Command/EventShell.php +++ b/app/Console/Command/EventShell.php @@ -328,6 +328,17 @@ class EventShell extends AppShell $task['Task']['next_execution_time'] = strtotime('+' . $task['Task']['timer'] . ' hours', $task['Task']['next_execution_time']); } $task['Task']['scheduled_time'] = $this->Task->breakTime($task['Task']['scheduled_time'], $task['Task']['timer']); + $task['Task']['scheduled_time'] = date('H:i', $task['Task']['next_execution_time']); + + // Now that we have figured out when the next execution should happen, it's time to enqueue it. + $process_id = CakeResque::enqueueAt( + $task['Task']['next_execution_time'], + 'default', + 'EventShell', + array('enqueueCaching', $task['Task']['next_execution_time']), + true + ); + $task['Task']['job_id'] = $process_id; $this->Task->save($task); } } diff --git a/app/Console/Command/ServerShell.php b/app/Console/Command/ServerShell.php index 146096138..c2a4c4865 100644 --- a/app/Console/Command/ServerShell.php +++ b/app/Console/Command/ServerShell.php @@ -18,7 +18,7 @@ class ServerShell extends AppShell $this->User->recursive = -1; $user = $this->User->read(null, $userId); $server = $this->Server->read(null, $serverId); - $result = $this->Server->pull($user['User'], null, $technique, $server, $jobId); + $result = $this->Server->pull($user['User'], $serverId, $technique, $server, $jobId); $this->Job->id = $jobId; $this->Job->save(array( 'id' => $jobId, @@ -53,20 +53,23 @@ class ServerShell extends AppShell $serverId = $this->args[0]; $technique = $this->args[1]; $jobId = $this->args[2]; + $userId = $this->args[3]; $this->Job->read(null, $jobId); $server = $this->Server->read(null, $serverId); App::uses('SyncTool', 'Tools'); $syncTool = new SyncTool(); $HttpSocket = $syncTool->setupHttpSocket($server); - $result = $this->Server->push($serverId, 'full', $jobId, $HttpSocket); + $this->User->recursive = -1; + $user = $this->User->read(array('id', 'org', 'email'), $userId); + $result = $this->Server->push($serverId, 'full', $jobId, $HttpSocket, $user['User']['email']); $this->Job->save(array( 'id' => $jobId, 'message' => 'Job done.', 'progress' => 100, 'status' => 4 )); - if (isset($this->args[3])) { - $this->Task->id = $this->args[4]; + if (isset($this->args[4])) { + $this->Task->id = $this->args[5]; $this->Task->saveField('message', 'Job(s) started at ' . date('d/m/Y - H:i:s') . '.'); } } @@ -91,14 +94,14 @@ class ServerShell extends AppShell 'job_input' => 'Server: ' . $server['Server']['id'], 'retries' => 0, 'org' => $user['User']['org'], - 'process_id' => $this->Task->data['Task']['job_id'], + 'process_id' => 'Part of scheduled pull', 'message' => 'Pushing.', ); $this->Job->save($data); $jobId = $this->Job->id; App::uses('SyncTool', 'Tools'); $syncTool = new SyncTool(); - $result = $this->Server->pull($user['User'], null, 'full', $server, $jobId); + $result = $this->Server->pull($user['User'], $server['Server']['id'], 'full', $server, $jobId); $this->Job->save(array( 'id' => $jobId, 'message' => 'Job done.', @@ -150,12 +153,14 @@ class ServerShell extends AppShell $timestamp = $this->args[0]; $taskId = $this->args[1]; $org = $this->args[2]; + $userId = $this->args[3]; $this->Task->id = $taskId; $task = $this->Task->read(null, $taskId); if ($timestamp != $task['Task']['next_execution_time']) { return; } - + $this->User->recursive = -1; + $user = $this->User->read(array('id', 'org', 'email'), $userId); $servers = $this->Server->find('all', array('recursive' => -1, 'conditions' => array('push' => 1))); $count = count($servers); foreach ($servers as $k => $server) { @@ -166,7 +171,7 @@ class ServerShell extends AppShell 'job_input' => 'Server: ' . $server['Server']['id'], 'retries' => 0, 'org' => $org, - 'process_id' => $this->Task->data['Task']['job_id'], + 'process_id' => 'Part of scheduled push', 'message' => 'Pushing.', ); $this->Job->save($data); @@ -174,7 +179,7 @@ class ServerShell extends AppShell App::uses('SyncTool', 'Tools'); $syncTool = new SyncTool(); $HttpSocket = $syncTool->setupHttpSocket($server); - $result = $this->Server->push($server['Server']['id'], 'full', $jobId, $HttpSocket); + $result = $this->Server->push($server['Server']['id'], 'full', $jobId, $HttpSocket, $user['User']['email']); } $task['Task']['message'] = count($servers) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '.'; if ($task['Task']['timer'] > 0) { @@ -191,7 +196,7 @@ class ServerShell extends AppShell $task['Task']['next_execution_time'], 'default', 'ServerShell', - array('enqueuePush', $task['Task']['next_execution_time'], $taskId, $org), + array('enqueuePush', $task['Task']['next_execution_time'], $taskId, $org, $userId), true ); $task['Task']['job_id'] = $process_id; diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index c33173353..b9674b3ca 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -116,6 +116,7 @@ class AppController extends Controller { // instead of using checkAction(), like we normally do from controllers when trying to find out about a permission flag, we can use getActions() // getActions returns all the flags in a single SQL query if ($this->Auth->user()) { + //$this->_refreshAuth(); $this->set('mispVersion', $this->mispVersion); $role = $this->getActions(); $this->set('me', $this->Auth->user()); diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 776c081da..58dfc30e0 100755 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -81,7 +81,7 @@ class AttributesController extends AppController { */ public function index() { $this->Attribute->recursive = 0; - $this->Attribute->contain = array('Event.id', 'Event.orgc', 'Event.org'); + $this->Attribute->contain = array('Event.id', 'Event.orgc', 'Event.org', 'Event.info'); $this->set('isSearch', 0); $this->set('attributes', $this->paginate()); $this->set('attrDescriptions', $this->Attribute->fieldDescriptions); @@ -939,7 +939,7 @@ class AttributesController extends AppController { // attachment will be deleted with the beforeDelete() function in the Model if ($this->Attribute->delete()) { // delete the attribute from remote servers - $this->__deleteAttributeFromServers($uuid); + //$this->__deleteAttributeFromServers($uuid); // We have just deleted the attribute, let's also check if there are any shadow attributes that were attached to it and delete them $this->loadModel('ShadowAttribute'); @@ -1088,12 +1088,14 @@ class AttributesController extends AppController { if ($this->request->is('post') && ($this->request->here == $fullAddress)) { $keyword = $this->request->data['Attribute']['keyword']; $keyword2 = $this->request->data['Attribute']['keyword2']; + $tags = $this->request->data['Attribute']['tags']; $org = $this->request->data['Attribute']['org']; $type = $this->request->data['Attribute']['type']; $ioc = $this->request->data['Attribute']['ioc']; $this->set('ioc', $ioc); $category = $this->request->data['Attribute']['category']; $this->set('keywordSearch', $keyword); + $this->set('tags', $tags); $keyWordText = null; $keyWordText2 = null; $keyWordText3 = null; @@ -1114,28 +1116,74 @@ class AttributesController extends AppController { $temp = array(); $temp2 = array(); foreach ($keywordArray as $keywordArrayElement) { - $saveWord = trim($keywordArrayElement); - $keywordArrayElement = '%' . trim($keywordArrayElement) . '%'; - if ($keywordArrayElement != '%%') { - if ($keywordArrayElement[1] == '!') { - if (preg_match('@^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(\d|[1-2]\d|3[0-2]))$@', substr($saveWord, 2))) { - $cidrresults = $this->Cidr->CIDR($saveWord); - foreach ($cidrresults as $result) { - array_push($temp2, array('Attribute.value NOT LIKE' => $result)); + $saveWord = trim(strtolower($keywordArrayElement)); + if ($saveWord != '') { + $toInclude = true; + if ($saveWord[0] == '!') { + $toInclude = false; + $saveWord = substr($saveWord, 1); + } + + if (preg_match('@^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(\d|[1-2]\d|3[0-2]))$@', $saveWord)) { + $cidrresults = $this->Cidr->CIDR($saveWord); + foreach ($cidrresults as $result) { + $result = strtolower($result); + if (strpos($result, '|')) { + $resultParts = explode('|', $result); + if (!toInclude) { + $temp2[] = array( + 'AND' => array( + 'LOWER(Attribute.value1) NOT LIKE' => $resultParts[0], + 'LOWER(Attribute.value2) NOT LIKE' => $resultParts[1], + )); + } else { + $temp[] = array( + 'AND' => array( + 'LOWER(Attribute.value1)' => $resultParts[0], + 'LOWER(Attribute.value2)' => $resultParts[1], + )); + } + } else { + if (!$toInclude) { + array_push($temp2, array('LOWER(Attribute.value1) NOT LIKE' => $result)); + array_push($temp2, array('LOWER(Attribute.value2) NOT LIKE' => $result)); + } else { + array_push($temp, array('LOWER(Attribute.value1) LIKE' => $result)); + array_push($temp, array('LOWER(Attribute.value2) LIKE' => $result)); + } } - } else { - array_push($temp2, array('Attribute.value NOT LIKE' => '%' . substr($keywordArrayElement, 2))); } } else { - if (preg_match('@^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(\d|[1-2]\d|3[0-2]))$@', $saveWord)) { - $cidrresults = $this->Cidr->CIDR($saveWord); - foreach ($cidrresults as $result) { - array_push($temp, array('Attribute.value LIKE' => $result)); + if (strpos($saveWord, '|')) { + $resultParts = explode('|', $saveWord); + if (!$toInclude) { + $temp2[] = array( + 'AND' => array( + 'LOWER(Attribute.value1) NOT LIKE' => '%' . $resultParts[0], + 'LOWER(Attribute.value2) NOT LIKE' => $resultParts[1] . '%', + )); + } else { + $temp2[] = array( + 'AND' => array( + 'LOWER(Attribute.value1)' => '%' . $resultParts[0], + 'LOWER(Attribute.value2)' => $resultParts[1] . '%', + )); } } else { - array_push($temp, array('Attribute.value LIKE' => $keywordArrayElement)); + if (!$toInclude) { + array_push($temp2, array('LOWER(Attribute.value1) NOT LIKE' => '%' . $saveWord . '%')); + array_push($temp2, array('LOWER(Attribute.value2) NOT LIKE' => '%' . $saveWord . '%')); + } else { + array_push($temp, array('LOWER(Attribute.value1) LIKE' => '%' . $saveWord . '%')); + array_push($temp, array('LOWER(Attribute.value2) LIKE' => '%' . $saveWord . '%')); + } } } + if ($toInclude) { + array_push($temp, array('LOWER(Attribute.comment) LIKE' => '%' . $saveWord . '%')); + } else { + array_push($temp2, array('LOWER(Attribute.comment) NOT LIKE' => '%' . $saveWord . '%')); + } } if ($i == 1 && $saveWord != '') $keyWordText = $saveWord; else if (($i > 1 && $i < 10) && $saveWord != '') $keyWordText = $keyWordText . ', ' . $saveWord; @@ -1175,6 +1223,19 @@ class AttributesController extends AppController { $conditions['AND'][] = $temp; } } + if (!empty($tags)) { + $include = array(); + $exclude = array(); + $keywordArray = explode("\n", $tags); + foreach ($keywordArray as $tagname) { + $tagname = trim($tagname); + if (substr($tagname, 0, 1) === '!') $exclude[] = substr($tagname, 1); + else $include[] = $tagname; + } + $this->loadModel('Tag'); + if (!empty($include)) $conditions['AND'][] = array('OR' => array('Attribute.event_id' => $this->Tag->findTags($include))); + if (!empty($exclude)) $conditions['AND'][] = array('Attribute.event_id !=' => $this->Tag->findTags($exclude)); + } if ($type != 'ALL') { $conditions['Attribute.type ='] = $type; } @@ -1204,7 +1265,6 @@ class AttributesController extends AppController { $conditions['AND'][] = $temp; } } - if ($this->request->data['Attribute']['alternate']) { $events = $this->searchAlternate($conditions); $this->set('events', $events); @@ -1215,7 +1275,7 @@ class AttributesController extends AppController { 'limit' => 60, 'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 attributes? 'conditions' => $conditions, - 'contain' => array('Event.orgc', 'Event.id', 'Event.org', 'Event.user_id') + 'contain' => array('Event.orgc', 'Event.id', 'Event.org', 'Event.user_id', 'Event.info') ); if (!$this->_isSiteAdmin()) { // merge in private conditions @@ -1244,7 +1304,6 @@ class AttributesController extends AppController { } } $this->set('attributes', $attributes); - // and store into session $this->Session->write('paginate_conditions', $this->paginate); $this->Session->write('paginate_conditions_keyword', $keyword); @@ -1252,6 +1311,7 @@ class AttributesController extends AppController { $this->Session->write('paginate_conditions_org', $org); $this->Session->write('paginate_conditions_type', $type); $this->Session->write('paginate_conditions_ioc', $ioc); + $this->Session->write('paginate_conditions_tags', $tags); $this->Session->write('paginate_conditions_category', $category); $this->Session->write('search_find_idlist', $idList); $this->Session->write('search_find_attributeidlist', $attributeIdList); @@ -1278,17 +1338,18 @@ class AttributesController extends AppController { $this->set('attrDescriptions', $this->Attribute->fieldDescriptions); $this->set('typeDefinitions', $this->Attribute->typeDefinitions); $this->set('categoryDefinitions', $this->Attribute->categoryDefinitions); - // get from Session $keyword = $this->Session->read('paginate_conditions_keyword'); $keyword2 = $this->Session->read('paginate_conditions_keyword2'); $org = $this->Session->read('paginate_conditions_org'); $type = $this->Session->read('paginate_conditions_type'); $category = $this->Session->read('paginate_conditions_category'); + $tags = $this->Session->read('paginate_conditions_tags'); $this->set('keywordSearch', $keyword); $this->set('keywordSearch2', $keyword2); $this->set('orgSearch', $org); $this->set('typeSearch', $type); + $this->set('tags', $tags); $this->set('isSearch', 1); $this->set('categorySearch', $category); @@ -1662,7 +1723,11 @@ class AttributesController extends AppController { $this->__downloadAttachment($this->Attribute->data['Attribute']); } - public function text($key='download', $type="", $tags='') { + public function text($key='download', $type='all', $tags=false, $eventId=false, $allowNonIDS=false) { + if ($eventId === 'null' || $eventId == '0' || $eventId === 'false') $eventId = false; + if ($allowNonIDS === 'null' || $allowNonIDS === '0' || $allowNonIDS === 'false') $allowNonIDS = false; + if ($type === 'null' || $type === '0' || $type === 'false') $type = 'all'; + if ($tags === 'null' || $tags === '0' || $tags === 'false') $tags = false; if ($key != 'download') { // check if the key is valid -> search for users based on key $user = $this->checkAuthUser($key); @@ -1677,7 +1742,7 @@ class AttributesController extends AppController { $this->response->type('txt'); // set the content type $this->header('Content-Disposition: download; filename="misp.' . $type . '.txt"'); $this->layout = 'text/default'; - $attributes = $this->Attribute->text($this->_checkOrg(), $this->_isSiteAdmin(), $type, $tags); + $attributes = $this->Attribute->text($this->_checkOrg(), $this->_isSiteAdmin(), $type, $tags, $eventId, $allowNonIDS); $this->loadModel('Whitelist'); $attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true); $this->set('attributes', $attributes); @@ -1693,9 +1758,33 @@ class AttributesController extends AppController { public function generateCorrelation() { if (!self::_isSiteAdmin()) throw new NotFoundException(); - $k = $this->Attribute->generateCorrelation(); - $this->Session->setFlash(__('All done. ' . $k . ' attributes processed.')); - $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); + if (!Configure::read('MISP.background_jobs')) { + $k = $this->Attribute->generateCorrelation(); + $this->Session->setFlash(__('All done. ' . $k . ' attributes processed.')); + $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); + } else { + $job = ClassRegistry::init('Job'); + $job->create(); + $data = array( + 'worker' => 'default', + 'job_type' => 'generate correlation', + 'job_input' => 'All attributes', + 'status' => 0, + 'retries' => 0, + 'org' => 'ADMIN', + 'message' => 'Job created.', + ); + $job->save($data); + $jobId = $job->id; + $process_id = CakeResque::enqueue( + 'default', + 'AdminShell', + array('jobGenerateCorrelation', $jobId) + ); + $job->saveField('process_id', $process_id); + $this->Session->setFlash(__('Job queued. You can view the progress if you navigate to the active jobs view (administration -> jobs).')); + $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); + } } public function fetchViewValue($id, $field = null) { @@ -1885,6 +1974,8 @@ class AttributesController extends AppController { 'recursive' => -1 )); $event['Event']['published'] = 0; + $date = new DateTime(); + $event['Event']['timestamp'] = $date->getTimestamp(); $this->Attribute->Event->save($event); } else { $message .= 'Update completed with some errors.'; diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 3287659b5..162f89142 100755 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -170,6 +170,8 @@ class EventsController extends AppController { } $includeIDs = array_keys($includeIDs); $excludeIDs = array_keys($excludeIDs); + // return -1 as the only value in includedIDs if both arrays are empty. This will mean that no events will be shown if there was no hit + if (empty($includeIDs) && empty($excludeIDs)) $includeIDs[] = -1; return array($includeIDs, $excludeIDs); } @@ -2819,6 +2821,7 @@ class EventsController extends AppController { 'email-dst' => 'Payload delivery', 'text' => 'Other', ); + $this->set('typeList', array_keys($this->Event->Attribute->typeDefinitions)); $this->set('defaultCategories', $defaultCategories); $this->set('typeCategoryMapping', $typeCategoryMapping); $this->set('resultArray', $resultArray); @@ -2841,26 +2844,42 @@ class EventsController extends AppController { $failed = 0; foreach ($this->request->data['Attribute'] as $k => $attribute) { if ($attribute['save'] == '1') { - $this->Event->Attribute->create(); - $attribute['distribution'] = $event['Event']['distribution']; - $attribute['comment'] = 'Imported via the freetext import.'; - $attribute['event_id'] = $id; - if ($this->Event->Attribute->save($attribute)) { - $saved++; + if ($attribute['type'] == 'ip-src/ip-dst') { + $types = array('ip-src', 'ip-dst'); } else { - $failed++; + $types = array($attribute['type']); + } + foreach ($types as $type) { + $this->Event->Attribute->create(); + $attribute['type'] = $type; + $attribute['distribution'] = $event['Event']['distribution']; + if (empty($attribute['comment'])) $attribute['comment'] = 'Imported via the freetext import.'; + $attribute['event_id'] = $id; + if ($this->Event->Attribute->save($attribute)) { + $saved++; + } else { + $failed++; + } } } } - if ($saved > 0 && $event['Event']['published'] == 1) { + if ($saved > 0) { $event = $this->Event->find('first', array( 'conditions' => array('Event.id' => $id), 'recursive' => -1 )); - $event['Event']['published'] = 0; + if ($event['Event']['published'] == 1) { + $event['Event']['published'] = 0; + } + $date = new DateTime(); + $event['Event']['timestamp'] = $date->getTimestamp(); $this->Event->save($event); } - $this->Session->setFlash($saved . ' attributes created. ' . $failed . ' attributes could not be saved. This may be due to attributes with similar values already existing.'); + if ($failed > 0) { + $this->Session->setFlash($saved . ' attributes created. ' . $failed . ' attributes could not be saved. This may be due to attributes with similar values already existing.'); + } else { + $this->Session->setFlash($saved . ' attributes created.'); + } $this->redirect(array('controller' => 'events', 'action' => 'view', $id)); } else { throw new MethodNotAllowedException(); @@ -3022,4 +3041,87 @@ class EventsController extends AppController { $this->set('_serialize', 'data'); } } -} + + public function exportChoice($id) { + $event = $this->Event->find('first' ,array( + 'conditions' => array('id' => $id), + 'recursive' => -1, + 'fields' => array('distribution', 'orgc','id', 'published'), + )); + if (empty($event) || (!$this->_isSiteAdmin() && $event['Event']['orgc'] != $this->Auth->user('org') && $event['Event']['distribution'] < 1)) throw new NotFoundException('Event not found or you are not authorised to view it.'); + $exports = array( + 'xml' => array( + 'url' => '/events/xml/download/' . $id, + 'text' => 'MISP XML (metadata + all attributes)', + 'requiresPublished' => false, + 'checkbox' => true, + 'checkbox_text' => 'Encode Attachments', + 'checkbox_set' => '/true' + ), + 'json' => array( + 'url' => '/events/view/' . $id . 'json', + 'text' => 'MISP JSON (metadata + all attributes)', + 'requiresPublished' => false, + 'checkbox' => false, + ), + 'openIOC' => array( + 'url' => '/events/downloadOpenIOCEvent/' . $id, + 'text' => 'OpenIOC (all indicators marked to IDS)', + 'requiresPublished' => true, + 'checkbox' => false, + ), + 'csv' => array( + 'url' => '/events/csv/download/' . $id . '/1', + 'text' => 'CSV', + 'requiresPublished' => true, + 'checkbox' => true, + 'checkbox_text' => 'Include non-IDS marked attributes', + 'checkbox_set' => '/1' + ), + 'stix_xml' => array( + 'url' => '/events/stix/download/' . $id . '.xml', + 'text' => 'STIX XML (metadata + all attributes)', + 'requiresPublished' => true, + 'checkbox' => true, + 'checkbox_text' => 'Encode Attachments', + 'checkbox_set' => '/true' + ), + 'stix_json' => array( + 'url' => '/events/stix/download/' . $id . '.json', + 'text' => 'STIX JSON (metadata + all attributes)', + 'requiresPublished' => true, + 'checkbox' => true, + 'checkbox_text' => 'Encode Attachments', + 'checkbox_set' => '/true' + ), + 'suricata' => array( + 'url' => '/events/nids/suricata/download/' . $id, + 'text' => 'Download Suricata rules', + 'requiresPublished' => true, + 'checkbox' => false, + ), + 'snort' => array( + 'url' => '/events/nids/snort/download/' . $id, + 'text' => 'Download Snort rules', + 'requiresPublished' => true, + 'checkbox' => false, + ), + 'text' => array( + 'url' => '/attributes/text/download/all/false/' . $id, + 'text' => 'Export all attribute values as a text file', + 'requiresPublished' => true, + 'checkbox' => true, + 'checkbox_text' => 'Include non-IDS marked attributes', + 'checkbox_set' => '/true' + ), + ); + if ($event['Event']['published'] == 0) { + foreach ($exports as $k => $export) { + if ($export['requiresPublished']) unset($exports[$k]); + } + } + $this->set('exports', $exports); + $this->set('id', $id); + $this->render('ajax/exportChoice'); + } +} \ No newline at end of file diff --git a/app/Controller/JobsController.php b/app/Controller/JobsController.php index 493dc368c..2ed87788b 100644 --- a/app/Controller/JobsController.php +++ b/app/Controller/JobsController.php @@ -54,7 +54,7 @@ class JobsController extends AppController { } public function getGenerateCorrelationProgress($id) { - //if (!self::_isSiteAdmin()) throw new NotFoundException(); + if (!self::_isSiteAdmin()) throw new NotFoundException(); $progress = $this->Job->findById($id); if (!$progress) { $progress = 0; diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index 91089bf94..36ffe81d1 100755 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -233,7 +233,7 @@ class ServersController extends AppController { App::uses('SyncTool', 'Tools'); $syncTool = new SyncTool(); $HttpSocket = $syncTool->setupHttpSocket($server); - $result = $this->Server->push($id, $technique, false, $HttpSocket); + $result = $this->Server->push($id, $technique, false, $HttpSocket, $this->Auth->user('email')); $this->set('successes', $result[0]); $this->set('fails', $result[1]); } else { @@ -253,7 +253,7 @@ class ServersController extends AppController { $process_id = CakeResque::enqueue( 'default', 'ServerShell', - array('push', $id, $technique, $jobId) + array('push', $id, $technique, $jobId, $this->Auth->user('id')) ); $this->Job->saveField('process_id', $process_id); $this->Session->setFlash('Push queued for background execution.'); diff --git a/app/Controller/ShadowAttributesController.php b/app/Controller/ShadowAttributesController.php index b7d50ce90..09bfcb2a0 100644 --- a/app/Controller/ShadowAttributesController.php +++ b/app/Controller/ShadowAttributesController.php @@ -160,6 +160,8 @@ class ShadowAttributesController extends AppController { $event['Event']['proposal_email_lock'] = 0; } $event['Event']['published'] = 0; + $date = new DateTime(); + $event['Event']['timestamp'] = $date->getTimestamp(); $this->autoRender = false; if ($this->Event->save($event, array('fieldList' => $fieldList))) { $this->Log = ClassRegistry::init('Log'); diff --git a/app/Controller/TasksController.php b/app/Controller/TasksController.php index 2017ad39c..a5c6a9333 100644 --- a/app/Controller/TasksController.php +++ b/app/Controller/TasksController.php @@ -99,7 +99,7 @@ class TasksController extends AppController { $timestamp, 'default', 'ServerShell', - array('enqueuePush', $timestamp, $id, $this->Auth->user('org')), + array('enqueuePush', $timestamp, $id, $this->Auth->user('org'), $this->Auth->user('id')), true ); $this->Task->id = $id; diff --git a/app/Controller/TemplatesController.php b/app/Controller/TemplatesController.php index 2cf9c9724..8117f5b88 100644 --- a/app/Controller/TemplatesController.php +++ b/app/Controller/TemplatesController.php @@ -351,6 +351,8 @@ class TemplatesController extends AppController { 'recursive' => -1 )); $event['Event']['published'] = 0; + $date = new DateTime(); + $event['Event']['timestamp'] = $date->getTimestamp(); $this->Event->save($event); if ($fails == 0) $this->Session->setFlash(__('Event populated, ' . $count . ' attributes successfully created.')); else $this->Session->setFlash(__('Event populated, but ' . $fails . ' attributes could not be saved.')); diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php index 0860e83b0..caf68704a 100755 --- a/app/Controller/UsersController.php +++ b/app/Controller/UsersController.php @@ -783,7 +783,7 @@ class UsersController extends AppController { throw new MethodNotAllowedException(); } $this->User->recursive = 0; - $temp = $this->User->find('all', array('fields' => array('email', 'gpgkey'))); + $temp = $this->User->find('all', array('fields' => array('email', 'gpgkey'), 'order' => array('email ASC'))); $emails = array(); $gpgKeys = array(); // save all the emails of the users and set it for the dropdown list in the form @@ -850,7 +850,6 @@ class UsersController extends AppController { } } } - // If the recipient is a user, and the action to create a password, create it and squeeze it between the main message and the signature if ($this->request->data['User']['recipient'] == 1) { $recipients[0] = $emails[$this->request->data['User']['recipientEmailList']]; @@ -866,6 +865,7 @@ class UsersController extends AppController { require_once 'Crypt/GPG.php'; $i = 0; + $this->Log = ClassRegistry::init('Log'); foreach ($recipients as $recipient) { if (!empty($recipientGPG[$i])) { $gpg = new Crypt_GPG(array('homedir' => Configure::read('GnuPG.homedir'))); // , 'debug' => true @@ -885,7 +885,6 @@ class UsersController extends AppController { } else { $encryptedMessage = $message[$i]; } - // prepare the email $this->Email->from = Configure::read('MISP.email'); $this->Email->to = $recipients[$i]; @@ -897,6 +896,28 @@ class UsersController extends AppController { // send it $result = $this->Email->send(); + $this->Log->create(); + if ($result) { + $this->Log->save(array( + 'org' => $this->Auth->user('org'), + 'model' => 'User', + 'model_id' => $this->Auth->user('id'), + 'email' => $this->Auth->user('email'), + 'action' => 'admin_email', + 'title' => 'Admin email to ' . $recipients[$i] . ' sent, titled "' . $subject . '".', + 'change' => null, + )); + } else { + $this->Log->save(array( + 'org' => $this->Auth->user('org'), + 'model' => 'User', + 'model_id' => $this->Auth->user('id'), + 'email' => $this->Auth->user('email'), + 'action' => 'admin_email', + 'title' => 'Admin email to ' . $recipients[$i] . ' failed.', + 'change' => null, + )); + } // if sending successful and action was a password change, update the user's password. if ($result && $this->request->data['User']['action'] == '1') { diff --git a/app/Lib/Tools/ComplexTypeTool.php b/app/Lib/Tools/ComplexTypeTool.php index 0ac74f1ff..4d79d3d60 100644 --- a/app/Lib/Tools/ComplexTypeTool.php +++ b/app/Lib/Tools/ComplexTypeTool.php @@ -84,11 +84,11 @@ class ComplexTypeTool { if (strlen($input) == 64 && preg_match("#[0-9a-f]{64}$#", $input)) return array('types' => array('sha256'), 'to_ids' => true, 'default_type' => 'sha256'); // check for IP - if (filter_var($input, FILTER_VALIDATE_IP)) return array('types' => array('ip-dst', 'ip-src'), 'to_ids' => true, 'default_type' => 'ip-dst'); + if (filter_var($input, FILTER_VALIDATE_IP)) return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'to_ids' => true, 'default_type' => 'ip-dst'); if (strpos($input, '/')) { $temp = explode('/', $input); if (count($temp == 2)) { - if (filter_var($temp[0], FILTER_VALIDATE_IP)) return array('types' => array('ip-dst', 'ip-src'), 'to_ids' => true, 'default_type' => 'ip-dst'); + if (filter_var($temp[0], FILTER_VALIDATE_IP)) return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'to_ids' => true, 'default_type' => 'ip-dst'); } } diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 36676a9fe..9523516b5 100755 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -1256,9 +1256,11 @@ class Attribute extends AppModel { return $rules; } - public function text($org, $isSiteAdmin, $type, $tags = '') { + public function text($org, $isSiteAdmin, $type, $tags = false, $eventId = false, $allowNonIDS = false) { //restricting to non-private or same org if the user is not a site-admin. - $conditions['AND'] = array('Attribute.type' => $type, 'Attribute.to_ids =' => 1, 'Event.published =' => 1); + $conditions['AND'] = array(); + if ($allowNonIDS === false) $conditions['AND'] = array('Attribute.to_ids =' => 1, 'Event.published =' => 1); + if ($type !== 'all') $conditions['AND']['Attribute.type'] = $type; if (!$isSiteAdmin) { $temp = array(); $distribution = array(); @@ -1267,8 +1269,10 @@ class Attribute extends AppModel { $conditions['OR'] = $temp; } - // If we sent any tags along, load the associated tag names for each attribute - if ($tags !== '') { + if ($eventId !== false) { + $conditions['AND'][] = array('Event.id' => $eventId); + } elseif ($tags !== false) { + // If we sent any tags along, load the associated tag names for each attribute $tag = ClassRegistry::init('Tag'); $args = $this->dissectArgs($tags); $tagArray = $tag->fetchEventTagIds($args[0], $args[1]); diff --git a/app/Model/Event.php b/app/Model/Event.php index e97fa1cf3..a55831750 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -1367,7 +1367,7 @@ class Event extends AppModel { * * @return bool true if success */ - public function _add(&$data, $fromXml, $user, $or='', $passAlong = null, $fromPull = false, $jobId = null) { + public function _add(&$data, $fromXml, $user, $org='', $passAlong = null, $fromPull = false, $jobId = null) { if ($jobId) { App::import('Component','Auth'); } @@ -1378,7 +1378,11 @@ class Event extends AppModel { //if ($this->checkAction('perm_sync')) $data['Event']['org'] = Configure::read('MISP.org'); //else $data['Event']['org'] = $auth->user('org'); - $data['Event']['org'] = $user['org']; + if ($fromPull) { + $data['Event']['org'] = $org; + } else { + $data['Event']['org'] = $user['org']; + } // set these fields if the event is freshly created and not pushed from another instance. // Moved out of if (!$fromXML), since we might get a restful event without the orgc/timestamp set if (!isset ($data['Event']['orgc'])) $data['Event']['orgc'] = $data['Event']['org']; diff --git a/app/Model/Job.php b/app/Model/Job.php index 7f922fb5c..ce246d766 100644 --- a/app/Model/Job.php +++ b/app/Model/Job.php @@ -7,7 +7,7 @@ App::uses('AppModel', 'Model'); */ class Job extends AppModel { - public function cache($type, $isSiteAdmin, $org, $target, $jobOrg, $sid) { + public function cache($type, $isSiteAdmin, $org, $target, $jobOrg, $sid = null) { $extra = null; $extra2 = null; $shell = 'Event'; diff --git a/app/Model/Log.php b/app/Model/Log.php index ee1230473..b618a24f3 100755 --- a/app/Model/Log.php +++ b/app/Model/Log.php @@ -22,7 +22,8 @@ class Log extends AppModel { 'discard', 'pull', 'push', - 'blacklisted' + 'blacklisted', + 'admin_email' )), 'message' => 'Options : ...' ) diff --git a/app/Model/Server.php b/app/Model/Server.php index c35dc67a5..cd88414c1 100755 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -373,6 +373,14 @@ class Server extends AppModel { 'test' => 'testForTermsFile', 'type' => 'string' ), + 'showorgalternate' => array( + 'level' => 2, + 'description' => 'True enables the alternate org fields for the event index (source org and member org) instead of the traditional way of showing only an org field. This allows users to see if an event was uploaded by a member organisation on their MISP instance, or if it originated on an interconnected instance.', + 'value' => '', + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), ), 'GnuPG' => array( 'branch' => 1, @@ -462,6 +470,9 @@ class Server extends AppModel { App::import('Component','Auth'); $this->Auth = new AuthComponent(new ComponentCollection()); $this->Auth->login($user); + $email = "Scheduled job"; + } else { + $email = $user['email']; } $eventModel = ClassRegistry::init('Event'); App::uses('HttpSocket', 'Network/Http'); @@ -537,7 +548,6 @@ class Server extends AppModel { unset($event['Event']['Attribute']); $event['Event']['Attribute'][0] = $tmp; } - if (is_array($event['Event']['Attribute'])) { $size = is_array($event['Event']['Attribute']) ? count($event['Event']['Attribute']) : 0; for ($i = 0; $i < $size; $i++) { @@ -575,7 +585,7 @@ class Server extends AppModel { if (!$existingEvent) { // add data for newly imported events $passAlong = $server['Server']['url']; - $result = $eventModel->_add($event, $fromXml = true, $user, $server['Server']['organization'], $passAlong, true, $jobId); + $result = $eventModel->_add($event, $fromXml = true, $user, $server['Server']['org'], $passAlong, true, $jobId); if ($result) $successes[] = $eventId; else { $fails[$eventId] = 'Failed (partially?) because of validation errors: '. print_r($eventModel->validationErrors, true); @@ -646,14 +656,14 @@ class Server extends AppModel { 'model_id' => $id, 'email' => $user['email'], 'action' => 'pull', - 'title' => 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $user['email'], + 'title' => 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $email, 'change' => count($successes) . ' events and ' . count($pulledProposals) . ' proposals pulled or updated. ' . count($fails) . ' events failed or didn\'t need an update.' )); if (!isset($lastpulledid)) $lastpulledid = 0; return array($successes, $fails, $pulledProposals, $lastpulledid); } - public function push($id = null, $technique=false, $jobId = false, $HttpSocket) { + public function push($id = null, $technique=false, $jobId = false, $HttpSocket, $email = "Scheduled job") { if ($jobId) { $job = ClassRegistry::init('Job'); $job->read(null, $jobId); @@ -737,8 +747,9 @@ class Server extends AppModel { $this->Log->save(array( 'model' => 'Server', 'model_id' => $id, + 'email' => $email, 'action' => 'push', - 'title' => 'Push to ' . $url . '.', + 'title' => 'Push to ' . $url . ' initiated by ' . $email, 'change' => count($successes) . ' events pushed or updated. ' . count($fails) . ' events failed or didn\'t need an update.' )); if ($jobId) { diff --git a/app/Model/Tag.php b/app/Model/Tag.php index 453353e96..27767fc1d 100644 --- a/app/Model/Tag.php +++ b/app/Model/Tag.php @@ -87,7 +87,7 @@ class Tag extends AppModel { public function findTags($array) { $ids = array(); foreach ($array as $a) { - $conditions['OR'][] = array('name like' => '%' . $a . '%'); + $conditions['OR'][] = array('LOWER(name) like' => '%' . strtolower($a) . '%'); } $params = array( 'recursive' => 1, diff --git a/app/View/Attributes/add.ctp b/app/View/Attributes/add.ctp index 6350643fa..d082e77d2 100755 --- a/app/View/Attributes/add.ctp +++ b/app/View/Attributes/add.ctp @@ -57,7 +57,7 @@ ?> - +

" id="warning-message">Warning: You are about to share data that is of a sensitive nature (Attribution / targeting data). Make sure that you are authorised to share this.

diff --git a/app/View/Attributes/add_attachment.ctp b/app/View/Attributes/add_attachment.ctp index 01ae8bcda..88536b84c 100755 --- a/app/View/Attributes/add_attachment.ctp +++ b/app/View/Attributes/add_attachment.ctp @@ -38,6 +38,7 @@ )); ?> +
Form->input('malware', array( 'type' => 'checkbox', diff --git a/app/View/Attributes/edit.ctp b/app/View/Attributes/edit.ctp index 0e55c5723..0c11097ad 100755 --- a/app/View/Attributes/edit.ctp +++ b/app/View/Attributes/edit.ctp @@ -43,7 +43,7 @@ $this->Js->get('#AttributeCategory')->event('change', 'formCategoryChanged("#AttributeCategory")'); ?> - +

" id="warning-message">Warning: You are about to share data that is of a sensitive nature (Attribution / targeting data). Make sure that you are authorised to share this.

Form->button('Submit', array('class' => 'btn btn-primary')); echo $this->Form->end(); diff --git a/app/View/Attributes/index.ctp b/app/View/Attributes/index.ctp index 2bb37d042..263c5db19 100755 --- a/app/View/Attributes/index.ctp +++ b/app/View/Attributes/index.ctp @@ -5,6 +5,7 @@ if ($isSearch == 1) { echo "

Results for all attributes"; if ($keywordSearch != null) echo " with the value containing \"" . h($keywordSearch) . "\""; if ($keywordSearch2 != null) echo " from the events \"" . h($keywordSearch2) . "\""; + if ($tags != null) echo " from events tagged \"" . h($tags) . "\""; if ($categorySearch != "ALL") echo " of category \"" . h($categorySearch) . "\""; if ($typeSearch != "ALL") echo " of type \"" . h($typeSearch) . "\""; if (isset($orgSearch) && $orgSearch != '' && $orgSearch != null) echo " created by the organisation \"" . h($orgSearch) . "\""; @@ -58,7 +59,7 @@ foreach ($attributes as $attribute): ?>

- - - - -
-
+
+ - +   +  ';"> -   + + Highlight->highlighter($sigDisplay, $replacePairs); + } + echo $sigDisplay; + ?>  +   Form->input('keyword', array('type' => 'textarea', 'label' => 'Containing the following expressions', 'div' => 'clear', 'class' => 'input-xxlarge')); echo $this->Form->input('keyword2', array('type' => 'textarea', 'label' => 'Being attributes of the following event IDs', 'div' => 'clear', 'class' => 'input-xxlarge')); + echo $this->Form->input('tags', array('type' => 'textarea', 'label' => 'Being an attribute of an event matching the following tags', 'div' => 'clear', 'class' => 'input-xxlarge')); + ?>
  • New User
  • List Users
    • +
  • Contact Users
  • New Role
  • List Roles
    • -
  • -
  • Contact Users
  • Administrative tools
  • Server settings
    • diff --git a/app/View/Elements/pivot.ctp b/app/View/Elements/pivot.ctp index b52669e09..6e8a2d131 100644 --- a/app/View/Elements/pivot.ctp +++ b/app/View/Elements/pivot.ctp @@ -3,9 +3,11 @@ ?>
    - Pivot->convertPivotToHTML($pivot, $currentEvent); - ?> + + Pivot->convertPivotToHTML($pivot, $currentEvent); + ?> +
    diff --git a/app/View/Elements/side_menu.ctp b/app/View/Elements/side_menu.ctp index 66528587b..d6d3d9903 100755 --- a/app/View/Elements/side_menu.ctp +++ b/app/View/Elements/side_menu.ctp @@ -43,13 +43,7 @@ class="publishButtons">Publish (no email)
  • >Contact Reporter
    • -
  • Download as XML
    • - - class="exportButtons">Download as IOC - class="exportButtons">Download as CSV - class="exportButtons">Download as STIX XML - class="exportButtons">Download as STIX JSON - +
  • Download as...
  • List Events
    • @@ -149,13 +143,13 @@ if ($isSiteAdmin): ?>
  • >Html->link('New User', array('controller' => 'users', 'action' => 'add', 'admin' => true)); ?>
  • >Html->link('List Users', array('controller' => 'users', 'action' => 'index', 'admin' => true)); ?>
    • +
  • >Html->link('Contact Users', array('controller' => 'users', 'action' => 'email', 'admin' => true)); ?>
  • >Html->link('New Role', array('controller' => 'roles', 'action' => 'add', 'admin' => true)); ?>
  • >Html->link('List Roles', array('controller' => 'roles', 'action' => 'index', 'admin' => true)); ?>
  • -
  • >Html->link('Contact Users', array('controller' => 'users', 'action' => 'email', 'admin' => true)); ?>
  • >Administrative Tools
  • >Server Settings
  • diff --git a/app/View/Events/ajax/exportChoice.ctp b/app/View/Events/ajax/exportChoice.ctp new file mode 100644 index 000000000..3a8c36b18 --- /dev/null +++ b/app/View/Events/ajax/exportChoice.ctp @@ -0,0 +1,32 @@ +
    + +
    + + $export): ?> + + + + + +
    + + + + +   + +
    +
    +
    Cancel
    +
    + \ No newline at end of file diff --git a/app/View/Events/automation.ctp b/app/View/Events/automation.ctp index 2b1acf0b5..6acb8f90d 100644 --- a/app/View/Events/automation.ctp +++ b/app/View/Events/automation.ctp @@ -77,7 +77,20 @@ foreach ($sigTypes as $sigType) {

    To restrict the results by tags, use the usual syntax. Please be aware the colons (:) cannot be used in the tag search. Use semicolons instead (the search will automatically search for colons instead). To get ip-src values from events tagged tag1 but not tag2 use:

     
+
    + +

    As of version 2.3.38, it is possible to restrict the text exports on two additional flags. The first allows the user to restrict based on event ID, whilst the second is a boolean switch allowing non IDS flagged attributes to be exported. Additionally, choosing "all" in the type field will return all eligible attributes.

    +
    +
+
    +

    For example, to retrieve all attributes for event #5, including non IDS marked attributes too, use the following line:

    +
    +
    diff --git a/app/View/Events/filter_event_index.ctp b/app/View/Events/filter_event_index.ctp index d97e826ec..268b20af0 100644 --- a/app/View/Events/filter_event_index.ctp +++ b/app/View/Events/filter_event_index.ctp @@ -119,6 +119,11 @@ endforeach; ?>
    + + + + +
    No filters set - add filter terms above.
    Form->create('Event', array('id' => 'test', 'url' => $baseurl . '/events/index'));?>
    diff --git a/app/View/Events/free_text_results.ctp b/app/View/Events/free_text_results.ctp index 8b684259b..b7d3fde45 100644 --- a/app/View/Events/free_text_results.ctp +++ b/app/View/Events/free_text_results.ctp @@ -7,13 +7,15 @@ Category Type IDS + Comment Actions Form->create('Attribute', array('url' => '/events/saveFreeText/' . $event_id)); foreach ($resultArray as $k => $item): ?> - + Form->input('Attribute.' . $k . '.save', array( 'label' => false, @@ -57,11 +59,13 @@ 'style' => 'padding:0px;height:20px;margin-bottom:0px;', 'options' => $item['types'], 'value' => $item['default_type'], + 'class' => 'typeToggle', )); + if (!in_array(array_keys($item['types']), $options)) $options[] = array_keys($item['types']); } ?> - + Form->input('Attribute.' . $k . '.to_ids', array( 'label' => false, @@ -70,19 +74,74 @@ )); ?> + + Form->input('Attribute.' . $k . '.comment', array( + 'label' => false, + 'style' => 'padding:0px;height:20px;margin-bottom:0px;', + 'type' => 'text', + 'placeholder' => 'Imported via the freetext import.', + )); + ?> + - + $element) { + $temp = $group; + unset ($temp[$k]); + if (!isset($optionsRearranged[$element])) $optionsRearranged[$element] = array(); + $optionsRearranged[$element] = array_merge($optionsRearranged[$element], $temp); + } + } ?> Form->button('Submit', array('class' => 'btn btn-inverse')); + echo $this->Form->button('Submit', array('class' => 'btn btn-primary')); echo $this->Form->end(); + if (!empty($optionsRearranged)): ?> + + + + + Change all + + + + element('side_menu', array('menuList' => 'regexp', 'menuItem' => 'index')); ?> diff --git a/app/View/Events/index.ctp b/app/View/Events/index.ctp index c17383778..3c313d39b 100755 --- a/app/View/Events/index.ctp +++ b/app/View/Events/index.ctp @@ -46,16 +46,24 @@ Paginator->sort('published');?> - Paginator->sort('org'); ?> - + Paginator->sort('org', 'Source org'); ?> + Paginator->sort('org', 'Member org'); ?> + + Paginator->sort('org'); ?> + + Paginator->sort('owner org');?> + - - - Paginator->sort('owner org');?> - - Paginator->sort('id');?> Tags @@ -78,7 +86,7 @@ > - + @@ -91,7 +99,7 @@ }?>  - + - - + + - +   - +   - +   - +   - +   - +   - + @@ -185,7 +193,6 @@ ?> -
    element('side_menu', array('menuList' => 'event-collection', 'menuItem' => 'index')); diff --git a/app/View/Events/view.ctp b/app/View/Events/view.ctp index 8f1b40ca4..29cdacd35 100755 --- a/app/View/Events/view.ctp +++ b/app/View/Events/view.ctp @@ -28,20 +28,38 @@ $mayPublish = ($isAclPublish && $event['Event']['orgc'] == $me['org']);   - -
    Org
    -
    - -   -
    - - -
    Owner org
    -
    - -   -
    - + +
    Source Organisation
    +
    + +   +
    +
    Member Organisation
    +
    + +   +
    + +
    Org
    +
    + +   +
    + + +
    Owner org
    +
    + +   +
    +
    Contributors
    1) echo $this->element('pivot'); ?> -
    -
    - +

    " id="warning-message">Warning: You are about to share data that is of a sensitive nature (Attribution / targeting data). Make sure that you are authorised to share this.

    Form->button('Propose', array('class' => 'btn btn-primary')); echo $this->Form->end(); diff --git a/app/View/Templates/view.ctp b/app/View/Templates/view.ctp index 759cbe96a..8bee6b2a5 100644 --- a/app/View/Templates/view.ctp +++ b/app/View/Templates/view.ctp @@ -45,8 +45,6 @@
    -
    -
    element('side_menu', array('menuList' => 'templates', 'menuItem' => 'view', 'mayModify' => $mayModify)); diff --git a/app/View/Users/admin_filter_user_index.ctp b/app/View/Users/admin_filter_user_index.ctp index 9e5799e11..8c9be8a87 100644 --- a/app/View/Users/admin_filter_user_index.ctp +++ b/app/View/Users/admin_filter_user_index.ctp @@ -76,6 +76,11 @@ endforeach; ?> + + + + +
    No filters set - add filter terms above.
    Form->create('User', array('id' => 'test', 'url' => $baseurl . '/admin/users/index'));?>
    diff --git a/app/View/Users/admin_index.ctp b/app/View/Users/admin_index.ctp index 4e72089d5..2ef53341a 100755 --- a/app/View/Users/admin_index.ctp +++ b/app/View/Users/admin_index.ctp @@ -3,13 +3,12 @@ -
    element('side_menu', array('menuList' => 'admin', 'menuItem' => 'indexUser')); diff --git a/app/files/scripts/misp2stix.py b/app/files/scripts/misp2stix.py index e6dacb306..ee708b83d 100644 --- a/app/files/scripts/misp2stix.py +++ b/app/files/scripts/misp2stix.py @@ -16,6 +16,13 @@ from stix.extensions.marking.tlp import TLPMarkingStructure from stix.common.related import * from stix.common.confidence import Confidence from stix.common.vocabs import IncidentStatus +from cybox.utils import Namespace + +namespace = ['https://github.com/MISP/MISP', 'MISP'] + +cybox.utils.idgen.set_id_namespace(Namespace(namespace[0], namespace[1])) +stix.utils.idgen.set_id_namespace({namespace[0]: namespace[1]}) + NS_DICT = { "http://cybox.mitre.org/common-2" : 'cyboxCommon', @@ -34,7 +41,6 @@ NS_DICT = { "http://cybox.mitre.org/objects#WinRegistryKeyObject-2" : 'WinRegistryKeyObj', "http://data-marking.mitre.org/Marking-1" : 'marking', "http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1" : 'tlpMarking', - "http://example.com" : 'example', "http://stix.mitre.org/ExploitTarget-1" : 'et', "http://stix.mitre.org/Incident-1" : 'incident', "http://stix.mitre.org/Indicator-2" : 'indicator', @@ -49,6 +55,7 @@ NS_DICT = { "urn:oasis:names:tc:ciq:xal:3" : 'xal', "urn:oasis:names:tc:ciq:xnl:3" : 'xnl', "urn:oasis:names:tc:ciq:xpil:3" : 'xpil', + namespace[0] : namespace[1] } SCHEMALOC_DICT = { @@ -86,7 +93,7 @@ SCHEMALOC_DICT = { # mappings status_mapping = {'0' : 'New', '1' : 'Open', '2' : 'Closed'} TLP_mapping = {'0' : 'AMBER', '1' : 'GREEN', '2' : 'GREEN', '3' : 'GREEN'} -confidence_mapping = {'0' : 'None', '1' : 'High'} +confidence_mapping = {False : 'None', True : 'High'} not_implemented_attributes = ['yara', 'pattern-in-traffic', 'pattern-in-memory'] @@ -126,7 +133,7 @@ def generateMainPackage(events): # generate a package for each event def generateEventPackage(event): - package_name = 'example:STIXPackage-' + event["Event"]["uuid"] + package_name = namespace[1] + ':STIXPackage-' + event["Event"]["uuid"] stix_package = STIXPackage(id_=package_name) stix_header = STIXHeader() stix_header.title="MISP event #" + event["Event"]["id"] + " uuid: " + event["Event"]["uuid"] @@ -142,7 +149,7 @@ def generateEventPackage(event): # generate the incident information. MISP events are currently mapped to incidents with the event metadata being stored in the incident information def generateSTIXObjects(event): - incident = Incident(id_ = "example:incident-" + event["Event"]["uuid"], description=event["Event"]["info"]) + incident = Incident(id_ = namespace[1] + ":incident-" + event["Event"]["uuid"], description=event["Event"]["info"]) setDates(incident, event["Event"]["date"], int(event["Event"]["publish_timestamp"])) addJournalEntry(incident, "Event Threat Level: " + event["ThreatLevel"]["name"]) ttps = [] @@ -227,7 +234,7 @@ def handleNonIndicatorAttribute(incident, ttps, attribute): # TTPs are only used to describe malware names currently (attribute with category Payload Type and type text/comment/other) def generateTTP(incident, attribute): ttp = TTP() - ttp.id_="example:ttp-" + attribute["uuid"] + ttp.id_= namespace[1] + ":ttp-" + attribute["uuid"] setTLP(ttp, attribute["distribution"]) ttp.title = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"] if attribute["type"] == "vulnerability": @@ -247,7 +254,7 @@ def generateTTP(incident, attribute): # Threat actors are currently only used for the category:attribution / type:(text|comment|other) attributes def generateThreatActor(attribute): ta = ThreatActor() - ta.id_="example:threatactor-" + attribute["uuid"] + ta.id_= namespace[1] + ":threatactor-" + attribute["uuid"] ta.title = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"] ta.description = attribute["value"] return ta @@ -255,7 +262,7 @@ def generateThreatActor(attribute): # generate the indicator and add the relevant information def generateIndicator(attribute): indicator = Indicator() - indicator.id_="example:indicator-" + attribute["uuid"] + indicator.id_= namespace[1] + ":indicator-" + attribute["uuid"] setTLP(indicator, attribute["distribution"]) indicator.title = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"] confidence_description = "Derived from MISP's IDS flag. If an attribute is marked for IDS exports, the confidence will be high, otherwise none" @@ -286,7 +293,7 @@ def addReference(target, reference): # takes an object and applies a TLP marking based on the distribution passed along to it def setTLP(target, distribution): marking_specification = MarkingSpecification() - marking_specification.controlled_structure = "../../../descendant-or-self()" + marking_specification.controlled_structure = "../../../descendant-or-self::node()" tlp = TLPMarkingStructure() colour = TLP_mapping.get(distribution, None) if colour is None: diff --git a/app/webroot/css/main.css b/app/webroot/css/main.css index 71ef97c2b..c646430c5 100755 --- a/app/webroot/css/main.css +++ b/app/webroot/css/main.css @@ -906,8 +906,9 @@ a.proposal_link_red:hover { .tabMenuFixedContainer { position:relative; left:10px; + height:20px; width:calc(100% - 10px); - bottom:-2px; + bottom:0px; } .tabMenuFixed { diff --git a/app/webroot/js/ajaxification.js b/app/webroot/js/ajaxification.js index ece046e5a..910ab1a71 100644 --- a/app/webroot/js/ajaxification.js +++ b/app/webroot/js/ajaxification.js @@ -897,9 +897,12 @@ function templateFileUploadTriggerBrowse(id) { $('#upload_' + id + '_file').click(); } -function freetextRemoveRow(id) { +function freetextRemoveRow(id, event_id) { $('#row_' + id).hide(); $('#Attribute' + id + 'Save').attr("value", "0"); + if ($(".freetext_row:visible").length == 0) { + window.location = "/events/" + event_id; + } } function indexEvaluateFiltering() { @@ -1098,8 +1101,8 @@ function indexSetTableVisibility() { if ($("[id^='value_']").text().trim()!="" && $("[id^='value_']").text().trim()!="-1") { visible = true; } - if (visible == true) $('#rule_table').show(); - else $('#rule_table').hide(); + if (visible == true) $('#FilterplaceholderTable').hide(); + else $('#FilterplaceholderTable').show(); } function indexRuleChange() { @@ -1275,4 +1278,32 @@ function serverSettingSubmitForm(name, setting, id) { $(name + '_field').unbind("keyup"); $(name + '_form').unbind("focusout"); return false; -}; +} + +function changeFreetextImportFrom() { + $('#changeTo').find('option').remove(); + options[$('#changeFrom').val()].forEach(function(element) { + $('#changeTo').append(''); + }); +} + +function changeFreetextImportExecute() { + var from = $('#changeFrom').val(); + var to = $('#changeTo').val(); + $('.typeToggle').each(function() { + if ($( this ).val() == from) { + if ($('#' + $(this).attr('id') + " option[value='" + from + "']").length > 0) { + $( this ).val(to); + } + } + }); +} + +function exportChoiceSelect(url, elementId, checkbox) { + if (checkbox == 1) { + if ($('#' + elementId + '_toggle').prop('checked')) { + url = url + $('#' + elementId + '_set').html(); + } + } + document.location.href = url; +} \ No newline at end of file