From 85754ccc83134db14b3184d31800f6af7db25309 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 15 Sep 2019 13:05:55 +0200 Subject: [PATCH 001/159] fix: [internal] Deleting multiple Redis keys --- app/Model/Feed.php | 4 ++-- app/Model/Server.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/Model/Feed.php b/app/Model/Feed.php index b8dc7897a..2e26a4eba 100644 --- a/app/Model/Feed.php +++ b/app/Model/Feed.php @@ -986,14 +986,14 @@ class Feed extends AppModel } elseif ($scope == 'freetext' || $scope == 'csv') { $params['conditions']['source_format'] = array('csv', 'freetext'); } elseif ($scope == 'misp') { - $redis->del('misp:feed_cache:event_uuid_lookup:'); + $redis->del($redis->keys('misp:feed_cache:event_uuid_lookup:*')); $params['conditions']['source_format'] = 'misp'; } else { throw new InvalidArgumentException("Invalid value for scope, it must be integer or 'freetext', 'csv', 'misp' or 'all' string."); } } else { $redis->del('misp:feed_cache:combined'); - $redis->del('misp:feed_cache:event_uuid_lookup:'); + $redis->del($redis->keys('misp:feed_cache:event_uuid_lookup:*')); } $feeds = $this->find('all', $params); $atLeastOneSuccess = false; diff --git a/app/Model/Server.php b/app/Model/Server.php index 6e7cc1c84..920e762ce 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -5141,7 +5141,7 @@ class Server extends AppModel $params['conditions']['Server.id'] = $id; } else { $redis->del('misp:server_cache:combined'); - $redis->del('misp:server_cache:event_uuid_lookup:'); + $redis->del($redis->keys('misp:server_cache:event_uuid_lookup:*')); } $servers = $this->find('all', $params); if ($jobId) { From 08c9337e6c69b27737fd609bcfc47453ff756cf3 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 23 Sep 2019 18:31:44 +0200 Subject: [PATCH 002/159] fix: [internal] Just site admin can force when saving freetext --- app/Controller/EventsController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 628000092..5e0912d83 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -4120,7 +4120,7 @@ class EventsController extends AppController $this->Event->insertLock($this->Auth->user(), $id); $attributes = json_decode($this->request->data['Attribute']['JsonObject'], true); $default_comment = $this->request->data['Attribute']['default_comment']; - $force = $this->request->data['Attribute']['force']; + $force = $this->_isSiteAdmin() && $this->request->data['Attribute']['force']; $flashMessage = $this->Event->processFreeTextDataRouter($this->Auth->user(), $attributes, $id, $default_comment, $force); $this->Flash->info($flashMessage); $this->redirect(array('controller' => 'events', 'action' => 'view', $id)); From 73b9513a3896ac7f1e05a50ebf2b9d557fa85b49 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 5 Oct 2019 15:40:34 +0200 Subject: [PATCH 003/159] chg: [internal] Refactoring malware handling --- app/Controller/ServersController.php | 6 +- app/Lib/Tools/MalwareTool.php | 227 +++++++++++++++++++++++++++ app/Model/Attribute.php | 116 +++++--------- 3 files changed, 268 insertions(+), 81 deletions(-) create mode 100644 app/Lib/Tools/MalwareTool.php diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index df2746803..166b65df9 100644 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -985,13 +985,15 @@ class ServersController extends AppController if ($tab == 'diagnostics' || $tab == 'download' || $this->_isRest()) { $php_ini = php_ini_loaded_file(); $this->set('php_ini', $php_ini); - $advanced_attachments = shell_exec($this->Server->getPythonVersion() . ' ' . APP . 'files/scripts/generate_file_objects.py -c'); + $malwareTool = new MalwareTool(); try { - $advanced_attachments = json_decode($advanced_attachments, true); + $advanced_attachments = $malwareTool->checkAdvancedExtractionStatus($this->Server->getPythonVersion()); } catch (Exception $e) { + $this->log($e->getMessage(), LOG_NOTICE); $advanced_attachments = false; } + $this->set('advanced_attachments', $advanced_attachments); // check if the current version of MISP is outdated or not $version = $this->__checkVersion(); diff --git a/app/Lib/Tools/MalwareTool.php b/app/Lib/Tools/MalwareTool.php new file mode 100644 index 000000000..5ed2a3e3b --- /dev/null +++ b/app/Lib/Tools/MalwareTool.php @@ -0,0 +1,227 @@ +encryptByExtension($originalFilename, $content, $md5); + } else { + return $this->encryptByCommand($originalFilename, $content, $md5); + } + } + + /** + * @param string $originalFilename + * @param string $content + * @param string $md5 + * @return string Content of zipped file + * @throws Exception + */ + private function encryptByCommand($originalFilename, $content, $md5) + { + $tempDir = $this->tempDir(); + + $contentsFile = new File($tempDir . DS . $md5, true); + if (!$contentsFile->write($content)) { + throw new Exception("Could not write content to file '{$contentsFile->path}'."); + } + $contentsFile->close(); + + $fileNameFile = new File($tempDir . DS . $md5 . '.filename.txt', true); + if (!$fileNameFile->write($originalFilename)) { + throw new Exception("Could not write original file name to file '{$fileNameFile->path}'."); + } + $fileNameFile->close(); + + $zipFile = new File($tempDir . DS . $md5 . '.zip'); + + $exec = [ + 'zip', + '-j', // junk (don't record) directory names + '-P', // use standard encryption + self::ZIP_PASSWORD, + escapeshellarg($zipFile->path), + escapeshellarg($contentsFile->path), + escapeshellarg($fileNameFile->path), + ]; + + try { + $this->execute($exec); + $zipContent = $zipFile->read(); + if ($zipContent === false) { + throw new Exception("Could not read content of newly created ZIP file."); + } + + return $zipContent; + + } catch (Exception $e) { + throw new Exception("Could not create encrypted ZIP file '{$zipFile->path}'.", 0, $e); + + } finally { + $fileNameFile->delete(); + $contentsFile->delete(); + $zipFile->delete(); + } + } + + /** + * @param string $originalFilename + * @param string $content + * @param string $md5 + * @return string Content of zipped file + * @throws Exception + */ + private function encryptByExtension($originalFilename, $content, $md5) + { + $zipFilePath = $this->tempFileName(); + + $zip = new ZipArchive(); + $result = $zip->open($zipFilePath, ZipArchive::CREATE); + if ($result === true) { + $zip->setPassword(self::ZIP_PASSWORD); + + $zip->addFromString($md5, $content); + $zip->setEncryptionName($md5, ZipArchive::EM_AES_128); + + $zip->addFromString("$md5.filename.txt", $originalFilename); + $zip->setEncryptionName("$md5.filename.txt", ZipArchive::EM_AES_128); + + $zip->close(); + } else { + throw new Exception("Could not create encrypted ZIP file '$zipFilePath'. Error code: $result"); + } + + $zipFile = new File($zipFilePath); + $zipContent = $zipFile->read(); + if ($zipContent === false) { + throw new Exception("Could not read content of newly created ZIP file."); + } + $zipFile->delete(); + + return $zipContent; + } + + /** + * @param string $content + * @param array $hashTypes + * @return array + * @throws InvalidArgumentException + */ + public function computeHashes($content, array $hashTypes = array()) + { + $validHashes = array('md5', 'sha1', 'sha256'); + $hashes = []; + foreach ($hashTypes as $hashType) { + if (!in_array($hashType, $validHashes)) { + throw new InvalidArgumentException("Hash type '$hashType' is not valid hash type."); + } + $hashes[$hashType] = hash($hashType, $content); + } + return $hashes; + } + + /** + * @param string $pythonBin + * @param string $filePath + * @return array + * @throws Exception + */ + public function advancedExtraction($pythonBin, $filePath) + { + return $this->executeAndParseJsonOutput([ + $pythonBin, + self::ADVANCED_EXTRACTION_SCRIPT_PATH, + '-p', + escapeshellarg($filePath), + ]); + } + + /** + * @param string $pythonBin + * @return array + * @throws Exception + */ + public function checkAdvancedExtractionStatus($pythonBin) + { + return $this->executeAndParseJsonOutput([$pythonBin, self::ADVANCED_EXTRACTION_SCRIPT_PATH, '-c']); + } + + private function tempFileName() + { + $randomName = (new RandomTool())->random_str(false, 12); + return $this->tempDir() . DS . $randomName; + } + + /** + * @return string + */ + private function tempDir() + { + return Configure::read('MISP.tmpdir') ?: sys_get_temp_dir(); + } + + /** + * @param array $command + * @return array + * @throws Exception + */ + private function executeAndParseJsonOutput(array $command) + { + $output = $this->execute($command); + + $json = json_decode($output, true); + if ($json === null) { + throw new Exception("Command output is not valid JSON: " . json_last_error_msg()); + } + return $json; + } + + /** + * This method is much more complicated than just `exec`, but it also provide stderr output, so Exceptions + * can be much more specific. + * + * @param array $command + * @return string + * @throws Exception + */ + private function execute(array $command) + { + $descriptorspec = [ + 1 => ["pipe", "w"], // stdout + 2 => ["pipe", "w"], // stderr + ]; + + $command = implode(' ', $command); + $process = proc_open($command, $descriptorspec, $pipes); + if (!$process) { + throw new Exception("Command '$command' could be started."); + } + + $stdout = stream_get_contents($pipes[1]); + if ($stdout === false) { + throw new Exception("Could not get STDOUT of command."); + } + fclose($pipes[1]); + + $stderr = stream_get_contents($pipes[2]); + fclose($pipes[2]); + + $returnCode = proc_close($process); + if ($returnCode !== 0) { + throw new Exception("Command '$command' return error code $returnCode. STDERR: '$stderr', STDOUT: '$stdout'"); + } + + return $stdout; + } +} diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 1bb13bd75..d57b668d2 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -5,6 +5,7 @@ App::uses('Folder', 'Utility'); App::uses('File', 'Utility'); App::uses('FinancialTool', 'Tools'); App::uses('RandomTool', 'Tools'); +App::uses('MalwareTool', 'Tools'); class Attribute extends AppModel { @@ -3494,66 +3495,22 @@ class Attribute extends AppModel if (!is_numeric($event_id)) { throw new Exception(__('Something went wrong. Received a non-numeric event ID while trying to create a zip archive of an uploaded malware sample.')); } - $attachments_dir = Configure::read('MISP.attachments_dir'); - if (empty($attachments_dir)) { - $attachments_dir = $this->getDefaultAttachments_dir(); + + $content = base64_decode($base64); + + $malwareTool = new MalwareTool(); + $hashes = $malwareTool->computeHashes($content, $hash_types); + try { + $encrypted = $malwareTool->encrypt($original_filename, $content, $hashes['md5']); + } catch (Exception $e) { + $this->logException("Could not create encrypted malware sample.", $e); + return array('success' => false); } - // If we've set attachments to S3, we can't write there - if ($this->attachmentDirIsS3()) { - $attachments_dir = Configure::read('MISP.tmpdir'); - // Sometimes it's not set? - if (empty($attachments_dir)) { - // Get a default tmpdir - $attachments_dir = $this->getDefaultTmp_dir(); - } - } - - if ($proposal) { - $dir = new Folder($attachments_dir . DS . $event_id . DS . 'shadow', true); - } else { - $dir = new Folder($attachments_dir . DS . $event_id, true); - } - $tmpFile = new File($dir->path . DS . $this->generateRandomFileName(), true, 0600); - $tmpFile->write(base64_decode($base64)); - $hashes = array(); - foreach ($hash_types as $hash) { - $hashes[$hash] = $this->__hashRouter($hash, $tmpFile->path); - } - $contentsFile = new File($dir->path . DS . $hashes['md5']); - rename($tmpFile->path, $contentsFile->path); - $fileNameFile = new File($dir->path . DS . $hashes['md5'] . '.filename.txt'); - $fileNameFile->write($original_filename); - $fileNameFile->close(); - $zipFile = new File($dir->path . DS . $hashes['md5'] . '.zip'); - exec('zip -j -P infected ' . escapeshellarg($zipFile->path) . ' ' . escapeshellarg($contentsFile->path) . ' ' . escapeshellarg($fileNameFile->path), $execOutput, $execRetval); - if ($execRetval != 0) { - $result = array('success' => false); - } else { - $result = array_merge(array('data' => base64_encode($zipFile->read()), 'success' => true), $hashes); - } - $fileNameFile->delete(); - $zipFile->delete(); - $contentsFile->delete(); + $result = array_merge(array('data' => base64_encode($encrypted), 'success' => true), $hashes); return $result; } - private function __hashRouter($hashType, $file) - { - $validHashes = array('md5', 'sha1', 'sha256'); - if (!in_array($hashType, $validHashes)) { - return false; - } - switch ($hashType) { - case 'md5': - case 'sha1': - case 'sha256': - return hash_file($hashType, $file); - break; - } - return false; - } - public function resolveHashType($hash) { $hashTypes = $this->hashTypes; @@ -3897,7 +3854,7 @@ class Attribute extends AppModel 'event_id' => $event_id, 'comment' => !empty($attribute_settings['comment']) ? $attribute_settings['comment'] : '' ); - $result = $this->Event->Attribute->handleMaliciousBase64($event_id, $filename, base64_encode($tmpfile->read()), $hashes); + $result = $this->handleMaliciousBase64($event_id, $filename, base64_encode($tmpfile->read()), $hashes); foreach ($attributes as $k => $v) { $attribute = array( 'distribution' => 5, @@ -3928,33 +3885,34 @@ class Attribute extends AppModel public function advancedAddMalwareSample($event_id, $attribute_settings, $filename, $tmpfile) { - $execRetval = ''; - $execOutput = array(); - $result = shell_exec($this->getPythonVersion() . ' ' . APP . 'files/scripts/generate_file_objects.py -p ' . $tmpfile->path); - if (!empty($result)) { - $result = json_decode($result, true); - if (isset($result['objects'])) { - $result['Object'] = $result['objects']; - unset($result['objects']); - } - if (isset($result['references'])) { - $result['ObjectReference'] = $result['references']; - unset($result['references']); - } - foreach ($result['Object'] as $k => $object) { - $result['Object'][$k]['distribution'] = $attribute_settings['distribution']; - $result['Object'][$k]['sharing_group_id'] = isset($attribute_settings['distribution']) ? $attribute_settings['distribution'] : 0; - if (!empty($result['Object'][$k]['Attribute'])) { - foreach ($result['Object'][$k]['Attribute'] as $k2 => $attribute) { - if ($attribute['value'] == $tmpfile->name) { - $result['Object'][$k]['Attribute'][$k2]['value'] = $filename; - } + $malwareTool = new MalwareTool(); + try { + $result = $malwareTool->advancedExtraction($this->getPythonVersion(), $tmpfile->path); + } catch (Exception $e) { + $this->logException("Could not finish advanced extraction", $e); + return $this->simpleAddMalwareSample($event_id, $attribute_settings, $filename, $tmpfile); + } + + if (isset($result['objects'])) { + $result['Object'] = $result['objects']; + unset($result['objects']); + } + if (isset($result['references'])) { + $result['ObjectReference'] = $result['references']; + unset($result['references']); + } + foreach ($result['Object'] as $k => $object) { + $result['Object'][$k]['distribution'] = $attribute_settings['distribution']; + $result['Object'][$k]['sharing_group_id'] = isset($attribute_settings['distribution']) ? $attribute_settings['distribution'] : 0; + if (!empty($result['Object'][$k]['Attribute'])) { + foreach ($result['Object'][$k]['Attribute'] as $k2 => $attribute) { + if ($attribute['value'] == $tmpfile->name) { + $result['Object'][$k]['Attribute'][$k2]['value'] = $filename; } } } - } else { - $result = $this->simpleAddMalwareSample($event_id, $attribute_settings, $filename, $tmpfile); } + return $result; } From 19808f23971f2ae80054da9fb7212e767d718746 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 6 Oct 2019 10:29:20 +0200 Subject: [PATCH 004/159] chg: [internal] Refactored AttributesController:add_attachment --- app/Controller/AttributesController.php | 92 +++++++++++++------------ 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 0aeb10f41..5bc591f0c 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -409,9 +409,9 @@ class AttributesController extends AppController public function add_attachment($eventId = null) { + $this->loadModel('Event'); + if ($this->request->is('post')) { - $hashes = array('md5' => 'malware-sample', 'sha1' => 'filename|sha1', 'sha256' => 'filename|sha256'); - $this->loadModel('Event'); $this->Event->id = $this->request->data['Attribute']['event_id']; $this->Event->recursive = -1; $event = $this->Event->read(); @@ -421,7 +421,6 @@ class AttributesController extends AppController if (!$this->_isSiteAdmin() && ($this->Event->data['Event']['orgc_id'] != $this->_checkOrg() || !$this->userRole['perm_modify'])) { throw new UnauthorizedException(__('You do not have permission to do that.')); } - $partialFails = array(); $fails = array(); $success = 0; @@ -449,11 +448,6 @@ class AttributesController extends AppController $filename, $tmpfile ); - if ($result) { - $success++; - } else { - $fails[] = $filename; - } } else { $result = $this->Attribute->simpleAddMalwareSample( $eventId, @@ -461,12 +455,14 @@ class AttributesController extends AppController $filename, $tmpfile ); - if ($result) { - $success++; - } else { - $fails[] = $filename; - } } + + if ($result) { + $success++; + } else { + $fails[] = $filename; + } + if (!empty($result)) { foreach ($result['Object'] as $object) { $this->loadModel('MispObject'); @@ -509,15 +505,12 @@ class AttributesController extends AppController } } $message = 'The attachment(s) have been uploaded.'; - if (!empty($partialFails)) { - $message .= ' Some of the attributes however could not be created.'; - } if (!empty($fails)) { $message = 'Some of the attachments failed to upload. The failed files were: ' . implode(', ', $fails) . ' - This can be caused by the attachments already existing in the event.'; } if (empty($success)) { if (empty($fails)) { - $message = 'The attachment(s) could not be saved. please contact your administrator.'; + $message = 'The attachment(s) could not be saved. Please contact your administrator.'; } } else { $this->Event->id = $this->request->data['Attribute']['event_id']; @@ -536,55 +529,40 @@ class AttributesController extends AppController // set the event_id in the form $this->request->data['Attribute']['event_id'] = $eventId; } + + $events = $this->Event->findById($eventId); + if (empty($events)) { + throw new NotFoundException(__('Invalid Event.')); + } + if (!$this->_isRest()) { $this->Attribute->Event->insertLock($this->Auth->user(), $eventId); } - // combobox for categories - $categories = array_keys($this->Attribute->categoryDefinitions); - // just get them with attachments.. + + // Filter categories that contains attachment type $selectedCategories = array(); - foreach ($categories as $category) { - $types = $this->Attribute->categoryDefinitions[$category]['types']; - $alreadySet = false; - foreach ($types as $type) { - if ($this->Attribute->typeIsAttachment($type) && !$alreadySet) { - // add to the whole.. + foreach ($this->Attribute->categoryDefinitions as $category => $values) { + foreach ($values['types'] as $type) { + if ($this->Attribute->typeIsAttachment($type)) { $selectedCategories[] = $category; - $alreadySet = true; - continue; + continue 2; } } } $categories = $this->_arrayToValuesIndexArray($selectedCategories); $this->set('categories', $categories); - $this->set('attrDescriptions', $this->Attribute->fieldDescriptions); - $this->set('typeDefinitions', $this->Attribute->typeDefinitions); $this->set('categoryDefinitions', $this->Attribute->categoryDefinitions); - $this->set('zippedDefinitions', $this->Attribute->zippedDefinitions); - $this->set('uploadDefinitions', $this->Attribute->uploadDefinitions); // combobox for distribution - $this->loadModel('Event'); $this->set('distributionLevels', $this->Event->Attribute->distributionLevels); - - foreach ($this->Attribute->categoryDefinitions as $key => $value) { - $info['category'][$key] = array('key' => $key, 'desc' => isset($value['formdesc'])? $value['formdesc'] : $value['desc']); - } - foreach ($this->Event->Attribute->distributionLevels as $key => $value) { - $info['distribution'][$key] = array('key' => $value, 'desc' => $this->Attribute->distributionDescriptions[$key]['formdesc']); - } - $this->set('info', $info); + $this->set('info', $this->getInfo()); $this->loadModel('SharingGroup'); $sgs = $this->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1); $this->set('sharingGroups', $sgs); - $events = $this->Event->findById($eventId); - if (empty($events)) { - throw new NotFoundException(__('Invalid Event.')); - } $this->set('currentDist', $events['Event']['distribution']); $this->set('published', $events['Event']['published']); } @@ -3210,4 +3188,28 @@ class AttributesController extends AppController return $this->RestResponse->viewData($final, $responseType, false, true, 'search.' . $type . '.' . $responseType); } } + + private function getInfo() + { + $info = array('category' => array(), 'type' => array(), 'distribution' => array()); + foreach ($this->Attribute->categoryDefinitions as $key => $value) { + $info['category'][$key] = array( + 'key' => $key, + 'desc' => isset($value['formdesc']) ? $value['formdesc'] : $value['desc'] + ); + } + foreach ($this->Attribute->typeDefinitions as $key => $value) { + $info['type'][$key] = array( + 'key' => $key, + 'desc' => isset($value['formdesc']) ? $value['formdesc'] : $value['desc'] + ); + } + foreach ($this->Attribute->distributionLevels as $key => $value) { + $info['distribution'][$key] = array( + 'key' => $value, + 'desc' => $this->Attribute->distributionDescriptions[$key]['formdesc'] + ); + } + return $info; + } } From ed6bb367e390d6aa5375dfe2c93c7bca18b82445 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 6 Oct 2019 13:05:29 +0200 Subject: [PATCH 005/159] chg: [UI] Disable Advanced extraction button if it is not installed --- app/Controller/AttributesController.php | 16 ++++++++++------ app/Model/Attribute.php | 21 +++++++++++++++++++++ app/View/Attributes/add_attachment.ctp | 4 +++- app/webroot/js/misp.js | 10 ++++++++++ 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 5bc591f0c..1544e94b6 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -3,6 +3,9 @@ App::uses('AppController', 'Controller'); App::uses('Folder', 'Utility'); App::uses('File', 'Utility'); +/** + * @property Attribute $Attribute + */ class AttributesController extends AppController { public $components = array('Security', 'RequestHandler', 'Cidr'); @@ -424,7 +427,7 @@ class AttributesController extends AppController $fails = array(); $success = 0; - foreach ($this->request->data['Attribute']['values'] as $k => $value) { + foreach ($this->request->data['Attribute']['values'] as $value) { // Check if there were problems with the file upload // only keep the last part of the filename, this should prevent directory attacks $filename = basename($value['name']); @@ -464,8 +467,8 @@ class AttributesController extends AppController } if (!empty($result)) { + $this->loadModel('MispObject'); foreach ($result['Object'] as $object) { - $this->loadModel('MispObject'); $object['distribution'] = $this->request->data['Attribute']['distribution']; if (!empty($this->request->data['sharing_group_id'])) { $object['sharing_group_id'] = $this->request->data['Attribute']['sharing_group_id']; @@ -530,8 +533,8 @@ class AttributesController extends AppController $this->request->data['Attribute']['event_id'] = $eventId; } - $events = $this->Event->findById($eventId); - if (empty($events)) { + $event = $this->Event->findById($eventId); + if (empty($event)) { throw new NotFoundException(__('Invalid Event.')); } @@ -554,6 +557,7 @@ class AttributesController extends AppController $this->set('categoryDefinitions', $this->Attribute->categoryDefinitions); $this->set('zippedDefinitions', $this->Attribute->zippedDefinitions); + $this->set('advancedExtractionAvailable', $this->Attribute->isAdvancedExtractionAvailable()); // combobox for distribution $this->set('distributionLevels', $this->Event->Attribute->distributionLevels); @@ -563,8 +567,8 @@ class AttributesController extends AppController $sgs = $this->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1); $this->set('sharingGroups', $sgs); - $this->set('currentDist', $events['Event']['distribution']); - $this->set('published', $events['Event']['published']); + $this->set('currentDist', $event['Event']['distribution']); + $this->set('published', $event['Event']['published']); } diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index d57b668d2..dc478fc47 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -3511,6 +3511,27 @@ class Attribute extends AppModel return $result; } + /** + * @return bool Return true if at least one advanced extraction tool is available + */ + public function isAdvancedExtractionAvailable() + { + $malwareTool = new MalwareTool(); + try { + $types = $malwareTool->checkAdvancedExtractionStatus($this->getPythonVersion()); + } catch (Exception $e) { + return false; + } + + foreach ($types as $type => $missing) { + if ($missing === false) { + return true; + } + } + + return false; + } + public function resolveHashType($hash) { $hashTypes = $this->hashTypes; diff --git a/app/View/Attributes/add_attachment.ctp b/app/View/Attributes/add_attachment.ctp index 7cfca776d..4453a1c80 100644 --- a/app/View/Attributes/add_attachment.ctp +++ b/app/View/Attributes/add_attachment.ctp @@ -68,8 +68,10 @@ echo $this->Form->input('advanced', array( 'type' => 'checkbox', 'checked' => false, + 'disabled' => !$advancedExtractionAvailable, + 'data-disabled-reason' => !$advancedExtractionAvailable ? __('Advanced extraction is not installed') : '', 'div' => array('id' => 'advanced_input', 'style' => 'display:none'), - 'label' => __('Advanced extraction (if installed)'), + 'label' => __('Advanced extraction'), )); ?> diff --git a/app/webroot/js/misp.js b/app/webroot/js/misp.js index 9b4010a5c..2e287458a 100644 --- a/app/webroot/js/misp.js +++ b/app/webroot/js/misp.js @@ -4414,6 +4414,16 @@ function checkNoticeList(type) { } $(document).ready(function() { + // Show popover for disabled input that contains `data-disabled-reason`. + $('input:disabled[data-disabled-reason]').popover("destroy").popover({ + placement: 'right', + html: 'true', + trigger: 'hover', + content: function () { + return $(this).data('disabled-reason'); + } + }); + $('#quickFilterField').bind("enterKey",function(e){ $('#quickFilterButton').trigger("click"); }); From e609ad6b77afd886add001c7c4bc598db6719bce Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 26 Jan 2020 18:17:49 +0100 Subject: [PATCH 006/159] chg: [internal] Log also previous exception --- app/Model/AppModel.php | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 6e358c2c9..8ca4b1e3e 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -2754,12 +2754,17 @@ class AppModel extends Model */ protected function logException($message, Exception $exception, $type = LOG_ERR) { - $message = sprintf("%s\n[%s] %s", - $message, - get_class($exception), - $exception->getMessage() - ); - $message .= "\nStack Trace:\n" . $exception->getTraceAsString(); + $message .= "\n"; + + do { + $message .= sprintf("[%s] %s", + get_class($exception), + $exception->getMessage() + ); + $message .= "\nStack Trace:\n" . $exception->getTraceAsString(); + $exception = $exception->getPrevious(); + } while ($exception !== null); + return $this->log($message, $type); } } From 110eabb08d7deaa3704dfab06920a1db6d271e5c Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 27 Jan 2020 22:02:08 +0100 Subject: [PATCH 007/159] chg: [internal] Cache result of AppController::_isRest method --- app/Controller/AppController.php | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index fbc589c9c..4c4f75249 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -57,6 +57,8 @@ class AppController extends Controller public $baseurl = ''; public $sql_dump = false; + private $isRest = null; + // Used for _isAutomation(), a check that returns true if the controller & action combo matches an action that is a non-xml and non-json automation method // This is used to allow authentication via headers for methods not covered by _isRest() - as that only checks for JSON and XML formats public $automationArray = array( @@ -629,6 +631,11 @@ class AppController extends Controller protected function _isRest() { + // This method is surprisingly slow and called many times for one request, so it make sense to cache the result. + if ($this->isRest !== null) { + return $this->isRest; + } + $api = $this->__isApiFunction($this->request->params['controller'], $this->request->params['action']); if (isset($this->RequestHandler) && ($api || $this->RequestHandler->isXml() || $this->_isJson() || $this->_isCsv())) { if ($this->_isJson()) { @@ -636,8 +643,10 @@ class AppController extends Controller throw new MethodNotAllowedException('Invalid JSON input. Make sure that the JSON input is a correctly formatted JSON string. This request has been blocked to avoid an unfiltered request.'); } } + $this->isRest = true; return true; } else { + $this->isRest = false; return false; } } From bba085a69977ff6a1a0b4ec60dceffd8e0c46b14 Mon Sep 17 00:00:00 2001 From: kscheetz Date: Fri, 20 Mar 2020 11:05:45 -0400 Subject: [PATCH 008/159] Fixes missing MySQL ignore table statements. --- INSTALL/MYSQL.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/INSTALL/MYSQL.sql b/INSTALL/MYSQL.sql index 44c940f98..9644ee485 100644 --- a/INSTALL/MYSQL.sql +++ b/INSTALL/MYSQL.sql @@ -916,7 +916,7 @@ CREATE TABLE IF NOT EXISTS `shadow_attribute_correlations` ( -- Table structure for table `sharing_group_orgs` -- -CREATE TABLE `sharing_group_orgs` ( +CREATE TABLE IF NOT EXISTS `sharing_group_orgs` ( `id` int(11) NOT NULL AUTO_INCREMENT, `sharing_group_id` int(11) NOT NULL, `org_id` int(11) NOT NULL, @@ -932,7 +932,7 @@ CREATE TABLE `sharing_group_orgs` ( -- Table structure for table `sharing_group_servers` -- -CREATE TABLE `sharing_group_servers` ( +CREATE TABLE IF NOT EXISTS `sharing_group_servers` ( `id` int(11) NOT NULL AUTO_INCREMENT, `sharing_group_id` int(11) NOT NULL, `server_id` int(11) NOT NULL, @@ -948,7 +948,7 @@ CREATE TABLE `sharing_group_servers` ( -- Table structure for table `sharing_groups` -- -CREATE TABLE `sharing_groups` ( +CREATE TABLE IF NOT EXISTS `sharing_groups` ( `id` int(11) NOT NULL AUTO_INCREMENT, `name` varchar(255) CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL, `releasability` text CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL, From 790483014333e7330086d0b7f52bdd05ebee00a7 Mon Sep 17 00:00:00 2001 From: kscheetz Date: Mon, 23 Mar 2020 16:42:38 -0400 Subject: [PATCH 009/159] Fixes failed insert on existing records. --- INSTALL/MYSQL.sql | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/INSTALL/MYSQL.sql b/INSTALL/MYSQL.sql index 9644ee485..ecc782b4a 100644 --- a/INSTALL/MYSQL.sql +++ b/INSTALL/MYSQL.sql @@ -1355,14 +1355,14 @@ CREATE TABLE IF NOT EXISTS `whitelist` ( -- Default values for initial installation -- -INSERT INTO `admin_settings` (`id`, `setting`, `value`) VALUES +INSERT IGNORE INTO `admin_settings` (`id`, `setting`, `value`) VALUES (1, 'db_version', '40'); -INSERT INTO `feeds` (`id`, `provider`, `name`, `url`, `distribution`, `default`, `enabled`) VALUES +INSERT IGNORE INTO `feeds` (`id`, `provider`, `name`, `url`, `distribution`, `default`, `enabled`) VALUES (1, 'CIRCL', 'CIRCL OSINT Feed', 'https://www.circl.lu/doc/misp/feed-osint', 3, 1, 0), (2, 'Botvrij.eu', 'The Botvrij.eu Data', 'https://www.botvrij.eu/data/feed-osint', 3, 1, 0); - INSERT INTO `regexp` (`id`, `regexp`, `replacement`, `type`) VALUES +INSERT IGNORE INTO `regexp` (`id`, `regexp`, `replacement`, `type`) VALUES (1, '/.:.ProgramData./i', '%ALLUSERSPROFILE%\\\\', 'ALL'), (2, '/.:.Documents and Settings.All Users./i', '%ALLUSERSPROFILE%\\\\', 'ALL'), (3, '/.:.Program Files.Common Files./i', '%COMMONPROGRAMFILES%\\\\', 'ALL'), @@ -1407,22 +1407,22 @@ INSERT INTO `feeds` (`id`, `provider`, `name`, `url`, `distribution`, `default`, -- 7. Read Only - read -- -INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`) +INSERT IGNORE INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`) VALUES (1, 'admin', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0); -INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`) +INSERT IGNORE INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`) VALUES (2, 'Org Admin', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0); -INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`) +INSERT IGNORE INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`) VALUES (3, 'User', NOW(), NOW(), 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1); -INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`) +INSERT IGNORE INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`) VALUES (4, 'Publisher', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0); -INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`) +INSERT IGNORE INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`) VALUES (5, 'Sync user', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0); -INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`) +INSERT IGNORE INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`) VALUES (6, 'Read Only', NOW(), NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); -- -------------------------------------------------------- @@ -1431,7 +1431,7 @@ VALUES (6, 'Read Only', NOW(), NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, -- Initial threat levels -- -INSERT INTO `threat_levels` (`id`, `name`, `description`, `form_description`) +INSERT IGNORE INTO `threat_levels` (`id`, `name`, `description`, `form_description`) VALUES (1, 'High', '*high* means sophisticated APT malware or 0-day attack', 'Sophisticated APT malware or 0-day attack'), (2, 'Medium', '*medium* means APT malware', 'APT malware'), @@ -1444,13 +1444,13 @@ VALUES -- Default templates -- -INSERT INTO `templates` (`id`, `name`, `description`, `org`, `share`) VALUES +INSERT IGNORE INTO `templates` (`id`, `name`, `description`, `org`, `share`) VALUES (1, 'Phishing E-mail', 'Create a MISP event about a Phishing E-mail.', 'MISP', 1), (2, 'Phishing E-mail with malicious attachment', 'A MISP event based on Spear-phishing containing a malicious attachment. This event can include anything from the description of the e-mail itself, the malicious attachment and its description as well as the results of the analysis done on the malicious f', 'MISP', 1), (3, 'Malware Report', 'This is a template for a generic malware report. ', 'MISP', 1), (4, 'Indicator List', 'A simple template for indicator lists.', 'MISP', 1); -INSERT INTO `template_elements` (`id`, `template_id`, `position`, `element_definition`) VALUES +INSERT IGNORE INTO `template_elements` (`id`, `template_id`, `position`, `element_definition`) VALUES (1, 1, 2, 'attribute'), (2, 1, 3, 'attribute'), (3, 1, 1, 'text'), @@ -1497,7 +1497,7 @@ INSERT INTO `template_elements` (`id`, `template_id`, `position`, `element_defin (46, 4, 2, 'attribute'), (47, 4, 3, 'attribute'); -INSERT INTO `template_element_attributes` (`id`, `template_element_id`, `name`, `description`, `to_ids`, `category`, `complex`, `type`, `mandatory`, `batch`) VALUES +INSERT IGNORE INTO `template_element_attributes` (`id`, `template_element_id`, `name`, `description`, `to_ids`, `category`, `complex`, `type`, `mandatory`, `batch`) VALUES (1, 1, 'From address', 'The source address from which the e-mail was sent.', 1, 'Payload delivery', 0, 'email-src', 1, 1), (2, 2, 'Malicious url', 'The malicious url in the e-mail body.', 1, 'Payload delivery', 0, 'url', 1, 1), (3, 4, 'E-mail subject', 'The subject line of the e-mail.', 0, 'Payload delivery', 0, 'email-subject', 1, 0), @@ -1529,13 +1529,13 @@ INSERT INTO `template_element_attributes` (`id`, `template_element_id`, `name`, (29, 46, 'Network Indicators', 'Paste any combination of IP addresses, hostnames, domains or URL', 1, 'Network activity', 1, 'CnC', 0, 1), (30, 47, 'File Indicators', 'Paste any file hashes that you have (MD5, SHA1, SHA256) or filenames below. You can also add filename and hash pairs by using the following syntax for each applicable column: filename|hash ', 1, 'Payload installation', 1, 'File', 0, 1); -INSERT INTO `template_element_files` (`id`, `template_element_id`, `name`, `description`, `category`, `malware`, `mandatory`, `batch`) VALUES +INSERT IGNORE INTO `template_element_files` (`id`, `template_element_id`, `name`, `description`, `category`, `malware`, `mandatory`, `batch`) VALUES (1, 14, 'Malicious Attachment', 'The file (or files) that was (were) attached to the e-mail itself.', 'Payload delivery', 1, 0, 1), (2, 21, 'Payload installation', 'Payload installation detected during the analysis', 'Payload installation', 1, 0, 1), (3, 30, 'Malware sample', 'The sample that the report is based on', 'Payload delivery', 1, 0, 0), (4, 40, 'Artifacts dropped (Sample)', 'Upload any files that were dropped during the analysis.', 'Artifacts dropped', 1, 0, 1); -INSERT INTO `template_element_texts` (`id`, `name`, `template_element_id`, `text`) VALUES +INSERT IGNORE INTO `template_element_texts` (`id`, `name`, `template_element_id`, `text`) VALUES (1, 'Required fields', 3, 'The fields below are mandatory.'), (2, 'Optional information', 5, 'All of the fields below are optional, please fill out anything that''s applicable.'), (4, 'Required Fields', 11, 'The following fields are mandatory'), @@ -1548,6 +1548,6 @@ INSERT INTO `template_element_texts` (`id`, `name`, `template_element_id`, `text (11, 'Persistence mechanism', 41, 'The following fields allow you to describe the persistence mechanism used by the malware'), (12, 'Indicators', 45, 'Just paste your list of indicators based on type into the appropriate field. All of the fields are optional, so inputting a list of IP addresses into the Network indicator field for example is sufficient to complete this template.'); -INSERT INTO `org_blacklists` (`org_uuid`, `created`, `org_name`, `comment`) VALUES +INSERT IGNORE INTO `org_blacklists` (`org_uuid`, `created`, `org_name`, `comment`) VALUES ('58d38339-7b24-4386-b4b4-4c0f950d210f', NOW(), 'Setec Astrononomy', 'default example'), ('58d38326-eda8-443a-9fa8-4e12950d210f', NOW(), 'Acme Finance', 'default example'); From 309bbc6814acf8ca1ee7e27b411b878893a7fb5c Mon Sep 17 00:00:00 2001 From: Golbark Date: Wed, 25 Mar 2020 07:45:09 -0700 Subject: [PATCH 010/159] new: usr: Implementation of email-based OTP --- app/Controller/AppController.php | 6 ++ app/Controller/Component/ACLComponent.php | 1 + app/Controller/UsersController.php | 93 +++++++++++++++++++++++ app/Model/Server.php | 47 ++++++++++++ app/View/Users/email_otp.ctp | 30 ++++++++ 5 files changed, 177 insertions(+) create mode 100644 app/View/Users/email_otp.ctp diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index ed302bc3c..f94010d10 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -406,6 +406,12 @@ class AppController extends Controller if (!$this->_isRest()) { $this->redirect(array('controller' => 'users', 'action' => 'change_pw', 'admin' => false)); } + } elseif (Configure::read('Security.email_otp_enabled') && !$this->_isRest() && !in_array($this->request->here, array($base_dir.'/users/terms', $base_dir.'/users/email_otp', $base_dir.'/users/change_pw', $base_dir.'/users/logout', $base_dir.'/users/login'))) { + $redis = $this->{$this->modelClass}->setupRedis(); + $otp_authed = $redis->get('misp:otp_authed:'.$this->Auth->user('id')); + if (empty($otp_authed)) { + $this->redirect(array('controller' => 'users', 'action' => 'email_otp', 'admin' => false)); + } } elseif (!$this->_isRest() && !($this->params['controller'] == 'news' && $this->params['action'] == 'index') && (!in_array($this->request->here, array($base_dir.'/users/terms', $base_dir.'/users/change_pw', $base_dir.'/users/logout', $base_dir.'/users/login')))) { $newsread = $this->User->field('newsread', array('User.id' => $this->Auth->user('id'))); $this->loadModel('News'); diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php index 2fb2be9b3..97d9aad3d 100644 --- a/app/Controller/Component/ACLComponent.php +++ b/app/Controller/Component/ACLComponent.php @@ -566,6 +566,7 @@ class ACLComponent extends Component 'delete' => array('perm_admin'), 'downloadTerms' => array('*'), 'edit' => array('*'), + 'email_otp' => array('*'), 'searchGpgKey' => array('*'), 'fetchGpgKey' => array('*'), 'histogram' => array('*'), diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php index 9ee5f1051..e706ddda8 100644 --- a/app/Controller/UsersController.php +++ b/app/Controller/UsersController.php @@ -1254,6 +1254,8 @@ class UsersController extends AppController 'recursive' => -1 )); unset($user['User']['password']); + $redis = $this->User->setupRedis(); + $redis->delete('misp:otp_authed:'.$this->Auth->user('id')); $user['User']['action'] = 'logout'; $this->User->save($user['User'], true, array('id')); $this->redirect($this->Auth->logout()); @@ -1651,6 +1653,97 @@ class UsersController extends AppController } } + public function email_otp() + { + $redis = $this->User->setupRedis(); + $user_id = $this->Auth->user('id'); + + if ($this->request->is('post') && isset($this->request->data['User']['otp'])) { + $stored_otp = $redis->get('misp:otp:'.$user_id); + if (!empty($stored_otp) && $this->request->data['User']['otp'] == $stored_otp) { + // we invalidate the previously generated OTP + $redis->delete('misp:otp:'.$user_id); + // We store in redis the success of the OTP step + $redis->set('misp:otp_authed:'.$user_id, 1); + // After this time, the user will need to redo the OTP step + // We use the same time as for the session expiration + $redis->expire('misp:otp_authed:'.$user_id, intval(Configure::read('Session.cookieTimeout')) * 60); + $this->Flash->success(__("You are now logged in.")); + $this->redirect($this->Auth->redirectUrl()); + } else { + $this->Flash->error("The OTP is incorrect or has expired"); + } + } else { + // GET Request + + // If the OTP is still valid, we redirect + if (!Configure::read('Security.email_otp_enabled') || !empty($redis->get('misp:otp_authed:'.$user_id))) { + $this->redirect($this->Auth->redirectUrl()); + } + + $user = $this->User->find('first', array( + 'recursive' => -1, + 'conditions' => array('User.id' => $user_id) + )); + + // We check for exceptions + $exception_list = Configure::read('Security.email_otp_exceptions'); + if (!empty($exception_list)) { + $exceptions = explode(",", $exception_list); + foreach ($exceptions as &$exception) { + if ($user['User']['email'] == trim($exception)) { + $redis->set('misp:otp_authed:'.$user_id, 1); + // It will take maximum this time (in seconds) to ask a OTP for someone removed from the exception list + $redis->expire('misp:otp_authed:'.$user_id, 3600); + $this->redirect($this->Auth->redirectUrl()); + } + } + } + $this->loadModel('Server'); + + // Generating the OTP + $digits = !empty(Configure::read('Security.email_otp_length')) ? Configure::read('Security.email_otp_length') : $this->Server->serverSettings['Security']['email_otp_length']['value']; + $otp = ""; + for ($i=0; $i<$digits; $i++) { + $otp.= random_int(0,9); + } + // We use Redis to cache the OTP + $redis->set('misp:otp:'.$user_id, $otp); + $validity = !empty(Configure::read('Security.email_otp_validity')) ? Configure::read('Security.email_otp_validity') : $this->Server->serverSettings['Security']['email_otp_validity']['value']; + $redis->expire('misp:otp:'.$user_id, intval($validity) * 60); + + // Email construction + $body = !empty(Configure::read('Security.email_otp_text')) ? Configure::read('Security.email_otp_text') : $this->Server->serverSettings['Security']['email_otp_text']['value']; + $body = str_replace('$misp', Configure::read('MISP.baseurl'), $body); + $body = str_replace('$org', Configure::read('MISP.org'), $body); + $body = str_replace('$contact', Configure::read('MISP.contact'), $body); + $body = str_replace('$validity', $validity, $body); + $body = str_replace('$otp', $otp, $body); + $body = str_replace('$ip', $this->_getClientIP(), $body); + $body = str_replace('$username', $user['User']['email'], $body); + $result = $this->User->sendEmail($user, $body, false, "[MISP] Email OTP"); + + if ( $result ) { + $this->Flash->success(__("An email containing a OTP has been sent.")); + } else { + $this->Flash->error("The email couldn't be sent, please reach out to your administrator."); + } + } + } + + + /** + * Helper function to determine the IP of a client (proxy aware) + */ + private function _getClientIP() { + if(!empty($_SERVER['HTTP_CLIENT_IP'])){ + return $_SERVER['HTTP_CLIENT_IP']; + }elseif(!empty($_SERVER['HTTP_X_FORWARDED_FOR'])){ + return $_SERVER['HTTP_X_FORWARDED_FOR']; + } + return $_SERVER['REMOTE_ADDR']; + } + // shows some statistics about the instance public function statistics($page = 'data') { diff --git a/app/Model/Server.php b/app/Model/Server.php index 51dc00d9d..ea1e6a3c0 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -1243,6 +1243,53 @@ class Server extends AppModel 'type' => 'boolean', 'null' => true ), + 'email_otp_enabled' => array( + 'level'=> 2, + 'description' => __('Enable two step authentication with a OTP sent by email.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'email_otp_length' => array ( + 'level' => 2, + 'description' => __('Define the length of the OTP code sent by email'), + 'value' => '6', + 'errorMessage' => '', + 'type' => 'numeric', + 'test' => 'testForNumeric', + 'null' => true, + ), + 'email_otp_validity' => array ( + 'level' => 2, + 'description' => __('Define the validity (in minutes) of the OTP code sent by email'), + 'value' => '5', + 'errorMessage' => '', + 'type' => 'numeric', + 'test' => 'testForNumeric', + 'null' => true, + ), + 'email_otp_text' => array( + 'level' => 2, + 'bigField' => true, + 'description' => __('The message sent to the user when a new OTP is requested. Use \\n for line-breaks. The following variables will be automatically replaced in the text: $otp = the new OTP generated by MISP, $username = the user\'s e-mail address, $org the Organisation managing the instance, $misp = the url of this instance, $contact = the e-mail address used to contact the support team (as set in MISP.contact), $ip the IP used to complete the first step of the login and $validity the validity time in minutes.'), + 'value' => 'Dear MISP user,\n\nYou have attempted to login to MISP ($misp) from $ip with username $username.\n\n Use the following OTP to log into MISP: $otp\n This code is valid for the next $validity minutes.\n\nIf you have any questions, don\'t hesitate to contact us at: $contact.\n\nBest regards,\nYour $org MISP support team', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'null' => true, + ), + 'email_otp_exceptions' => array( + 'level' => 2, + 'bigField' => true, + 'description' => __('A comma separated list of emails for which the OTP is disabled.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'null' => true, + ), 'password_policy_length' => array( 'level' => 2, 'description' => __('Password length requirement. If it is not set or it is set to 0, then the default value is assumed (12).'), diff --git a/app/View/Users/email_otp.ctp b/app/View/Users/email_otp.ctp new file mode 100644 index 000000000..16dc9483d --- /dev/null +++ b/app/View/Users/email_otp.ctp @@ -0,0 +1,30 @@ +Flash->render(); ?> + +
+
+

Your administrator has turned on an additional authentication step which + requires you to enter a OTP (one time password) you have received via email. +

+

Make sure to check your SPAM folder.

+ +
+
+ +element('/genericElements/Form/genericForm', array( + "form" => $this->Form, + "data" => array( + "title" => "Validate your OTP", + "fields" => array( + array( + "field" => "otp", + "label" => "One Time Password", + "type" => "text", + "placeholder" => __("Enter your OTP here"), + ), + ), + "submit" => array ( + "action" => "EmailOtp", + ), +))); +?> From d254d04365d8d403f81d92a4900e62cdcc4224a3 Mon Sep 17 00:00:00 2001 From: Golbark Date: Thu, 26 Mar 2020 02:55:14 -0700 Subject: [PATCH 011/159] Rely on session_id instead of user_id and address minor comments --- app/Controller/AppController.php | 2 +- app/Controller/UsersController.php | 29 +++++++++++++++++------------ app/View/Users/email_otp.ctp | 2 +- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index f94010d10..26dbc1589 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -408,7 +408,7 @@ class AppController extends Controller } } elseif (Configure::read('Security.email_otp_enabled') && !$this->_isRest() && !in_array($this->request->here, array($base_dir.'/users/terms', $base_dir.'/users/email_otp', $base_dir.'/users/change_pw', $base_dir.'/users/logout', $base_dir.'/users/login'))) { $redis = $this->{$this->modelClass}->setupRedis(); - $otp_authed = $redis->get('misp:otp_authed:'.$this->Auth->user('id')); + $otp_authed = $redis->get('misp:otp_authed:'.session_id()); if (empty($otp_authed)) { $this->redirect(array('controller' => 'users', 'action' => 'email_otp', 'admin' => false)); } diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php index e706ddda8..fa07f7093 100644 --- a/app/Controller/UsersController.php +++ b/app/Controller/UsersController.php @@ -1255,7 +1255,7 @@ class UsersController extends AppController )); unset($user['User']['password']); $redis = $this->User->setupRedis(); - $redis->delete('misp:otp_authed:'.$this->Auth->user('id')); + $redis->delete('misp:otp_authed:'.session_id()); $user['User']['action'] = 'logout'; $this->User->save($user['User'], true, array('id')); $this->redirect($this->Auth->logout()); @@ -1657,6 +1657,7 @@ class UsersController extends AppController { $redis = $this->User->setupRedis(); $user_id = $this->Auth->user('id'); + $session_id = session_id(); if ($this->request->is('post') && isset($this->request->data['User']['otp'])) { $stored_otp = $redis->get('misp:otp:'.$user_id); @@ -1664,10 +1665,10 @@ class UsersController extends AppController // we invalidate the previously generated OTP $redis->delete('misp:otp:'.$user_id); // We store in redis the success of the OTP step - $redis->set('misp:otp_authed:'.$user_id, 1); + $redis->set('misp:otp_authed:'.$session_id, 1); // After this time, the user will need to redo the OTP step // We use the same time as for the session expiration - $redis->expire('misp:otp_authed:'.$user_id, intval(Configure::read('Session.cookieTimeout')) * 60); + $redis->expire('misp:otp_authed:'.$session_id, (int) Configure::read('Session.cookieTimeout') * 60); $this->Flash->success(__("You are now logged in.")); $this->redirect($this->Auth->redirectUrl()); } else { @@ -1677,7 +1678,7 @@ class UsersController extends AppController // GET Request // If the OTP is still valid, we redirect - if (!Configure::read('Security.email_otp_enabled') || !empty($redis->get('misp:otp_authed:'.$user_id))) { + if (!Configure::read('Security.email_otp_enabled') || !empty($redis->get('misp:otp_authed:'.$session_id))) { $this->redirect($this->Auth->redirectUrl()); } @@ -1692,9 +1693,9 @@ class UsersController extends AppController $exceptions = explode(",", $exception_list); foreach ($exceptions as &$exception) { if ($user['User']['email'] == trim($exception)) { - $redis->set('misp:otp_authed:'.$user_id, 1); + $redis->set('misp:otp_authed:'.$session_id, 1); // It will take maximum this time (in seconds) to ask a OTP for someone removed from the exception list - $redis->expire('misp:otp_authed:'.$user_id, 3600); + $redis->expire('misp:otp_authed:'.$session_id, 3600); $this->redirect($this->Auth->redirectUrl()); } } @@ -1710,7 +1711,7 @@ class UsersController extends AppController // We use Redis to cache the OTP $redis->set('misp:otp:'.$user_id, $otp); $validity = !empty(Configure::read('Security.email_otp_validity')) ? Configure::read('Security.email_otp_validity') : $this->Server->serverSettings['Security']['email_otp_validity']['value']; - $redis->expire('misp:otp:'.$user_id, intval($validity) * 60); + $redis->expire('misp:otp:'.$user_id, (int) $validity * 60); // Email construction $body = !empty(Configure::read('Security.email_otp_text')) ? Configure::read('Security.email_otp_text') : $this->Server->serverSettings['Security']['email_otp_text']['value']; @@ -1736,12 +1737,16 @@ class UsersController extends AppController * Helper function to determine the IP of a client (proxy aware) */ private function _getClientIP() { - if(!empty($_SERVER['HTTP_CLIENT_IP'])){ - return $_SERVER['HTTP_CLIENT_IP']; - }elseif(!empty($_SERVER['HTTP_X_FORWARDED_FOR'])){ - return $_SERVER['HTTP_X_FORWARDED_FOR']; + $x_forwarded = filter_input(INPUT_SERVER, 'HTTP_X_FORWARDED_FOR', FILTER_SANITIZE_STRING); + $client_ip = filter_input(INPUT_SERVER, 'HTTP_CLIENT_IP', FILTER_SANITIZE_STRING); + if (!empty($x_forwarded)) { + $x_forwarded = explode(",", $x_forwarded); + return $x_forwarded[0]; + } elseif(!empty($client_ip)){ + return $_client_ip; + } else { + return filter_input(INPUT_SERVER, 'REMOTE_ADDR', FILTER_SANITIZE_STRING); } - return $_SERVER['REMOTE_ADDR']; } // shows some statistics about the instance diff --git a/app/View/Users/email_otp.ctp b/app/View/Users/email_otp.ctp index 16dc9483d..5b57ab065 100644 --- a/app/View/Users/email_otp.ctp +++ b/app/View/Users/email_otp.ctp @@ -6,7 +6,7 @@ requires you to enter a OTP (one time password) you have received via email.

Make sure to check your SPAM folder.

- + From 9062881469f8a309a6dfca5a38b7baa34f1e13d7 Mon Sep 17 00:00:00 2001 From: Golbark Date: Thu, 26 Mar 2020 07:18:22 -0700 Subject: [PATCH 012/159] Add consistent i18n support for all strings. --- app/Controller/UsersController.php | 4 ++-- app/View/Users/email_otp.ctp | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php index fa07f7093..c00cf08d7 100644 --- a/app/Controller/UsersController.php +++ b/app/Controller/UsersController.php @@ -1672,7 +1672,7 @@ class UsersController extends AppController $this->Flash->success(__("You are now logged in.")); $this->redirect($this->Auth->redirectUrl()); } else { - $this->Flash->error("The OTP is incorrect or has expired"); + $this->Flash->error(__("The OTP is incorrect or has expired")); } } else { // GET Request @@ -1727,7 +1727,7 @@ class UsersController extends AppController if ( $result ) { $this->Flash->success(__("An email containing a OTP has been sent.")); } else { - $this->Flash->error("The email couldn't be sent, please reach out to your administrator."); + $this->Flash->error(__("The email couldn't be sent, please reach out to your administrator.")); } } } diff --git a/app/View/Users/email_otp.ctp b/app/View/Users/email_otp.ctp index 5b57ab065..50d5894cb 100644 --- a/app/View/Users/email_otp.ctp +++ b/app/View/Users/email_otp.ctp @@ -2,11 +2,11 @@
-

Your administrator has turned on an additional authentication step which - requires you to enter a OTP (one time password) you have received via email. +

-

Make sure to check your SPAM folder.

- +

+
@@ -14,11 +14,11 @@ echo $this->element('/genericElements/Form/genericForm', array( "form" => $this->Form, "data" => array( - "title" => "Validate your OTP", + "title" => __("Validate your OTP"), "fields" => array( array( "field" => "otp", - "label" => "One Time Password", + "label" => __("One Time Password"), "type" => "text", "placeholder" => __("Enter your OTP here"), ), From 7cd21755dd512c15575c1a1ce824b896fe2bed69 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Fri, 17 Apr 2020 11:22:15 +0200 Subject: [PATCH 013/159] fix: [event:fetchEvent] Block viewing the event if user does not belong to the sharing_group Even if the event belongs to the user. This scenario can happen if a remote sync is badly configured where the remote sync user have site_admin right, thus allowing the user to see the event even though he is not part of the SG --- app/Model/Event.php | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/Model/Event.php b/app/Model/Event.php index c1cddda5a..c962886d3 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -2150,6 +2150,22 @@ class Event extends AppModel 'Object' => array('name', 'meta-category') ); foreach ($results as $eventKey => &$event) { + if ($event['Event']['distribution'] == 4 && !in_array($event['Event']['sharing_group_id'], $sgids)) { + $this->Log = ClassRegistry::init('Log'); + $this->Log->create(); + $this->Log->save(array( + 'org' => $user['Organisation']['name'], + 'model' => 'Event', + 'model_id' => $event['Event']['id'], + 'email' => $user['email'], + 'action' => 'fetchEvent', + 'user_id' => $user['id'], + 'title' => 'User was able to fetch the event but not the sharing_group it belongs to', + 'change' => '' + )); + unset($results[$eventKey]); // Current user cannot access sharing_group associated to this event + continue; + } $this->__attachReferences($user, $event, $sgids, $fields); $event = $this->Orgc->attachOrgsToEvent($event, $fieldsOrg); if (!$options['sgReferenceOnly'] && $event['Event']['sharing_group_id']) { From c9481b23140d5f2ef3460e02574dae233493aacb Mon Sep 17 00:00:00 2001 From: mokaddem Date: Fri, 17 Apr 2020 11:26:22 +0200 Subject: [PATCH 014/159] fix: [event:fetchEvent] Block viewing Objects/Attributes if the user does not belong to the sharing_group Even if these elements belong to the user. Similar explanation than for 7cd2175 --- app/Model/Event.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index c962886d3..93e008f8d 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -2464,7 +2464,11 @@ class Event extends AppModel } foreach ($data as $k => $v) { if ($v['distribution'] == 4) { - $data[$k]['SharingGroup'] = $sharingGroupData[$v['sharing_group_id']]['SharingGroup']; + if (isset($sharingGroupData[$v['sharing_group_id']])) { + $data[$k]['SharingGroup'] = $sharingGroupData[$v['sharing_group_id']]['SharingGroup']; + } else { + unset($data[$k]); // current user could not fetch the sharing_group + } } } return $data; From 3547a8a888c899f43c8ce7c603119787db511f03 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Fri, 17 Apr 2020 11:29:09 +0200 Subject: [PATCH 015/159] fix: [correlations] Update correlations on Attribute or Event `distribution` change --- app/Model/Attribute.php | 6 ++++-- app/Model/Event.php | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 6f6b5e982..74972365c 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -698,7 +698,7 @@ class Attribute extends AppModel * Only recorrelate if: * - We are dealing with a new attribute OR * - The existing attribute's previous state is known AND - * value, type or disable correlation have changed + * value, type, disable correlation or distribution have changed * This will avoid recorrelations when it's not really needed, such as adding a tag */ if (!$created) { @@ -706,7 +706,9 @@ class Attribute extends AppModel empty($this->old) || $this->data['Attribute']['value'] != $this->old['Attribute']['value'] || $this->data['Attribute']['disable_correlation'] != $this->old['Attribute']['disable_correlation'] || - $this->data['Attribute']['type'] != $this->old['Attribute']['type'] + $this->data['Attribute']['type'] != $this->old['Attribute']['type'] || + $this->data['Attribute']['distribution'] != $this->old['Attribute']['distribution'] || + $this->data['Attribute']['sharing_group_id'] != $this->old['Attribute']['sharing_group_id'] ) { $this->__beforeSaveCorrelation($this->data['Attribute']); $this->__afterSaveCorrelation($this->data['Attribute'], false, $passedEvent); diff --git a/app/Model/Event.php b/app/Model/Event.php index 93e008f8d..8fe3d4cfe 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -641,6 +641,12 @@ class Event extends AppModel if (isset($this->data['Event']['info'])) { $this->Correlation->updateAll(array('Correlation.info' => $db->value($this->data['Event']['info'])), array('Correlation.event_id' => intval($this->data['Event']['id']))); } + if (isset($this->data['Event']['distribution'])) { + $this->Correlation->updateAll(array('Correlation.distribution' => $db->value($this->data['Event']['distribution'])), array('Correlation.event_id' => intval($this->data['Event']['id']))); + } + if (isset($this->data['Event']['sharing_group_id'])) { + $this->Correlation->updateAll(array('Correlation.sharing_group_id' => $db->value($this->data['Event']['sharing_group_id'])), array('Correlation.event_id' => intval($this->data['Event']['id']))); + } } if (empty($this->data['Event']['unpublishAction']) && empty($this->data['Event']['skip_zmq']) && Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) { $pubSubTool = $this->getPubSubTool(); From 549028c7af7fcacf1f6d5113583155ab02a45eed Mon Sep 17 00:00:00 2001 From: mokaddem Date: Fri, 17 Apr 2020 14:59:25 +0200 Subject: [PATCH 016/159] fix: [event:view] Restored disabled_correlation toggle --- app/View/Events/view.ctp | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/app/View/Events/view.ctp b/app/View/Events/view.ctp index baee82ee6..cc35bd3f0 100644 --- a/app/View/Events/view.ctp +++ b/app/View/Events/view.ctp @@ -301,27 +301,27 @@ ) ) ); - if (!Configure::read('MISP.completely_disable_correlation') && Configure::read('MISP.allow_disabling_correlation')) { - $table_data[] = array( - 'key' => __('Correlation'), - 'class' => $event['Event']['disable_correlation'] ? 'background-red bold' : '', - 'html' => sprintf( - '%s%s', - $event['Event']['disable_correlation'] ? __('Disabled') : __('Enabled'), - (!$mayModify && !$isSiteAdmin) ? '' : sprintf( + } + if (!Configure::read('MISP.completely_disable_correlation') && Configure::read('MISP.allow_disabling_correlation')) { + $table_data[] = array( + 'key' => __('Correlation'), + 'class' => $event['Event']['disable_correlation'] ? 'background-red bold' : '', + 'html' => sprintf( + '%s%s', + $event['Event']['disable_correlation'] ? __('Disabled') : __('Enabled'), + (!$mayModify && !$isSiteAdmin) ? '' : sprintf( + sprintf( + ' (%s)', sprintf( - ' (%s)', - sprintf( - "'%s', 'events', 'toggleCorrelation', '', '#confirmation_box'", - h($event['Event']['id']) - ), - $event['Event']['disable_correlation'] ? 'color:white;' : '', - $event['Event']['disable_correlation'] ? __('enable') : __('disable') - ) + "'%s', 'events', 'toggleCorrelation', '', '#confirmation_box'", + h($event['Event']['id']) + ), + $event['Event']['disable_correlation'] ? 'color:white;' : '', + $event['Event']['disable_correlation'] ? __('enable') : __('disable') ) ) - ); - } + ) + ); } ?> From e9dc28fda7292fd49113e6b027f2fd3b88f81222 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 20 Apr 2020 08:51:01 +0200 Subject: [PATCH 017/159] chg: [sharingGroup:capture] Prevent capture of SG in some specific cases - Need more testing Should fix #5784 --- app/Model/Event.php | 81 +++++++++++++++++++++----------------- app/Model/SharingGroup.php | 21 +++++++++- 2 files changed, 65 insertions(+), 37 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index 8fe3d4cfe..d5ad62957 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -3263,10 +3263,10 @@ class Event extends AppModel return array($bodyevent, $body); } - private function __captureSGForElement($element, $user) + private function __captureSGForElement($element, $user, $syncLocal=false) { if (isset($element['SharingGroup'])) { - $sg = $this->SharingGroup->captureSG($element['SharingGroup'], $user); + $sg = $this->SharingGroup->captureSG($element['SharingGroup'], $user, $syncLocal); unset($element['SharingGroup']); } elseif (isset($element['sharing_group_id'])) { $sg = $this->SharingGroup->checkIfAuthorised($user, $element['sharing_group_id']) ? $element['sharing_group_id'] : false; @@ -3283,17 +3283,17 @@ class Event extends AppModel // When we receive an event via REST, we might end up with organisations, sharing groups, tags that we do not know // or which we need to update. All of that is controlled in this method. - private function __captureObjects($data, $user) + private function __captureObjects($data, $user, $syncLocal=false) { // First we need to check whether the event or any attributes are tied to a sharing group and whether the user is even allowed to create the sharing group / is part of it if (isset($data['Event']['distribution']) && $data['Event']['distribution'] == 4) { - $data['Event'] = $this->__captureSGForElement($data['Event'], $user); + $data['Event'] = $this->__captureSGForElement($data['Event'], $user, $syncLocal); } if (!empty($data['Event']['Attribute'])) { foreach ($data['Event']['Attribute'] as $k => $a) { unset($data['Event']['Attribute']['id']); if (isset($a['distribution']) && $a['distribution'] == 4) { - $data['Event']['Attribute'][$k] = $this->__captureSGForElement($a, $user); + $data['Event']['Attribute'][$k] = $this->__captureSGForElement($a, $user, $syncLocal); if ($data['Event']['Attribute'][$k] === false) { unset($data['Event']['Attribute']); } @@ -3303,7 +3303,7 @@ class Event extends AppModel if (!empty($data['Event']['Object'])) { foreach ($data['Event']['Object'] as $k => $o) { if (isset($o['distribution']) && $o['distribution'] == 4) { - $data['Event']['Object'][$k] = $this->__captureSGForElement($o, $user); + $data['Event']['Object'][$k] = $this->__captureSGForElement($o, $user, $syncLocal); if ($data['Event']['Object'][$k] === false) { unset($data['Event']['Object'][$k]); continue; @@ -3311,7 +3311,7 @@ class Event extends AppModel } foreach ($o['Attribute'] as $k2 => $a) { if (isset($a['distribution']) && $a['distribution'] == 4) { - $data['Event']['Object'][$k]['Attribute'][$k2] = $this->__captureSGForElement($a, $user); + $data['Event']['Object'][$k]['Attribute'][$k2] = $this->__captureSGForElement($a, $user, $syncLocal); if ($data['Event']['Object'][$k]['Attribute'][$k2] === false) { unset($data['Event']['Object'][$k]['Attribute'][$k2]); } @@ -3479,6 +3479,24 @@ class Event extends AppModel return 'blocked'; } } + if ($passAlong) { + $this->Server = ClassRegistry::init('Server'); + $server = $this->Server->find('first', array( + 'conditions' => array( + 'Server.id' => $passAlong + ), + 'recursive' => -1, + 'fields' => array( + 'Server.name', + 'Server.id', + 'Server.unpublish_event', + 'Server.publish_without_email', + 'Server.internal' + ) + )); + } else { + $server['Server']['internal'] = false; + } if ($fromXml) { // Workaround for different structure in XML/array than what CakePHP expects $data = $this->cleanupEventArrayFromXML($data); @@ -3505,7 +3523,7 @@ class Event extends AppModel return $existingEvent['Event']['id']; } else { if ($fromXml) { - $data = $this->__captureObjects($data, $user); + $data = $this->__captureObjects($data, $user, $server['Server']['internal']); } if ($data === false) { $failedCapture = true; @@ -3513,7 +3531,7 @@ class Event extends AppModel } } else { if ($fromXml) { - $data = $this->__captureObjects($data, $user); + $data = $this->__captureObjects($data, $user, $server['Server']['internal']); } if ($data === false) { $failedCapture = true; @@ -3574,19 +3592,6 @@ class Event extends AppModel $this->Log = ClassRegistry::init('Log'); if ($saveResult) { if ($passAlong) { - $this->Server = ClassRegistry::init('Server'); - $server = $this->Server->find('first', array( - 'conditions' => array( - 'Server.id' => $passAlong - ), - 'recursive' => -1, - 'fields' => array( - 'Server.name', - 'Server.id', - 'Server.unpublish_event', - 'Server.publish_without_email' - ) - )); if ($server['Server']['publish_without_email'] == 0) { $st = "enabled"; } else { @@ -3729,6 +3734,23 @@ class Event extends AppModel } else { $existingEvent = $this->findById($id); } + if ($passAlong) { + $this->Server = ClassRegistry::init('Server'); + $server = $this->Server->find('first', array( + 'conditions' => array( + 'Server.id' => $passAlong + ), + 'recursive' => -1, + 'fields' => array( + 'Server.name', + 'Server.id', + 'Server.unpublish_event', + 'Server.publish_without_email' + ) + )); + } else { + $server['Server']['internal'] = false; + } // If the event exists... $dateObj = new DateTime(); $date = $dateObj->getTimestamp(); @@ -3751,7 +3773,7 @@ class Event extends AppModel return(array('error' => 'Event could not be saved: Invalid sharing group or you don\'t have access to that sharing group.')); } } else { - $data['Event']['sharing_group_id'] = $this->SharingGroup->captureSG($data['Event']['SharingGroup'], $user); + $data['Event']['sharing_group_id'] = $this->SharingGroup->captureSG($data['Event']['SharingGroup'], $user, $server['Server']['internal']); unset($data['Event']['SharingGroup']); if ($data['Event']['sharing_group_id'] === false) { return (array('error' => 'Event could not be saved: User not authorised to create the associated sharing group.')); @@ -3872,19 +3894,6 @@ class Event extends AppModel if ((!empty($data['Event']['published']) && 1 == $data['Event']['published'])) { // The edited event is from a remote server ? if ($passAlong) { - $this->Server = ClassRegistry::init('Server'); - $server = $this->Server->find('first', array( - 'conditions' => array( - 'Server.id' => $passAlong - ), - 'recursive' => -1, - 'fields' => array( - 'Server.name', - 'Server.id', - 'Server.unpublish_event', - 'Server.publish_without_email' - ) - )); if ($server['Server']['publish_without_email'] == 0) { $st = "enabled"; } else { diff --git a/app/Model/SharingGroup.php b/app/Model/SharingGroup.php index 89c51a1aa..2e15f5182 100644 --- a/app/Model/SharingGroup.php +++ b/app/Model/SharingGroup.php @@ -485,7 +485,7 @@ class SharingGroup extends AppModel return $results; } - public function captureSG($sg, $user) + public function captureSG($sg, $user, $syncLocal=false) { $existingSG = !isset($sg['uuid']) ? null : $this->find('first', array( 'recursive' => -1, @@ -501,6 +501,25 @@ class SharingGroup extends AppModel if (!$user['Role']['perm_sharing_group']) { return false; } + // check if current user is contained in the SG and we are in a local sync setup + $authorizedToSave = $this->checkIfAuthorisedToSave($user, $sg); + if (!$user['Role']['perm_site_admin'] && + !($user['Role']['perm_sync'] && $syncLocal ) && + !$authorizedToSave + ) { + $this->Log->create(); + $entry = array( + 'org' => $user['Organisation']['name'], + 'model' => 'SharingGroup', + 'model_id' => $sg['SharingGroup']['uuid'], + 'email' => $user['email'], + 'action' => 'error', + 'user_id' => $user['id'], + 'title' => 'Tried to save a sharing group but the user does not belong to it.' + ); + $this->Log->save($entry); + return false; + } $this->create(); $newSG = array(); $attributes = array( From a99c96adcfdfbadc652a14cab392dc61638073b1 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 20 Apr 2020 09:43:53 +0200 Subject: [PATCH 018/159] fix: [attribute:add] Prevent save for invalid sharing_groups ids --- app/Controller/AttributesController.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 80006607d..13d89201e 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -148,6 +148,12 @@ class AttributesController extends AppController if (!isset($this->request->data['Attribute'])) { $this->request->data = array('Attribute' => $this->request->data); } + if ($this->request->data['Attribute']['distribution'] == 4) { + $sg = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1, $this->request->data['Attribute']['sharing_group_id']); + if (empty($sg)) { + throw new MethodNotAllowedException(__('Invalid Sharing Group or not authorised.')); + } + } // // multiple attributes in batch import // From f29474325d6e2d94302373474662dab0d9ce444c Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 20 Apr 2020 09:49:12 +0200 Subject: [PATCH 019/159] fix: [attribute:edit] Prevent save for invalid sharing_groups ids --- app/Controller/AttributesController.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 13d89201e..95766179a 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -858,6 +858,12 @@ class AttributesController extends AppController if (!isset($this->request->data['Attribute'])) { $this->request->data = array('Attribute' => $this->request->data); } + if ($this->request->data['Attribute']['distribution'] == 4) { + $sg = $this->Attribute->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1, $this->request->data['Attribute']['sharing_group_id']); + if (empty($sg)) { + throw new MethodNotAllowedException(__('Invalid Sharing Group or not authorised.')); + } + } $existingAttribute = $this->Attribute->findByUuid($this->Attribute->data['Attribute']['uuid']); // check if the attribute has a timestamp already set (from a previous instance that is trying to edit via synchronisation) // check which attribute is newer From 93ba84fd026d0b3f84ae265a1eadd51caeb03ea2 Mon Sep 17 00:00:00 2001 From: Golbark Date: Mon, 20 Apr 2020 12:24:47 +0200 Subject: [PATCH 020/159] Hook into native authentication flow instead of beforefilter which prevents any after-auth bypass and rely on framework session management. --- app/Controller/AppController.php | 8 +-- app/Controller/UsersController.php | 108 ++++++++++++++--------------- 2 files changed, 55 insertions(+), 61 deletions(-) diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index e9451f4c2..1e8c9c14b 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -354,7 +354,7 @@ class AppController extends Controller } } } else { - if ($this->params['controller'] !== 'users' || !in_array($this->params['action'], array('login', 'register'))) { + if ($this->params['controller'] !== 'users' || !in_array($this->params['action'], array('login', 'register', 'email_otp'))) { if (!$this->request->is('ajax')) { $this->Session->write('pre_login_requested_url', $this->here); } @@ -408,12 +408,6 @@ class AppController extends Controller if (!$this->_isRest()) { $this->redirect(array('controller' => 'users', 'action' => 'change_pw', 'admin' => false)); } - } elseif (Configure::read('Security.email_otp_enabled') && !$this->_isRest() && !in_array($this->request->here, array($base_dir.'/users/terms', $base_dir.'/users/email_otp', $base_dir.'/users/change_pw', $base_dir.'/users/logout', $base_dir.'/users/login'))) { - $redis = $this->{$this->modelClass}->setupRedis(); - $otp_authed = $redis->get('misp:otp_authed:'.session_id()); - if (empty($otp_authed)) { - $this->redirect(array('controller' => 'users', 'action' => 'email_otp', 'admin' => false)); - } } elseif (!$this->_isRest() && !($this->params['controller'] == 'news' && $this->params['action'] == 'index') && (!in_array($this->request->here, array($base_dir.'/users/terms', $base_dir.'/users/change_pw', $base_dir.'/users/logout', $base_dir.'/users/login')))) { $newsread = $this->User->field('newsread', array('User.id' => $this->Auth->user('id'))); $this->loadModel('News'); diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php index b95cc55db..2da9da633 100644 --- a/app/Controller/UsersController.php +++ b/app/Controller/UsersController.php @@ -31,6 +31,9 @@ class UsersController extends AppController // what pages are allowed for non-logged-in users $allowedActions = array('login', 'logout'); + if(!empty(Configure::read('Security.email_otp_enabled'))) { + $allowedActions[] = 'email_otp'; + } if (!empty(Configure::read('Security.allow_self_registration'))) { $allowedActions[] = 'register'; } @@ -1116,33 +1119,15 @@ class UsersController extends AppController $this->Auth->constructAuthenticate(); } } + if ($this->request->is('post') && Configure::read('Security.email_otp_enabled')) { + $user = $this->Auth->identify($this->request, $this->response); + if ($user) { + $this->Session->write('email_otp_user', $user); + return $this->redirect('email_otp'); + } + } if ($this->Auth->login()) { - $this->User->extralog($this->Auth->user(), "login"); - $this->User->Behaviors->disable('SysLogLogable.SysLogLogable'); - $this->User->id = $this->Auth->user('id'); - $user = $this->User->find('first', array( - 'conditions' => array( - 'User.id' => $this->Auth->user('id') - ), - 'recursive' => -1 - )); - $lastUserLogin = $user['User']['last_login']; - unset($user['User']['password']); - $user['User']['action'] = 'login'; - $user['User']['last_login'] = $this->Auth->user('current_login'); - $user['User']['current_login'] = time(); - $this->User->save($user['User'], true, array('id', 'last_login', 'current_login')); - if (empty($this->Auth->authenticate['Form']['passwordHasher']) && !empty($passwordToSave)) { - $this->User->saveField('password', $passwordToSave); - } - $this->User->Behaviors->enable('SysLogLogable.SysLogLogable'); - if ($lastUserLogin) { - $readableDatetime = (new DateTime())->setTimestamp($lastUserLogin)->format('D, d M y H:i:s O'); // RFC822 - $this->Flash->info(sprintf('Welcome! Last login was on %s', $readableDatetime)); - } - // no state changes are ever done via GET requests, so it is safe to return to the original page: - $this->redirect($this->Auth->redirectUrl()); - // $this->redirect(array('controller' => 'events', 'action' => 'index')); + $this->_postlogin(); } else { $dataSourceConfig = ConnectionManager::getDataSource('default')->config; $dataSource = $dataSourceConfig['datasource']; @@ -1224,6 +1209,35 @@ class UsersController extends AppController } } + private function _postlogin() + { + $this->User->extralog($this->Auth->user(), "login"); + $this->User->Behaviors->disable('SysLogLogable.SysLogLogable'); + $this->User->id = $this->Auth->user('id'); + $user = $this->User->find('first', array( + 'conditions' => array( + 'User.id' => $this->Auth->user('id') + ), + 'recursive' => -1 + )); + $lastUserLogin = $user['User']['last_login']; + unset($user['User']['password']); + $user['User']['action'] = 'login'; + $user['User']['last_login'] = $this->Auth->user('current_login'); + $user['User']['current_login'] = time(); + $this->User->save($user['User'], true, array('id', 'last_login', 'current_login')); + if (empty($this->Auth->authenticate['Form']['passwordHasher']) && !empty($passwordToSave)) { + $this->User->saveField('password', $passwordToSave); + } + $this->User->Behaviors->enable('SysLogLogable.SysLogLogable'); + if ($lastUserLogin) { + $readableDatetime = (new DateTime())->setTimestamp($lastUserLogin)->format('D, d M y H:i:s O'); // RFC822 + $this->Flash->info(sprintf('Welcome! Last login was on %s', $readableDatetime)); + } + // no state changes are ever done via GET requests, so it is safe to return to the original page: + $this->redirect($this->Auth->redirectUrl()); + } + public function routeafterlogin() { // Events list @@ -1259,8 +1273,6 @@ class UsersController extends AppController 'recursive' => -1 )); unset($user['User']['password']); - $redis = $this->User->setupRedis(); - $redis->delete('misp:otp_authed:'.session_id()); $user['User']['action'] = 'logout'; $this->User->save($user['User'], true, array('id')); $this->redirect($this->Auth->logout()); @@ -1660,48 +1672,36 @@ class UsersController extends AppController public function email_otp() { + $user = $this->Session->read('email_otp_user'); + if(empty($user)) { + $this->redirect('login'); + } $redis = $this->User->setupRedis(); - $user_id = $this->Auth->user('id'); - $session_id = session_id(); + $user_id = $user['id']; if ($this->request->is('post') && isset($this->request->data['User']['otp'])) { $stored_otp = $redis->get('misp:otp:'.$user_id); if (!empty($stored_otp) && $this->request->data['User']['otp'] == $stored_otp) { // we invalidate the previously generated OTP $redis->delete('misp:otp:'.$user_id); - // We store in redis the success of the OTP step - $redis->set('misp:otp_authed:'.$session_id, 1); - // After this time, the user will need to redo the OTP step - // We use the same time as for the session expiration - $redis->expire('misp:otp_authed:'.$session_id, (int) Configure::read('Session.cookieTimeout') * 60); - $this->Flash->success(__("You are now logged in.")); - $this->redirect($this->Auth->redirectUrl()); + // We login the user with CakePHP + $this->Auth->login($user); + $this->_postlogin(); } else { $this->Flash->error(__("The OTP is incorrect or has expired")); } } else { // GET Request - // If the OTP is still valid, we redirect - if (!Configure::read('Security.email_otp_enabled') || !empty($redis->get('misp:otp_authed:'.$session_id))) { - $this->redirect($this->Auth->redirectUrl()); - } - - $user = $this->User->find('first', array( - 'recursive' => -1, - 'conditions' => array('User.id' => $user_id) - )); - // We check for exceptions $exception_list = Configure::read('Security.email_otp_exceptions'); if (!empty($exception_list)) { $exceptions = explode(",", $exception_list); foreach ($exceptions as &$exception) { - if ($user['User']['email'] == trim($exception)) { - $redis->set('misp:otp_authed:'.$session_id, 1); - // It will take maximum this time (in seconds) to ask a OTP for someone removed from the exception list - $redis->expire('misp:otp_authed:'.$session_id, 3600); - $this->redirect($this->Auth->redirectUrl()); + if ($user['email'] == trim($exception)) { + // We login the user with CakePHP + $this->Auth->login($user); + $this->_postlogin(); } } } @@ -1726,8 +1726,8 @@ class UsersController extends AppController $body = str_replace('$validity', $validity, $body); $body = str_replace('$otp', $otp, $body); $body = str_replace('$ip', $this->_getClientIP(), $body); - $body = str_replace('$username', $user['User']['email'], $body); - $result = $this->User->sendEmail($user, $body, false, "[MISP] Email OTP"); + $body = str_replace('$username', $user['email'], $body); + $result = $this->User->sendEmail(array('User' => $user), $body, false, "[MISP] Email OTP"); if ( $result ) { $this->Flash->success(__("An email containing a OTP has been sent.")); From 93bd5eddba6b53b34e3aefad064ee6a849203056 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 23 Apr 2020 10:08:34 +0200 Subject: [PATCH 021/159] chg: [event:timeline] Added Sightings visualisation --- app/Controller/EventsController.php | 8 +- app/Lib/Tools/EventTimelineTool.php | 141 ++++++++++++++++++++++++++++ app/webroot/css/event-timeline.css | 9 ++ app/webroot/js/event-timeline.js | 57 +++++++++-- 4 files changed, 207 insertions(+), 8 deletions(-) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 220f1989c..f13107bc3 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -4514,16 +4514,20 @@ class EventsController extends AppController if (!in_array($type, $validTools)) { throw new MethodNotAllowedException('Invalid type.'); } - App::uses('EventTimelineTool', 'Tools'); $grapher = new EventTimelineTool(); $data = $this->request->is('post') ? $this->request->data : array(); $dataFiltering = array_key_exists('filtering', $data) ? $data['filtering'] : array(); + $scope = isset($data['scope']) ? $data['scope'] : 'seen'; $extended = isset($this->params['named']['extended']) ? 1 : 0; $grapher->construct($this->Event, $this->Auth->user(), $dataFiltering, $extended); - $json = $grapher->get_timeline($id); + if ($scope == 'seen') { + $json = $grapher->get_timeline($id); + } elseif ($scope == 'sightings') { + $json = $grapher->get_sighting_timeline($id); + } array_walk_recursive($json, function (&$item, $key) { if (!mb_detect_encoding($item, 'utf-8', true)) { diff --git a/app/Lib/Tools/EventTimelineTool.php b/app/Lib/Tools/EventTimelineTool.php index d2eb44943..46fef4604 100644 --- a/app/Lib/Tools/EventTimelineTool.php +++ b/app/Lib/Tools/EventTimelineTool.php @@ -135,4 +135,145 @@ return $this->__json; } + + /* + * Extrapolation strategy: + * - If only positive sightings: Will be from first to last sighting + * - If both positive and false positive: False positive get priority. It will be marked as false positive until next positive sighting + */ + public function get_sighting_timeline($id) + { + $event = $this->__eventModel->fetchEvent($this->__user, array( + 'eventid' => $id, + 'flatten' => 1, + 'includeTagRelations' => 1, + 'extended' => $this->__extended_view + )); + $this->__json['items'] = array(); + + if (empty($event)) { + return $this->__json; + } else { + $event = $event[0]; + } + + $lookupAttribute = array(); + foreach ($event['Attribute'] as $k => $attribute) { + $lookupAttribute[$attribute['id']] = &$event['Attribute'][$k]; + } + + // regroup sightings per attribute + $regroupedSightings = array(); + foreach ($event['Sighting'] as $k => $sighting) { + $event['Sighting'][$k]['date_sighting'] *= 1000; // adapt to use micro + $regroupedSightings[$sighting['attribute_id']][] = &$event['Sighting'][$k]; + } + // make sure sightings are ordered + uksort($regroupedSightings, function ($a, $b) { + return $a['date_sighting'] > $b['date_sighting']; + }); + // generate extrapolation + $now = time()*1000; + foreach ($regroupedSightings as $attributeId => $sightings) { + $i = 0; + while ($i < count($sightings)) { + $sighting = $sightings[$i]; + $attribute = $lookupAttribute[$attributeId]; + $fpSightingIndex = $this->getNextFalsePositiveSightingIndex($sightings, $i+1); + if ($fpSightingIndex === false) { // No next FP, extrapolate to now + $this->__json['items'][] = array( + 'attribute_id' => $attributeId, + 'id' => sprintf('%s-%s', $attributeId, $sighting['id']), + 'uuid' => $sighting['uuid'], + 'content' => $attribute['value'], + 'event_id' => $attribute['event_id'], + 'group' => 'sighting_positive', + 'timestamp' => $attribute['timestamp'], + 'first_seen' => $sighting['date_sighting'], + 'last_seen' => $now, + ); + break; + } else { + // set up until last positive + $pSightingIndex = $fpSightingIndex - 1; + $halfTime = 0; + if ($pSightingIndex == $i) { + // we have only one positive sighting, thus the UP time should be take from a pooling frequence + // for now, consider it UP only for half the time until the next FP + $halfTime = ($sightings[$i+1]['date_sighting'] - $sighting['date_sighting'])/2; + } + $pSighting = $sightings[$pSightingIndex]; + $this->__json['items'][] = array( + 'attribute_id' => $attributeId, + 'id' => sprintf('%s-%s', $attributeId, $sighting['id']), + 'uuid' => $sighting['uuid'], + 'content' => $attribute['value'], + 'event_id' => $attribute['event_id'], + 'group' => 'sighting_positive', + 'timestamp' => $attribute['timestamp'], + 'first_seen' => $sighting['date_sighting'], + 'last_seen' => $pSighting['date_sighting'] + $halfTime, + ); + // No next FP, extrapolate to now + $fpSighting = $sightings[$fpSightingIndex]; + $secondNextPSightingIndex = $this->getNextPositiveSightingIndex($sightings, $fpSightingIndex+1); + if ($secondNextPSightingIndex === false) { // No next P, extrapolate to now + $this->__json['items'][] = array( + 'attribute_id' => $attributeId, + 'id' => sprintf('%s-%s', $attributeId, $sighting['id']), + 'uuid' => $sighting['uuid'], + 'content' => $attribute['value'], + 'event_id' => $attribute['event_id'], + 'group' => 'sighting_negative', + 'timestamp' => $attribute['timestamp'], + 'first_seen' => $pSighting['date_sighting'] - $halfTime, + 'last_seen' => $now, + ); + break; + } else { + if ($halfTime > 0) { // We need to fake a previous P + $pSightingIndex = $pSightingIndex+1; + $pSighting = $sightings[$pSightingIndex]; + } + // set down until next postive + $secondNextPSighting = $sightings[$secondNextPSightingIndex]; + $this->__json['items'][] = array( + 'attribute_id' => $attributeId, + 'id' => sprintf('%s-%s', $attributeId, $sighting['id']), + 'uuid' => $pSighting['uuid'], + 'content' => $attribute['value'], + 'event_id' => $attribute['event_id'], + 'group' => 'sighting_negative', + 'timestamp' => $attribute['timestamp'], + 'first_seen' => $pSighting['date_sighting'] - $halfTime, + 'last_seen' => $secondNextPSighting['date_sighting'], + ); + $i = $secondNextPSightingIndex; + } + } + } + } + return $this->__json; + } + + private function getNextFalsePositiveSightingIndex($sightings, $startIndex) + { + for ($i=$startIndex; $i < count($sightings) ; $i++) { + $sighting = $sightings[$i]; + if ($sighting['type'] == 1) { // is false positive + return $i; + } + } + return false; + } + private function getNextPositiveSightingIndex($sightings, $startIndex) + { + for ($i=$startIndex; $i < count($sightings) ; $i++) { + $sighting = $sightings[$i]; + if ($sighting['type'] == 0) { // is false positive + return $i; + } + } + return false; + } } diff --git a/app/webroot/css/event-timeline.css b/app/webroot/css/event-timeline.css index d5653e776..b82318aa7 100644 --- a/app/webroot/css/event-timeline.css +++ b/app/webroot/css/event-timeline.css @@ -59,6 +59,15 @@ box-shadow: 0 0 20px rgba(82, 168, 236, 1);` } +.vis-item.sighting_positive { + background-color: green; + border-color: white; +} +.vis-item.sighting_negative { + background-color: red; + border-color: white; +} + .vis-item.object { background-color: #3465a4; border-color: black; diff --git a/app/webroot/js/event-timeline.js b/app/webroot/js/event-timeline.js index 3d810143b..2e00638a1 100644 --- a/app/webroot/js/event-timeline.js +++ b/app/webroot/js/event-timeline.js @@ -29,10 +29,16 @@ var options = { return build_object_template(item); case "object_attribute": - console.log('Error'); + console.log('Error: Group not valid'); break; default: + if (item.className == "sighting_positive" || item.className == "sighting_negative") { + return build_sighting_template(item); + } else { + console.log(item) + console.log('Error: Unkown group'); + } break; } }, @@ -199,6 +205,13 @@ function build_object_template(obj) { return html; } +function build_sighting_template(attr){ + var span = $(''); + span.text(attr.content); + var html = span[0].outerHTML; + return html; +} + function contain_seen_attribute(obj) { if (obj['Attribute'] === undefined) { return false; @@ -372,6 +385,8 @@ function map_scope(val) { return 'seen'; case 'Object relationship': return 'relationship'; + case 'Sightings': + return 'sightings'; default: return 'seen'; } @@ -400,7 +415,8 @@ function update_badge() { function reload_timeline() { update_badge(); - var payload = {scope: map_scope($('#select_timeline_scope').val())}; + var selectedScope = map_scope($('#select_timeline_scope').val()); + var payload = {scope: selectedScope}; $.ajax({ url: "/events/"+"getEventTimeline"+"/"+scope_id+"/"+extended_text+"event.json", dataType: 'json', @@ -413,6 +429,8 @@ function reload_timeline() { }, success: function( data, textStatus, jQxhr ){ items_timeline.clear(); + mapping_text_to_id = new Map(); + var itemIds = {}; for (var item of data.items) { item.className = item.group; item.orig_id = item.id; @@ -420,16 +438,43 @@ function reload_timeline() { set_spanned_time(item); if (item.group == 'object') { for (var attr of item.Attribute) { - mapping_text_to_id.set(attr.contentType+': '+attr.content+' ('+item.orig_id+')', item.id); + if (selectedScope == 'sightings') { + var k = attr.contentType+': '+attr.content+' ('+item.orig_id.split('-')[0]+')' + if (!mapping_text_to_id.get(k)) { + mapping_text_to_id.set(k, item.id); + } + } else { + mapping_text_to_id.set(attr.contentType+': '+attr.content+' ('+item.orig_id+')', item.id); + } adjust_text_length(attr); } } else { - mapping_text_to_id.set(item.content+' ('+item.orig_id+')', item.id); + if (selectedScope == 'sightings') { + var k = item.content+' ('+item.orig_id.split('-')[0]+')' + if (!mapping_text_to_id.get(k)) { + mapping_text_to_id.set(k, item.id); + } + } else { + mapping_text_to_id.set(item.content+' ('+item.orig_id+')', item.id); + } adjust_text_length(item); } + itemIds[item.attribute_id] = item.content; + if (selectedScope == 'sightings') { + item.group = item.attribute_id; + item.content = ''; + } } items_timeline.add(data.items); handle_not_seen_enabled($('#checkbox_timeline_display_hide_not_seen_enabled').prop('checked'), false) + if (selectedScope == 'sightings') { + var groups = Object.keys(itemIds).map(function(id) { + return {id: id, content: itemIds[id]} + }) + eventTimeline.setGroups(groups); + } else { + eventTimeline.setGroups([]); + } }, error: function( jqXhr, textStatus, errorThrown ){ console.log( errorThrown ); @@ -610,11 +655,11 @@ function init_popover() { label: "Scope", tooltip: "The time scope represented by the timeline", event: function(value) { - if (value == "First seen/Last seen") { + if (value == "First seen/Last seen" || value == "Sightings") { reload_timeline(); } }, - options: ["First seen/Last seen"], + options: ["First seen/Last seen", "Sightings"], default: "First seen/Last seen" }); From 5034c1c798f22fc2054f46569f550609dffe443a Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 23 Apr 2020 10:18:34 +0200 Subject: [PATCH 022/159] chg: [event:timeline] Prevent item selection while in the sighting context --- app/webroot/js/event-timeline.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/webroot/js/event-timeline.js b/app/webroot/js/event-timeline.js index 2e00638a1..787b056aa 100644 --- a/app/webroot/js/event-timeline.js +++ b/app/webroot/js/event-timeline.js @@ -472,7 +472,9 @@ function reload_timeline() { return {id: id, content: itemIds[id]} }) eventTimeline.setGroups(groups); + eventTimeline.setOptions({selectable: false}); } else { + eventTimeline.setOptions({selectable: true}); eventTimeline.setGroups([]); } }, From c3fd04e5e333afa8a597586281c7b8422adcfe2b Mon Sep 17 00:00:00 2001 From: Applenice Date: Fri, 24 Apr 2020 22:20:00 +0800 Subject: [PATCH 023/159] Modify the default parsing settings of Phishtank feed --- app/files/feed-metadata/defaults.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/feed-metadata/defaults.json b/app/files/feed-metadata/defaults.json index 4965e1da9..33ff72ece 100644 --- a/app/files/feed-metadata/defaults.json +++ b/app/files/feed-metadata/defaults.json @@ -232,7 +232,7 @@ "event_id": "5655", "publish": false, "override_ids": false, - "settings": "{\"csv\":{\"value\":\"\",\"delimiter\":\"\"},\"common\":{\"excluderegex\":\"\\/^http:\\\\\\/\\\\\\/www.phishtank.com\\/i\"}}", + "settings": "{\"csv\":{\"value\":\"2\",\"delimiter\":\",\"},\"common\":{\"excluderegex\":\"\\/^http:\\\\\\/\\\\\\/www.phishtank.com\\/i\"}}", "input_source": "network", "delete_local_file": false, "lookup_visible": false, From 6950db8eb6f1cafa3ed25c8e39156f58ef4b2c60 Mon Sep 17 00:00:00 2001 From: kscheetz Date: Fri, 24 Apr 2020 13:10:30 -0400 Subject: [PATCH 024/159] Stix2 importer naming change. --- app/files/scripts/stix2/stix2misp.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/app/files/scripts/stix2/stix2misp.py b/app/files/scripts/stix2/stix2misp.py index f3ab86084..322e08290 100644 --- a/app/files/scripts/stix2/stix2misp.py +++ b/app/files/scripts/stix2/stix2misp.py @@ -130,12 +130,17 @@ class StixParser(): def build_from_STIX_with_report(self): report_attributes = defaultdict(set) + report_attributes['name'] = None + for ruuid, report in self.report.items(): try: report_attributes['orgs'].add(report.created_by_ref.split('--')[1]) except AttributeError: pass - report_attributes['name'].add(report.name) + + if report_attributes['name'] is None: + report_attributes['name'] = report.name + if report.get('published'): report_attributes['published'].add(report.published) if 'labels' in report: @@ -155,10 +160,14 @@ class StixParser(): self.misp_event['Org'] = {'name': identity['name']} if len(report_attributes['published']) == 1: self.misp_event.publish_timestamp = self.getTimestampfromDate(report_attributes['published'].pop()) - if len(report_attributes['name']) == 1: - self.misp_event.info = report_attributes['name'].pop() + + if report_attributes['name'] is None: + self.misp_event.info = "Imported with MISP import script for {} from {}.".format(self.stix_version, + os.path.basename( + self.filename)) else: - self.misp_event.info = "Imported with MISP import script for {}.".format(self.stix_version) + self.misp_event.info = report_attributes['name'] + for l in report_attributes['labels']: self.misp_event.add_tag(l) From 84bdfbc8d6a3a320f9b25f9393063a2043d483e5 Mon Sep 17 00:00:00 2001 From: kscheetz Date: Fri, 24 Apr 2020 13:13:55 -0400 Subject: [PATCH 025/159] Preserve report order. --- app/files/scripts/stix2/stix2misp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/files/scripts/stix2/stix2misp.py b/app/files/scripts/stix2/stix2misp.py index 322e08290..980b59ad3 100644 --- a/app/files/scripts/stix2/stix2misp.py +++ b/app/files/scripts/stix2/stix2misp.py @@ -24,7 +24,7 @@ import io import re import stix2 from stix2misp_mapping import * -from collections import defaultdict +from collections import defaultdict, OrderedDict _MISP_dir = "/".join([p for p in os.path.dirname(os.path.realpath(__file__)).split('/')[:-4]]) _PyMISP_dir = '{_MISP_dir}/PyMISP'.format(_MISP_dir=_MISP_dir) @@ -107,7 +107,7 @@ class StixParser(): try: self.report[parsed_object['id'].split('--')[1]] = parsed_object except AttributeError: - self.report = {parsed_object['id'].split('--')[1]: parsed_object} + self.report = OrderedDict({parsed_object['id'].split('--')[1]: parsed_object}) def _load_usual_object(self, parsed_object): self.event[parsed_object._type][parsed_object['id'].split('--')[1]] = parsed_object From 9b45aac8101aff0736469df9023cd8cca19403d8 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 26 Apr 2020 10:22:27 +0200 Subject: [PATCH 026/159] fix: Remove unused variable --- app/Model/Server.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/Model/Server.php b/app/Model/Server.php index ea5aef715..f3621ddea 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -3901,7 +3901,6 @@ class Server extends AppModel } else { $serverSettings = $this->serverSettings; } - $relevantSettings = (array_intersect_key(Configure::read(), $serverSettings)); $setting = false; foreach ($serverSettings as $k => $s) { if (isset($s['branch'])) { From 37f8699a919b3f75be326d6e034949fa91688e4a Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 26 Apr 2020 10:57:55 +0200 Subject: [PATCH 027/159] fix: [internal] Remove unused code --- app/Model/Server.php | 9 --------- 1 file changed, 9 deletions(-) diff --git a/app/Model/Server.php b/app/Model/Server.php index f3621ddea..9a19c859b 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -3113,15 +3113,6 @@ class Server extends AppModel private function readModuleSettings($serverSettings, $moduleTypes) { $this->Module = ClassRegistry::init('Module'); - $orgs = $this->Organisation->find('list', array( - 'conditions' => array( - 'Organisation.local' => 1 - ), - 'fields' => array( - 'Organisation.id', 'Organisation.name' - ) - )); - $orgs = array_merge(array('Unrestricted'), $orgs); foreach ($moduleTypes as $moduleType) { if (Configure::read('Plugin.' . $moduleType . '_services_enable')) { $results = $this->Module->getModuleSettings($moduleType); From f0ada4196372f7b4b683121f68f3299c58366114 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 27 Jan 2020 21:22:18 +0100 Subject: [PATCH 028/159] chg: [internal] Speed up of loading event page --- app/Controller/EventsController.php | 24 ++++++++---------------- app/Model/Attribute.php | 24 ++++++++++++++++++++++++ app/Model/Event.php | 23 +++++++++++++++++++++++ 3 files changed, 55 insertions(+), 16 deletions(-) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 220f1989c..ed56eb208 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -1403,12 +1403,8 @@ class EventsController extends AppController $this->set($alias, $currentModel->{$variable}); } } - $cluster_names = $this->GalaxyCluster->find('list', array('fields' => array('GalaxyCluster.tag_name'), 'group' => array('GalaxyCluster.tag_name', 'GalaxyCluster.id'))); - foreach ($event['EventTag'] as $k => $eventTag) { - if (in_array($eventTag['Tag']['name'], $cluster_names)) { - unset($event['EventTag'][$k]); - } - } + + $this->Event->removeGalaxyClusterTags($event); $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($event['EventTag']); foreach ($tagConflicts['global'] as $tagConflict) { @@ -1430,11 +1426,9 @@ class EventsController extends AppController } $modDate = date("Y-m-d", $attribute['timestamp']); $modificationMap[$modDate] = empty($modificationMap[$modDate])? 1 : $modificationMap[date("Y-m-d", $attribute['timestamp'])] + 1; - foreach ($attribute['AttributeTag'] as $k2 => $attributeTag) { - if (in_array($attributeTag['Tag']['name'], $cluster_names)) { - unset($event['Attribute'][$k]['AttributeTag'][$k2]); - } - } + + $this->Event->Attribute->removeGalaxyClusterTags($event['Attribute'][$k]); + $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); foreach ($tagConflicts['global'] as $tagConflict) { $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; @@ -1463,11 +1457,9 @@ class EventsController extends AppController } $modDate = date("Y-m-d", $attribute['timestamp']); $modificationMap[$modDate] = empty($modificationMap[$modDate])? 1 : $modificationMap[date("Y-m-d", $attribute['timestamp'])] + 1; - foreach ($attribute['AttributeTag'] as $k3 => $attributeTag) { - if (in_array($attributeTag['Tag']['name'], $cluster_names)) { - unset($event['Object'][$k]['Attribute'][$k2]['AttributeTag'][$k3]); - } - } + + $this->Event->Attribute->removeGalaxyClusterTags($event['Object'][$k]['Attribute'][$k2]); + $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']); foreach ($tagConflicts['global'] as $tagConflict) { $warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy']; diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 6f6b5e982..ae2b50723 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -4639,4 +4639,28 @@ class Attribute extends AppModel } return $conditions; } + + /** + * @param array $attribute + */ + public function removeGalaxyClusterTags(array &$attribute) + { + $galaxyTagIds = array(); + foreach ($attribute['Galaxy'] as $galaxy) { + foreach ($galaxy['GalaxyCluster'] as $galaxyCluster) { + $galaxyTagIds[$galaxyCluster['tag_id']] = true; + } + } + + if (empty($galaxyTagIds)) { + return; + } + + foreach ($attribute['AttributeTag'] as $k => $attributeTag) { + $tagId = $attributeTag['Tag']['id']; + if (isset($galaxyTagIds[$tagId])) { + unset($attribute['AttributeTag'][$k]); + } + } + } } diff --git a/app/Model/Event.php b/app/Model/Event.php index c1cddda5a..ea3a8fc75 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -6948,4 +6948,27 @@ class Event extends AppModel } return $filters; } + + /** + * @param array $event + */ + public function removeGalaxyClusterTags(array &$event) + { + $galaxyTagIds = array(); + foreach ($event['Galaxy'] as $galaxy) { + foreach ($galaxy['GalaxyCluster'] as $galaxyCluster) { + $galaxyTagIds[$galaxyCluster['tag_id']] = true; + } + } + + if (empty($galaxyTagIds)) { + return; + } + + foreach ($event['EventTag'] as $k => $eventTag) { + if (isset($galaxyTagIds[$eventTag['tag_id']])) { + unset($event['EventTag'][$k]); + } + } + } } From a2933030b6ab93a0a03616ec59be5bee3c31391b Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 27 Apr 2020 18:19:29 +0200 Subject: [PATCH 029/159] fix: [internal] syslog shouldn't end with new line Because then two lines are logged --- app/Plugin/SysLog/Lib/SysLog.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Plugin/SysLog/Lib/SysLog.php b/app/Plugin/SysLog/Lib/SysLog.php index 6f23144b2..cf0a71dd2 100644 --- a/app/Plugin/SysLog/Lib/SysLog.php +++ b/app/Plugin/SysLog/Lib/SysLog.php @@ -68,10 +68,10 @@ class SysLog { } else if (in_array($type, $debugTypes)) { $priority = LOG_DEBUG; } - $output = date('Y-m-d H:i:s') . ' ' . ucfirst($type) . ': ' . $message . "\n"; if (!openlog($this->_ident, LOG_PID | LOG_PERROR, $this->_facility)) { return false; } + $output = date('Y-m-d H:i:s') . ' ' . ucfirst($type) . ': ' . $message; $result = syslog($priority, $output); closelog(); return $result; From 3c5e44fa8d1d802e1905ceabc89ab5090762bf75 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Mon, 27 Apr 2020 23:30:27 +0200 Subject: [PATCH 030/159] chg: [internal] Removed unused function This function has typo in name `beforeValid*e*te`, so its never called. And because everything works, I think it is safe to remove it. --- app/Model/Log.php | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/app/Model/Log.php b/app/Model/Log.php index 2f3d9474b..c0dc91ed8 100644 --- a/app/Model/Log.php +++ b/app/Model/Log.php @@ -96,18 +96,6 @@ class Log extends AppModel 'email' => array('values' => array('admin_email')) ); - public function beforeValidete() - { - parent::beforeValidate(); - if (!isset($this->data['Log']['org']) || empty($this->data['Log']['org'])) { - $this->data['Log']['org'] = 'SYSTEM'; - } - // truncate the description if it would exceed the allowed size in mysql - if (!empty($this->data['Log']['description'] && strlen($this->data['Log']['description']) > 65536)) { - $this->data['Log']['description'] = substr($this->data['Log']['description'], 0, 65535); - } - } - public function beforeSave($options = array()) { if (!empty(Configure::read('MISP.log_skip_db_logs_completely'))) { From ad439e0b4d50b0093465b791faa81d9465769ff7 Mon Sep 17 00:00:00 2001 From: iglocska Date: Tue, 28 Apr 2020 09:28:42 +0200 Subject: [PATCH 031/159] fix: [pagination] Fixed bottom pagination links on the bottom --- .../Elements/genericElements/IndexTable/index_table.ctp | 3 ++- .../Elements/genericElements/IndexTable/pagination.ctp | 6 ------ .../genericElements/IndexTable/pagination_links.ctp | 7 +++++++ 3 files changed, 9 insertions(+), 7 deletions(-) create mode 100644 app/View/Elements/genericElements/IndexTable/pagination_links.ctp diff --git a/app/View/Elements/genericElements/IndexTable/index_table.ctp b/app/View/Elements/genericElements/IndexTable/index_table.ctp index c6ead8ee9..7d4bdd66a 100644 --- a/app/View/Elements/genericElements/IndexTable/index_table.ctp +++ b/app/View/Elements/genericElements/IndexTable/index_table.ctp @@ -32,6 +32,7 @@ if (!$skipPagination) { $paginationData = !empty($data['paginatorOptions']) ? $data['paginatorOptions'] : array(); echo $this->element('/genericElements/IndexTable/pagination', array('paginationOptions' => $paginationData)); + echo $this->element('/genericElements/IndexTable/pagination_links'); } if (!empty($data['top_bar'])) { echo $this->element('/genericElements/ListTopBar/scaffold', array('data' => $data['top_bar'])); @@ -81,7 +82,7 @@ echo ''; if (!$skipPagination) { echo $this->element('/genericElements/IndexTable/pagination_counter', $paginationData); - echo $this->element('/genericElements/IndexTable/pagination', $paginationData); + echo $this->element('/genericElements/IndexTable/pagination_links'); } ?>