diff --git a/.travis.yml b/.travis.yml index 5509ecb9e..dc3c8986f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -147,6 +147,7 @@ before_script: - popd script: + - ./app/Vendor/bin/phpunit app/Test/ComplexTypeToolTest.php - pushd tests - ./curl_tests.sh $AUTH - popd diff --git a/INSTALL/INSTALL.sh b/INSTALL/INSTALL.sh index f5e479d2d..2541158a3 100755 --- a/INSTALL/INSTALL.sh +++ b/INSTALL/INSTALL.sh @@ -83,27 +83,33 @@ MISPvars () { # RHEL/CentOS if [[ -f "/etc/redhat-release" ]]; then WWW_USER='apache' + SUDO_WWW="sudo -H -u ${WWW_USER} " # Debian flavoured elif [[ -f "/etc/debian_version" ]]; then WWW_USER="www-data" + SUDO_WWW="sudo -H -u ${WWW_USER} " # OpenBSD elif [[ "$(uname -s)" == "OpenBSD" ]]; then WWW_USER="www" PATH_TO_MISP="/var/www/htdocs/MISP" + SUDO_WWW="doas -u www " + SUDO_CMD="doas " # NetBSD elif [[ "$(uname -s)" == "NetBSD" ]]; then WWW_USER="www" PATH_TO_MISP="/usr/pkg/share/httpd/htdocs/MISP" + SUDO_WWW="sudo -H -u ${WWW_USER} " else # I am feeling lucky WWW_USER="www-data" + SUDO_WWW="sudo -H -u ${WWW_USER} " fi - if [ -z "$FQDN" ]; then + if [ -z "${FQDN}" ]; then FQDN="misp.local" fi - if [ -z "$MISP_BASEURL" ]; then + if [ -z "${MISP_BASEURL}" ]; then MISP_BASEURL='""' fi @@ -118,13 +124,13 @@ MISPvars () { DBPASSWORD_MISP="$(openssl rand -hex 32)" # OpenSSL configuration - OPENSSL_CN=$FQDN + OPENSSL_CN=${FQDN} OPENSSL_C='LU' OPENSSL_ST='State' OPENSSL_L='Location' OPENSSL_O='Organization' OPENSSL_OU='Organizational Unit' - OPENSSL_EMAILADDRESS="info@$FQDN" + OPENSSL_EMAILADDRESS="info@${FQDN}" # GPG configuration GPG_REAL_NAME='Autogenerated Key' @@ -147,7 +153,7 @@ MISPvars () { max_execution_time=300 memory_limit=2048M - CAKE="$PATH_TO_MISP/app/Console/cake" + CAKE="${PATH_TO_MISP}/app/Console/cake" # sudo config to run $LUSER commands if [[ "$(groups ${MISP_USER} |grep -o 'staff')" == "staff" ]]; then @@ -155,8 +161,7 @@ MISPvars () { else SUDO_CMD="sudo -H -u ${MISP_USER}" fi - SUDO_WWW="sudo -H -u ${WWW_USER} " - + echo "The following DB Passwords were generated..." echo "Admin (${DBUSER_ADMIN}) DB Password: ${DBPASSWORD_ADMIN}" echo "User (${DBUSER_MISP}) DB Password: ${DBPASSWORD_MISP}" @@ -236,6 +241,7 @@ setOpt () { ("-U") echo "upgrade"; UPGRADE=1 ;; ("-N") echo "nuke"; NUKE=1 ;; ("-u") echo "unattended"; UNATTENDED=1 ;; + ("-ni") echo "noninteractive"; NONINTERACTIVE=1 ;; ("-f") echo "force"; FORCE=1 ;; (*) echo "$o is not a valid argument"; exit 1 ;; esac @@ -1718,9 +1724,9 @@ mispmodules () { sudo apt-get install cmake libcaca-dev liblua5.3-dev -y ## TODO: checkUsrLocalSrc in main doc debug "Cloning misp-modules" - $SUDO_CMD git clone https://github.com/MISP/misp-modules.git - $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git - $SUDO_CMD git clone git://github.com/stricaud/faup.git + false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone https://github.com/MISP/misp-modules.git; done + [[ ! -d "faup" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone git://github.com/stricaud/faup.git faup; done + [[ ! -d "gtcaca" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca; done sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca # Install gtcaca cd gtcaca @@ -1805,7 +1811,7 @@ mispDashboard () { sudo mkdir misp-dashboard sudo chown $WWW_USER:$WWW_USER misp-dashboard - $SUDO_WWW git clone https://github.com/MISP/misp-dashboard.git + false; while [[ $? -ne 0 ]]; do $SUDO_WWW git clone https://github.com/MISP/misp-dashboard.git; done cd misp-dashboard sudo -H /var/www/misp-dashboard/install_dependencies.sh sudo sed -i "s/^host\ =\ localhost/host\ =\ 0.0.0.0/g" /var/www/misp-dashboard/config/config.cfg @@ -1889,9 +1895,10 @@ mail2misp () { debug "Installing Mail2${LBLUE}MISP${NC}" cd /usr/local/src/ sudo apt-get install cmake libcaca-dev liblua5.3-dev -y - $SUDO_CMD git clone https://github.com/MISP/mail_to_misp.git - [[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup - [[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca + false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone https://github.com/MISP/mail_to_misp.git; done + ## TODO: The below fails miserably (obviously) if faup/gtcac dirs exist, let's just make the dangerous assumption (for the sake of the installer, that they exist) + ##[[ ! -d "faup" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone git://github.com/stricaud/faup.git faup; done + ##[[ ! -d "gtcaca" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca; done sudo chown -R ${MISP_USER}:${MISP_USER} faup mail_to_misp gtcaca cd gtcaca $SUDO_CMD mkdir -p build @@ -1949,8 +1956,8 @@ viper () { fi fi echo "Cloning Viper" - $SUDO_CMD git clone https://github.com/viper-framework/viper.git - $SUDO_CMD git clone https://github.com/viper-framework/viper-web.git + false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone https://github.com/viper-framework/viper.git; done + false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone https://github.com/viper-framework/viper-web.git; done sudo chown -R $MISP_USER:$MISP_USER viper sudo chown -R $MISP_USER:$MISP_USER viper-web cd viper @@ -2506,7 +2513,7 @@ mispmodulesRHEL () { sudo chmod 2777 /usr/local/src sudo chown root:users /usr/local/src cd /usr/local/src/ - $SUDO_WWW git clone https://github.com/MISP/misp-modules.git + false; while [[ $? -ne 0 ]]; do $SUDO_WWW git clone https://github.com/MISP/misp-modules.git; done cd misp-modules # pip install $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U -I -r REQUIREMENTS diff --git a/INSTALL/INSTALL.sh.sfv b/INSTALL/INSTALL.sh.sfv index de7070b5c..4c0e18f69 100644 --- a/INSTALL/INSTALL.sh.sfv +++ b/INSTALL/INSTALL.sh.sfv @@ -1,5 +1,5 @@ -; Generated by RHash v1.3.9 on 2020-05-22 at 16:25.08 +; Generated by RHash v1.3.8 on 2020-07-15 at 04:14.11 ; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/ ; -; 133066 16:25.07 2020-05-22 INSTALL.sh -INSTALL.sh 79EF825B019669270DBCA0DD922C1E3DE6DA3D89 22A82CD073DA3312DF51089884DE4F3AF88ECD0E359D4C048915178C366327EC 4DA7D94FB036B2CC02120C1AC5AEBA9B57E4200FFC2940CB5BF8D9FE8C8600C72888DD2093590E7E77BB3A9F38D7F656 AA0CFD458A4B5CD84103EB641F59FFBBBB740890CB433108C6E0B8912F795DC521E3C75BD563496088862798D90A5C4D20B862ABA5152A84F62C037E889C3ED3 +; 133908 04:14.11 2020-07-15 INSTALL.sh +INSTALL.sh 1048857D0C71A2FB6029A448090BC88E008AA499 2E3E878D35568521B5DEC1E7F6DA3193FE3C51049E4BCC127068659E5375939E ED7092DC612C51D7B81969418B4EEA90CE5E990DDE693A3CE83566DEC11E1CF456452DCF103689F05B7E6CCB63F9BC45 2D4CC5D6E02135B337541CE00CDDB205E10EDE924B89B1EEFE069DD1FCE7CE552970AB65A7342838A3BB41E0DCDAB6460E7F96AB7F63EA554D9A1DB61116AE2A diff --git a/INSTALL/INSTALL.sh.sha1 b/INSTALL/INSTALL.sh.sha1 index 9a11ea658..df3855a20 100644 --- a/INSTALL/INSTALL.sh.sha1 +++ b/INSTALL/INSTALL.sh.sha1 @@ -1 +1 @@ -79ef825b019669270dbca0dd922c1e3de6da3d89 INSTALL.sh +1048857d0c71a2fb6029a448090bc88e008aa499 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha256 b/INSTALL/INSTALL.sh.sha256 index 85d3feb5c..37a9d0798 100644 --- a/INSTALL/INSTALL.sh.sha256 +++ b/INSTALL/INSTALL.sh.sha256 @@ -1 +1 @@ -22a82cd073da3312df51089884de4f3af88ecd0e359d4c048915178c366327ec INSTALL.sh +2e3e878d35568521b5dec1e7f6da3193fe3c51049e4bcc127068659e5375939e INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha384 b/INSTALL/INSTALL.sh.sha384 index 3f3bd4f09..aa3276e7f 100644 --- a/INSTALL/INSTALL.sh.sha384 +++ b/INSTALL/INSTALL.sh.sha384 @@ -1 +1 @@ -4da7d94fb036b2cc02120c1ac5aeba9b57e4200ffc2940cb5bf8d9fe8c8600c72888dd2093590e7e77bb3a9f38d7f656 INSTALL.sh +ed7092dc612c51d7b81969418b4eea90ce5e990dde693a3ce83566dec11e1cf456452dcf103689f05b7e6ccb63f9bc45 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha512 b/INSTALL/INSTALL.sh.sha512 index ab6748ed5..4e3fca072 100644 --- a/INSTALL/INSTALL.sh.sha512 +++ b/INSTALL/INSTALL.sh.sha512 @@ -1 +1 @@ -aa0cfd458a4b5cd84103eb641f59ffbbbb740890cb433108c6e0b8912f795dc521e3c75bd563496088862798d90a5c4d20b862aba5152a84f62c037e889c3ed3 INSTALL.sh +2d4cc5d6e02135b337541ce00cddb205e10ede924b89b1eefe069dd1fce7ce552970ab65a7342838a3bb41e0dcdab6460e7f96ab7f63ea554d9a1db61116ae2a INSTALL.sh diff --git a/VERSION.json b/VERSION.json index 7f1adaf86..c92e69f03 100644 --- a/VERSION.json +++ b/VERSION.json @@ -1 +1 @@ -{"major":2, "minor":4, "hotfix":128} +{"major":2, "minor":4, "hotfix":129} diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index 958eae3b1..e8f6003ff 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -46,7 +46,7 @@ class AppController extends Controller public $helpers = array('Utility', 'OrgImg', 'FontAwesome', 'UserName', 'DataPathCollector'); - private $__queryVersion = '106'; + private $__queryVersion = '107'; public $pyMispVersion = '2.4.128'; public $phpmin = '7.2'; public $phprec = '7.4'; diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 384772138..d5c10a85c 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -8,7 +8,7 @@ App::uses('File', 'Utility'); */ class AttributesController extends AppController { - public $components = array('Security', 'RequestHandler', 'Cidr'); + public $components = array('Security', 'RequestHandler'); public $paginate = array( 'limit' => 60, diff --git a/app/Controller/Component/CidrComponent.php b/app/Controller/Component/CidrComponent.php deleted file mode 100644 index 17fa54c8d..000000000 --- a/app/Controller/Component/CidrComponent.php +++ /dev/null @@ -1,62 +0,0 @@ -Toolbox->findIdByUuid($this->Event, $id); $this->Event->id = $id; - if (!$this->Event->exists()) { - throw new NotFoundException(__('Invalid event')); - } $conditions = array('eventid' => $id); if (!$this->_isRest()) { $conditions['includeAllTags'] = true; @@ -2535,9 +2531,6 @@ class EventsController extends AppController { $id = $this->Toolbox->findIdByUuid($this->Event, $id); $this->Event->id = $id; - if (!$this->Event->exists()) { - throw new NotFoundException(__('Invalid event')); - } $this->Event->recursive = -1; $event = $this->Event->read(null, $id); if (!$this->_isSiteAdmin()) { @@ -2638,9 +2631,6 @@ class EventsController extends AppController { $id = $this->Toolbox->findIdByUuid($this->Event, $id); $this->Event->id = $id; - if (!$this->Event->exists()) { - throw new NotFoundException(__('Invalid event')); - } // update the event and set the from field to the current instance's organisation from the bootstrap. We also need to save id and info for the logs. $this->Event->recursive = -1; $event = $this->Event->read(null, $id); @@ -3686,14 +3676,6 @@ class EventsController extends AppController $resultArray[$key]['types'] = $temp; } - // remove all duplicates - foreach ($resultArray as $k => $v) { - for ($i = 0; $i < $k; $i++) { - if (isset($resultArray[$i]) && $v == $resultArray[$i]) { - unset($resultArray[$k]); - } - } - } if ($this->_isRest()) { if ($returnMetaAttributes || !empty($this->request->data['Attribute']['returnMetaAttributes'])) { return $this->RestResponse->viewData($resultArray, $this->response->type()); diff --git a/app/Controller/GalaxyClustersController.php b/app/Controller/GalaxyClustersController.php index 370aac885..2c32924b3 100644 --- a/app/Controller/GalaxyClustersController.php +++ b/app/Controller/GalaxyClustersController.php @@ -425,7 +425,7 @@ class GalaxyClustersController extends AppController ), 'fields' => array('Tag.name, COUNT(DISTINCT event_id) as tag_count'), 'recursive' => -1, - 'group' => array('Tag.name') + 'group' => array('Tag.name', 'Tag.id') )); // fetch all related tags belonging to attack pattern or belonging to an event having this cluster @@ -440,7 +440,7 @@ class GalaxyClustersController extends AppController ), 'fields' => array('Tag.name, COUNT(DISTINCT event_id) as tag_count'), 'recursive' => -1, - 'group' => array('Tag.name') + 'group' => array('Tag.name', 'Tag.id') )); $scores = array(); diff --git a/app/Controller/LogsController.php b/app/Controller/LogsController.php index a72aa3f74..b80de5eb1 100644 --- a/app/Controller/LogsController.php +++ b/app/Controller/LogsController.php @@ -146,32 +146,18 @@ class LogsController extends AppController } // Shows a minimalistic history for the currently selected event - public function event_index($id, $org = null) + public function event_index($id) { // check if the user has access to this event... $mayModify = false; - $mineOrAdmin = false; $this->loadModel('Event'); - if (!is_numeric($id) || $id < 1) { - $id = -1; - } $event = $this->Event->fetchEvent($this->Auth->user(), array( 'eventid' => $id, 'includeAllTags' => 1, 'sgReferenceOnly' => 1, - 'deleted' => 1, + 'deleted' => [0, 1], 'deleted_proposals' => 1 )); - $conditions = array( - 'OR' => array( - array( - 'AND' => array( - 'Log.model' => 'Event', - 'Log.model_id' => $id - ) - ) - ) - ); if (empty($event)) { throw new NotFoundException('Invalid event.'); } @@ -179,8 +165,6 @@ class LogsController extends AppController $attribute_ids = array(); $object_ids = array(); $proposal_ids = array(); - $event_tag_ids = array(); - $attribute_tag_ids = array(); if (!empty($event['Attribute'])) { foreach ($event['Attribute'] as $aa) { $attribute_ids[] = $aa['id']; diff --git a/app/Controller/ShadowAttributesController.php b/app/Controller/ShadowAttributesController.php index 249b66530..632d67e47 100644 --- a/app/Controller/ShadowAttributesController.php +++ b/app/Controller/ShadowAttributesController.php @@ -803,20 +803,18 @@ class ShadowAttributesController extends AppController public function delete($id) { - if (strlen($id) == 36) { - $this->ShadowAttribute->Event->recursive = -1; - $temp = $this->ShadowAttribute->Event->Attribute->find('first', array('recursive' => -1, 'conditions' => array('Attribute.uuid' => $id), 'fields' => array('id'))); - if ($temp == null) { - throw new NotFoundException(__('Invalid attribute')); - } - $id = $temp['Attribute']['id']; - } - - $existingAttribute = $this->ShadowAttribute->Event->Attribute->fetchAttributes($this->Auth->user(), array('conditions' => array('Attribute.id' => $id))); - if (empty($existingAttribute)) { - throw new NotFoundException(__('Invalid attribute.')); + if (is_numeric($id)) { + $conditions = ['Attribute.id' => $id]; + } else if (Validation::uuid($id)) { + $conditions = ['Attribute.uuid' => $id]; + } else { + throw new NotFoundException(__('Invalid attribute')); } + $existingAttribute = $this->ShadowAttribute->Event->Attribute->fetchAttributes( + $this->Auth->user(), + array('conditions' => $conditions, 'flatten' => true) + ); if ($this->request->is('post')) { if (empty($existingAttribute)) { return new CakeResponse(array('body'=> json_encode(array('false' => true, 'errors' => 'Invalid Attribute.')), 'status'=>200, 'type' => 'json')); @@ -850,7 +848,7 @@ class ShadowAttributesController extends AppController } } else { if (empty($existingAttribute)) { - throw new NotFoundException(__('Invalid Attribute')); + throw new NotFoundException(__('Invalid attribute')); } $existingAttribute = $existingAttribute[0]; $this->set('id', $id); diff --git a/app/Controller/UserSettingsController.php b/app/Controller/UserSettingsController.php index 20f519d5b..e092c8ebb 100644 --- a/app/Controller/UserSettingsController.php +++ b/app/Controller/UserSettingsController.php @@ -28,7 +28,6 @@ class UserSettingsController extends AppController public function beforeFilter() { parent::beforeFilter(); - $this->Security->unlockedActions = array_merge($this->Security->unlockedActions, array('setHomePage')); } public function index() @@ -325,23 +324,27 @@ class UserSettingsController extends AppController public function setHomePage() { - if (!$this->request->is('post')) { - throw new MethodNotAllowedException(__('This endpoint only aaccepts POST requests.')); + if ($this->request->is('post')) { + if (isset($this->request->data['UserSetting'])) { + $this->request->data = $this->request->data['UserSetting']; + } + if (!isset($this->request->data['path'])) { + $this->request->data = array('path' => $this->request->data); + } + if (empty($this->request->data['path'])) { + throw new InvalidArgumentException(__('No path POSTed.')); + } + $setting = array( + 'UserSetting' => array( + 'user_id' => $this->Auth->user('id'), + 'setting' => 'homepage', + 'value' => json_encode(array('path' => $this->request->data['path'])) + ) + ); + $result = $this->UserSetting->setSetting($this->Auth->user(), $setting); + return $this->RestResponse->saveSuccessResponse('UserSettings', 'setHomePage', false, 'json', 'Homepage set to ' . $this->request->data['path']); + } else { + $this->layout = false; } - if (empty($this->request->data['path'])) { - $this->request->data = array('path' => $this->request->data); - } - if (empty($this->request->data['path'])) { - throw new InvalidArgumentException(__('No path POSTed.')); - } - $setting = array( - 'UserSetting' => array( - 'user_id' => $this->Auth->user('id'), - 'setting' => 'homepage', - 'value' => json_encode(array('path' => $this->request->data['path'])) - ) - ); - $result = $this->UserSetting->setSetting($this->Auth->user(), $setting); - return $this->RestResponse->saveSuccessResponse('UserSettings', 'setHomePage', false, $this->response->type(), 'Homepage set to ' . $this->request->data['path']); } } diff --git a/app/Lib/Export/StixExport.php b/app/Lib/Export/StixExport.php index c66b38a12..33e2c2878 100644 --- a/app/Lib/Export/StixExport.php +++ b/app/Lib/Export/StixExport.php @@ -97,7 +97,7 @@ class StixExport $stix_event = ($this->__return_type == 'stix') ? $file->read() : substr($file->read(), 1, -1); $file->close(); $file->delete(); - unlink($this->__tmp_dir . $filename); + @unlink($this->__tmp_dir . $filename); $this->__stix_file->append($stix_event . $this->__framing['separator']); unset($stix_event); } @@ -131,7 +131,7 @@ class StixExport { foreach ($this->__filenames as $f => $filename) { if ($index >= $f) { - unlink($this->__tmp_dir . $filename); + @unlink($this->__tmp_dir . $filename); } } $this->__stix_file->close(); diff --git a/app/Lib/Tools/CIDRTool.php b/app/Lib/Tools/CIDRTool.php deleted file mode 100644 index 44b17e886..000000000 --- a/app/Lib/Tools/CIDRTool.php +++ /dev/null @@ -1,62 +0,0 @@ - '/^hxxp/i', + 'from' => '/^(hxxp|hxtp|htxp|meow|h\[tt\]p)/i', 'to' => 'http', 'types' => array('link', 'url') ), array( - 'from' => '/^meow/i', - 'to' => 'http', - 'types' => array('link', 'url') - ), - array( - 'from' => '/^h\[tt\]p/i', - 'to' => 'http', - 'types' => array('link', 'url') - ), - array( - 'from' => '/\[\.\]/', - 'to' => '.', - 'types' => array('link', 'url', 'ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname') - ), - array( - 'from' => '/\[dot\]/', - 'to' => '.', - 'types' => array('link', 'url', 'ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname') - ), - array( - 'from' => '/\(dot\)/', - 'to' => '.', - 'types' => array('link', 'url', 'ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname') - ), - array( - 'from' => '/\\\\\./', + 'from' => '/(\[\.\]|\[dot\]|\(dot\)|\\\\\.)/', 'to' => '.', 'types' => array('link', 'url', 'ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname') ), @@ -49,7 +24,7 @@ class ComplexTypeTool 'types' => array('link', 'url') ), array( - 'from' => '/[\@]/', + 'from' => '/[\@]|\[at\]/', 'to' => '@', 'types' => array('email-src', 'email-dst') ), @@ -60,24 +35,24 @@ class ComplexTypeTool ) ); + private $__tlds = null; + public function refangValue($value, $type) { foreach ($this->__refangRegexTable as $regex) { - if (!isset($regex['types']) || in_array($type, $regex['types'])) { + if (in_array($type, $regex['types'])) { $value = preg_replace($regex['from'], $regex['to'], $value); } } return $value; } - private $__tlds = array(); - public function setTLDs($tlds = array()) { - if (!empty($tlds)) { - $this->__tlds = $tlds; + $this->__tlds = []; + foreach ($tlds as $tld) { + $this->__tlds[$tld] = true; } - return true; } public function checkComplexRouter($input, $type, $settings = array()) @@ -85,17 +60,13 @@ class ComplexTypeTool switch ($type) { case 'File': return $this->checkComplexFile($input); - break; case 'CnC': return $this->checkComplexCnC($input); - break; case 'freetext': case 'FreeText': return $this->checkFreeText($input, $settings); - break; case 'csv': return $this->checkCSV($input, $settings); - break; default: return false; } @@ -181,6 +152,7 @@ class ComplexTypeTool { $delimiter = !empty($settings['delimiter']) ? $settings['delimiter'] : ","; $rows = str_getcsv($input, "\n"); + unset($input); $data = array(); foreach ($rows as $k => $row) { if (empty($row[0]) || $row[0] === '#') { @@ -193,7 +165,6 @@ class ComplexTypeTool } } unset($rows); - unset($input); $values = !empty($settings['value']) ? $settings['value'] : array(); if (!is_array($values)) { $values = explode(',', $values); @@ -223,7 +194,7 @@ class ComplexTypeTool public function checkFreeText($input, $settings = array()) { - $charactersToTrim = array('\'', '"', ',', '(', ')', ' '); + $charactersToTrim = '\'",() ' . "\t\n\r\0\x0B"; // custom + default PHP trim $iocArray = preg_split("/\r\n|\n|\r|\s|\s+|,|\<|\>|;/", $input); $quotedText = explode('"', $input); foreach ($quotedText as $k => $temp) { @@ -236,34 +207,27 @@ class ComplexTypeTool } $iocArray = array_merge($iocArray, $this->__returnOddElements($quotedText)); $resultArray = array(); - if (!empty($iocArray)) { - foreach ($iocArray as $ioc) { - $ioc = trim($ioc); - $ioc = str_replace("\xc2\xa0", '', $ioc); - foreach ($charactersToTrim as $c) { - $ioc = trim($ioc, $c); - } - $ioc = preg_replace('/\p{C}+/u', '', $ioc); - if (empty($ioc)) { - continue; - } - if (isset($settings['excluderegex']) && !empty($settings['excluderegex'])) { - if (preg_match($settings['excluderegex'], $ioc)) { - continue; - } - } - $typeArray = $this->__resolveType($ioc); - if ($typeArray === false) { - continue; - } - $temp = $typeArray; - if (!isset($temp['value'])) { - $temp['value'] = $ioc; - } - $resultArray[] = $temp; + foreach ($iocArray as $ioc) { + $ioc = str_replace("\xc2\xa0", '', $ioc); // remove non breaking space + $ioc = trim($ioc, $charactersToTrim); + $ioc = preg_replace('/\p{C}+/u', '', $ioc); + if (empty($ioc)) { + continue; } + if (!empty($settings['excluderegex']) && preg_match($settings['excluderegex'], $ioc)) { + continue; + } + $typeArray = $this->__resolveType($ioc); + if ($typeArray === false) { + continue; + } + // Remove duplicates + if (isset($resultArray[$typeArray['value']])) { + continue; + } + $resultArray[$typeArray['value']] = $typeArray; } - return $resultArray; + return array_values($resultArray); } private $__hexHashTypes = array( @@ -295,13 +259,13 @@ class ComplexTypeTool return false; } - private function __checkForBTC($input) - { - if (preg_match("#^[13][a-km-zA-HJ-NP-Z1-9]{25,34}$#i", $input['raw'])) { - return array('types' => array('btc'), 'categories' => array('Financial fraud'), 'to_ids' => true, 'default_type' => 'btc', 'value' => $input['raw']); + private function __checkForBTC($input) + { + if (preg_match("#^[13][a-km-zA-HJ-NP-Z1-9]{25,34}$#i", $input['raw'])) { + return array('types' => array('btc'), 'categories' => array('Financial fraud'), 'to_ids' => true, 'default_type' => 'btc', 'value' => $input['raw']); } - return false; - } + return false; + } private function __checkForEmail($input) { @@ -314,13 +278,14 @@ class ComplexTypeTool return false; } - private function __checkForAS($input) - { - if (preg_match('#^as[0-9]+$#i', $input['raw'])) { - $input['raw'] = strtoupper($input['raw']); - return array('types' => array('AS'), 'to_ids' => false, 'default_type' => 'AS', 'value' => $input['raw']); - } - } + private function __checkForAS($input) + { + if (preg_match('#^as[0-9]+$#i', $input['raw'])) { + $input['raw'] = strtoupper($input['raw']); + return array('types' => array('AS'), 'to_ids' => false, 'default_type' => 'AS', 'value' => $input['raw']); + } + return false; + } private function __checkForHashes($input) { @@ -329,12 +294,11 @@ class ComplexTypeTool $compositeParts = explode('|', $input['raw']); if (count($compositeParts) == 2) { if ($this->__resolveFilename($compositeParts[0])) { - foreach ($this->__hexHashTypes as $k => $v) { - if (strlen($compositeParts[1]) == $k && preg_match("#[0-9a-f]{" . $k . "}$#i", $compositeParts[1])) { - return array('types' => $v['composite'], 'to_ids' => true, 'default_type' => $v['composite'][0], 'value' => $input['raw']); - } + $hash = $this->__resolveHash($compositeParts[1]); + if ($hash) { + return array('types' => $hash['composite'], 'to_ids' => true, 'default_type' => $hash['composite'][0], 'value' => $input['raw']); } - if (preg_match('#^[0-9]+:[0-9a-zA-Z\/\+]+:[0-9a-zA-Z\/\+]+$#', $compositeParts[1]) && !preg_match('#^[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$#', $compositeParts[1])) { + if ($this->__resolveSsdeep($compositeParts[1])) { return array('types' => array('filename|ssdeep'), 'to_ids' => true, 'default_type' => 'filename|ssdeep', 'value' => $input['raw']); } } @@ -342,17 +306,16 @@ class ComplexTypeTool } // check for hashes - foreach ($this->__hexHashTypes as $k => $v) { - if (strlen($input['raw']) == $k && preg_match("#[0-9a-f]{" . $k . "}$#i", $input['raw'])) { - $types = $v['single']; - if (!empty($this->__checkForBTC($input))) { - $types[] = 'btc'; - } - return array('types' => $types, 'to_ids' => true, 'default_type' => $v['single'][0], 'value' => $input['raw']); + $hash = $this->__resolveHash($input['raw']); + if ($hash) { + $types = $hash['single']; + if ($this->__checkForBTC($input)) { + $types[] = 'btc'; } + return array('types' => $types, 'to_ids' => true, 'default_type' => $hash['single'][0], 'value' => $input['raw']); } // ssdeep has a different pattern - if (preg_match('#^[0-9]+:[0-9a-zA-Z\/\+]+:[0-9a-zA-Z\/\+]+$#', $input['raw']) && !preg_match('#^[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$#', $input['raw'])) { + if ($this->__resolveSsdeep($input['raw'])) { return array('types' => array('ssdeep'), 'to_ids' => true, 'default_type' => 'ssdeep', 'value' => $input['raw']); } return false; @@ -362,13 +325,12 @@ class ComplexTypeTool { // note down and remove the port if it's a url / domain name / hostname / ip // input2 from here on is the variable containing the original input with the port removed. It is only used by url / domain name / hostname / ip - $input['comment'] = false; - if (preg_match('/(:[0-9]{2,5})$/', $input['refanged'], $input['port'])) { - $input['comment'] = 'On port ' . substr($input['port'][0], 1); - $input['refanged_no_port'] = str_replace($input['port'][0], '', $input['refanged']); - $input['port'] = substr($input['port'][0], 1); + if (preg_match('/(:[0-9]{2,5})$/', $input['refanged'], $port)) { + $input['comment'] = 'On port ' . substr($port[0], 1); + $input['refanged_no_port'] = str_replace($port[0], '', $input['refanged']); + $input['port'] = substr($port[0], 1); } else { - unset($input['port']); + $input['comment'] = false; $input['refanged_no_port'] = $input['refanged']; } return $input; @@ -381,13 +343,13 @@ class ComplexTypeTool $input['refanged'] = preg_replace($regex['from'], $regex['to'], $input['refanged']); } $input['refanged'] = rtrim($input['refanged'], "."); - $input['refanged'] = preg_replace_callback( - '/\[.\]/', - function ($matches) { - return trim($matches[0], '[]'); - }, - $input['refanged'] - ); + $input['refanged'] = preg_replace_callback( + '/\[.\]/', + function ($matches) { + return trim($matches[0], '[]'); + }, + $input['refanged'] + ); return $input; } @@ -397,17 +359,16 @@ class ComplexTypeTool if (preg_match("#^cve-[0-9]{4}-[0-9]{4,9}$#i", $input['raw'])) { return array('types' => array('vulnerability'), 'categories' => array('External analysis'), 'to_ids' => false, 'default_type' => 'vulnerability', 'value' => $input['raw']); } - // Phone numbers - for automatic recognition, needs to start with + or include dashes - if (!empty($input['raw'])) { - if ($input['raw'][0] === '+' || strpos($input['raw'], '-')) { - if (!preg_match('#^[0-9]{4}-[0-9]{2}-[0-9]{2}$#i', $input['raw']) && preg_match("#^(\+)?([0-9]{1,3}(\(0\))?)?[0-9\/\-]{5,}[0-9]$#i", $input['raw'])) { - return array('types' => array('phone-number', 'prtn', 'whois-registrant-phone'), 'categories' => array('Other'), 'to_ids' => false, 'default_type' => 'phone-number', 'value' => $input['raw']); - } - } - } + // Phone numbers - for automatic recognition, needs to start with + or include dashes + if ($input['raw'][0] === '+' || strpos($input['raw'], '-')) { + if (!preg_match('#^[0-9]{4}-[0-9]{2}-[0-9]{2}$#i', $input['raw']) && preg_match("#^(\+)?([0-9]{1,3}(\(0\))?)?[0-9\/\-]{5,}[0-9]$#i", $input['raw'])) { + return array('types' => array('phone-number', 'prtn', 'whois-registrant-phone'), 'categories' => array('Other'), 'to_ids' => false, 'default_type' => 'phone-number', 'value' => $input['raw']); + } + } + return false; } - private function __checkForIP($input) + private function __checkForIP(array $input) { if (filter_var($input['refanged_no_port'], FILTER_VALIDATE_IP)) { if (isset($input['port'])) { @@ -416,30 +377,48 @@ class ComplexTypeTool return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'to_ids' => true, 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']); } } + // IPv6 address that is considered as IP address with port + if (filter_var($input['refanged'], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) { + return [ + 'types' => ['ip-dst', 'ip-src', 'ip-src/ip-dst'], + 'to_ids' => true, + 'default_type' => 'ip-dst', + 'comment' => '', + 'value' => $input['refanged'], + ]; + } + // IPv6 with port in `[1fff:0:a88:85a3::ac1f]:8001` format + if (isset($input['port']) && + !empty($input['refanged_no_port']) && + $input['refanged_no_port'][0] === '[' && + filter_var(substr($input['refanged_no_port'], 1, -1), FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) + ) { + $value = substr($input['refanged_no_port'], 1, -1); // remove brackets + return [ + 'types' => ['ip-dst|port', 'ip-src|port', 'ip-src|port/ip-dst|port'], + 'to_ids' => true, + 'default_type' => 'ip-dst|port', + 'comment' => $input['comment'], + 'value' => "$value|{$input['port']}", + ]; + } // it could still be a CIDR block if (strpos($input['refanged_no_port'], '/')) { $temp = explode('/', $input['refanged_no_port']); - if (count($temp) == 2) { - if (filter_var($temp[0], FILTER_VALIDATE_IP) && is_numeric($temp[1])) { - return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'to_ids' => true, 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']); - } + if (count($temp) === 2 && filter_var($temp[0], FILTER_VALIDATE_IP) && is_numeric($temp[1])) { + return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'to_ids' => true, 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']); } } + return false; } - private function __checkForDomainOrFilename($input) + private function __checkForDomainOrFilename(array $input) { - if (strpos($input['refanged'], '.') !== false) { - $temp = explode('.', $input['refanged']); - // TODO: use a more flexible matching approach, like the one below (that still doesn't support non-ASCII domains) - //if (filter_var($input, FILTER_VALIDATE_URL)) { + if (strpos($input['refanged_no_port'], '.') !== false) { + $temp = explode('.', $input['refanged_no_port']); $domainDetection = true; - if (preg_match('/^([-\pL\pN]+\.)+[a-z]+(:[0-9]{2,5})?$/iu', $input['refanged'])) { - if (empty($this->__tlds) || count($this->__tlds) == 1) { - $this->__generateTLDList(); - } - $tldExploded = explode(':', $temp[count($temp)-1]); - if (!in_array(strtolower($tldExploded[0]), $this->__tlds)) { + if (preg_match('/^([-\pL\pN]+\.)+[a-z0-9-]+$/iu', $input['refanged_no_port'])) { + if (!$this->isTld(end($temp))) { $domainDetection = false; } } else { @@ -457,12 +436,9 @@ class ComplexTypeTool if (count($temp) > 1 && (filter_var($input['refanged_no_port'], FILTER_VALIDATE_URL) || filter_var('http://' . $input['refanged_no_port'], FILTER_VALIDATE_URL))) { // Even though some domains are valid, we want to exclude them as they are known security vendors / etc // TODO, replace that with the appropriate warninglist. - if (preg_match('/^https:\/\/(www.)?virustotal.com\//i', $input['refanged_no_port'])) { + if (preg_match('/^(https:\/\/(www.)?virustotal.com\/|https:\/\/www\.hybrid-analysis\.com\/)/i', $input['refanged_no_port'])) { return array('types' => array('link'), 'to_ids' => false, 'default_type' => 'link', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']); } - if (preg_match('/^https:\/\/www\.hybrid-analysis\.com\//i', $input['refanged_no_port'])) { - return array('types' => array('link'), 'categories' => array('External analysis'), 'to_ids' => false, 'default_type' => 'link', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']); - } if (strpos($input['refanged_no_port'], '/')) { return array('types' => array('url'), 'to_ids' => true, 'default_type' => 'url', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']); } @@ -474,11 +450,11 @@ class ComplexTypeTool } if (strpos($input['raw'], '\\') !== false) { $temp = explode('\\', $input['raw']); - if (strpos($temp[count($temp)-1], '.') || preg_match('/^.:/i', $temp[0])) { - if ($this->__resolveFilename($temp[count($temp)-1])) { + if (strpos(end($temp), '.') || preg_match('/^.:/i', $temp[0])) { + if ($this->__resolveFilename(end($temp))) { return array('types' => array('filename'), 'categories' => array('Payload installation'), 'to_ids' => true, 'default_type' => 'filename', 'value' => $input['raw']); } - } else { + } else if (!empty($temp[0])) { return array('types' => array('regkey'), 'to_ids' => false, 'default_type' => 'regkey', 'value' => $input['raw']); } } @@ -487,26 +463,61 @@ class ComplexTypeTool private function __resolveFilename($param) { - if ((preg_match('/^.:/', $param) || strpos($param, '.') !=0)) { + if ((preg_match('/^.:/', $param) || strpos($param, '.') != 0)) { $parts = explode('.', $param); - if (!is_numeric($parts[count($parts)-1]) && ctype_alnum($parts[count($parts)-1])) { + if (!is_numeric(end($parts)) && ctype_alnum(end($parts))) { return true; } } return false; } + /** + * @param string $value + * @return bool + */ + private function __resolveSsdeep($value) + { + return preg_match('#^[0-9]+:[0-9a-zA-Z\/\+]+:[0-9a-zA-Z\/\+]+$#', $value) && !preg_match('#^[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$#', $value); + } + + /** + * @param string $value + * @return bool|string[][] + */ + private function __resolveHash($value) + { + $strlen = strlen($value); + if (isset($this->__hexHashTypes[$strlen]) && preg_match("#[0-9a-f]{" . $strlen . "}$#i", $value)) { + return $this->__hexHashTypes[$strlen]; + } + return false; + } + + /** + * @param string $tld + * @return bool + */ + private function isTld($tld) + { + if ($this->__tlds === null) { + $this->setTLDs($this->__generateTLDList()); + } + return isset($this->__tlds[strtolower($tld)]); + } + private function __generateTLDList() { - $this->__tlds = array('biz', 'cat', 'com', 'edu', 'gov', 'int', 'mil', 'net', 'org', 'pro', 'tel', 'aero', 'arpa', 'asia', 'coop', 'info', 'jobs', 'mobi', 'name', 'museum', 'travel', 'onion'); + $tlds = array('biz', 'cat', 'com', 'edu', 'gov', 'int', 'mil', 'net', 'org', 'pro', 'tel', 'aero', 'arpa', 'asia', 'coop', 'info', 'jobs', 'mobi', 'name', 'museum', 'travel', 'onion'); $char1 = $char2 = 'a'; for ($i = 0; $i < 26; $i++) { for ($j = 0; $j < 26; $j++) { - $this->__tlds[] = $char1 . $char2; + $tlds[] = $char1 . $char2; $char2++; } $char1++; $char2 = 'a'; } + return $tlds; } } diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 51543f0a6..58cee3a41 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -823,6 +823,10 @@ class Attribute extends AppModel ), false ); + if ($this->data['Attribute']['type'] === 'ssdeep') { + $this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep'); + $this->FuzzyCorrelateSsdeep->purge(null, $this->data['Attribute']['id']); + } } } @@ -1926,11 +1930,9 @@ class Attribute extends AppModel 'Correlation.attribute_id' => $a['id'])) ); } - if ($a['type'] == 'ssdeep') { + if ($a['type'] === 'ssdeep') { $this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep'); - $this->FuzzyCorrelateSsdeep->deleteAll( - array('FuzzyCorrelateSsdeep.attribute_id' => $a['id']) - ); + $this->FuzzyCorrelateSsdeep->purge(null, $a['id']); } } @@ -2806,6 +2808,10 @@ class Attribute extends AppModel { $this->Correlation = ClassRegistry::init('Correlation'); $this->purgeCorrelations($eventId); + + $this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep'); + $this->FuzzyCorrelateSsdeep->purge($eventId, $attributeId); + // get all attributes.. if (!$eventId) { $eventIds = $this->Event->find('list', array('recursive' => -1, 'fields' => array('Event.id'))); diff --git a/app/Model/Event.php b/app/Model/Event.php index e25181044..9d46dfc19 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -5575,99 +5575,6 @@ class Event extends AppModel return array('data' => array(), 'csv' => array()); } - public function setSimpleConditions($parameterKey, $parameterValue, $conditions, $restrictScopeToEvents = false) - { - if (is_array($parameterValue)) { - $elements = $parameterValue; - } else { - $elements = explode('&&', $parameterValue); - } - App::uses('CIDRTool', 'Tools'); - $cidr = new CIDRTool(); - $subcondition = array(); - foreach ($elements as $v) { - if ($v === '') { - continue; - } - if (substr($v, 0, 1) === '!') { - // check for an IPv4 address and subnet in CIDR notation (e.g. 127.0.0.1/8) - if ($parameterKey === 'value' && $cidr->checkCIDR(substr($v, 1), 4)) { - $cidrresults = $cidr->CIDR(substr($v, 1)); - foreach ($cidrresults as $result) { - $subcondition['AND'][] = array('Attribute.value NOT LIKE' => $result); - } - } else { - if ($parameterKey === 'org') { - $found_orgs = $this->Org->find('all', array( - 'recursive' => -1, - 'conditions' => array('name' => substr($v, 1)), - )); - foreach ($found_orgs as $o) { - $subcondition['AND'][] = array('Event.orgc_id !=' => $o['Org']['id']); - } - } elseif ($parameterKey === 'eventid') { - if ($restrictScopeToEvents) { - $subcondition['AND'][] = array('Event.id !=' => substr($v, 1)); - } else { - $subcondition['AND'][] = array('Attribute.event_id !=' => substr($v, 1)); - } - } elseif ($parameterKey === 'uuid') { - $subcondition['AND'][] = array('Event.uuid !=' => substr($v, 1)); - $subcondition['AND'][] = array('Attribute.uuid !=' => substr($v, 1)); - } else { - $lookup = substr($v, 1); - if (strlen($lookup) != strlen(trim($lookup, '%'))) { - $subcondition['AND'][] = array('Attribute.' . $parameterKey . ' NOT LIKE' => $lookup); - } else { - $subcondition['AND'][] = array('NOT' => array('Attribute.' . $parameterKey => $lookup)); - } - } - } - } else { - // check for an IPv4 address and subnet in CIDR notation (e.g. 127.0.0.1/8) - if ($parameterKey === 'value' && $cidr->checkCIDR($v, 4)) { - $cidrresults = $cidr->CIDR($v); - foreach ($cidrresults as $result) { - if (!empty($result)) { - $subcondition['OR'][] = array('Attribute.value LIKE' => $result); - } - } - } else { - if ($parameterKey === 'org') { - $found_orgs = $this->Org->find('all', array( - 'recursive' => -1, - 'conditions' => array('name' => $v), - )); - foreach ($found_orgs as $o) { - $subcondition['OR'][] = array('Event.orgc_id' => $o['Org']['id']); - } - } elseif ($parameterKey === 'eventid') { - if ($restrictScopeToEvents) { - $subcondition['OR'][] = array('Event.id' => $v); - } else { - $subcondition['OR'][] = array('Attribute.event_id' => $v); - } - } elseif ($parameterKey === 'uuid') { - $subcondition['OR'][] = array('Attribute.uuid' => $v); - $subcondition['OR'][] = array('Event.uuid' => $v); - } else { - if (!empty($v)) { - if (strlen($v) != strlen(trim($v, '%'))) { - $subcondition['AND'][] = array('Attribute.' . $parameterKey . ' LIKE' => $v); - } else { - $subcondition['AND'][] = array('Attribute.' . $parameterKey => $v); - } - } - } - } - } - } - if (!empty($subcondition)) { - array_push($conditions['AND'], $subcondition); - } - return $conditions; - } - public function cacheSgids($user, $useCache = false) { if ($useCache && isset($this->__assetCache['sgids'])) { diff --git a/app/Model/FuzzyCorrelateSsdeep.php b/app/Model/FuzzyCorrelateSsdeep.php index bab041d13..e7a93aafc 100644 --- a/app/Model/FuzzyCorrelateSsdeep.php +++ b/app/Model/FuzzyCorrelateSsdeep.php @@ -89,4 +89,27 @@ class FuzzyCorrelateSsdeep extends AppModel $this->saveAll($to_save); return $result; } + + /** + * @param int|null $eventId + * @param int|null $attributeId + * @return bool True on success, false on failure + */ + public function purge($eventId = null, $attributeId = null) + { + if (!$eventId && !$attributeId) { + $this->query('TRUNCATE TABLE fuzzy_correlate_ssdeep;'); + } elseif (!$attributeId) { + $this->Attribute = ClassRegistry::init('Attribute'); + $attributeId = $this->Attribute->find('list', array( + 'conditions' => array( + 'Attribute.event_id' => $eventId, + 'Attribute.type' => 'ssdeep', + ), + 'fields' => 'Attribute.id', + )); + } + + return $this->deleteAll(array('FuzzyCorrelateSsdeep.attribute_id' => $attributeId)); + } } diff --git a/app/Test/ComplexTypeToolTest.php b/app/Test/ComplexTypeToolTest.php new file mode 100644 index 000000000..28a4967f2 --- /dev/null +++ b/app/Test/ComplexTypeToolTest.php @@ -0,0 +1,428 @@ +checkFreeText(<<; "127.0.0.4" '127.0.0.5' +EOT +); + $this->assertCount(5, $results); + $this->assertEquals('127.0.0.1', $results[0]['value']); + $this->assertEquals('ip-dst', $results[0]['default_type']); + $this->assertEquals('127.0.0.2', $results[1]['value']); + $this->assertEquals('ip-dst', $results[1]['default_type']); + $this->assertEquals('127.0.0.3', $results[2]['value']); + $this->assertEquals('ip-dst', $results[2]['default_type']); + $this->assertEquals('127.0.0.4', $results[3]['value']); + $this->assertEquals('ip-dst', $results[3]['default_type']); + $this->assertEquals('127.0.0.5', $results[4]['value']); + $this->assertEquals('ip-dst', $results[4]['default_type']); + } + + public function testCheckFreeTextIpv4(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('127.0.0.1'); + $this->assertCount(1, $results); + $this->assertEquals('127.0.0.1', $results[0]['value']); + $this->assertEquals('ip-dst', $results[0]['default_type']); + } + + public function testCheckFreeTextIpv4WithPort(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('127.0.0.1:8080'); + $this->assertCount(1, $results); + $this->assertEquals('127.0.0.1|8080', $results[0]['value']); + $this->assertEquals('ip-dst|port', $results[0]['default_type']); + $this->assertEquals('On port 8080', $results[0]['comment']); + } + + public function testCheckFreeTextIpv4Cidr(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('127.0.0.1/32'); + $this->assertCount(1, $results); + $this->assertEquals('127.0.0.1/32', $results[0]['value']); + $this->assertEquals('ip-dst', $results[0]['default_type']); + } + + // Issue https://github.com/MISP/MISP/issues/6009 + public function testCheckFreeTextIpv6(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('2a00:1450:4005:80a::2003'); + $this->assertCount(1, $results); + $this->assertEquals('2a00:1450:4005:80a::2003', $results[0]['value']); + $this->assertEquals('ip-dst', $results[0]['default_type']); + } + + // Issue https://github.com/MISP/MISP/issues/3383 + public function testCheckFreeTextIpv6Invalid(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('fe80:0000:f2cd:7d80:3f37:52c6'); + $this->assertCount(0, $results); + } + + public function testCheckFreeTextIpv6Cidr(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('2a00:1450:4005:80a::2003/128'); + $this->assertCount(1, $results); + $this->assertEquals('2a00:1450:4005:80a::2003/128', $results[0]['value']); + $this->assertEquals('ip-dst', $results[0]['default_type']); + } + + public function testCheckFreeTextIpv6WithPort(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('[1fff:0:a88:85a3::ac1f]:8001'); + $this->assertCount(1, $results); + $this->assertEquals('1fff:0:a88:85a3::ac1f|8001', $results[0]['value']); + $this->assertEquals('ip-dst|port', $results[0]['default_type']); + $this->assertEquals('On port 8001', $results[0]['comment']); + } + + public function testCheckFreeTextDomain(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('example.com'); + $this->assertCount(1, $results); + $this->assertEquals('example.com', $results[0]['value']); + $this->assertEquals('domain', $results[0]['default_type']); + } + + public function testCheckFreeTextDomainThirdLevel(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('example.example.com'); + $this->assertCount(1, $results); + $this->assertEquals('example.example.com', $results[0]['value']); + $this->assertEquals('hostname', $results[0]['default_type']); + } + + public function testCheckFreeTextDomainDot(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('example.com.'); + $this->assertCount(1, $results); + $this->assertEquals('example.com', $results[0]['value']); + $this->assertEquals('domain', $results[0]['default_type']); + } + + public function testCheckFreeTextDomainNotExistsTld(): void + { + $complexTypeTool = new ComplexTypeTool(); + $complexTypeTool->setTLDs(['com']); + $results = $complexTypeTool->checkFreeText('example.example'); + $this->assertCount(1, $results); + $this->assertEquals('example.example', $results[0]['value']); + $this->assertEquals('filename', $results[0]['default_type']); + } + + public function testCheckFreeTextFilenameMultipleExt(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('example.txt.zip'); + $this->assertCount(1, $results); + $this->assertEquals('example.txt.zip', $results[0]['value']); + $this->assertEquals('filename', $results[0]['default_type']); + } + + public function testCheckFreeTextFilenameWithPathUnix(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('/var/log/example.txt.zip'); + $this->assertCount(1, $results); + $this->assertEquals('/var/log/example.txt.zip', $results[0]['value']); + $this->assertEquals('filename', $results[0]['default_type']); + } + + public function testCheckFreeTextFilenameWithPathWindows(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('C:\example.txt.zip'); + $this->assertCount(1, $results); + $this->assertEquals('C:\example.txt.zip', $results[0]['value']); + $this->assertEquals('filename', $results[0]['default_type']); + } + + public function testCheckFreeTextRegkey(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion'); + $this->assertCount(1, $results); + $this->assertEquals('HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion', $results[0]['value']); + $this->assertEquals('regkey', $results[0]['default_type']); + } + + public function testCheckFreeTextDomainWithPort(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('example.com:80'); + $this->assertCount(1, $results); + $this->assertEquals('example.com', $results[0]['value']); + $this->assertEquals('domain', $results[0]['default_type']); + $this->assertEquals('On port 80', $results[0]['comment']); + } + + public function testCheckFreeTextDomainUppercase(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('EXAMPLE.COM'); + $this->assertCount(1, $results); + $this->assertEquals('EXAMPLE.COM', $results[0]['value']); + $this->assertEquals('domain', $results[0]['default_type']); + } + + // Issue https://github.com/MISP/MISP/issues/657 + public function testCheckFreeTextPunycode(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('xn--ghq549cb2anjl2suxo.com'); + $this->assertCount(1, $results); + $this->assertEquals('xn--ghq549cb2anjl2suxo.com', $results[0]['value']); + $this->assertEquals('domain', $results[0]['default_type']); + } + + // Issue https://github.com/MISP/MISP/issues/657 + public function testCheckFreeTextPunycodeTld(): void + { + $complexTypeTool = new ComplexTypeTool(); + $complexTypeTool->setTLDs(['xn--fiqs8s']); + $results = $complexTypeTool->checkFreeText('xn--lbrs59br5a.xn--fiqs8s'); + $this->assertCount(1, $results); + $this->assertEquals('xn--lbrs59br5a.xn--fiqs8s', $results[0]['value']); + $this->assertEquals('domain', $results[0]['default_type']); + } + + // Issue https://github.com/MISP/MISP/issues/3580 + public function testCheckFreeTextDate(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('2018-08-21'); + $this->assertCount(0, $results); + } + + public function testCheckFreeTextEmail(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('test@example.com'); + $this->assertCount(1, $results); + $this->assertEquals('test@example.com', $results[0]['value']); + $this->assertEquals('email-src', $results[0]['default_type']); + } + + public function testCheckFreeTextEmailBracket(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('test[@]example.com'); + $this->assertCount(1, $results); + $this->assertEquals('test@example.com', $results[0]['value']); + $this->assertEquals('email-src', $results[0]['default_type']); + } + + // Issue https://github.com/MISP/MISP/issues/4805 + public function testCheckFreeTextEmailBracketAt(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('test[at]example.com'); + $this->assertCount(1, $results); + $this->assertEquals('test@example.com', $results[0]['value']); + $this->assertEquals('email-src', $results[0]['default_type']); + } + + public function testCheckFreeTextUrlHttp(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('http://example.com'); + $this->assertCount(1, $results); + $this->assertEquals('http://example.com', $results[0]['value']); + $this->assertEquals('url', $results[0]['default_type']); + } + + public function testCheckFreeTextUrlHttps(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('https://example.com'); + $this->assertCount(1, $results); + $this->assertEquals('https://example.com', $results[0]['value']); + $this->assertEquals('url', $results[0]['default_type']); + } + + public function testCheckFreeTextUrlWithoutProtocol(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('github.com/MISP/MISP'); + $this->assertCount(1, $results); + $this->assertEquals('github.com/MISP/MISP', $results[0]['value']); + $this->assertEquals('url', $results[0]['default_type']); + } + + public function testCheckFreeTextUrlVirusTotal(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('https://www.virustotal.com/example https://virustotal.com/example'); + $this->assertCount(2, $results); + + $this->assertEquals('https://www.virustotal.com/example', $results[0]['value']); + $this->assertEquals('link', $results[0]['default_type']); + $this->assertFalse($results[0]['to_ids']); + + $this->assertEquals('https://virustotal.com/example', $results[1]['value']); + $this->assertEquals('link', $results[1]['default_type']); + $this->assertFalse($results[1]['to_ids']); + } + + public function testCheckFreeTextUrlHybridAnalysis(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('https://www.hybrid-analysis.com/example'); + $this->assertCount(1, $results); + $this->assertEquals('https://www.hybrid-analysis.com/example', $results[0]['value']); + $this->assertEquals('link', $results[0]['default_type']); + $this->assertFalse($results[0]['to_ids']); + } + + // Issue https://github.com/MISP/MISP/issues/4908 + public function testCheckFreeTextUrlReplace(): void + { + $complexTypeTool = new ComplexTypeTool(); + foreach (['hxxp://example.com', 'hxtp://example.com', 'htxp://example.com'] as $test) { + $results = $complexTypeTool->checkFreeText($test); + $this->assertCount(1, $results); + $this->assertEquals('http://example.com', $results[0]['value']); + $this->assertEquals('url', $results[0]['default_type']); + } + } + + // Issue https://github.com/MISP/MISP/issues/4908 + public function testCheckFreeTextUrlReplaceHttps(): void + { + $complexTypeTool = new ComplexTypeTool(); + foreach (['hxxps://example.com', 'hxtps://example.com', 'htxps://example.com'] as $test) { + $results = $complexTypeTool->checkFreeText($test); + $this->assertCount(1, $results); + $this->assertEquals('https://example.com', $results[0]['value']); + $this->assertEquals('url', $results[0]['default_type']); + } + } + + public function testCheckFreeTextBtc(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa'); + $this->assertCount(1, $results); + $this->assertEquals('1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa', $results[0]['value']); + $this->assertEquals('btc', $results[0]['default_type']); + } + + public function testCheckFreeTextSsdeep(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('24:VGXGP7L5e/Ixt3af/WKPPaYpzg4m3XWMCsXNCRs0:kYDxcfPZpelCs9Cm0'); + $this->assertCount(1, $results); + $this->assertEquals('24:VGXGP7L5e/Ixt3af/WKPPaYpzg4m3XWMCsXNCRs0:kYDxcfPZpelCs9Cm0', $results[0]['value']); + $this->assertEquals('ssdeep', $results[0]['default_type']); + } + + public function testCheckFreeTextCve(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('CVE-2019-16202'); + $this->assertCount(1, $results); + $this->assertEquals('CVE-2019-16202', $results[0]['value']); + $this->assertEquals('vulnerability', $results[0]['default_type']); + } + + public function testCheckFreeTextAs(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('as0 AS0'); + $this->assertCount(1, $results); + $this->assertEquals('AS0', $results[0]['value']); + $this->assertEquals('AS', $results[0]['default_type']); + } + + public function testCheckFreeTextMd5(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('9e107d9d372bb6826bd81d3542a419d6'); + $this->assertCount(1, $results); + $this->assertEquals('9e107d9d372bb6826bd81d3542a419d6', $results[0]['value']); + $this->assertEquals('md5', $results[0]['default_type']); + } + + public function testCheckFreeTextSha1(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('da39a3ee5e6b4b0d3255bfef95601890afd80709'); + $this->assertCount(1, $results); + $this->assertEquals('da39a3ee5e6b4b0d3255bfef95601890afd80709', $results[0]['value']); + $this->assertEquals('sha1', $results[0]['default_type']); + } + + public function testCheckFreeTextFilenameWithMd5(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('ahoj.txt|9e107d9d372bb6826bd81d3542a419d6'); + $this->assertCount(1, $results); + $this->assertEquals('ahoj.txt|9e107d9d372bb6826bd81d3542a419d6', $results[0]['value']); + $this->assertEquals('filename|md5', $results[0]['default_type']); + } + + public function testCheckFreeTextRandomString(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('cK753n3MVw'); + $this->assertCount(0, $results); + } + + public function testCheckFreeTextEmpty(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText(''); + $this->assertCount(0, $results); + } + + public function testCheckFreeTextEmptyValues(): void + { + $complexTypeTool = new ComplexTypeTool(); + foreach (['|', '&', '$', '0', ':80', '1.2', '[]:80', '\.', '.', ':', 'a:b', 'a:b:c'] as $char) { + $results = $complexTypeTool->checkFreeText($char); + $this->assertCount(0, $results); + } + } + + public function testCheckFreeTextRemoveDuplicates(): void + { + $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText('1.2.3.4 1.2.3.4'); + $this->assertCount(1, $results); + } + + public function testRefangValueUrl(): void + { + $complexTypeTool = new ComplexTypeTool(); + foreach (['meow://example.com', 'h[tt]p://example.com'] as $test) { + $this->assertEquals('http://example.com', $complexTypeTool->refangValue($test, 'url')); + $this->assertEquals('http://example.com', $complexTypeTool->refangValue($test, 'link')); + } + } + + public function testRefangValueDot(): void + { + $complexTypeTool = new ComplexTypeTool(); + foreach (['127.0.0.1', '127[.]0.0.1', '127[.]0[.]0[.]1', '127[dot]0[dot]0[dot]1', '127(dot)0(dot)0(dot)1', '127\.0.0.1'] as $test) { + $this->assertEquals('127.0.0.1', $complexTypeTool->refangValue($test, 'ip-src')); + } + } +} diff --git a/app/View/Attributes/ajax/attributeEditMassForm.ctp b/app/View/Attributes/ajax/attributeEditMassForm.ctp index dde8bcb51..bd25abc3a 100644 --- a/app/View/Attributes/ajax/attributeEditMassForm.ctp +++ b/app/View/Attributes/ajax/attributeEditMassForm.ctp @@ -36,7 +36,7 @@ echo $this->Form->input('is_proposal', array( 'type' => 'checkbox', 'label' => __('Create proposals'), - 'checked' => true + 'checked' => false )); ?>
diff --git a/app/View/Elements/global_menu.ctp b/app/View/Elements/global_menu.ctp index 6bb141a74..5cfa1ebb5 100755 --- a/app/View/Elements/global_menu.ctp +++ b/app/View/Elements/global_menu.ctp @@ -413,8 +413,9 @@ 'type' => 'root', 'url' => '#', 'html' => sprintf( - '', - (!empty($homepage['path']) && $homepage['path'] === $this->here) ? 'orange' : '' + '', + (!empty($homepage['path']) && $homepage['path'] === $this->here) ? 'orange' : '', + $this->here ) ), array( diff --git a/app/View/Events/add.ctp b/app/View/Events/add.ctp index c8e921341..7fc71ac1a 100644 --- a/app/View/Events/add.ctp +++ b/app/View/Events/add.ctp @@ -65,7 +65,11 @@ ) ) )); - echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event-collection', 'menuItem' => $action === 'add' ? 'add' : 'editEvent')); + echo $this->element('/genericElements/SideMenu/side_menu', array( + 'menuList' => $action === 'add' ? 'event-collection' : 'event', + 'menuItem' => $action === 'add' ? 'add' : 'editEvent', + 'event' => isset($event) ? $event : null, + )); ?>