diff --git a/INSTALL/INSTALL.sh b/INSTALL/INSTALL.sh index 7ef744c64..606ea434b 100644 --- a/INSTALL/INSTALL.sh +++ b/INSTALL/INSTALL.sh @@ -120,9 +120,11 @@ MISPvars () { # GPG configuration GPG_REAL_NAME='Autogenerated Key' + # On a REAL install, please do not set a comment, see here for why: https://www.debian-administration.org/users/dkg/weblog/97 GPG_COMMENT='WARNING: MISP AutoGenerated Key consider this Key VOID!' GPG_EMAIL_ADDRESS='admin@admin.test' - GPG_KEY_LENGTH='2048' + # 3072 bits used as per suggestions here: https://riseup.net/en/security/message-security/openpgp/best-practices + GPG_KEY_LENGTH='3072' GPG_PASSPHRASE="$(openssl rand -hex 32)" # debug alias to make sure people are not confused when blindly copy pasting blobs of code @@ -247,7 +249,7 @@ checkCoreOS () { if [[ -f "/etc/redhat-release" ]]; then echo "This is some redhat flavour" REDHAT=1 - RHfla=$(cat /etc/redhat-release | cut -f 1 -d\ | tr [A-Z] [a-z]) + RHfla=$(cat /etc/redhat-release | cut -f 1 -d\ | tr '[:upper:]' '[:lower:]') fi } @@ -259,7 +261,7 @@ checkFlavour () { sudo apt install lsb-release dialog -y fi - FLAVOUR=$(lsb_release -s -i |tr [A-Z] [a-z]) + FLAVOUR=$(lsb_release -s -i |tr '[:upper:]' '[:lower:]') if [ FLAVOUR == "ubuntu" ]; then RELEASE=$(lsb_release -s -r) debug "We detected the following Linux flavour: ${YELLOW}$(tr '[:lower:]' '[:upper:]' <<< ${FLAVOUR:0:1})${FLAVOUR:1} ${RELEASE}${NC}" @@ -377,6 +379,27 @@ checkFail () { fi } +ask_o () { + + ANSWER="" + + if [ -z "${1}" ]; then + echo "This function needs at least 1 parameter." + exit 1 + fi + + [ -z "${2}" ] && OPT1="y" || OPT1="${2}" + [ -z "${3}" ] && OPT2="n" || OPT2="${3}" + + while true; do + case "${ANSWER}" in "${OPT1}" | "${OPT2}") break ;; esac + echo -e -n "${1} (${OPT1}/${OPT2}) " + read ANSWER + ANSWER=$(echo "${ANSWER}" | tr '[:upper:]' '[:lower:]') + done + +} + # Check if misp user is present and if run as root checkID () { debug "Checking if run as root and $MISP_USER is present" @@ -387,7 +410,7 @@ checkID () { if [[ "$UNATTENDED" != "1" ]]; then echo "There is NO user called '$MISP_USER' create a user '$MISP_USER' (y) or continue as $USER (n)? (y/n) " read ANSWER - ANSWER=$(echo $ANSWER |tr [A-Z] [a-z]) + ANSWER=$(echo $ANSWER |tr '[:upper:]' '[:lower:]') else ANSWER="y" fi @@ -530,7 +553,7 @@ setBaseURL () { echo "You can now enter your own MISP_BASEURL, if you wish to NOT do that, the MISP_BASEURL will be empty, which will work, but ideally you configure it afterwards." echo "Do you want to change it now? (y/n) " read ANSWER - ANSWER=$(echo $ANSWER |tr [A-Z] [a-z]) + ANSWER=$(echo $ANSWER |tr '[:upper:]' '[:lower:]') if [[ "$ANSWER" == "y" ]]; then if [[ ! -z $IP ]]; then echo "It seems you have an interface called $IFACE UP with the following IP: $IP - FYI" @@ -854,6 +877,7 @@ genRCLOCAL () { sed -i -e '$i \echo never > /sys/kernel/mm/transparent_hugepage/enabled\n' /etc/rc.local sed -i -e '$i \echo 1024 > /proc/sys/net/core/somaxconn\n' /etc/rc.local sed -i -e '$i \sysctl vm.overcommit_memory=1\n' /etc/rc.local + sed -i -e '$i \[ -f /etc/init.d/firstBoot ] && bash /etc/init.d/firstBoot\n' /etc/rc.local } # Run PyMISP tests @@ -943,7 +967,16 @@ aptUpgrade () { debug "Upgrading system" checkAptLock sudo apt-get update - sudo apt-get upgrade -qy + + # If we run in non-interactive mode, make sure we do not stop all of a sudden + if [[ "${PACKER}" == "1" || "${UNATTENDED}" == "1" ]]; then + export DEBIAN_FRONTEND=noninteractive + export DEBIAN_PRIORITY=critical + sudo -E apt-get -qy -o "Dpkg::Options::=--force-confdef" -o "Dpkg::Options::=--force-confold" upgrade + sudo -E apt-get -qy autoclean + else + sudo apt-get upgrade -qy + fi } # check if sudo is installed @@ -2117,6 +2150,9 @@ installMISPonKali () { # install python-magic $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install python-magic 2> /dev/null > /dev/null + # install plyara + $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install plyara 2> /dev/null > /dev/null + # install zmq needed by mispzmq $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install zmq 2> /dev/null > /dev/null @@ -2318,7 +2354,7 @@ fi # If Ubuntu is detected, figure out which release it is and run the according scripts if [ "${FLAVOUR}" == "ubuntu" ]; then - RELEASE=$(lsb_release -s -r| tr [A-Z] [a-z]) + RELEASE=$(lsb_release -s -r| tr '[:upper:]' '[:lower:]') if [ "${RELEASE}" == "18.04" ]; then echo "Install on Ubuntu 18.04 LTS fully supported." echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues" @@ -2344,7 +2380,7 @@ fi # If Debian is detected, figure out which release it is and run the according scripts if [ "${FLAVOUR}" == "debian" ]; then - CODE=$(lsb_release -s -c| tr [A-Z] [a-z]) + CODE=$(lsb_release -s -c| tr '[:upper:]' '[:lower:]') if [ "${CODE}" == "buster" ]; then echo "Install on Debian testing fully supported." echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues" @@ -2366,7 +2402,7 @@ fi # If Tsurugi is detected, figure out which release it is and run the according scripts if [ "${FLAVOUR}" == "tsurugi" ]; then - CODE=$(lsb_release -s -c| tr [A-Z] [a-z]) + CODE=$(lsb_release -s -c| tr '[:upper:]' '[:lower:]') if [ "${CODE}" == "bamboo" ]; then echo "Install on Tsurugi Lab partially supported." echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues" diff --git a/INSTALL/INSTALL.sh.sfv b/INSTALL/INSTALL.sh.sfv index 0c668962c..cbc9ddd71 100644 --- a/INSTALL/INSTALL.sh.sfv +++ b/INSTALL/INSTALL.sh.sfv @@ -1,5 +1,5 @@ -; Generated by RHash v1.3.8 on 2019-06-13 at 09:09.32 +; Generated by RHash v1.3.8 on 2019-06-20 at 14:00.36 ; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/ ; -; 94127 09:09.32 2019-06-13 INSTALL.sh -INSTALL.sh BE7DE7EC5AA24D277F5F752E59356BAD26DC6EF7 E3ACE0081A95A94771EFE51AB8EDEE10CE5687F07A4C36DFD791B9B5F7E128DA 715AE909E454D2A74EEAF8A83D38F82534CB17B9C2BBDF5927251F08BD34651CF46D3260506ACC2BD3F26370E3A89696 24C56E075B7D4415517614C35A255A1D8217495A6A658F6B344A5D98AC24BD9A903E85EC3DC0C25CA89070EF0C0FB82D114ED7FA96D8120179DF4AEF0B1914ED +; 95437 14:00.36 2019-06-20 INSTALL.sh +INSTALL.sh 8CFDF8FC14572C9AA51673D7449C354257C6CE67 61B8976709B44B3F2C0491EDB6FA013AC15BA4833F074929810C65C93C274FCB 75D5538717565401F180E7C777BE41A167B2365473806141C9357E75220DD198BE68DC2B59452F39396B42FB50208194 460D731E91EAA0F589B96D13CC451BCB7014C683A357099B13492E7683A459ECB7C2A43AFFB237C966CE67DD06140199686E968B518A33ED73C84135948E0749 diff --git a/INSTALL/INSTALL.sh.sha1 b/INSTALL/INSTALL.sh.sha1 index 14d559c83..716062ee5 100644 --- a/INSTALL/INSTALL.sh.sha1 +++ b/INSTALL/INSTALL.sh.sha1 @@ -1 +1 @@ -be7de7ec5aa24d277f5f752e59356bad26dc6ef7 INSTALL.sh +8cfdf8fc14572c9aa51673d7449c354257c6ce67 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha256 b/INSTALL/INSTALL.sh.sha256 index 981af954b..bd3388041 100644 --- a/INSTALL/INSTALL.sh.sha256 +++ b/INSTALL/INSTALL.sh.sha256 @@ -1 +1 @@ -e3ace0081a95a94771efe51ab8edee10ce5687f07a4c36dfd791b9b5f7e128da INSTALL.sh +61b8976709b44b3f2c0491edb6fa013ac15ba4833f074929810c65c93c274fcb INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha384 b/INSTALL/INSTALL.sh.sha384 index 8b18ad67d..98ebc9458 100644 --- a/INSTALL/INSTALL.sh.sha384 +++ b/INSTALL/INSTALL.sh.sha384 @@ -1 +1 @@ -715ae909e454d2a74eeaf8a83d38f82534cb17b9c2bbdf5927251f08bd34651cf46d3260506acc2bd3f26370e3a89696 INSTALL.sh +75d5538717565401f180e7c777be41a167b2365473806141c9357e75220dd198be68dc2b59452f39396b42fb50208194 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha512 b/INSTALL/INSTALL.sh.sha512 index 71f848293..3e622b45f 100644 --- a/INSTALL/INSTALL.sh.sha512 +++ b/INSTALL/INSTALL.sh.sha512 @@ -1 +1 @@ -24c56e075b7d4415517614c35a255a1d8217495a6a658f6b344a5d98ac24bd9a903e85ec3dc0c25ca89070ef0c0fb82d114ed7fa96d8120179df4aef0b1914ed INSTALL.sh +460d731e91eaa0f589b96d13cc451bcb7014c683a357099b13492e7683a459ecb7c2a43affb237c966ce67dd06140199686e968b518a33ed73c84135948e0749 INSTALL.sh diff --git a/INSTALL/INSTALL.tpl.sh b/INSTALL/INSTALL.tpl.sh index e79f3d6f7..d0d1fdd9e 100755 --- a/INSTALL/INSTALL.tpl.sh +++ b/INSTALL/INSTALL.tpl.sh @@ -465,6 +465,9 @@ installMISPonKali () { # install python-magic $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install python-magic 2> /dev/null > /dev/null + # install plyara + $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install plyara 2> /dev/null > /dev/null + # install zmq needed by mispzmq $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install zmq 2> /dev/null > /dev/null @@ -666,7 +669,7 @@ fi # If Ubuntu is detected, figure out which release it is and run the according scripts if [ "${FLAVOUR}" == "ubuntu" ]; then - RELEASE=$(lsb_release -s -r| tr [A-Z] [a-z]) + RELEASE=$(lsb_release -s -r| tr '[:upper:]' '[:lower:]') if [ "${RELEASE}" == "18.04" ]; then echo "Install on Ubuntu 18.04 LTS fully supported." echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues" @@ -692,7 +695,7 @@ fi # If Debian is detected, figure out which release it is and run the according scripts if [ "${FLAVOUR}" == "debian" ]; then - CODE=$(lsb_release -s -c| tr [A-Z] [a-z]) + CODE=$(lsb_release -s -c| tr '[:upper:]' '[:lower:]') if [ "${CODE}" == "buster" ]; then echo "Install on Debian testing fully supported." echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues" @@ -714,7 +717,7 @@ fi # If Tsurugi is detected, figure out which release it is and run the according scripts if [ "${FLAVOUR}" == "tsurugi" ]; then - CODE=$(lsb_release -s -c| tr [A-Z] [a-z]) + CODE=$(lsb_release -s -c| tr '[:upper:]' '[:lower:]') if [ "${CODE}" == "bamboo" ]; then echo "Install on Tsurugi Lab partially supported." echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues" diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index 557597f1f..153cc85f6 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -46,7 +46,7 @@ class AppController extends Controller public $helpers = array('Utility', 'OrgImg', 'FontAwesome'); - private $__queryVersion = '76'; + private $__queryVersion = '77'; public $pyMispVersion = '2.4.106'; public $phpmin = '7.0'; public $phprec = '7.2'; @@ -525,7 +525,7 @@ class AppController extends Controller private function __convertEmailToName($email) { - $name = explode('@', $email); + $name = explode('@', (string)$email); $name = explode('.', $name[0]); foreach ($name as $key => $value) { $name[$key] = ucfirst($value); @@ -668,7 +668,7 @@ class AppController extends Controller foreach ($options['paramArray'] as $p) { if ( isset($options['ordered_url_params'][$p]) && - (!in_array(strtolower($options['ordered_url_params'][$p]), array('null', '0', false, 'false', null))) + (!in_array(strtolower((string)$options['ordered_url_params'][$p]), array('null', '0', false, 'false', null))) ) { $data[$p] = $options['ordered_url_params'][$p]; $data[$p] = str_replace(';', ':', $data[$p]); diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php index f954a9d09..d50b3c01d 100644 --- a/app/Controller/Component/ACLComponent.php +++ b/app/Controller/Component/ACLComponent.php @@ -343,6 +343,7 @@ class ACLComponent extends Component 'getPyMISPVersion' => array('*'), 'getSubmodulesStatus' => array('perm_site_admin'), 'getSubmoduleQuickUpdateForm' => array('perm_site_admin'), + 'getWorkers' => array(), 'getVersion' => array('*'), 'import' => ('perm_site_admin'), 'index' => array('OR' => array('perm_sync', 'perm_admin')), diff --git a/app/Controller/EventBlacklistsController.php b/app/Controller/EventBlacklistsController.php index 1080f8e97..a42bb636d 100644 --- a/app/Controller/EventBlacklistsController.php +++ b/app/Controller/EventBlacklistsController.php @@ -77,7 +77,7 @@ class EventBlacklistsController extends AppController if ($this->_isRest()) { return $this->RestResponse->saveSuccessResponse('EventBlacklist', 'Deleted', $ids, $this->response->type()); } else { - $this->Flash->success('Event deleted.'); + $this->Flash->success('Blacklist entry removed'); $this->redirect(array('controller' => 'eventBlacklists', 'action' => 'index')); } } else { diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index e285e38ec..505a2c1be 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -1549,6 +1549,8 @@ class EventsController extends AppController $conditions['includeFeedCorrelations'] = 1; if (!$this->_isRest()) { $conditions['includeGranularCorrelations'] = 1; + } else if (!empty($this->params['named']['includeGranularCorrelations'])) { + $conditions['includeGranularCorrelations'] = 1; } if (!isset($this->params['named']['includeServerCorrelations'])) { $conditions['includeServerCorrelations'] = 1; @@ -1559,6 +1561,28 @@ class EventsController extends AppController $conditions['includeServerCorrelations'] = $this->params['named']['includeServerCorrelations']; } $results = $this->Event->fetchEvent($this->Auth->user(), $conditions); + if (!empty($this->params['named']['includeGranularCorrelations'])) { + foreach ($results as $k => $event) { + if (!empty($event['RelatedAttribute'])) { + foreach ($event['RelatedAttribute'] as $attribute_id => $relation) { + foreach ($event['Attribute'] as $k2 => $attribute) { + if ((int)$attribute['id'] == $attribute_id) { + $results[$k]['Attribute'][$k2]['RelatedAttribute'][] = $relation; + break 2; + } + } + foreach ($event['Object'] as $k2 => $object) { + foreach ($object['Attribute'] as $k3 => $attribute) { + if ((int)$attribute['id'] == $attribute_id) { + $results[$k]['Object'][$k2]['Attribute'][$k3]['RelatedAttribute'][] = $relation; + break 3; + } + } + } + } + } + } + } if (empty($results)) { throw new NotFoundException(__('Invalid event')); } diff --git a/app/Controller/ObjectTemplatesController.php b/app/Controller/ObjectTemplatesController.php index d1bcb66dc..b4f6c8031 100644 --- a/app/Controller/ObjectTemplatesController.php +++ b/app/Controller/ObjectTemplatesController.php @@ -163,12 +163,22 @@ class ObjectTemplatesController extends AppController public function index($all = false) { + $passedArgsArray = array(); + $passedArgs = $this->passedArgs; if (!$all || !$this->_isSiteAdmin()) { $this->paginate['conditions'][] = array('ObjectTemplate.active' => 1); $this->set('all', false); } else { $this->set('all', true); } + if (!empty($this->params['named']['searchall'])) { + $this->paginate['conditions']['AND']['OR'] = array( + 'ObjectTemplate.uuid LIKE' => '%' . strtolower($this->params['named']['searchall']) . '%', + 'LOWER(ObjectTemplate.name) LIKE' => '%' . strtolower($this->params['named']['searchall']) . '%', + 'ObjectTemplate.meta-category LIKE' => '%' . strtolower($this->params['named']['searchall']) . '%', + 'LOWER(ObjectTemplate.description) LIKE' => '%' . strtolower($this->params['named']['searchall']) . '%' + ); + } if ($this->_isRest()) { $rules = $this->paginate; unset($rules['limit']); @@ -180,6 +190,8 @@ class ObjectTemplatesController extends AppController $objectTemplates = $this->paginate(); $this->set('list', $objectTemplates); } + $this->set('passedArgs', json_encode($passedArgs)); + $this->set('passedArgsArray', $passedArgsArray); } public function update($type = false, $force = false) diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index 102ae2b32..8de5bbab0 100644 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -1103,7 +1103,13 @@ class ServersController extends AppController } else { shell_exec($prepend . APP . 'Console' . DS . 'cake CakeResque.CakeResque startscheduler -i 5 > /dev/null 2>&1 &'); } - $this->redirect('/servers/serverSettings/workers'); + $message = __('Worker start signal sent'); + if ($this->_isRest()) { + return $this->RestResponse->saveSuccessResponse('Servers', 'startWorker', $type, $this->response->type(), $message); + } else { + $this->Flash->info($message); + $this->redirect('/servers/serverSettings/workers'); + } } public function stopWorker($pid) @@ -1112,7 +1118,20 @@ class ServersController extends AppController throw new MethodNotAllowedException(); } $this->Server->killWorker($pid, $this->Auth->user()); - $this->redirect('/servers/serverSettings/workers'); + $message = __('Worker stop signal sent'); + if ($this->_isRest()) { + return $this->RestResponse->saveSuccessResponse('Servers', 'stopWorker', $pid, $this->response->type(), $message); + } else { + $this->Flash->info($message); + $this->redirect('/servers/serverSettings/workers'); + } + } + + public function getWorkers() + { + $issues = 0; + $worker_array = $this->Server->workerDiagnostics($issues); + return $this->RestResponse->viewData($worker_array); } private function __checkVersion() @@ -1168,6 +1187,9 @@ class ServersController extends AppController } $setting = $this->Server->getSettingData($setting_name); + if (!empty($setting['cli_only'])) { + throw new MethodNotAllowedException(__('This setting can only be edited via the CLI.')); + } if ($this->request->is('get')) { if ($setting != null) { $value = Configure::read($setting['name']); diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 889ae0b60..0939e46b1 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -1477,9 +1477,6 @@ class Attribute extends AppModel $value = strtolower($value); str_replace(':', '|', $value); break; - case 'float': - $value = floatval($value); - break; case 'hex': $value = strtoupper($value); break; diff --git a/app/Model/MispObject.php b/app/Model/MispObject.php index 1610a0e6c..f06b31a19 100644 --- a/app/Model/MispObject.php +++ b/app/Model/MispObject.php @@ -747,11 +747,10 @@ class MispObject extends AppModel } $attribute_types = array_keys($attribute_types); - $potential_templates = $this->ObjectTemplate->find('all', array( + $potential_templates = $this->ObjectTemplate->find('list', array( 'recursive' => -1, 'fields' => array( 'ObjectTemplate.id', - 'ObjectTemplate.name', 'COUNT(ObjectTemplateElement.type) as type_count' ), 'conditions' => array( @@ -767,11 +766,11 @@ class MispObject extends AppModel 'conditions' => array('ObjectTemplate.id = ObjectTemplateElement.object_template_id') ) ), - 'group' => 'ObjectTemplate.name', + 'group' => 'ObjectTemplate.id', 'order' => 'type_count DESC' )); - $potential_template_ids = Hash::extract($potential_templates, '{n}.ObjectTemplate.id'); + $potential_template_ids = array_keys($potential_templates); $templates = $this->ObjectTemplate->find('all', array( 'recursive' => -1, 'conditions' => array('id' => $potential_template_ids), diff --git a/app/Model/Server.php b/app/Model/Server.php index 072505b1e..ab902a7ed 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -192,6 +192,15 @@ class Server extends AppModel 'type' => 'boolean', 'null' => true ), + 'server_settings_skip_backup_rotate' => array( + 'level' => 1, + 'description' => __('Enable this setting to directly save the config.php file without first creating a temporary file and moving it to avoid concurency issues. Generally not recommended, but useful when for example other tools modify/maintain the config.php file.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), 'python_bin' => array( 'level' => 1, 'description' => __('It is highly recommended to install all the python dependencies in a virtualenv. The recommended location is: %s/venv', ROOT), @@ -201,6 +210,7 @@ class Server extends AppModel 'test' => 'testForBinExec', 'beforeHook' => 'beforeHookBinExec', 'type' => 'string', + 'cli_only' => 1 ), 'disable_auto_logout' => array( 'level' => 1, @@ -456,6 +466,7 @@ class Server extends AppModel 'null' => false, 'test' => 'testForWritableDir', 'type' => 'string', + 'cli_only' => 1 ), 'cached_attachments' => array( 'level' => 1, @@ -847,6 +858,7 @@ class Server extends AppModel 'test' => 'testForPath', 'type' => 'string', 'null' => true, + 'cli_only' => 1 ), 'custom_css' => array( 'level' => 2, @@ -970,6 +982,7 @@ class Server extends AppModel 'errorMessage' => '', 'test' => 'testForGPGBinary', 'type' => 'string', + 'cli_only' => 1 ), 'onlyencrypted' => array( 'level' => 0, @@ -3076,6 +3089,9 @@ class Server extends AppModel public function testForBinExec($value) { + if (substr($value, 0, 7) === "phar://") { + return 'Phar protocol not allowed.'; + } $finfo = finfo_open(FILEINFO_MIME_TYPE); if ($value === '') { return true; @@ -3094,6 +3110,9 @@ class Server extends AppModel public function testForWritableDir($value) { + if (substr($value, 0, 7) === "phar://") { + return 'Phar protocol not allowed.'; + } if (!is_dir($value)) { return 'Not a valid directory.'; } @@ -3724,26 +3743,30 @@ class Server extends AppModel if (function_exists('opcache_reset')) { opcache_reset(); } - $randomFilename = $this->generateRandomFileName(); - // To protect us from 2 admin users having a concurent file write to the config file, solar flares and the bogeyman - file_put_contents(APP . 'Config' . DS . $randomFilename, $settingsString); - rename(APP . 'Config' . DS . $randomFilename, APP . 'Config' . DS . 'config.php'); - $config_saved = file_get_contents(APP . 'Config' . DS . 'config.php'); - // if the saved config file is empty, restore the backup. - if (strlen($config_saved) < 20) { - copy(APP . 'Config' . DS . 'config.php.bk', APP . 'Config' . DS . 'config.php'); - $this->Log = ClassRegistry::init('Log'); - $this->Log->create(); - $this->Log->save(array( - 'org' => 'SYSTEM', - 'model' => 'Server', - 'model_id' => $id, - 'email' => 'SYSTEM', - 'action' => 'error', - 'user_id' => 0, - 'title' => 'Error: Something went wrong saving the config file, reverted to backup file.', - )); - return false; + if (empty(Configure::read('MISP.server_settings_skip_backup_rotate'))) { + $randomFilename = $this->generateRandomFileName(); + // To protect us from 2 admin users having a concurent file write to the config file, solar flares and the bogeyman + file_put_contents(APP . 'Config' . DS . $randomFilename, $settingsString); + rename(APP . 'Config' . DS . $randomFilename, APP . 'Config' . DS . 'config.php'); + $config_saved = file_get_contents(APP . 'Config' . DS . 'config.php'); + // if the saved config file is empty, restore the backup. + if (strlen($config_saved) < 20) { + copy(APP . 'Config' . DS . 'config.php.bk', APP . 'Config' . DS . 'config.php'); + $this->Log = ClassRegistry::init('Log'); + $this->Log->create(); + $this->Log->save(array( + 'org' => 'SYSTEM', + 'model' => 'Server', + 'model_id' => $id, + 'email' => 'SYSTEM', + 'action' => 'error', + 'user_id' => 0, + 'title' => 'Error: Something went wrong saving the config file, reverted to backup file.', + )); + return false; + } + } else { + file_put_contents(APP . 'Config' . DS . 'config.php', $settingsString); } return true; } diff --git a/app/View/Elements/ajaxTags.ctp b/app/View/Elements/ajaxTags.ctp index 3b4afadef..04502f06c 100644 --- a/app/View/Elements/ajaxTags.ctp +++ b/app/View/Elements/ajaxTags.ctp @@ -21,14 +21,24 @@ ?> TextColour->getTextColour($tag['Tag']['colour']) . ';'; $aClass = $full ? 'tagFirstHalf' : 'tag'; $aText = h($tag['Tag']['name']); - $aSearchTagUrl = $baseurl . '/events/index/searchtag: ' . h($tag['Tag']['id']); - $span1 = sprintf('%s', $aSearchTagUrl, $aStyle, $aClass, $aText); + if (!empty($tag['Tag']['id'])) { + $aSearchTagUrl = $baseurl . '/events/index/searchtag: ' . h($tag['Tag']['id']); + $span1 = sprintf('%s', $aSearchTagUrl, $aStyle, $aClass, $aText); + } else { + $span1 = sprintf('%s', $aStyle, $aClass, $aText); + } $span2 = ''; if ($full) { $spanClass = "tagSecondHalf useCursorPointer noPrint"; diff --git a/app/View/Elements/eventattribute.ctp b/app/View/Elements/eventattribute.ctp index fceb11d5d..a157f8c7b 100644 --- a/app/View/Elements/eventattribute.ctp +++ b/app/View/Elements/eventattribute.ctp @@ -315,6 +315,32 @@ attributes or the appropriate distribution level. If you think there is a mistak url = "" + "/sightings/advanced/" + object_id + "/" + object_context; genericPopup(url, '#popover_box'); }); + $(".eventViewAttributeHover").mouseenter(function() { + $('#' + currentPopover).popover('destroy'); + var type = $(this).attr('data-object-type'); + var id = $(this).attr('data-object-id'); + + if (type + "_" + id in ajaxResults["hover"]) { + var element = $('#' + type + '_' + id + '_container'); + element.popover({ + title: attributeHoverTitle(id, type), + content: ajaxResults["hover"][type + "_" + id], + placement: attributeHoverPlacement(element), + html: true, + trigger: 'manual', + container: 'body' + }).popover('show'); + currentPopover = type + '_' + id + '_container'; + } else { + timer = setTimeout(function () { + runHoverLookup(type, id) + }, + 500 + ); + } + }).mouseout(function() { + clearTimeout(timer); + }); }); $('#attributesFilterField').bind("keydown", function(e) { var eventid = $('#attributesFilterField').data("eventid"); diff --git a/app/View/Elements/healthElements/settings_row.ctp b/app/View/Elements/healthElements/settings_row.ctp index a575b5296..3be1d1d28 100644 --- a/app/View/Elements/healthElements/settings_row.ctp +++ b/app/View/Elements/healthElements/settings_row.ctp @@ -28,7 +28,7 @@ 'value_passive' => array( 'html' => nl2br(h($setting['value'])), 'class' => 'inline-field-solid live_filter_target', - 'requirement' => ((isset($setting['editable']) && !$setting['editable'])), + 'requirement' => ((isset($setting['editable']) && !$setting['editable']) || !empty($setting['cli_only'])), 'style' => 'width:500px;', 'id' => sprintf( 'setting_%s_%s_passive', @@ -39,7 +39,7 @@ 'value_solid' => array( 'html' => nl2br(h($setting['value'])), 'class' => 'inline-field-solid live_filter_target', - 'requirement' => ((!isset($setting['editable']) || $setting['editable'])), + 'requirement' => ((!isset($setting['editable']) || $setting['editable']) && empty($setting['cli_only'])), 'style' => 'width:500px;', 'id' => sprintf( 'setting_%s_%s_solid', @@ -51,7 +51,7 @@ ), 'value_placeholder' => array( 'class' => 'inline-field-placeholder hidden', - 'requirement' => ((!isset($setting['editable']) || $setting['editable'])), + 'requirement' => ((!isset($setting['editable']) || $setting['editable']) && empty($setting['cli_only'])), 'style' => 'width:500px;', 'id' => sprintf( 'setting_%s_%s_placeholder', @@ -60,7 +60,11 @@ ) ), 'description' => array( - 'html' => h($setting['description']), + 'html' => sprintf( + '%s%s', + !empty($setting['cli_only']) ? sprintf('[%s] ', __('CLI only')) : '', + h($setting['description']) + ), 'class' => 'live_filter_target' ), 'error' => array( diff --git a/app/View/Events/automation.ctp b/app/View/Events/automation.ctp index 6829f587f..893026975 100644 --- a/app/View/Events/automation.ctp +++ b/app/View/Events/automation.ctp @@ -329,6 +329,17 @@ echo sprintf('

%s

', $data['title']); echo sprintf('

%s

', implode(" ", $data['description'])); echo sprintf("
%s
", implode("\n", $data['url'])); + $data = array( + 'title' => __('Administering the background workers via the API.'), + 'description' => array( + __('You can start/stop and view the bacground workers via the API.'), + sprintf('
%s: %s/servers/%s
', __('Add worker'), $baseurl, 'startWorker/[queue_name]'), + sprintf('%s: %s/servers/%s
', __('Stop worker'), $baseurl, 'stopWorker/[worker_pid]'), + sprintf('%s: %s/servers/%s
', __('Get worker info'), $baseurl, 'getWorkers') + ) + ); + echo sprintf('

%s

', $data['title']); + echo sprintf('

%s

', implode(" ", $data['description'])); foreach ($command_line_functions as $clusterRef => $cluster) { echo sprintf('

%s

', $clusterRef, $cluster['header']); echo sprintf('

%s:
', $cluster['description']); diff --git a/app/View/Events/view_graph.ctp b/app/View/Events/view_graph.ctp index 8647f5d57..5288607e6 100644 --- a/app/View/Events/view_graph.ctp +++ b/app/View/Events/view_graph.ctp @@ -12,7 +12,7 @@ -

+
@@ -30,6 +30,7 @@
  • + diff --git a/app/View/ObjectTemplates/index.ctp b/app/View/ObjectTemplates/index.ctp index 3482d4b9d..3c1f77a94 100644 --- a/app/View/ObjectTemplates/index.ctp +++ b/app/View/ObjectTemplates/index.ctp @@ -41,6 +41,12 @@ 'active' => $all ) ) + ), + array( + 'type' => 'search', + 'button' => __('Filter'), + 'placeholder' => __('Enter value to search'), + 'data' => '', ) ) ); @@ -141,5 +147,17 @@ endforeach; ?>
    + element('/genericElements/SideMenu/side_menu', array('menuList' => 'objectTemplates', 'menuItem' => 'index')); diff --git a/app/View/Users/edit.ctp b/app/View/Users/edit.ctp index bf77caf44..51e508682 100755 --- a/app/View/Users/edit.ctp +++ b/app/View/Users/edit.ctp @@ -20,7 +20,7 @@ ?>
    Form->input('gpgkey', array('label' => __('GnuPG key'), 'div' => 'clear', 'class' => 'input-xxlarge')); + echo $this->Form->input('gpgkey', array('label' => __('GnuPG key'), 'div' => 'clear', 'class' => 'input-xxlarge', 'placeholder' => __('Paste the user\'s GnuPG key here or try to retrieve it from the MIT key server by clicking on "Fetch GnuPG key" below.'))); ?>
    ``` diff --git a/docs/generic/globalVariables.md b/docs/generic/globalVariables.md index 412c542bf..420befbe1 100644 --- a/docs/generic/globalVariables.md +++ b/docs/generic/globalVariables.md @@ -57,9 +57,11 @@ MISPvars () { # GPG configuration GPG_REAL_NAME='Autogenerated Key' + # On a REAL install, please do not set a comment, see here for why: https://www.debian-administration.org/users/dkg/weblog/97 GPG_COMMENT='WARNING: MISP AutoGenerated Key consider this Key VOID!' GPG_EMAIL_ADDRESS='admin@admin.test' - GPG_KEY_LENGTH='2048' + # 3072 bits used as per suggestions here: https://riseup.net/en/security/message-security/openpgp/best-practices + GPG_KEY_LENGTH='3072' GPG_PASSPHRASE="$(openssl rand -hex 32)" # debug alias to make sure people are not confused when blindly copy pasting blobs of code diff --git a/docs/generic/supportFunctions.md b/docs/generic/supportFunctions.md index 56882d7cb..8ba6c2512 100644 --- a/docs/generic/supportFunctions.md +++ b/docs/generic/supportFunctions.md @@ -95,7 +95,7 @@ checkCoreOS () { if [[ -f "/etc/redhat-release" ]]; then echo "This is some redhat flavour" REDHAT=1 - RHfla=$(cat /etc/redhat-release | cut -f 1 -d\ | tr [A-Z] [a-z]) + RHfla=$(cat /etc/redhat-release | cut -f 1 -d\ | tr '[:upper:]' '[:lower:]') fi } @@ -107,7 +107,7 @@ checkFlavour () { sudo apt install lsb-release dialog -y fi - FLAVOUR=$(lsb_release -s -i |tr [A-Z] [a-z]) + FLAVOUR=$(lsb_release -s -i |tr '[:upper:]' '[:lower:]') if [ FLAVOUR == "ubuntu" ]; then RELEASE=$(lsb_release -s -r) debug "We detected the following Linux flavour: ${YELLOW}$(tr '[:lower:]' '[:upper:]' <<< ${FLAVOUR:0:1})${FLAVOUR:1} ${RELEASE}${NC}" @@ -225,6 +225,27 @@ checkFail () { fi } +ask_o () { + + ANSWER="" + + if [ -z "${1}" ]; then + echo "This function needs at least 1 parameter." + exit 1 + fi + + [ -z "${2}" ] && OPT1="y" || OPT1="${2}" + [ -z "${3}" ] && OPT2="n" || OPT2="${3}" + + while true; do + case "${ANSWER}" in "${OPT1}" | "${OPT2}") break ;; esac + echo -e -n "${1} (${OPT1}/${OPT2}) " + read ANSWER + ANSWER=$(echo "${ANSWER}" | tr '[:upper:]' '[:lower:]') + done + +} + # Check if misp user is present and if run as root checkID () { debug "Checking if run as root and $MISP_USER is present" @@ -235,7 +256,7 @@ checkID () { if [[ "$UNATTENDED" != "1" ]]; then echo "There is NO user called '$MISP_USER' create a user '$MISP_USER' (y) or continue as $USER (n)? (y/n) " read ANSWER - ANSWER=$(echo $ANSWER |tr [A-Z] [a-z]) + ANSWER=$(echo $ANSWER |tr '[:upper:]' '[:lower:]') else ANSWER="y" fi @@ -378,7 +399,7 @@ setBaseURL () { echo "You can now enter your own MISP_BASEURL, if you wish to NOT do that, the MISP_BASEURL will be empty, which will work, but ideally you configure it afterwards." echo "Do you want to change it now? (y/n) " read ANSWER - ANSWER=$(echo $ANSWER |tr [A-Z] [a-z]) + ANSWER=$(echo $ANSWER |tr '[:upper:]' '[:lower:]') if [[ "$ANSWER" == "y" ]]; then if [[ ! -z $IP ]]; then echo "It seems you have an interface called $IFACE UP with the following IP: $IP - FYI" @@ -706,6 +727,7 @@ genRCLOCAL () { sed -i -e '$i \echo never > /sys/kernel/mm/transparent_hugepage/enabled\n' /etc/rc.local sed -i -e '$i \echo 1024 > /proc/sys/net/core/somaxconn\n' /etc/rc.local sed -i -e '$i \sysctl vm.overcommit_memory=1\n' /etc/rc.local + sed -i -e '$i \[ -f /etc/init.d/firstBoot ] && bash /etc/init.d/firstBoot\n' /etc/rc.local } # Run PyMISP tests diff --git a/tools/misp-backup/misp-backup.sh b/tools/misp-backup/misp-backup.sh index 49d9fac7d..5a535217e 100755 --- a/tools/misp-backup/misp-backup.sh +++ b/tools/misp-backup/misp-backup.sh @@ -26,11 +26,14 @@ ## vi misp-backup.conf # adjust values ## sudo bash misp-backup.sh 2>&1 | tee misp-backup.log ## -## TODO: Target directory, rudimentary free space check: stat -f --format="%a" OutputDirName -## TODO: Make sure no directories are blank +## TODO: Make sure no directories are blank, $OutputDirName==Done ## TODO: Review how much sense it makes to ask fo MySQL credentials when most of the script does auto detection anyway. ## +# This makes use of the standard variables used by the installer +eval "$(curl -fsSL https://raw.githubusercontent.com/MISP/MISP/2.4/docs/generic/globalVariables.md | grep -v \`\`\`)" +MISPvars > /dev/null 2>&1 + # Leave empty for NO debug messages, if run with set -x or bash -x it will enable DEBUG by default DEBUG= @@ -59,6 +62,10 @@ checkDiskFree () { if [[ ! -e $1 ]]; then echo "$1 does not exist, creating" mkdir -p $1 + if [[ $? != 0 ]]; then + echo "Could not create $1, please fix permissions." + exit 126 + fi fi threshhold=90 free=$(df -l --sync --output=pcent $1 |tail -1|cut -f 1 -d% | tr -d \ ) diff --git a/tools/misp-backup/misp-restore.sh b/tools/misp-backup/misp-restore.sh index ea750a9ed..7d02bf0e5 100755 --- a/tools/misp-backup/misp-restore.sh +++ b/tools/misp-backup/misp-restore.sh @@ -27,6 +27,10 @@ # TODO: Move DB, check DB? # TODO: Check db user exists. +# This makes use of the standard variables used by the installer +eval "$(curl -fsSL https://raw.githubusercontent.com/MISP/MISP/2.4/docs/generic/globalVariables.md | grep -v \`\`\`)" +MISPvars > /dev/null 2>&1 + # Leave empty for NO debug messages, if run with set -x or bash -x it will enable DEBUG by default DEBUG= diff --git a/tools/misp-wipe/misp-wipe.conf.sample b/tools/misp-wipe/misp-wipe.conf.sample index 69bc79c45..58ed36825 100644 --- a/tools/misp-wipe/misp-wipe.conf.sample +++ b/tools/misp-wipe/misp-wipe.conf.sample @@ -1 +1 @@ -MISPPath=/var/www/MISP +PATH_TO_MISP=/var/www/MISP diff --git a/tools/misp-wipe/misp-wipe.sh b/tools/misp-wipe/misp-wipe.sh index ab5635551..4183893fd 100755 --- a/tools/misp-wipe/misp-wipe.sh +++ b/tools/misp-wipe/misp-wipe.sh @@ -25,6 +25,10 @@ ## Time to set some variables ## +# This makes use of the standard variables used by the installer +eval "$(curl -fsSL https://raw.githubusercontent.com/MISP/MISP/2.4/docs/generic/globalVariables.md | grep -v \`\`\`)" +MISPvars > /dev/null 2>&1 + LUSER_ID="$(id -u)" if [[ "${LUSER_ID}" > "0" ]]; then @@ -54,17 +58,17 @@ fi # Fill in any missing values with defaults # MISP path -MISPPath=${MISPPath:-$(locate MISP/app/webroot/index.php|sed 's/\/app\/webroot\/index\.php//')} +PATH_TO_MISP=${PATH_TO_MISP:-$(locate MISP/app/webroot/index.php|sed 's/\/app\/webroot\/index\.php//')} # database.php -MySQLUUser=$(grep -o -P "(?<='login' => ').*(?=')" $MISPPath/app/Config/database.php) -MySQLUPass=$(grep -o -P "(?<='password' => ').*(?=')" $MISPPath/app/Config/database.php) -MISPDB=$(grep -o -P "(?<='database' => ').*(?=')" $MISPPath/app/Config/database.php) -DB_Port=$(grep -o -P "(?<='port' => ).*(?=,)" $MISPPath/app/Config/database.php) -MISPDBHost=$(grep -o -P "(?<='host' => ').*(?=')" $MISPPath/app/Config/database.php) +MySQLUUser=$(grep -o -P "(?<='login' => ').*(?=')" $PATH_TO_MISP/app/Config/database.php) +MySQLUPass=$(grep -o -P "(?<='password' => ').*(?=')" $PATH_TO_MISP/app/Config/database.php) +MISPDB=$(grep -o -P "(?<='database' => ').*(?=')" $PATH_TO_MISP/app/Config/database.php) +DB_Port=$(grep -o -P "(?<='port' => ).*(?=,)" $PATH_TO_MISP/app/Config/database.php) +MISPDBHost=$(grep -o -P "(?<='host' => ').*(?=')" $PATH_TO_MISP/app/Config/database.php) echo "Clearing data model cache files" -rm -f $MISPPath/app/tmp/cache/models/myapp_* -rm -f $MISPPath/app/tmp/cache/persistent/myapp_* +rm -f $PATH_TO_MISP/app/tmp/cache/models/myapp_* +rm -f $PATH_TO_MISP/app/tmp/cache/persistent/myapp_* echo "Wiping MySQL tables" echo "Removes all users and organizations, except default (id=1)" @@ -76,7 +80,7 @@ mysql --host $MISPDBHost -u $MySQLRUser -p$MySQLRPass $MISPDB < $SQL echo "Inserting default values to MySQL tables" TMP=/tmp/misp-wipe-$$.sql -cd $MISPPath +cd $PATH_TO_MISP sed -n '/Default values for initial installation/ { s///; :a; n; p; ba; }' INSTALL/MYSQL.sql | egrep -v '(admin_settings|db_version)' > $TMP mysql --host $MISPDBHost -u $MySQLRUser -p$MySQLRPass $MISPDB < $TMP rm -f $TMP @@ -88,7 +92,7 @@ git clean -f -d -x app/tmp git clean -f -d -x app/files echo "Updating taxonomies" -baseurl=$(grep -o -P "(?<='baseurl' => ').*(?=')" $MISPPath/app/Config/config.php) +baseurl=$(grep -o -P "(?<='baseurl' => ').*(?=')" $PATH_TO_MISP/app/Config/config.php) AuthKey=$(echo 'select authkey from users where role_id = 1 order by id limit 1;' | mysql -u $MySQLRUser -p$MySQLRPass $MISPDB 2>/dev/null | tail -1) curl --header "Authorization: $AuthKey" --header "Accept: application/json" --header "Content-Type: application/json" -o /dev/null -s -X POST ${baseurl}/taxonomies/update