From 33dc9bc904da5f75d1ede648b144d1697ecc79e9 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 25 Apr 2019 19:43:12 +0900 Subject: [PATCH 01/26] chg: [doc] Debian stable install doc still not working, until Python3.6 will be default. Debian 10 will fix that. --- docs/xINSTALL.debian9.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/xINSTALL.debian9.md b/docs/xINSTALL.debian9.md index 31b088fef..1597a409f 100644 --- a/docs/xINSTALL.debian9.md +++ b/docs/xINSTALL.debian9.md @@ -5,13 +5,13 @@ ------------------------------------ !!! notice - Maintained and tested by @SteveClement on 20190405 + Maintained and tested by @SteveClement on 20190425 !!! warning - This install document is NOT working as expected. There are Python issues as we "only" have python 3.5 but need at least python 3.6 + This install document is **NOT** working as expected. There are Python issues as we "only" have python 3.5 but need at least python 3.6 This guide effectively converts your "stretch" install into a partial "testing" install. Thus following the "testing" install guide is a better choice, but not for production. - One manual work-around is to install Python >3.5 from source. + One manual work-around is to install Python >3.5 from source and leaving apt untouched. ### 1/ Minimal Debian install ------------------------- @@ -80,10 +80,11 @@ sudo apt -t testing install -y \ mariadb-client \ mariadb-server +# /!\ # This is maybe needed. If mysql does not start and you find a solution, please contribute. # What did work for me was running mysqld interactively: sudo mysqld -mkdir -p /var/run/mysqld -chown mysql /var/run/mysqld +sudo mkdir -p /var/run/mysqld +sudo chown mysql /var/run/mysqld sudo /etc/init.d/mysql restart sudo apt -t testing install -y jupyter-notebook From ee735f00d46854fe5790fad75342988f481a6313 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Fri, 26 Apr 2019 09:45:03 +0200 Subject: [PATCH 02/26] new: [update] Injected update-related files/changes from zoidberg --- app/Console/Command/ServerShell.php | 14 + app/Controller/AppController.php | 12 +- app/Controller/Component/ACLComponent.php | 2 + app/Controller/ServersController.php | 64 ++++ app/Model/AppModel.php | 298 +++++++++++++++++- app/Model/Server.php | 10 +- app/View/Elements/Flash/error.ctp | 7 + .../Elements/healthElements/diagnostics.ctp | 10 +- app/View/Servers/ondemand_action.ctp | 87 +++++ app/View/Servers/update_progress.ctp | 156 +++++++++ app/webroot/css/main.css | 77 +++++ app/webroot/js/update_progress.js | 182 +++++++++++ 12 files changed, 895 insertions(+), 24 deletions(-) create mode 100644 app/View/Servers/ondemand_action.ctp create mode 100644 app/View/Servers/update_progress.ctp create mode 100644 app/webroot/js/update_progress.js diff --git a/app/Console/Command/ServerShell.php b/app/Console/Command/ServerShell.php index 67cf048dc..495684469 100644 --- a/app/Console/Command/ServerShell.php +++ b/app/Console/Command/ServerShell.php @@ -446,4 +446,18 @@ class ServerShell extends AppShell $this->Task->id = $task['Task']['id']; $this->Task->saveField('message', count($servers) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '.'); } + + public function updateApp() { + $processId = $this->args[0]; + $job = $this->Job->read(null, $processId); + $command = $this->args[1]; + $liveOff = $this->args[2]; + $exitOnError = $this->args[3]; + $useWorker = $this->args[4]; + $result = $this->Server->updateDatabase($command, $liveOff, $exitOnError, $useWorker); + $job['Job']['progress'] = 100; + $job['Job']['message'] = 'Update done'; + $this->Job->save($job); + } + } diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index 8ff97d661..a86a5ab2c 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -372,7 +372,7 @@ class AppController extends Controller $this->Auth->logout(); throw new MethodNotAllowedException($message);//todo this should pb be removed? } else { - $this->Flash->error('Warning: MISP is currently disabled for all users. Enable it in Server Settings (Administration -> Server Settings -> MISP tab -> live)', array('clear' => 1)); + $this->Flash->error('Warning: MISP is currently disabled for all users. Enable it in Server Settings (Administration -> Server Settings -> MISP tab -> live). An update might also be in progress, you can see the progress in ' , array('params' => array('url' => $baseurl . '/servers/advancedUpdate/', 'urlName' => 'Advanced Update'), 'clear' => 1)); } } @@ -824,7 +824,7 @@ class AppController extends Controller $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); } - public function updateDatabase($command) + public function updateDatabase($command, $liveOff=false, $exitOnError=false) { if (!$this->_isSiteAdmin() || !$this->request->is('post')) { throw new MethodNotAllowedException(); @@ -833,9 +833,13 @@ class AppController extends Controller if (is_numeric($command)) { $command = intval($command); } - $this->Server->updateDatabase($command); + $this->Server->updateDatabase($command, $liveOff, $exitOnError); $this->Flash->success('Done.'); - $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); + if ($liveOff) { + $this->redirect(array('controller' => 'servers', 'action' => 'updateProgress')); + } else { + $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); + } } public function upgrade2324() diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php index e492ad3b4..b7094bad4 100644 --- a/app/Controller/Component/ACLComponent.php +++ b/app/Controller/Component/ACLComponent.php @@ -340,6 +340,7 @@ class ACLComponent extends Component 'getSubmoduleQuickUpdateForm' => array('perm_site_admin'), 'getVersion' => array('*'), 'index' => array('OR' => array('perm_sync', 'perm_admin')), + 'ondemandAction' => array(), 'postTest' => array('perm_sync'), 'previewEvent' => array(), 'previewIndex' => array(), @@ -359,6 +360,7 @@ class ACLComponent extends Component 'testConnection' => array('perm_sync'), 'update' => array(), 'updateJSON' => array(), + 'updateProgress' => array(), 'updateSubmodule' => array(), 'uploadFile' => array(), 'clearWorkerQueue' => array() diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index 70bc5e127..62641784e 100644 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -1542,6 +1542,70 @@ class ServersController extends AppController } } + public function ondemandAction() + { + if (!$this->_isSiteAdmin()) { + throw new MethodNotAllowedException('You are not authorised to do that.'); + } + $this->AdminSetting = ClassRegistry::init('AdminSetting'); + //$actions = $this->Server->advanced_updates_description; + $actions = $this->Server->actions_description; + $default_fields = array( + 'title' => '', + 'description' => '', + 'liveOff' => false, + 'recommendBackup' => false, + 'exitOnError' => false, + 'requirements' => '', + 'url' => '/' + ); + foreach($actions as $id => $action) { + foreach($default_fields as $field => $value) { + if (!isset($action[$field])) { + $actions[$id][$field] = $value; + } + } + $done = $this->AdminSetting->getSetting($id); + $actions[$id]['done'] = $done !== false && $done == '1' ? true : false; + } + $this->set('actions', $actions); + $this->set('updateLocked', $this->Server->isUpdateLocked()); + } + + public function updateProgress() + { + if (!$this->_isSiteAdmin()) { + throw new MethodNotAllowedException('You are not authorised to do that.'); + } + $updateProgress = $this->Server->getUpdateProgress(); + $curIndex = $updateProgress['cur']; + $curCommand = !isset($updateProgress['cmd'][$curIndex]) ? '' : $updateProgress['cmd'][$curIndex]; + $lookupString = preg_replace('/\s{2,}/', '', substr($curCommand, 0, -1)); + $sqlInfo = $this->Server->query("SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;"); + if (empty($sqlInfo)) { + $updateProgress['process_list'] = array(); + } else { + // retreive current update process + foreach($sqlInfo as $row) { + if (preg_replace('/\s{2,}/', '', $row['PROCESSLIST']['INFO']) == $lookupString) { + $sqlInfo = $row['PROCESSLIST']; + break; + } + } + $updateProgress['process_list'] = array(); + $updateProgress['process_list']['STATE'] = isset($sqlInfo['STATE']) ? $sqlInfo['STATE'] : ''; + $updateProgress['process_list']['PROGRESS'] = isset($sqlInfo['PROGRESS']) ? $sqlInfo['PROGRESS'] : 0; + $updateProgress['process_list']['STAGE'] = isset($sqlInfo['STAGE']) ? $sqlInfo['STAGE'] : 0; + $updateProgress['process_list']['MAX_STAGE'] = isset($sqlInfo['MAX_STAGE']) ? $sqlInfo['MAX_STAGE'] : 0; + } + if ($this->request->is('ajax')) { + return $this->RestResponse->viewData(h($updateProgress), $this->response->type()); + } else { + $this->set('updateProgress', $updateProgress); + } + } + + public function getSubmoduleQuickUpdateForm($submodule_path=false) { $this->set('submodule', base64_decode($submodule_path)); $this->render('ajax/submodule_quick_update_form'); diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 4fcadca66..cb443c02b 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -78,6 +78,21 @@ class AppModel extends Model 33 => false, 34 => false ); + public $advanced_updates_description = array( + ); + public $actions_description = array( + 'verifyGnuPGkeys' => array( + 'title' => 'Verify GnuPG keys', + 'description' => "Run a full validation of all GnuPG keys within this instance's userbase. The script will try to identify possible issues with each key and report back on the results.", + 'url' => '/users/verifyGPG/' + ), + 'databaseCleanupScripts' => array( + 'title' => 'Database Cleanup Scripts', + 'description' => 'If you run into an issue with an infinite upgrade loop (when upgrading from version ~2.4.50) that ends up filling your database with upgrade script log messages, run the following script.', + 'url' => '/logs/pruneUpdateLogs/' + ) + ); + public function afterSave($created, $options = array()) { if ($created) { @@ -209,8 +224,39 @@ class AppModel extends Model } // SQL scripts for updates - public function updateDatabase($command) + public function updateDatabase($command, $liveOff=false, $exitOnError=false, $useWorker=true) { + // Exit if updates are locked + if ($this->isUpdateLocked()) { + return false; + } + $this->__resetUpdateProgress(); + // restart this function by a worker + if ($useWorker && Configure::read('MISP.background_jobs')) { + $job = ClassRegistry::init('Job'); + $job->create(); + $data = array( + 'worker' => 'prio', + 'job_type' => 'update_app', + 'job_input' => 'command: ' . $command, + 'status' => 0, + 'retries' => 0, + 'org_id' => '', + 'org' => '', + 'message' => 'Updating.', + ); + $job->save($data); + $jobId = $job->id; + $process_id = CakeResque::enqueue( + 'prio', + 'ServerShell', + array('updateApp', $jobId, $command, $liveOff, $exitOnError, false), + true + ); + $job->saveField('process_id', $process_id); + return true; + } + $dataSourceConfig = ConnectionManager::getDataSource('default')->config; $dataSource = $dataSourceConfig['datasource']; $sqlArray = array(); @@ -1150,8 +1196,41 @@ class AppModel extends Model return false; break; } - foreach ($sqlArray as $sql) { + + $now = new DateTime(); + $this->__changeLockState($now->format('Y-m-d H:i:s')); + // switch MISP instance live to false + if ($liveOff) { + $this->Server = Classregistry::init('Server'); + $liveSetting = 'MISP.live'; + $this->Server->serverSettingsSaveValue($liveSetting, false); + } + $SqlUpdateCount = count($sqlArray); + $IndexUpdateCount = count($indexArray); + $totupdatecount = $SqlUpdateCount + $IndexUpdateCount; + $this->__setUpdateProgress(0, $totupdatecount); + $strIndexArray = array(); + foreach($indexArray as $toIndex) { + $strIndexArray[] = __('Indexing ') . implode($toIndex, '->'); + } + $this->__setUpdateCmdMessages(array_merge($sqlArray, $strIndexArray)); + $flagStop = false; + $errorCount = 0; + foreach ($sqlArray as $i => $sql) { try { + $this->__setUpdateProgress($i, false); + // execute test before update. Exit if it fails + if (isset($this->advanced_updates_description[$command]['preUpdate'])) { + $funName = $this->advanced_updates_description[$command]['preUpdate']; + try { + $this->{$funName}(); + } catch (Exception $e) { + $exitOnError = true; + $this->__setPreUpdateTestState(false); + throw new Exception($e->getMessage()); + } + } + $this->__setPreUpdateTestState(true); $this->query($sql); $this->Log->create(); $this->Log->save(array( @@ -1164,6 +1243,7 @@ class AppModel extends Model 'title' => 'Successfuly executed the SQL query for ' . $command, 'change' => 'The executed SQL query was: ' . $sql )); + $this->__setUpdateResMessages($i, 'Successfuly executed the SQL query for ' . $command); } catch (Exception $e) { $this->Log->create(); $this->Log->save(array( @@ -1176,26 +1256,55 @@ class AppModel extends Model 'title' => 'Issues executing the SQL query for ' . $command, 'change' => 'The executed SQL query was: ' . $sql . PHP_EOL . ' The returned error is: ' . $e->getMessage() )); - } - } - if (!empty($indexArray)) { - if ($clean) { - $this->cleanCacheFiles(); - } - foreach ($indexArray as $iA) { - if (isset($iA[2])) { - $this->__addIndex($iA[0], $iA[1], $iA[2]); - } else { - $this->__addIndex($iA[0], $iA[1]); + $this->__setUpdateResMessages($i, 'Issues executing the SQL query for ' . $command . '. The returned error is: ' . PHP_EOL . $e->getMessage()); + $this->__setUpdateError($i); + $errorCount++; + if ($exitOnError) { + $flagStop = true; + break; } } } + if (!$flagStop) { + if (!empty($indexArray)) { + if ($clean) { + $this->cleanCacheFiles(); + } + foreach ($indexArray as $i => $iA) { + $this->__setUpdateProgress(count($sqlArray)+$i, false); + if (isset($iA[2])) { + $this->__addIndex($iA[0], $iA[1], $iA[2]); + } else { + $this->__addIndex($iA[0], $iA[1]); + } + $this->__setUpdateResMessages(count($sqlArray)+$i, 'Successfuly indexed ' . implode($iA, '->')); + } + } + $this->__setUpdateProgress(count($sqlArray)+count($indexArray), false); + } if ($clean) { $this->cleanCacheFiles(); } + if ($liveOff) { + $liveSetting = 'MISP.live'; + $this->Server->serverSettingsSaveValue($liveSetting, true); + } + if (!$flagStop && $errorCount == 0) { + $this->__postUpdate($command); + } + $this->__changeLockState(false); return true; } + // check whether the adminSetting should be updated after the update + private function __postUpdate($command) { + if (isset($this->advanced_updates_description[$command]['record'])) { + if($this->advanced_updates_description[$command]['record']) { + $this->AdminSetting->changeSetting($command, 1); + } + } + } + private function __dropIndex($table, $field) { $dataSourceConfig = ConnectionManager::getDataSource('default')->config; @@ -1284,6 +1393,43 @@ class AppModel extends Model } } + /* + * Given an array, dynamically add contextual fields. An historical example is given with the *_seen fields + public function addFieldsBasedOnUpdate(&$fieldsAtt, $context = null) { + if (is_null($context)) { + $alias = ''; + } else if ($context === true) { + $alias = $this->alias; + } else { + $alias = $context; + } + if ($this->additionalFeatureEnabled('seenOnAttributeAndObject')) { + // DB have *_seen columns + $fs = (strlen($alias) > 0 ? $alias . '.' : '') . 'first_seen'; + $ls = (strlen($alias) > 0 ? $alias . '.' : '') . 'last_seen'; + array_push($fieldsAtt, $fs, $ls); + } + } + */ + /* + * Check whether a feature is enabled (based on an update) or not. An historical example is given with the *_seen update + public function additionalFeatureEnabled($featureName) { + if (!isset($this->AdminSetting)) { + $this->AdminSetting = ClassRegistry::init('AdminSetting'); + } + $value = $this->AdminSetting->getSetting('seenOnAttributeAndObject'); + if($value === false) { + return false; + } else if ($value === "0") { + return false; + } else if ($value === "1") { + return true; + } else { + return $value; + } + } + */ + public function checkMISPVersion() { App::uses('Folder', 'Utility'); @@ -1390,6 +1536,132 @@ class AppModel extends Model if ($requiresLogout) { $this->updateDatabase('destroyAllSessions'); } + return true; + } + + private function __setUpdateProgress($current, $total=false) + { + $updateProgress = $this->getUpdateProgress(); + $updateProgress['cur'] = $current; + if ($total !== false) { + $updateProgress['tot'] = $total; + } else { + $now = new DateTime(); + $updateProgress['time']['started'][$current] = $now->format('Y-m-d H:i:s'); + } + $this->__saveUpdateProgress($updateProgress); + } + + private function __setPreUpdateTestState($state) + { + $updateProgress = $this->getUpdateProgress(); + $updateProgress['preTestSuccess'] = $state; + $this->__saveUpdateProgress($updateProgress); + } + + private function __setUpdateError($index) + { + $updateProgress = $this->getUpdateProgress(); + $updateProgress['failed_num'][] = $index; + $this->__saveUpdateProgress($updateProgress); + } + + private function __resetUpdateProgress() + { + $updateProgress = array( + 'cmd' => array(), + 'res' => array(), + 'time' => array('started' => array(), 'elapsed' => array()), + 'cur' => '', + 'tot' => '', + 'failed_num' => array() + ); + $this->__saveUpdateProgress($updateProgress); + } + + private function __setUpdateCmdMessages($messages) + { + $updateProgress = $this->getUpdateProgress(); + $updateProgress['cmd'] = $messages; + $this->__saveUpdateProgress($updateProgress); + } + + private function __setUpdateResMessages($index, $message) + { + $updateProgress = $this->getUpdateProgress(); + $updateProgress['res'][$index] = $message; + $temp = new DateTime(); + $diff = $temp->diff(new DateTime($updateProgress['time']['started'][$index])); + $updateProgress['time']['elapsed'][$index] = $diff->format('%H:%I:%S'); + $this->__saveUpdateProgress($updateProgress); + } + + public function getUpdateProgress() + { + if (!isset($this->AdminSetting)) { + $this->AdminSetting = ClassRegistry::init('AdminSetting'); + } + $updateProgress = $this->AdminSetting->getSetting('update_progress'); + if ($updateProgress !== false) { + $updateProgress = json_decode($updateProgress, true); + } else { + $this->__resetUpdateProgress(); + $updateProgress = $this->AdminSetting->getSetting('update_progress'); + $updateProgress = json_decode($updateProgress, true); + } + foreach($updateProgress as $setting => $value) { + if (!is_array($value)) { + $value = $value !== false && $value !== '' ? intval($value) : 0; + } + $updateProgress[$setting] = $value; + } + return $updateProgress; + } + + private function __saveUpdateProgress($updateProgress) + { + $data = json_encode($updateProgress); + $this->AdminSetting->changeSetting('update_progress', $data); + } + + private function __changeLockState($locked) + { + $this->AdminSetting->changeSetting('update_locked', $locked); + } + + private function getUpdateLockState() + { + if (!isset($this->AdminSetting)) { + $this->AdminSetting = ClassRegistry::init('AdminSetting'); + } + $locked = $this->AdminSetting->getSetting('update_locked'); + return is_null($locked) ? false : $locked; + } + + public function isUpdateLocked() + { + $lockState = $this->getUpdateLockState(); + $lockState = $lockState === false ? false : $lockState; + $lockState = $lockState === '' ? false : $lockState; + if ($lockState !== false) { + // if lock is old, still allows the update + // This can be useful if the update process crashes + $lockDate = (new DateTime($lockState))->format('U'); + $now = (new DateTime())->format('U'); + $diffSec = $now - $lockDate; + if (Configure::read('MISP.updateTimeThreshold')) { + $updateWaitThreshold = intval(Configure::read('MISP.updateTimeThreshold')); + } else { + if (!isset($this->Server)) { + $this->Server = ClassRegistry::init('Server'); + } + $updateWaitThreshold = intval($this->Server->serverSettings['MISP']['updateTimeThreshold']['value']); + } + if ($diffSec < $updateWaitThreshold) { + return true; + } + } + return false; } private function __queueCleanDB() diff --git a/app/Model/Server.php b/app/Model/Server.php index 938b45a93..ad7ba1c4c 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -915,7 +915,15 @@ class Server extends AppModel 'test' => 'testBool', 'type' => 'boolean', 'null' => true - ) + ), + 'updateTimeThreshold' => array( + 'level' => 1, + 'description' => __('Sets the minimum time before being able to re-trigger an update if the previous one failed. (safe guard to avoid starting the same update multiple time)'), + 'value' => '600', + 'test' => 'testForNumeric', + 'type' => 'numeric', + 'null' => true + ) ), 'GnuPG' => array( 'branch' => 1, diff --git a/app/View/Elements/Flash/error.ctp b/app/View/Elements/Flash/error.ctp index 823cab6c5..50b3e7659 100644 --- a/app/View/Elements/Flash/error.ctp +++ b/app/View/Elements/Flash/error.ctp @@ -6,6 +6,13 @@ $message = str_replace('$flashErrorMessage', 'here', $message); } echo $message; + if (isset($params['url'])) { + if (isset($params['urlName'])) { + echo '' . h($params['urlName']) . ''; + } else { + echo '' . h($params['url']) . ''; + } + } if ($this->Session->read('flashErrorMessage')) { echo sprintf('', $this->element('flashErrorMessage', array('message' => $this->Session->read('flashErrorMessage')))); } diff --git a/app/View/Elements/healthElements/diagnostics.ctp b/app/View/Elements/healthElements/diagnostics.ctp index b50f7003c..2fd0236a8 100644 --- a/app/View/Elements/healthElements/diagnostics.ctp +++ b/app/View/Elements/healthElements/diagnostics.ctp @@ -61,6 +61,7 @@ +

@@ -347,12 +348,9 @@


Form->postButton(__('Remove orphaned attributes'), $baseurl . '/attributes/pruneOrphanedAttributes', $options = array('class' => 'btn btn-primary', 'style' => 'padding-top:1px;padding-bottom:1px;')); ?> -

-

- () -

-

- Form->postButton(__('Prune upgrade logs'), $baseurl . '/logs/pruneUpdateLogs', $options = array('class' => 'btn btn-primary', 'style' => 'padding-top:1px;padding-bottom:1px;')); ?> +

+

+

diff --git a/app/View/Servers/ondemand_action.ctp b/app/View/Servers/ondemand_action.ctp new file mode 100644 index 000000000..3244892f2 --- /dev/null +++ b/app/View/Servers/ondemand_action.ctp @@ -0,0 +1,87 @@ + + +
+

+ + +
+ +
+ + +
+ +
+ + + $action): ?> +
+

+
+
+
+ + + + +
+ +
+ + + +
+ back your database up before running it.'); ?> +
+ + + +
+ +
+ + + Form->create(false, array( 'url' => $baseurl . $action['url'] . $url_param )); + ?> + + + + Form->end(); + ?> + +
+ +
+ +
+ + +
+element('side_menu', array('menuList' => 'admin', 'menuItem' => 'adminTools')); +?> + + diff --git a/app/View/Servers/update_progress.ctp b/app/View/Servers/update_progress.ctp new file mode 100644 index 000000000..7858cd330 --- /dev/null +++ b/app/View/Servers/update_progress.ctp @@ -0,0 +1,156 @@ + +
+ 0): ?> +

+
+ +
+
Pre update test status:
+ + +
+ +
+
%
+
+
+ + + + + + + + + + $cmd): + if (isset($updateProgress['res'][$i])) { + $res = $updateProgress['res'][$i]; + } else { + $res = false; + } + $rowDone = $i < $updateProgress['cur']; + $rowCurrent = $i === $updateProgress['cur']; + $rowFail = in_array($i, $updateProgress['failed_num']); + $rowClass = ''; + $rowIcon = ''; + if ($rowDone) { + $rowClass = 'class="alert alert-success"'; + $rowIcon = ''; + } + if ($rowCurrent && !$rowFail) { + $rowClass = 'class="alert alert-info"'; + $rowIcon = ''; + } else if ($rowFail) { + $rowClass = 'class="alert alert-danger"'; + $rowIcon = ''; + } + + if (isset($updateProgress['time']['started'][$i])) { + $datetimeStart = $updateProgress['time']['started'][$i]; + if (isset($updateProgress['time']['elapsed'][$i])) { + $updateDuration = $updateProgress['time']['elapsed'][$i]; + } else { // compute elapsed based on started + $temp = new DateTime(); + $diff = $temp->diff(new DateTime($datetimeStart)); + $updateDuration = $diff->format('%H:%I:%S'); + } + } else { + $datetimeStart = ''; + $updateDuration = ''; + } + ?> + > + + + + + +
Update command
+
+ + + + 60 ? '[...]' : '' );?> + + + + + + + + + + +
+
+ $line) { + $pad = $j > 0 ? '30' : '0'; + if ($line !== '') { + echo '' . h($line) . ''; + } + } + ?> +
+
+ +
+ $line) { + $pad = $j > 0 ? '30' : '0'; + if ($line !== '') { + echo '' . h($line) . ''; + } + } + } + ?> +
+
+
+
+ + +
+
+ +

+ + + + Html->script('update_progress'); ?> +
diff --git a/app/webroot/css/main.css b/app/webroot/css/main.css index 3b70d121a..34ec47762 100644 --- a/app/webroot/css/main.css +++ b/app/webroot/css/main.css @@ -2270,3 +2270,80 @@ table tr:hover .down-expand-button { margin-left:2px; color: black; } + +.headerUpdateBlock { + border: #ccc solid 1px; + border-bottom: 1px solid transparent; + background: #e6e6e6; + padding: 0px 6px; + border-top-left-radius: 5px; + border-top-right-radius: 5px; +} + +.bodyUpdateBlock { + border: #ccc solid 1px; + border-bottom-left-radius: 5px; + border-bottom-right-radius: 5px; + background: #fbfbfb; + padding: 6px; + margin-bottom: 10px; +} + +.div-terminal { + background-color: #303030; + color: white; + font-family: Monaco, Menlo, Consolas, "Courier New", monospace; + padding: 10px; + border-radius: 5px; +} + +.div-terminal > span { + display: block; +} + +.terminal-res { + margin-top: 5px; + margin-left: 25px; + max-height: 500px; + overflow-y: auto; +} + +.terminal-res-icon { + position: absolute; + transform: rotate(90deg); + font-size: large; + margin-right: 5px; + margin-left: 5px; +} + +table.updateProgressTable > tbody > tr > td:first-child { + width: 40px; + text-align: center; +} + +table.updateProgressTable .small-pb-in-td { + width: calc(100% + 16px); + height: 4px; + position: relative; + left: -8px; + top: 26px; + background-color: #f3f3f3; +} + +table.updateProgressTable .small-state-text-in-td { + position: relative; + display: inline-block; + float: right; + +} + +.back-and-forth-animation { + position: relative; + animation: backandforthAnim 2.0s infinite; +} + +@keyframes backandforthAnim { + 0% {left: 0;} + 50% {left: calc(95%);} + 100% {left: 0;} +} diff --git a/app/webroot/js/update_progress.js b/app/webroot/js/update_progress.js new file mode 100644 index 000000000..b18407ec7 --- /dev/null +++ b/app/webroot/js/update_progress.js @@ -0,0 +1,182 @@ +function toggleVisiblity(termId, auto, show) { + var term = $('div[data-terminalid='+termId+']') + if (auto === true) { + if (term.data('manual') !== true) { // show if manual is not set + if (show === true) { + term.show(); + } else if (show === false) { + term.hide(); + } else { + term.toggle(); + } + } + } else { + term.data('manual', true); + if (show === true) { + term.show(); + } else if (show === false) { + term.hide(); + } else { + term.toggle(); + } + } +} + +var pooler; +var poolerInterval = 3000; +$(document).ready(function() { + pooler = setInterval(function() { update_state(); }, poolerInterval); +}); + + +function update_state() { + $.getJSON(urlGetProgress, function(data) { + var tot = parseInt(data['tot']); + var cur = parseInt(data['cur']); + var failArray = data['failed_num']; + for (var i=0; i 0) { + var percFail = Math.round(failArray.length/tot*100); + var perc = Math.round((cur)/tot*100); + update_pb(perc, percFail); + } + + if ((cur+1) >= tot || failArray.indexOf(cur) != -1) { + clearInterval(pooler); + $('.single-update-progress').hide(); + } + }); +} + + +function update_messages(messages) { + if (messages.cmd === undefined) { + return; + } + messages.cmd.forEach(function(msg, i) { + var div = $('#termcmd-'+i); + create_spans_from_message(div, msg); + }); + messages.res.forEach(function(msg, i) { + var div = $('#termres-'+i); + div.css('display', ''); + create_spans_from_message(div, msg); + }); + messages.time.started.forEach(function(startedText, i) { + var elapsedText = messages.time.elapsed[i]; + if (elapsedText === undefined) { + var diff = new Date((new Date()).getTime() - (new Date(startedText)).getTime()); + elapsedText = pad(diff.getUTCHours(), 2) + + ':' + pad(diff.getUTCMinutes(), 2) + + ':' + pad(diff.getUTCSeconds(), 2); + } + update_times(i, startedText, elapsedText) + }); +} + +function create_spans_from_message(toAppendto, msg) { + toAppendto.empty(); + // create span for each line of text + msg = msg.replace(/^\n*\s+/, ''); + var lines = msg.split(/\s{2,}/m) + lines.forEach(function(line, j) { + var pad = j > 0 ? '30' : '0'; + if (line !== '') { + var span = $('' + line + ''); + toAppendto.append(span); + } + }); +} + +function update_row_state(i, state) { + var icon = $('#icon-'+i); + var row = $('#row-'+i); + switch(state) { + case 0: // success + row.removeClass('alert-danger alert-info'); + row.addClass('alert-success'); + icon.removeClass('fa-times-circle-o fa-cogs'); + icon.addClass('fa-check-circle-o'); + break; + case 1: // current + row.removeClass('alert-success alert-danger'); + row.addClass('alert-info'); + icon.removeClass('fa-check-circle-o', 'fa-times-circle-o'); + icon.addClass('fa-cogs'); + break; + case 2: //fail + row.removeClass('alert-success alert-info'); + row.addClass('alert-danger'); + icon.removeClass('fa-check-circle-o fa-cogs'); + icon.addClass('fa-times-circle-o'); + break; + case 3: //no state + default: + row.removeClass('alert-success alert-info alert-danger'); + icon.removeClass('fa-check-circle-o fa-times-circle-o fa-cogs'); + break; + } +} + +function update_pb(perc, percFail) { + var pb = $('#pb-progress'); + pb.css('width', perc+'%'); + pb.text(perc+'%'); + var pbF = $('#pb-fail'); + pbF.css('width', percFail+'%'); +} + +function update_times(i, startedText, elapsedText) { + var started = $('#startedTime-'+i); + var elapsed = $('#elapsedTime-'+i); + started.text(startedText); + elapsed.text(elapsedText); +} + +function update_single_update_progress(i, data) { + $('.single-update-progress').hide(); + var div = $('#single-update-progress-'+i); + var pb = div.find('#single-update-pb-'+i); + var state = div.find('#small-state-text-'+i); + div.show(); + var perc = parseInt(data['process_list']['PROGRESS']); + if (data['process_list']['MAX_STAGE'] == 0) { // if MAX_STAGE == 0, progress could not be determined + perc = 5; + if (data['failed_num'].indexOf(data['cur']) >= 0) { // do not animate if failed + state.text('Failed'); + perc = 0; + } else { + state.text('Unkown or No state'); + pb.addClass('back-and-forth-animation'); + } + } else { + perc = perc == 0 ? 1 : perc; // for UI, always set min progress to 1 + pb.removeClass('back-and-forth-animation'); + state.text(data['process_list']['STATE']); + } + pb.css('width', perc+'%'); +} + +function pad(num, size){ return ('000000000' + num).substr(-size); } From 924923c93b0525cb0f1fe3191b3e5f417ecb955d Mon Sep 17 00:00:00 2001 From: mokaddem Date: Fri, 26 Apr 2019 09:50:28 +0200 Subject: [PATCH 03/26] chg: [update] repaired badly merged file --- app/View/Elements/healthElements/diagnostics.ctp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/View/Elements/healthElements/diagnostics.ctp b/app/View/Elements/healthElements/diagnostics.ctp index 2fd0236a8..6e8583a10 100644 --- a/app/View/Elements/healthElements/diagnostics.ctp +++ b/app/View/Elements/healthElements/diagnostics.ctp @@ -60,8 +60,8 @@
+ -

From 57bd054615138df821c3f9dd2a4ec00fd4881d43 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Fri, 26 Apr 2019 10:06:32 +0200 Subject: [PATCH 04/26] chg: [onDemandAction] Redirect on updateProgress page is no longueur de default behavior --- app/View/Servers/ondemand_action.ctp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/View/Servers/ondemand_action.ctp b/app/View/Servers/ondemand_action.ctp index 3244892f2..1e58b2b80 100644 --- a/app/View/Servers/ondemand_action.ctp +++ b/app/View/Servers/ondemand_action.ctp @@ -50,7 +50,7 @@ $disabledBtnText = $updateLocked ? 'title="' . __('An action is already in progr echo $this->Form->create(false, array( 'url' => $baseurl . $action['url'] . $url_param )); ?> - + Form->end(); @@ -70,7 +70,7 @@ $disabledBtnText = $updateLocked ? 'title="' . __('An action is already in progr - Html->script('update_progress'); ?> + element('genericElements/assetLoader', array( + 'css' => array('update_progress'), + 'js' => array('update_progress') + )); + ?> diff --git a/app/webroot/css/main.css b/app/webroot/css/main.css index 34ec47762..3b70d121a 100644 --- a/app/webroot/css/main.css +++ b/app/webroot/css/main.css @@ -2270,80 +2270,3 @@ table tr:hover .down-expand-button { margin-left:2px; color: black; } - -.headerUpdateBlock { - border: #ccc solid 1px; - border-bottom: 1px solid transparent; - background: #e6e6e6; - padding: 0px 6px; - border-top-left-radius: 5px; - border-top-right-radius: 5px; -} - -.bodyUpdateBlock { - border: #ccc solid 1px; - border-bottom-left-radius: 5px; - border-bottom-right-radius: 5px; - background: #fbfbfb; - padding: 6px; - margin-bottom: 10px; -} - -.div-terminal { - background-color: #303030; - color: white; - font-family: Monaco, Menlo, Consolas, "Courier New", monospace; - padding: 10px; - border-radius: 5px; -} - -.div-terminal > span { - display: block; -} - -.terminal-res { - margin-top: 5px; - margin-left: 25px; - max-height: 500px; - overflow-y: auto; -} - -.terminal-res-icon { - position: absolute; - transform: rotate(90deg); - font-size: large; - margin-right: 5px; - margin-left: 5px; -} - -table.updateProgressTable > tbody > tr > td:first-child { - width: 40px; - text-align: center; -} - -table.updateProgressTable .small-pb-in-td { - width: calc(100% + 16px); - height: 4px; - position: relative; - left: -8px; - top: 26px; - background-color: #f3f3f3; -} - -table.updateProgressTable .small-state-text-in-td { - position: relative; - display: inline-block; - float: right; - -} - -.back-and-forth-animation { - position: relative; - animation: backandforthAnim 2.0s infinite; -} - -@keyframes backandforthAnim { - 0% {left: 0;} - 50% {left: calc(95%);} - 100% {left: 0;} -} diff --git a/app/webroot/css/update_progress.css b/app/webroot/css/update_progress.css new file mode 100644 index 000000000..97535700d --- /dev/null +++ b/app/webroot/css/update_progress.css @@ -0,0 +1,76 @@ +.headerUpdateBlock { + border: #ccc solid 1px; + border-bottom: 1px solid transparent; + background: #e6e6e6; + padding: 0px 6px; + border-top-left-radius: 5px; + border-top-right-radius: 5px; +} + +.bodyUpdateBlock { + border: #ccc solid 1px; + border-bottom-left-radius: 5px; + border-bottom-right-radius: 5px; + background: #fbfbfb; + padding: 6px; + margin-bottom: 10px; +} + +.div-terminal { + background-color: #303030; + color: white; + font-family: Monaco, Menlo, Consolas, "Courier New", monospace; + padding: 10px; + border-radius: 5px; +} + +.div-terminal > span { + display: block; +} + +.terminal-res { + margin-top: 5px; + margin-left: 25px; + max-height: 500px; + overflow-y: auto; +} + +.terminal-res-icon { + position: absolute; + transform: rotate(90deg); + font-size: large; + margin-right: 5px; + margin-left: 5px; +} + +table.updateProgressTable > tbody > tr > td:first-child { + width: 40px; + text-align: center; +} + +table.updateProgressTable .small-pb-in-td { + width: calc(100% + 16px); + height: 4px; + position: relative; + left: -8px; + top: 26px; + background-color: #f3f3f3; +} + +table.updateProgressTable .small-state-text-in-td { + position: relative; + display: inline-block; + float: right; + +} + +.back-and-forth-animation { + position: relative; + animation: backandforthAnim 2.0s infinite; +} + +@keyframes backandforthAnim { + 0% {left: 0;} + 50% {left: calc(95%);} + 100% {left: 0;} +} From d6be8023e5b72a97e4d0be0e18c27593aecde1f3 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Fri, 26 Apr 2019 14:10:26 +0200 Subject: [PATCH 06/26] chg: [updateProgress] bit of cleanup --- app/Controller/ServersController.php | 1 - app/View/Servers/update_progress.ctp | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index 62641784e..e8386b5b2 100644 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -1548,7 +1548,6 @@ class ServersController extends AppController throw new MethodNotAllowedException('You are not authorised to do that.'); } $this->AdminSetting = ClassRegistry::init('AdminSetting'); - //$actions = $this->Server->advanced_updates_description; $actions = $this->Server->actions_description; $default_fields = array( 'title' => '', diff --git a/app/View/Servers/update_progress.ctp b/app/View/Servers/update_progress.ctp index f91f73c7a..69ec38f95 100644 --- a/app/View/Servers/update_progress.ctp +++ b/app/View/Servers/update_progress.ctp @@ -15,7 +15,7 @@ if (isset($updateProgress['preTestSuccess']) && $updateProgress['preTestSuccess' ?>
0): ?> -

+

From 47e13c8369bcfc9b6d8eb1e73b6757a1d7886f72 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 29 Apr 2019 11:09:04 +0200 Subject: [PATCH 07/26] chg: [updates] Implented changes requested by the PR's review #4534. --- app/Console/Command/AdminShell.php | 45 ++++++++- app/Console/Command/ServerShell.php | 13 --- app/Controller/AppController.php | 4 +- app/Controller/ServersController.php | 36 +++---- app/Model/AppModel.php | 140 ++++++++++----------------- app/Model/Server.php | 2 +- app/View/Servers/update_progress.ctp | 18 ++-- app/webroot/js/update_progress.js | 26 ++--- 8 files changed, 136 insertions(+), 148 deletions(-) diff --git a/app/Console/Command/AdminShell.php b/app/Console/Command/AdminShell.php index 9dc666372..5c128020c 100644 --- a/app/Console/Command/AdminShell.php +++ b/app/Console/Command/AdminShell.php @@ -308,11 +308,48 @@ class AdminShell extends AppShell } public function updateDatabase() { - echo 'Executing all updates to bring the database up to date with the current version.' . PHP_EOL; - $this->Server->runUpdates(true); - echo 'All updates completed.' . PHP_EOL; + $whoami = exec('whoami'); + if ($whoami === 'httpd' || $whoami === 'www-data') { + echo 'Executing all updates to bring the database up to date with the current version.' . PHP_EOL; + $this->Server->runUpdates(true); + echo 'All updates completed.' . PHP_EOL; + } else { + die('This OS user is not allowed to run this command.'. PHP_EOL. 'Run it under `www-data` or `httpd`.' . PHP_EOL); + } } + public function updateApp() { + $whoami = exec('whoami'); + if ($whoami === 'httpd' || $whoami === 'www-data') { + $command = $this->args[0]; + if (!empty($this->args[1])) { + $processId = $this->args[1]; + $job = $this->Job->read(null, $processId); + } else { // create worker + $this->Job->create(); + $job_data = array( + 'worker' => 'prio', + 'job_type' => 'update_app', + 'job_input' => 'command: ' . $command, + 'status' => 0, + 'retries' => 0, + 'org_id' => '', + 'org' => '', + 'message' => 'Updating.', + ); + $this->Job->save($job_data); + $job = $this->Job->read(null, $this->Job->id); + } + $result = $this->Server->updateDatabase($command, false); + $job['Job']['progress'] = 100; + $job['Job']['message'] = 'Update done'; + $this->Job->save($job); + } else { + die('This OS user is not allowed to run this command.'. PHP_EOL. 'Run it under `www-data` or `httpd`.' . PHP_EOL); + } + } + + public function getAuthkey() { if (empty($this->args[0])) { echo 'Invalid parameters. Usage: ' . APP . 'Console/cake Admin getAuthkey [user_email]' . PHP_EOL; @@ -404,7 +441,7 @@ class AdminShell extends AppShell } echo 'Updated, new key:' . PHP_EOL . $authKey . PHP_EOL; } - + public function getOptionParser() { $parser = parent::getOptionParser(); $parser->addSubcommand('updateJSON', array( diff --git a/app/Console/Command/ServerShell.php b/app/Console/Command/ServerShell.php index 495684469..015d5e2b8 100644 --- a/app/Console/Command/ServerShell.php +++ b/app/Console/Command/ServerShell.php @@ -447,17 +447,4 @@ class ServerShell extends AppShell $this->Task->saveField('message', count($servers) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '.'); } - public function updateApp() { - $processId = $this->args[0]; - $job = $this->Job->read(null, $processId); - $command = $this->args[1]; - $liveOff = $this->args[2]; - $exitOnError = $this->args[3]; - $useWorker = $this->args[4]; - $result = $this->Server->updateDatabase($command, $liveOff, $exitOnError, $useWorker); - $job['Job']['progress'] = 100; - $job['Job']['message'] = 'Update done'; - $this->Job->save($job); - } - } diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index a86a5ab2c..0d11c28a6 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -824,7 +824,7 @@ class AppController extends Controller $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration')); } - public function updateDatabase($command, $liveOff=false, $exitOnError=false) + public function updateDatabase($command) { if (!$this->_isSiteAdmin() || !$this->request->is('post')) { throw new MethodNotAllowedException(); @@ -833,7 +833,7 @@ class AppController extends Controller if (is_numeric($command)) { $command = intval($command); } - $this->Server->updateDatabase($command, $liveOff, $exitOnError); + $this->Server->updateDatabase($command); $this->Flash->success('Done.'); if ($liveOff) { $this->redirect(array('controller' => 'servers', 'action' => 'updateProgress')); diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index e8386b5b2..20095e979 100644 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -1565,7 +1565,7 @@ class ServersController extends AppController } } $done = $this->AdminSetting->getSetting($id); - $actions[$id]['done'] = $done !== false && $done == '1' ? true : false; + $actions[$id]['done'] = ($done == '1'); } $this->set('actions', $actions); $this->set('updateLocked', $this->Server->isUpdateLocked()); @@ -1576,31 +1576,31 @@ class ServersController extends AppController if (!$this->_isSiteAdmin()) { throw new MethodNotAllowedException('You are not authorised to do that.'); } - $updateProgress = $this->Server->getUpdateProgress(); - $curIndex = $updateProgress['cur']; - $curCommand = !isset($updateProgress['cmd'][$curIndex]) ? '' : $updateProgress['cmd'][$curIndex]; - $lookupString = preg_replace('/\s{2,}/', '', substr($curCommand, 0, -1)); - $sqlInfo = $this->Server->query("SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;"); - if (empty($sqlInfo)) { - $updateProgress['process_list'] = array(); + $update_progress = $this->Server->getUpdateProgress(); + $current_index = $update_progress['current']; + $current_command = !isset($update_progress['commands'][$current_index]) ? '' : $update_progress['commands'][$current_index]; + $lookup_string = preg_replace('/\s{2,}/', '', substr($current_command, 0, -1)); + $sql_info = $this->Server->query("SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;"); + if (empty($sql_info)) { + $update_progress['process_list'] = array(); } else { // retreive current update process - foreach($sqlInfo as $row) { - if (preg_replace('/\s{2,}/', '', $row['PROCESSLIST']['INFO']) == $lookupString) { - $sqlInfo = $row['PROCESSLIST']; + foreach($sql_info as $row) { + if (preg_replace('/\s{2,}/', '', $row['PROCESSLIST']['INFO']) == $lookup_string) { + $sql_info = $row['PROCESSLIST']; break; } } - $updateProgress['process_list'] = array(); - $updateProgress['process_list']['STATE'] = isset($sqlInfo['STATE']) ? $sqlInfo['STATE'] : ''; - $updateProgress['process_list']['PROGRESS'] = isset($sqlInfo['PROGRESS']) ? $sqlInfo['PROGRESS'] : 0; - $updateProgress['process_list']['STAGE'] = isset($sqlInfo['STAGE']) ? $sqlInfo['STAGE'] : 0; - $updateProgress['process_list']['MAX_STAGE'] = isset($sqlInfo['MAX_STAGE']) ? $sqlInfo['MAX_STAGE'] : 0; + $update_progress['process_list'] = array(); + $update_progress['process_list']['STATE'] = isset($sql_info['STATE']) ? $sql_info['STATE'] : ''; + $update_progress['process_list']['PROGRESS'] = isset($sql_info['PROGRESS']) ? $sql_info['PROGRESS'] : 0; + $update_progress['process_list']['STAGE'] = isset($sql_info['STAGE']) ? $sql_info['STAGE'] : 0; + $update_progress['process_list']['MAX_STAGE'] = isset($sql_info['MAX_STAGE']) ? $sql_info['MAX_STAGE'] : 0; } if ($this->request->is('ajax')) { - return $this->RestResponse->viewData(h($updateProgress), $this->response->type()); + return $this->RestResponse->viewData(h($update_progress), $this->response->type()); } else { - $this->set('updateProgress', $updateProgress); + $this->set('updateProgress', $update_progress); } } diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index cb443c02b..5a1587340 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -224,7 +224,7 @@ class AppModel extends Model } // SQL scripts for updates - public function updateDatabase($command, $liveOff=false, $exitOnError=false, $useWorker=true) + public function updateDatabase($command, $useWorker=true) { // Exit if updates are locked if ($this->isUpdateLocked()) { @@ -236,27 +236,34 @@ class AppModel extends Model $job = ClassRegistry::init('Job'); $job->create(); $data = array( - 'worker' => 'prio', - 'job_type' => 'update_app', - 'job_input' => 'command: ' . $command, - 'status' => 0, - 'retries' => 0, - 'org_id' => '', - 'org' => '', - 'message' => 'Updating.', + 'worker' => 'prio', + 'job_type' => 'update_app', + 'job_input' => 'command: ' . $command, + 'status' => 0, + 'retries' => 0, + 'org_id' => '', + 'org' => '', + 'message' => 'Updating.', ); $job->save($data); $jobId = $job->id; $process_id = CakeResque::enqueue( 'prio', - 'ServerShell', - array('updateApp', $jobId, $command, $liveOff, $exitOnError, false), + 'AdminShell', + array('updateApp', $command, $jobId), true ); $job->saveField('process_id', $process_id); return true; } + $liveOff = false; + $exitOnError = false; + if (isset($advanced_updates_description[$command])) { + $liveOff = isset($advanced_updates_description[$command]['liveOff']) ? $advanced_updates_description[$command]['liveOff'] : $liveOff; + $exitOnError = isset($advanced_updates_description[$command]['exitOnError']) ? $advanced_updates_description[$command]['exitOnError'] : $exitOnError; + } + $dataSourceConfig = ConnectionManager::getDataSource('default')->config; $dataSource = $dataSourceConfig['datasource']; $sqlArray = array(); @@ -1198,32 +1205,32 @@ class AppModel extends Model } $now = new DateTime(); - $this->__changeLockState($now->format('Y-m-d H:i:s')); + $this->__changeLockState(time()); // switch MISP instance live to false if ($liveOff) { $this->Server = Classregistry::init('Server'); $liveSetting = 'MISP.live'; $this->Server->serverSettingsSaveValue($liveSetting, false); } - $SqlUpdateCount = count($sqlArray); - $IndexUpdateCount = count($indexArray); - $totupdatecount = $SqlUpdateCount + $IndexUpdateCount; - $this->__setUpdateProgress(0, $totupdatecount); - $strIndexArray = array(); + $sql_update_count = count($sqlArray); + $index_update_count = count($indexArray); + $total_update_count = $sql_update_count + $index_update_count; + $this->__setUpdateProgress(0, $total_update_count); + $str_index_array = array(); foreach($indexArray as $toIndex) { - $strIndexArray[] = __('Indexing ') . implode($toIndex, '->'); + $str_index_array[] = __('Indexing ') . implode($toIndex, '->'); } - $this->__setUpdateCmdMessages(array_merge($sqlArray, $strIndexArray)); - $flagStop = false; - $errorCount = 0; + $this->__setUpdateCmdMessages(array_merge($sqlArray, $str_index_array)); + $flag_stop = false; + $error_count = 0; foreach ($sqlArray as $i => $sql) { try { $this->__setUpdateProgress($i, false); // execute test before update. Exit if it fails if (isset($this->advanced_updates_description[$command]['preUpdate'])) { - $funName = $this->advanced_updates_description[$command]['preUpdate']; + $function_name = $this->advanced_updates_description[$command]['preUpdate']; try { - $this->{$funName}(); + $this->{$function_name}(); } catch (Exception $e) { $exitOnError = true; $this->__setPreUpdateTestState(false); @@ -1240,10 +1247,10 @@ class AppModel extends Model 'email' => 'SYSTEM', 'action' => 'update_database', 'user_id' => 0, - 'title' => 'Successfuly executed the SQL query for ' . $command, - 'change' => 'The executed SQL query was: ' . $sql + 'title' => __('Successfuly executed the SQL query for ') . $command, + 'change' => __('The executed SQL query was: ') . $sql )); - $this->__setUpdateResMessages($i, 'Successfuly executed the SQL query for ' . $command); + $this->__setUpdateResMessages($i, __('Successfuly executed the SQL query for ') . $command); } catch (Exception $e) { $this->Log->create(); $this->Log->save(array( @@ -1253,19 +1260,19 @@ class AppModel extends Model 'email' => 'SYSTEM', 'action' => 'update_database', 'user_id' => 0, - 'title' => 'Issues executing the SQL query for ' . $command, - 'change' => 'The executed SQL query was: ' . $sql . PHP_EOL . ' The returned error is: ' . $e->getMessage() + 'title' => __('Issues executing the SQL query for ') . $command, + 'change' => __('The executed SQL query was: ') . $sql . PHP_EOL . __(' The returned error is: ') . $e->getMessage() )); - $this->__setUpdateResMessages($i, 'Issues executing the SQL query for ' . $command . '. The returned error is: ' . PHP_EOL . $e->getMessage()); + $this->__setUpdateResMessages($i, __('Issues executing the SQL query for ') . $command . __('. The returned error is: ') . PHP_EOL . $e->getMessage()); $this->__setUpdateError($i); - $errorCount++; + $error_count++; if ($exitOnError) { - $flagStop = true; + $flag_stop = true; break; } } } - if (!$flagStop) { + if (!$flag_stop) { if (!empty($indexArray)) { if ($clean) { $this->cleanCacheFiles(); @@ -1277,7 +1284,7 @@ class AppModel extends Model } else { $this->__addIndex($iA[0], $iA[1]); } - $this->__setUpdateResMessages(count($sqlArray)+$i, 'Successfuly indexed ' . implode($iA, '->')); + $this->__setUpdateResMessages(count($sqlArray)+$i, __('Successfuly indexed ') . implode($iA, '->')); } } $this->__setUpdateProgress(count($sqlArray)+count($indexArray), false); @@ -1289,7 +1296,7 @@ class AppModel extends Model $liveSetting = 'MISP.live'; $this->Server->serverSettingsSaveValue($liveSetting, true); } - if (!$flagStop && $errorCount == 0) { + if (!$flag_stop && $error_count == 0) { $this->__postUpdate($command); } $this->__changeLockState(false); @@ -1393,43 +1400,6 @@ class AppModel extends Model } } - /* - * Given an array, dynamically add contextual fields. An historical example is given with the *_seen fields - public function addFieldsBasedOnUpdate(&$fieldsAtt, $context = null) { - if (is_null($context)) { - $alias = ''; - } else if ($context === true) { - $alias = $this->alias; - } else { - $alias = $context; - } - if ($this->additionalFeatureEnabled('seenOnAttributeAndObject')) { - // DB have *_seen columns - $fs = (strlen($alias) > 0 ? $alias . '.' : '') . 'first_seen'; - $ls = (strlen($alias) > 0 ? $alias . '.' : '') . 'last_seen'; - array_push($fieldsAtt, $fs, $ls); - } - } - */ - /* - * Check whether a feature is enabled (based on an update) or not. An historical example is given with the *_seen update - public function additionalFeatureEnabled($featureName) { - if (!isset($this->AdminSetting)) { - $this->AdminSetting = ClassRegistry::init('AdminSetting'); - } - $value = $this->AdminSetting->getSetting('seenOnAttributeAndObject'); - if($value === false) { - return false; - } else if ($value === "0") { - return false; - } else if ($value === "1") { - return true; - } else { - return $value; - } - } - */ - public function checkMISPVersion() { App::uses('Folder', 'Utility'); @@ -1542,9 +1512,9 @@ class AppModel extends Model private function __setUpdateProgress($current, $total=false) { $updateProgress = $this->getUpdateProgress(); - $updateProgress['cur'] = $current; + $updateProgress['current'] = $current; if ($total !== false) { - $updateProgress['tot'] = $total; + $updateProgress['total'] = $total; } else { $now = new DateTime(); $updateProgress['time']['started'][$current] = $now->format('Y-m-d H:i:s'); @@ -1569,11 +1539,11 @@ class AppModel extends Model private function __resetUpdateProgress() { $updateProgress = array( - 'cmd' => array(), - 'res' => array(), + 'commands' => array(), + 'results' => array(), 'time' => array('started' => array(), 'elapsed' => array()), - 'cur' => '', - 'tot' => '', + 'current' => '', + 'total' => '', 'failed_num' => array() ); $this->__saveUpdateProgress($updateProgress); @@ -1582,14 +1552,14 @@ class AppModel extends Model private function __setUpdateCmdMessages($messages) { $updateProgress = $this->getUpdateProgress(); - $updateProgress['cmd'] = $messages; + $updateProgress['commands'] = $messages; $this->__saveUpdateProgress($updateProgress); } private function __setUpdateResMessages($index, $message) { $updateProgress = $this->getUpdateProgress(); - $updateProgress['res'][$index] = $message; + $updateProgress['results'][$index] = $message; $temp = new DateTime(); $diff = $temp->diff(new DateTime($updateProgress['time']['started'][$index])); $updateProgress['time']['elapsed'][$index] = $diff->format('%H:%I:%S'); @@ -1641,20 +1611,14 @@ class AppModel extends Model public function isUpdateLocked() { $lockState = $this->getUpdateLockState(); - $lockState = $lockState === false ? false : $lockState; - $lockState = $lockState === '' ? false : $lockState; - if ($lockState !== false) { + if ($lockState !== false && $lockState !== '') { // if lock is old, still allows the update // This can be useful if the update process crashes - $lockDate = (new DateTime($lockState))->format('U'); - $now = (new DateTime())->format('U'); - $diffSec = $now - $lockDate; + $diffSec = time() - intval($lockState); if (Configure::read('MISP.updateTimeThreshold')) { $updateWaitThreshold = intval(Configure::read('MISP.updateTimeThreshold')); } else { - if (!isset($this->Server)) { - $this->Server = ClassRegistry::init('Server'); - } + $this->Server = ClassRegistry::init('Server'); $updateWaitThreshold = intval($this->Server->serverSettings['MISP']['updateTimeThreshold']['value']); } if ($diffSec < $updateWaitThreshold) { diff --git a/app/Model/Server.php b/app/Model/Server.php index ad7ba1c4c..11f297e06 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -919,7 +919,7 @@ class Server extends AppModel 'updateTimeThreshold' => array( 'level' => 1, 'description' => __('Sets the minimum time before being able to re-trigger an update if the previous one failed. (safe guard to avoid starting the same update multiple time)'), - 'value' => '600', + 'value' => '7200', 'test' => 'testForNumeric', 'type' => 'numeric', 'null' => true diff --git a/app/View/Servers/update_progress.ctp b/app/View/Servers/update_progress.ctp index 69ec38f95..33437bf1b 100644 --- a/app/View/Servers/update_progress.ctp +++ b/app/View/Servers/update_progress.ctp @@ -1,8 +1,8 @@
- 0): ?> + 0): ?>

@@ -39,14 +39,14 @@ if (isset($updateProgress['preTestSuccess']) && $updateProgress['preTestSuccess' - $cmd): - if (isset($updateProgress['res'][$i])) { - $res = $updateProgress['res'][$i]; + $cmd): + if (isset($updateProgress['results'][$i])) { + $res = $updateProgress['results'][$i]; } else { $res = false; } - $rowDone = $i < $updateProgress['cur']; - $rowCurrent = $i === $updateProgress['cur']; + $rowDone = $i < $updateProgress['current']; + $rowCurrent = $i === $updateProgress['current']; $rowFail = in_array($i, $updateProgress['failed_num']); $rowClass = ''; $rowIcon = ''; diff --git a/app/webroot/js/update_progress.js b/app/webroot/js/update_progress.js index b18407ec7..087dca913 100644 --- a/app/webroot/js/update_progress.js +++ b/app/webroot/js/update_progress.js @@ -31,19 +31,19 @@ $(document).ready(function() { function update_state() { $.getJSON(urlGetProgress, function(data) { - var tot = parseInt(data['tot']); - var cur = parseInt(data['cur']); + var total = parseInt(data['total']); + var current = parseInt(data['current']); var failArray = data['failed_num']; - for (var i=0; i 0) { - var percFail = Math.round(failArray.length/tot*100); - var perc = Math.round((cur)/tot*100); + if (total > 0) { + var percFail = Math.round(failArray.length/total*100); + var perc = Math.round(current/total*100); update_pb(perc, percFail); } - if ((cur+1) >= tot || failArray.indexOf(cur) != -1) { + if ((current+1) >= total || failArray.indexOf(current) != -1) { clearInterval(pooler); $('.single-update-progress').hide(); } @@ -72,14 +72,14 @@ function update_state() { function update_messages(messages) { - if (messages.cmd === undefined) { + if (messages.commands === undefined) { return; } - messages.cmd.forEach(function(msg, i) { + messages.commands.forEach(function(msg, i) { var div = $('#termcmd-'+i); create_spans_from_message(div, msg); }); - messages.res.forEach(function(msg, i) { + messages.results.forEach(function(msg, i) { var div = $('#termres-'+i); div.css('display', ''); create_spans_from_message(div, msg); @@ -164,7 +164,7 @@ function update_single_update_progress(i, data) { var perc = parseInt(data['process_list']['PROGRESS']); if (data['process_list']['MAX_STAGE'] == 0) { // if MAX_STAGE == 0, progress could not be determined perc = 5; - if (data['failed_num'].indexOf(data['cur']) >= 0) { // do not animate if failed + if (data['failed_num'].indexOf(data['current']) >= 0) { // do not animate if failed state.text('Failed'); perc = 0; } else { From 3794dad160d03dc64b30fd657b16c06f5053ccfa Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 29 Apr 2019 11:26:55 +0200 Subject: [PATCH 08/26] chg: [update] Avoid executing pre-update test multiple times --- app/Model/AppModel.php | 66 +++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 5a1587340..d44bbf8be 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -1223,24 +1223,31 @@ class AppModel extends Model $this->__setUpdateCmdMessages(array_merge($sqlArray, $str_index_array)); $flag_stop = false; $error_count = 0; - foreach ($sqlArray as $i => $sql) { + + // execute test before update. Exit if it fails + if (isset($this->advanced_updates_description[$command]['preUpdate'])) { + $function_name = $this->advanced_updates_description[$command]['preUpdate']; try { - $this->__setUpdateProgress($i, false); - // execute test before update. Exit if it fails - if (isset($this->advanced_updates_description[$command]['preUpdate'])) { - $function_name = $this->advanced_updates_description[$command]['preUpdate']; - try { - $this->{$function_name}(); - } catch (Exception $e) { - $exitOnError = true; - $this->__setPreUpdateTestState(false); - throw new Exception($e->getMessage()); - } - } - $this->__setPreUpdateTestState(true); - $this->query($sql); - $this->Log->create(); - $this->Log->save(array( + $this->{$function_name}(); + } catch (Exception $e) { + $this->__setPreUpdateTestState(false); + $this->__setUpdateProgress(0, false); + $this->__setUpdateResMessages(0, __('Issues executing the pre-update test `') . $function_name . __('`. The returned error is: ') . PHP_EOL . $e->getMessage()); + $this->__setUpdateError(0); + $error_count++; + $exitOnError = true; + $flag_stop = true; + } + } + + if (!$flag_stop) { + $this->__setPreUpdateTestState(true); + foreach ($sqlArray as $i => $sql) { + try { + $this->__setUpdateProgress($i, false); + $this->query($sql); + $this->Log->create(); + $this->Log->save(array( 'org' => 'SYSTEM', 'model' => 'Server', 'model_id' => 0, @@ -1249,11 +1256,11 @@ class AppModel extends Model 'user_id' => 0, 'title' => __('Successfuly executed the SQL query for ') . $command, 'change' => __('The executed SQL query was: ') . $sql - )); - $this->__setUpdateResMessages($i, __('Successfuly executed the SQL query for ') . $command); - } catch (Exception $e) { - $this->Log->create(); - $this->Log->save(array( + )); + $this->__setUpdateResMessages($i, __('Successfuly executed the SQL query for ') . $command); + } catch (Exception $e) { + $this->Log->create(); + $this->Log->save(array( 'org' => 'SYSTEM', 'model' => 'Server', 'model_id' => 0, @@ -1262,13 +1269,14 @@ class AppModel extends Model 'user_id' => 0, 'title' => __('Issues executing the SQL query for ') . $command, 'change' => __('The executed SQL query was: ') . $sql . PHP_EOL . __(' The returned error is: ') . $e->getMessage() - )); - $this->__setUpdateResMessages($i, __('Issues executing the SQL query for ') . $command . __('. The returned error is: ') . PHP_EOL . $e->getMessage()); - $this->__setUpdateError($i); - $error_count++; - if ($exitOnError) { - $flag_stop = true; - break; + )); + $this->__setUpdateResMessages($i, __('Issues executing the SQL query for ') . $command . __('. The returned error is: ') . PHP_EOL . $e->getMessage()); + $this->__setUpdateError($i); + $error_count++; + if ($exitOnError) { + $flag_stop = true; + break; + } } } } From 14feb57e2fd2bbec97c903afbd2fc9aef52fb340 Mon Sep 17 00:00:00 2001 From: 4ekin Date: Mon, 29 Apr 2019 16:12:55 +0300 Subject: [PATCH 09/26] fix: Typos in controllers --- app/Controller/AttributesController.php | 2 +- app/Controller/EventsController.php | 2 +- app/Controller/FeedsController.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 12a9add7a..750bcdd87 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -2246,7 +2246,7 @@ class AttributesController extends AppController $data = $this->request->data; } if (empty($data)) { - throw new BadRequestException(__('Either specify the search terms in the url, or POST a json array / xml (with the root element being "request" and specify the correct headers based on content type.')); + throw new BadRequestException(__('Either specify the search terms in the url, or POST a json array / xml (with the root element being "request" and specify the correct headers based on content type).')); } $paramArray = array('eventId', 'tags', 'from', 'to', 'policy', 'walled_garden', 'ns', 'email', 'serial', 'refresh', 'retry', 'expiry', 'minimum_ttl', 'ttl', 'enforceWarninglist', 'ns_alt'); foreach ($paramArray as $p) { diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 9a0c73432..d28fe079c 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -3947,7 +3947,7 @@ class EventsController extends AppController } if ($this->request->is('post')) { if (empty($this->request->data)) { - throw new BadRequestException(__('Either specify the search terms in the url, or POST an xml (with the root element being "request".')); + throw new BadRequestException(__('Either specify the search terms in the url, or POST an xml (with the root element being "request").')); } else { $data = $this->request->data; } diff --git a/app/Controller/FeedsController.php b/app/Controller/FeedsController.php index 0a39c85db..1242d0284 100644 --- a/app/Controller/FeedsController.php +++ b/app/Controller/FeedsController.php @@ -433,7 +433,7 @@ class FeedsController extends AppController $this->redirect(array('action' => 'index')); } } - $message = __('Fetching the feed has successfuly completed.'); + $message = __('Fetching the feed has successfully completed.'); if ($this->Feed->data['Feed']['source_format'] == 'misp') { if (isset($result['add'])) { $message .= ' Downloaded ' . count($result['add']) . ' new event(s).'; From 9bb893344508ca209f20052dee15e843abac5478 Mon Sep 17 00:00:00 2001 From: 4ekin Date: Mon, 29 Apr 2019 16:14:14 +0300 Subject: [PATCH 10/26] fix: Fixed i18n strings in Event controller, model and view --- app/Controller/EventsController.php | 2 +- app/Model/Event.php | 187 +++++++++++++------------- app/View/Events/ajax/enrich_event.ctp | 2 +- app/View/Events/merge.ctp | 2 +- 4 files changed, 100 insertions(+), 93 deletions(-) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index d28fe079c..25ce4e351 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -2614,7 +2614,7 @@ class EventsController extends AppController { $filesize_units = array('B', 'KB', 'MB', 'GB', 'TB'); if ($this->_isSiteAdmin()) { - $this->Flash->info('Warning, you are logged in as a site admin, any export that you generate will contain the FULL UNRESTRICTED data-set. If you would like to generate an export for your own organisation, please log in with a different user.'); + $this->Flash->info(__('Warning, you are logged in as a site admin, any export that you generate will contain the FULL UNRESTRICTED data-set. If you would like to generate an export for your own organisation, please log in with a different user.')); } // Check if the background jobs are enabled - if not, fall back to old export page. if (Configure::read('MISP.background_jobs') && !Configure::read('MISP.disable_cached_exports')) { diff --git a/app/Model/Event.php b/app/Model/Event.php index 0918d4edf..1f46f038b 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -58,96 +58,7 @@ class Event extends AppModel private $__assetCache = array(); - public $export_types = array( - 'json' => array( - 'extension' => '.json', - 'type' => 'JSON', - 'scope' => 'Event', - 'requiresPublished' => 0, - 'params' => array('includeAttachments' => 1, 'ignore' => 1, 'returnFormat' => 'json'), - 'description' => 'Click this to download all events and attributes that you have access to in MISP JSON format.', - ), - 'xml' => array( - 'extension' => '.xml', - 'type' => 'XML', - 'scope' => 'Event', - 'params' => array('includeAttachments' => 1, 'ignore' => 1, 'returnFormat' => 'xml'), - 'requiresPublished' => 0, - 'description' => 'Click this to download all events and attributes that you have access to in MISP XML format.', - ), - 'csv_sig' => array( - 'extension' => '.csv', - 'type' => 'CSV_Sig', - 'scope' => 'Event', - 'requiresPublished' => 1, - 'params' => array('published' => 1, 'to_ids' => 1, 'returnFormat' => 'csv'), - 'description' => 'Click this to download all attributes that are indicators and that you have access to (except file attachments) in CSV format.', - ), - 'csv_all' => array( - 'extension' => '.csv', - 'type' => 'CSV_All', - 'scope' => 'Event', - 'requiresPublished' => 0, - 'params' => array('ignore' => 1, 'returnFormat' => 'csv'), - 'description' => 'Click this to download all attributes that you have access to (except file attachments) in CSV format.', - ), - 'suricata' => array( - 'extension' => '.rules', - 'type' => 'Suricata', - 'scope' => 'Attribute', - 'requiresPublished' => 1, - 'params' => array('returnFormat' => 'suricata'), - 'description' => 'Click this to download all network related attributes that you have access to under the Suricata rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a whitelist containing host, domain name and IP numbers to exclude from the NIDS export.', - ), - 'snort' => array( - 'extension' => '.rules', - 'type' => 'Snort', - 'scope' => 'Attribute', - 'requiresPublished' => 1, - 'params' => array('returnFormat' => 'snort'), - 'description' => 'Click this to download all network related attributes that you have access to under the Snort rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a whitelist containing host, domain name and IP numbers to exclude from the NIDS export.', - ), - 'bro' => array( - 'extension' => '.intel', - 'type' => 'Bro', - 'scope' => 'Attribute', - 'requiresPublished' => 1, - 'params' => array('returnFormat' => 'bro'), - 'description' => 'Click this to download all network related attributes that you have access to under the Bro rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a whitelist containing host, domain name and IP numbers to exclude from the NIDS export.', - ), - 'stix' => array( - 'extension' => '.xml', - 'type' => 'STIX', - 'scope' => 'Event', - 'requiresPublished' => 1, - 'params' => array('returnFormat' => 'stix', 'includeAttachments' => 1), - 'description' => 'Click this to download an a STIX document containing the STIX version of all events and attributes that you have access to.' - ), - 'stix2' => array( - 'extension' => '.json', - 'type' => 'STIX2', - 'scope' => 'Event', - 'requiresPublished' => 1, - 'params' => array('returnFormat' => 'stix2', 'includeAttachments' => 1), - 'description' => 'Click this to download an a STIX2 document containing the STIX2 version of all events and attributes that you have access to.' - ), - 'rpz' => array( - 'extension' => '.txt', - 'type' => 'RPZ', - 'scope' => 'Attribute', - 'requiresPublished' => 1, - 'params' => array('returnFormat' => 'rpz'), - 'description' => 'Click this to download an RPZ Zone file generated from all ip-src/ip-dst, hostname, domain attributes. This can be useful for DNS level firewalling. Only published events and attributes marked as IDS Signature are exported.' - ), - 'text' => array( - 'extension' => '.txt', - 'type' => 'TEXT', - 'scope' => 'Attribute', - 'requiresPublished' => 1, - 'params' => array('returnFormat' => 'text', 'includeAttachments' => 1), - 'description' => 'Click on one of the buttons below to download all the attributes with the matching type. This list can be used to feed forensic software when searching for susipicious files. Only published events and attributes marked as IDS Signature are exported.' - ), - ); + public $export_types = array(); public $validFormats = array( 'json' => array('json', 'JsonExport', 'json'), @@ -345,6 +256,102 @@ class Event extends AppModel ) ); + public function __construct($id = false, $table = null, $ds = null) + { + parent::__construct($id, $table, $ds); + + $this->export_types = array( + 'json' => array( + 'extension' => '.json', + 'type' => 'JSON', + 'scope' => 'Event', + 'requiresPublished' => 0, + 'params' => array('includeAttachments' => 1, 'ignore' => 1, 'returnFormat' => 'json'), + 'description' => __('Click this to download all events and attributes that you have access to in MISP JSON format.'), + ), + 'xml' => array( + 'extension' => '.xml', + 'type' => 'XML', + 'scope' => 'Event', + 'params' => array('includeAttachments' => 1, 'ignore' => 1, 'returnFormat' => 'xml'), + 'requiresPublished' => 0, + 'description' => __('Click this to download all events and attributes that you have access to in MISP XML format.'), + ), + 'csv_sig' => array( + 'extension' => '.csv', + 'type' => 'CSV_Sig', + 'scope' => 'Event', + 'requiresPublished' => 1, + 'params' => array('published' => 1, 'to_ids' => 1, 'returnFormat' => 'csv'), + 'description' => __('Click this to download all attributes that are indicators and that you have access to (except file attachments) in CSV format.'), + ), + 'csv_all' => array( + 'extension' => '.csv', + 'type' => 'CSV_All', + 'scope' => 'Event', + 'requiresPublished' => 0, + 'params' => array('ignore' => 1, 'returnFormat' => 'csv'), + 'description' => __('Click this to download all attributes that you have access to (except file attachments) in CSV format.'), + ), + 'suricata' => array( + 'extension' => '.rules', + 'type' => 'Suricata', + 'scope' => 'Attribute', + 'requiresPublished' => 1, + 'params' => array('returnFormat' => 'suricata'), + 'description' => __('Click this to download all network related attributes that you have access to under the Suricata rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a whitelist containing host, domain name and IP numbers to exclude from the NIDS export.'), + ), + 'snort' => array( + 'extension' => '.rules', + 'type' => 'Snort', + 'scope' => 'Attribute', + 'requiresPublished' => 1, + 'params' => array('returnFormat' => 'snort'), + 'description' => __('Click this to download all network related attributes that you have access to under the Snort rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a whitelist containing host, domain name and IP numbers to exclude from the NIDS export.'), + ), + 'bro' => array( + 'extension' => '.intel', + 'type' => 'Bro', + 'scope' => 'Attribute', + 'requiresPublished' => 1, + 'params' => array('returnFormat' => 'bro'), + 'description' => __('Click this to download all network related attributes that you have access to under the Bro rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a whitelist containing host, domain name and IP numbers to exclude from the NIDS export.'), + ), + 'stix' => array( + 'extension' => '.xml', + 'type' => 'STIX', + 'scope' => 'Event', + 'requiresPublished' => 1, + 'params' => array('returnFormat' => 'stix', 'includeAttachments' => 1), + 'description' => __('Click this to download an a STIX document containing the STIX version of all events and attributes that you have access to.') + ), + 'stix2' => array( + 'extension' => '.json', + 'type' => 'STIX2', + 'scope' => 'Event', + 'requiresPublished' => 1, + 'params' => array('returnFormat' => 'stix2', 'includeAttachments' => 1), + 'description' => __('Click this to download an a STIX2 document containing the STIX2 version of all events and attributes that you have access to.') + ), + 'rpz' => array( + 'extension' => '.txt', + 'type' => 'RPZ', + 'scope' => 'Attribute', + 'requiresPublished' => 1, + 'params' => array('returnFormat' => 'rpz'), + 'description' => __('Click this to download an RPZ Zone file generated from all ip-src/ip-dst, hostname, domain attributes. This can be useful for DNS level firewalling. Only published events and attributes marked as IDS Signature are exported.') + ), + 'text' => array( + 'extension' => '.txt', + 'type' => 'TEXT', + 'scope' => 'Attribute', + 'requiresPublished' => 1, + 'params' => array('returnFormat' => 'text', 'includeAttachments' => 1), + 'description' => __('Click on one of the buttons below to download all the attributes with the matching type. This list can be used to feed forensic software when searching for susipicious files. Only published events and attributes marked as IDS Signature are exported.') + ), + ); + } + public function beforeDelete($cascade = true) { // blacklist the event UUID if the feature is enabled diff --git a/app/View/Events/ajax/enrich_event.ctp b/app/View/Events/ajax/enrich_event.ctp index d75535be2..d1c278b2f 100644 --- a/app/View/Events/ajax/enrich_event.ctp +++ b/app/View/Events/ajax/enrich_event.ctp @@ -14,7 +14,7 @@ Form->submit('Enrich', array('class' => 'btn btn-primary')); + echo $this->Form->submit(__('Enrich'), array('class' => 'btn btn-primary')); ?> diff --git a/app/View/Events/merge.ctp b/app/View/Events/merge.ctp index af9d4f312..03067b788 100644 --- a/app/View/Events/merge.ctp +++ b/app/View/Events/merge.ctp @@ -23,7 +23,7 @@ Form->button('Merge', array('class' => 'btn btn-primary')); +echo $this->Form->button(__('Merge'), array('class' => 'btn btn-primary')); echo $this->Form->end(); ?>
From 1d86e420efc477cc50b1f80bffb03dfc61f43389 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Tue, 30 Apr 2019 15:58:44 +0200 Subject: [PATCH 11/26] chg: [object:revise] Improved text and added a back button --- app/View/Elements/Objects/object_similarities.ctp | 8 ++++---- app/View/Objects/add.ctp | 3 +++ app/View/Objects/revise_object.ctp | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/app/View/Elements/Objects/object_similarities.ctp b/app/View/Elements/Objects/object_similarities.ctp index 547673bdb..e06e3e399 100644 --- a/app/View/Elements/Objects/object_similarities.ctp +++ b/app/View/Elements/Objects/object_similarities.ctp @@ -78,7 +78,7 @@ if (!isset($simple_flattened_attribute_noval) || !isset($simple_flattened_attrib $btn_style = 'btn-danger'; $temp_text = __('Can\'t merge due to template version'); } else { - $temp_text = __('Merge'); + $temp_text = __('Review merge'); $btn_style = 'btn-success'; } ?> @@ -156,20 +156,20 @@ if (!isset($simple_flattened_attribute_noval) || !isset($simple_flattened_attrib && isset($multiple_attribute_allowed[$attribute['object_relation'] . ':' . $attribute['type']]) ) { // Multiple allowed $classname = 'warning'; - $title = __('This attribute is also contained by the revised object. However, as multiple instanciation is allowed by the template, the two attributes will be keept.'); + $title = __('This attribute is also contained in the revised object. However, as multiple instantiations are allowed by the template, both attributes will be kept.'); $to_highlight = $simple_flattened_similar_attribute_noval; } else if ( isset($simple_flattened_attribute_noval[$simple_flattened_similar_attribute_noval]) && !isset($simple_flattened_attribute[$simple_flattened_similar_attribute]) ) { // Not overridable attribute $classname = 'error'; - $title = __('This attribute is conflicting with the one in the revised object, manual merge will be required.'); + $title = __('This attribute is conflicting with the one in the revised object. Manual merge will be required.'); $to_highlight = $simple_flattened_similar_attribute_noval; } else if ( !isset($simple_flattened_attribute[$simple_flattened_similar_attribute]) ) { // Attribute not present in the revised object $classname = 'info'; - $title = __('This attribute is only contained by this similar object. It will remain untouched.'); + $title = __('This attribute is only contained in this matching object. It will remain untouched.'); } else { // Attributes are basically the same $classname = ''; $title = __('This attribute has the same value as the one in the revised object.'); diff --git a/app/View/Objects/add.ctp b/app/View/Objects/add.ctp index 59b738935..75a6af4dd 100644 --- a/app/View/Objects/add.ctp +++ b/app/View/Objects/add.ctp @@ -167,6 +167,9 @@ Form->button('Submit', array('class' => 'btn btn-primary')); + ?> + + Form->end(); ?> diff --git a/app/View/Objects/revise_object.ctp b/app/View/Objects/revise_object.ctp index dcd2abd5b..70c19dd0d 100644 --- a/app/View/Objects/revise_object.ctp +++ b/app/View/Objects/revise_object.ctp @@ -100,12 +100,12 @@
- Form->button(__('Submit'), array('class' => 'btn btn-primary')); ?> + Form->button(__('Create new object'), array('class' => 'btn btn-primary')); ?> ' . __('This event contains similar objects.') . '

'; ?> - ' . __('Would you like to merge your new object into one of the following?') . ''; ?> + ' . __('Instead of creating a new object, would you like to merge your new object into one of the following?') . ''; ?>
Date: Tue, 30 Apr 2019 20:18:54 +0200 Subject: [PATCH 12/26] chg: [misp-galaxy] updated to the latest version of ATT&CK --- app/files/misp-galaxy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy index 094f0e068..94466d819 160000 --- a/app/files/misp-galaxy +++ b/app/files/misp-galaxy @@ -1 +1 @@ -Subproject commit 094f0e0684efa8857944aed6e70a29b33b5065d8 +Subproject commit 94466d8196dce3dafd2a41942d02e4c5362dbe51 From 937253cba745edb2093a4512ac38d8527d03923b Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 1 May 2019 10:21:30 +0900 Subject: [PATCH 13/26] chg: [doc] Added updates to rhel8, which partially works with Fedora Server 30 --- docs/xINSTALL.rhel8.md | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/docs/xINSTALL.rhel8.md b/docs/xINSTALL.rhel8.md index c4e9e8ad0..521c94f8f 100644 --- a/docs/xINSTALL.rhel8.md +++ b/docs/xINSTALL.rhel8.md @@ -1,4 +1,4 @@ -# INSTALLATION INSTRUCTIONS for RHEL 8.x (beta) +# INSTALLATION INSTRUCTIONS for RHEL 8.x (beta) and partially Fedora Server 30 ------------------------- ### -1/ Installer and Manual install instructions @@ -129,6 +129,7 @@ sudo yum install gcc git zip \ mod_ssl \ redis \ mariadb \ + mariadb-server \ python3-devel python3-pip python3-virtualenv \ libxslt-devel zlib-devel ssdeep-devel -y sudo alternatives --set python /usr/bin/python3 @@ -159,11 +160,6 @@ sudo yum install php php-fpm php-devel php-pear \ sudo systemctl enable --now php-fpm.service ``` -## 2.07/ Start redis service and enable to start on boot -```bash -sudo systemctl enable --now redis.service -``` - ```bash # # GPG needs lots of entropy, haveged provides entropy @@ -237,7 +233,8 @@ $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U zmq $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U redis # lief needs manual compilation -sudo yum install devtoolset-8 cmake3 cppcheck -y +sudo yum groupinstall "Development Tools" -y +sudo yum install cmake3 cppcheck -y cd $PATH_TO_MISP/app/files/scripts/lief $SUDO_WWW mkdir build @@ -289,10 +286,10 @@ installCake_RHEL () sudo chown $WWW_USER:$WWW_USER /usr/share/httpd/.composer cd $PATH_TO_MISP/app # Update composer.phar (optional) - #$SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" - #$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === '48e3236262b34d30969dca3c37281b3b4bbe3221bda826ac6a9a62d6444cdb0dcd0615698a5cbe587c3f0fe57a54d8f5') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;" - #$SUDO_WWW php composer-setup.php - #$SUDO_WWW php -r "unlink('composer-setup.php');" + $SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" + $SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === '48e3236262b34d30969dca3c37281b3b4bbe3221bda826ac6a9a62d6444cdb0dcd0615698a5cbe587c3f0fe57a54d8f5') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;" + $SUDO_WWW php composer-setup.php + $SUDO_WWW php -r "unlink('composer-setup.php');" $SUDO_WWW php composer.phar require kamisama/cake-resque:4.1.2 $SUDO_WWW php composer.phar config vendor-dir Vendor $SUDO_WWW php composer.phar install @@ -650,7 +647,7 @@ Make the workers' script executable and reload the systemd units : ```bash sudo chmod +x /var/www/MISP/app/Console/worker/start.sh sudo systemctl daemon-reload -sudo checkmodule -M -m -o /tmp/workerstartsh.mod $PATH_TO_MISP/INSTALL/workerstartsh.te +sudo checkmodule -M -m -o /tmp/workerstartsh.mod $PATH_TO_MISP/INSTALL/worker/startsh.te sudo semodule_package -o /tmp/workerstartsh.pp -m /tmp/workerstartsh.mod sudo semodule -i /tmp/workerstartsh.pp ``` @@ -670,10 +667,10 @@ sudo chown root:users /usr/local/src cd /usr/local/src/ $SUDO_WWW git clone https://github.com/MISP/misp-modules.git cd misp-modules +sudo yum install rubygem-rouge rubygem-asciidoctor zbar-devel opencv-core poppler-cpp-devel -y # pip install $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U -I -r REQUIREMENTS $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U . -sudo yum install rubygem-rouge rubygem-asciidoctor zbar-devel opencv-core -y echo "[Unit] Description=MISP's modules From 234902a52acfe54675d99ca96682e50b2f41dcc5 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 1 May 2019 10:23:11 +0900 Subject: [PATCH 14/26] chg: [INSTALL] Updated installer. --- INSTALL/INSTALL.sh | 6 ++---- INSTALL/INSTALL.sh.sha1 | 2 +- INSTALL/INSTALL.sh.sha256 | 2 +- INSTALL/INSTALL.sh.sha384 | 2 +- INSTALL/INSTALL.sh.sha512 | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/INSTALL/INSTALL.sh b/INSTALL/INSTALL.sh index ff57aa35a..5eb939907 100755 --- a/INSTALL/INSTALL.sh +++ b/INSTALL/INSTALL.sh @@ -1350,18 +1350,16 @@ mispmodules () { $SUDO_USER git clone https://github.com/MISP/misp-modules.git cd misp-modules # some misp-modules dependencies - sudo apt-get install libpq5 libjpeg-dev libfuzzy-dev -y + sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr libpoppler-cpp-dev imagemagick virtualenv libopencv-dev zbar-tools libzbar0 libzbar-dev libfuzzy-dev -y # If you build an egg, the user you build it as need write permissions in the CWD sudo chgrp $WWW_USER . sudo chmod g+w . $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I -r REQUIREMENTS sudo chgrp staff . $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I . - sudo apt install ruby-pygments.rb -y + sudo apt install ruby-pygments.rb libopencv-dev zbar-tools -y sudo gem install asciidoctor-pdf --pre - # install additional dependencies for extended object generation and extraction - $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install wand yara pathlib # Start misp-modules as a service sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/ sudo systemctl daemon-reload diff --git a/INSTALL/INSTALL.sh.sha1 b/INSTALL/INSTALL.sh.sha1 index 65f6db3de..30d65e71c 100644 --- a/INSTALL/INSTALL.sh.sha1 +++ b/INSTALL/INSTALL.sh.sha1 @@ -1 +1 @@ -931fae16c4462a0f8aab1e7428fe4c168480d33a INSTALL.sh +1df66d811e99de4e2bc864707e6f3028c594e515 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha256 b/INSTALL/INSTALL.sh.sha256 index e5ccefaf7..420d1c0c3 100644 --- a/INSTALL/INSTALL.sh.sha256 +++ b/INSTALL/INSTALL.sh.sha256 @@ -1 +1 @@ -01c7ad59ac9105c2fb8e8cd86e3794d71df0091e9d6f2110a302891561b2af65 INSTALL.sh +ec15b8f47e26dd0604165c14d38728c5ef02db625d9b1fbe6b8e238f09447dd2 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha384 b/INSTALL/INSTALL.sh.sha384 index baf64d457..b4b7403f7 100644 --- a/INSTALL/INSTALL.sh.sha384 +++ b/INSTALL/INSTALL.sh.sha384 @@ -1 +1 @@ -0f0f2e4005833dd563ec0a71e2fcf520da798c081c6c5df3d8f5d5684c09d9eb194097a721c6c7a0687c820f0f0a8dcd INSTALL.sh +ed1af1ecd1140d330d1f775121bd5192f82f9a92f75f6fe0c611ca11968e81fd9c24de4af54635811bfaeadae8f55f70 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha512 b/INSTALL/INSTALL.sh.sha512 index 56dd74ed2..e7061e43e 100644 --- a/INSTALL/INSTALL.sh.sha512 +++ b/INSTALL/INSTALL.sh.sha512 @@ -1 +1 @@ -0ea0ea8f1de9a03a85bf475732446388a096aac0c54e8ebd7203f5119731521ef30ca99cc998637037a8f50587fa4c0bcd0f3994249898ed164729f08bdf3c7d INSTALL.sh +a5c87c821aaae9b23cb5842cb9cb907e4ac0b918e4ac3abe333fa38d88f08074d5389df05a945fbd52cfe5d83989d44a7a22e2bd9a9c45ddfc5ab16f1878fb25 INSTALL.sh From b2b0399d990a308ef0f6cd0ef9e38d40c04ddf3b Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 1 May 2019 10:51:58 +0900 Subject: [PATCH 15/26] chg: [doc] Added plyara --- docs/INSTALL.ubuntu1804.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/INSTALL.ubuntu1804.md b/docs/INSTALL.ubuntu1804.md index 2b8948876..32d94939a 100644 --- a/docs/INSTALL.ubuntu1804.md +++ b/docs/INSTALL.ubuntu1804.md @@ -175,6 +175,9 @@ installCore () { # install python-magic $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install python-magic + # install plyara + $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install plyara + # Install Crypt_GPG and Console_CommandLine sudo pear install ${PATH_TO_MISP}/INSTALL/dependencies/Console_CommandLine/package.xml sudo pear install ${PATH_TO_MISP}/INSTALL/dependencies/Crypt_GPG/package.xml From 082cf53225910a4e692f9af33e74392a863fa00b Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 1 May 2019 13:47:44 +0900 Subject: [PATCH 16/26] chg: [AdminShell] return the name of the setting change and what we changed it to. --- app/Console/Command/AdminShell.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Console/Command/AdminShell.php b/app/Console/Command/AdminShell.php index 9dc666372..1cfa96813 100644 --- a/app/Console/Command/AdminShell.php +++ b/app/Console/Command/AdminShell.php @@ -283,7 +283,7 @@ class AdminShell extends AppShell } $result = $this->Server->serverSettingsEditValue($cli_user, $setting, $value); if ($result === true) { - echo 'Setting changed.'; + echo 'Setting "' . $setting_name . '" changed to ' . $value; } else { echo $result; } From 80fe2bc43309c38c6884a84ef61c3afa5669e930 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 1 May 2019 14:11:50 +0900 Subject: [PATCH 17/26] fix: [install] Fixed the endless loop in viper db update --- INSTALL/INSTALL.sh | 9 +++++++++ INSTALL/INSTALL.sh.sha1 | 2 +- INSTALL/INSTALL.sh.sha256 | 2 +- INSTALL/INSTALL.sh.sha384 | 2 +- INSTALL/INSTALL.sh.sha512 | 2 +- docs/generic/viper-debian.md | 6 ++++++ 6 files changed, 19 insertions(+), 4 deletions(-) diff --git a/INSTALL/INSTALL.sh b/INSTALL/INSTALL.sh index 5eb939907..9f79ef596 100755 --- a/INSTALL/INSTALL.sh +++ b/INSTALL/INSTALL.sh @@ -1084,6 +1084,9 @@ installCore () { # install python-magic $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install python-magic + # install plyara + $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install plyara + # Install Crypt_GPG and Console_CommandLine sudo pear install ${PATH_TO_MISP}/INSTALL/dependencies/Console_CommandLine/package.xml sudo pear install ${PATH_TO_MISP}/INSTALL/dependencies/Crypt_GPG/package.xml @@ -1581,11 +1584,17 @@ viper () { $SUDO_USER sed -i "s/^misp_key\ =/misp_key\ =\ $AUTH_KEY/g" ${VIPER_HOME}/viper.conf # Reset admin password to: admin/Password1234 echo "Fixing admin.db with default password" + VIPER_COUNT=0 while [ "$(sudo sqlite3 ${VIPER_HOME}/admin.db 'UPDATE auth_user SET password="pbkdf2_sha256$100000$iXgEJh8hz7Cf$vfdDAwLX8tko1t0M1TLTtGlxERkNnltUnMhbv56wK/U="'; echo $?)" -ne "0" ]; do # FIXME This might lead to a race condition, the while loop is sub-par sudo chown $MISP_USER:$MISP_USER ${VIPER_HOME}/admin.db echo "Updating viper-web admin password, giving process time to start-up, sleeping 5, 4, 3,…" sleep 6 + VIPER_COUNT=$[$VIPER_COUNT+1] + if [[ "$VIPER_COUNT" > '10' ]]; then + echo "Something is wrong with updating viper. Continuing without db update." + break + fi done # Add viper-web to rc.local to be started on boot diff --git a/INSTALL/INSTALL.sh.sha1 b/INSTALL/INSTALL.sh.sha1 index 30d65e71c..3e95e65f5 100644 --- a/INSTALL/INSTALL.sh.sha1 +++ b/INSTALL/INSTALL.sh.sha1 @@ -1 +1 @@ -1df66d811e99de4e2bc864707e6f3028c594e515 INSTALL.sh +81a19d45ae7f499cfe34e35e2c3af93af58ecc9f INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha256 b/INSTALL/INSTALL.sh.sha256 index 420d1c0c3..a6833fa19 100644 --- a/INSTALL/INSTALL.sh.sha256 +++ b/INSTALL/INSTALL.sh.sha256 @@ -1 +1 @@ -ec15b8f47e26dd0604165c14d38728c5ef02db625d9b1fbe6b8e238f09447dd2 INSTALL.sh +cb9ab4292ed6dac59884e4cf51f4a83a5525e499dfbc76a637f1d49811382280 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha384 b/INSTALL/INSTALL.sh.sha384 index b4b7403f7..cd48dbbef 100644 --- a/INSTALL/INSTALL.sh.sha384 +++ b/INSTALL/INSTALL.sh.sha384 @@ -1 +1 @@ -ed1af1ecd1140d330d1f775121bd5192f82f9a92f75f6fe0c611ca11968e81fd9c24de4af54635811bfaeadae8f55f70 INSTALL.sh +0a807abd32d6d8f894d1466a0854b449c1bc53fde9b0fbb43582d007f48749c103c4b607f385a75c548a3422b5a9fd57 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha512 b/INSTALL/INSTALL.sh.sha512 index e7061e43e..dc77c1f12 100644 --- a/INSTALL/INSTALL.sh.sha512 +++ b/INSTALL/INSTALL.sh.sha512 @@ -1 +1 @@ -a5c87c821aaae9b23cb5842cb9cb907e4ac0b918e4ac3abe333fa38d88f08074d5389df05a945fbd52cfe5d83989d44a7a22e2bd9a9c45ddfc5ab16f1878fb25 INSTALL.sh +4fa55166257b1438e503323d28ffc35cab19a7503709bac699393870ff2225899204f7263b583512e15e9214b912457ab52ab9281a957720c611bbdbc3b58815 INSTALL.sh diff --git a/docs/generic/viper-debian.md b/docs/generic/viper-debian.md index 729830abc..55041670b 100644 --- a/docs/generic/viper-debian.md +++ b/docs/generic/viper-debian.md @@ -49,11 +49,17 @@ viper () { $SUDO_USER sed -i "s/^misp_key\ =/misp_key\ =\ $AUTH_KEY/g" ${VIPER_HOME}/viper.conf # Reset admin password to: admin/Password1234 echo "Fixing admin.db with default password" + VIPER_COUNT=0 while [ "$(sudo sqlite3 ${VIPER_HOME}/admin.db 'UPDATE auth_user SET password="pbkdf2_sha256$100000$iXgEJh8hz7Cf$vfdDAwLX8tko1t0M1TLTtGlxERkNnltUnMhbv56wK/U="'; echo $?)" -ne "0" ]; do # FIXME This might lead to a race condition, the while loop is sub-par sudo chown $MISP_USER:$MISP_USER ${VIPER_HOME}/admin.db echo "Updating viper-web admin password, giving process time to start-up, sleeping 5, 4, 3,…" sleep 6 + VIPER_COUNT=$[$VIPER_COUNT+1] + if [[ "$VIPER_COUNT" > '10' ]]; then + echo "Something is wrong with updating viper. Continuing without db update." + break + fi done # Add viper-web to rc.local to be started on boot From 750429797e46a212e2e9c7a011a82f4ff33a6e3a Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 1 May 2019 18:01:48 +0900 Subject: [PATCH 18/26] fix: [i18n] Added yara/yara-export --- app/Model/Event.php | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/app/Model/Event.php b/app/Model/Event.php index 1f46f038b..e6034c251 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -71,6 +71,8 @@ class Event extends AppModel 'csv' => array('csv', 'CsvExport', 'csv'), 'stix' => array('xml', 'Stix1Export', 'xml'), 'stix2' => array('json', 'Stix2Export', 'json'), + 'yara' => array('txt', 'YaraExport', 'yara'), + 'yara-json' => array('json', 'YaraExport', 'json'), 'cache' => array('txt', 'CacheExport', 'cache') ); @@ -349,6 +351,22 @@ class Event extends AppModel 'params' => array('returnFormat' => 'text', 'includeAttachments' => 1), 'description' => __('Click on one of the buttons below to download all the attributes with the matching type. This list can be used to feed forensic software when searching for susipicious files. Only published events and attributes marked as IDS Signature are exported.') ), + 'yara' => array( + 'extension' => '.yara', + 'type' => 'Yara', + 'scope' => 'Event', + 'requiresPublished' => 1, + 'params' => array('returnFormat' => 'yara'), + 'description' => __('Click this to download Yara rules generated from all relevant attributes.') + ), + 'yara-json' => array( + 'extension' => '.json', + 'type' => 'Yara', + 'scope' => 'Event', + 'requiresPublished' => 1, + 'params' => array('returnFormat' => 'yara-json'), + 'description' => __('Click this to download Yara rules generated from all relevant attributes. Rules are returned in a JSON format with information about origin (generated or parsed) and validity.') + ), ); } From 9bccd9382658edff92935d6a8263205d1cb507af Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 1 May 2019 22:27:38 +0900 Subject: [PATCH 19/26] chg: [AdminShell] Adde PHP_EOLs where it made sense. (QoL enhancement) --- app/Console/Command/AdminShell.php | 40 +++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/app/Console/Command/AdminShell.php b/app/Console/Command/AdminShell.php index 1cfa96813..d75a722b8 100644 --- a/app/Console/Command/AdminShell.php +++ b/app/Console/Command/AdminShell.php @@ -65,7 +65,7 @@ class AdminShell extends AppShell public function restartWorker() { if (empty($this->args[0]) || !is_numeric($this->args[0])) { - echo 'Usage: ' . APP . '/cake ' . 'Admin restartWorker [PID]'; + echo 'Usage: ' . APP . '/cake ' . 'Admin restartWorker [PID]' . PHP_EOL; } $pid = $this->args[0]; $result = $this->Server->restartWorker($pid); @@ -85,7 +85,7 @@ class AdminShell extends AppShell public function killWorker() { if (empty($this->args[0]) || !is_numeric($this->args[0])) { - echo 'Usage: ' . APP . '/cake ' . 'Admin killWorker [PID]'; + echo 'Usage: ' . APP . '/cake ' . 'Admin killWorker [PID]' . PHP_EOL; die(); } $pid = $this->args[0]; @@ -101,7 +101,7 @@ class AdminShell extends AppShell public function startWorker() { if (empty($this->args[0])) { - echo 'Usage: ' . APP . '/cake ' . 'Admin startWorker [queue]'; + echo 'Usage: ' . APP . '/cake ' . 'Admin startWorker [queue]' . PHP_EOL; die(); } $queue = $this->args[0]; @@ -143,9 +143,9 @@ class AdminShell extends AppShell $force = $value; $result = $this->Galaxy->update($force); if ($result) { - echo 'Galaxies updated'; + echo 'Galaxies updated' . PHP_EOL; } else { - echo 'Could not update Galaxies'; + echo 'Could not update Galaxies' . PHP_EOL; } } @@ -153,34 +153,34 @@ class AdminShell extends AppShell public function updateTaxonomies() { $result = $this->Taxonomy->update(); if ($result) { - echo 'Taxonomies updated'; + echo 'Taxonomies updated' . PHP_EOL; } else { - echo 'Could not update Taxonomies'; + echo 'Could not update Taxonomies' . PHP_EOL; } } public function updateWarningLists() { $result = $this->Galaxy->update(); if ($result) { - echo 'Warning lists updated'; + echo 'Warning lists updated' . PHP_EOL; } else { - echo 'Could not update warning lists'; + echo 'Could not update warning lists' . PHP_EOL; } } public function updateNoticeLists() { $result = $this->Noticelist->update(); if ($result) { - echo 'Notice lists updated'; + echo 'Notice lists updated' . PHP_EOL; } else { - echo 'Could not update notice lists'; + echo 'Could not update notice lists' . PHP_EOL; } } # FIXME: Debug and make it work, fails to pass userId/orgId properly public function updateObjectTemplates() { if (empty($this->args[0])) { - echo 'Usage: ' . APP . '/cake ' . 'Admin updateNoticeLists [user_id]'; + echo 'Usage: ' . APP . '/cake ' . 'Admin updateNoticeLists [user_id]' . PHP_EOL; } else { $userId = $this->args[0]; $user = $this->User->find('first', array( @@ -191,13 +191,13 @@ class AdminShell extends AppShell 'fields' => array('User.id', 'User.org_id') )); if (empty($user)) { - echo 'User not found'; + echo 'User not found' . PHP_EOL; } else { $result = $this->ObjectTemplate->update($user, false,false); if ($result) { - echo 'Object templates updated'; + echo 'Object templates updated' . PHP_EOL; } else { - echo 'Could not update object templates'; + echo 'Could not update object templates' . PHP_EOL; } } } @@ -275,15 +275,15 @@ class AdminShell extends AppShell if ($value === 'true') $value = 1; $cli_user = array('id' => 0, 'email' => 'SYSTEM', 'Organisation' => array('name' => 'SYSTEM')); if (empty($setting_name) || $value === null) { - echo 'Invalid parameters. Usage: ' . APP . 'Console/cake Admin setSetting [setting_name] [setting_value]'; + echo 'Invalid parameters. Usage: ' . APP . 'Console/cake Admin setSetting [setting_name] [setting_value]' . PHP_EOL; } else { $setting = $this->Server->getSettingData($setting_name); if (empty($setting)) { - echo 'Invalid setting. Please make sure that the setting that you are attempting to change exists.'; + echo 'Invalid setting "' . $setting_name . '". Please make sure that the setting that you are attempting to change exists and if a module parameter, the modules are running.' . PHP_EOL; } $result = $this->Server->serverSettingsEditValue($cli_user, $setting, $value); if ($result === true) { - echo 'Setting "' . $setting_name . '" changed to ' . $value; + echo 'Setting "' . $setting_name . '" changed to ' . $value . PHP_EOL; } else { echo $result; } @@ -379,7 +379,7 @@ class AdminShell extends AppShell public function change_authkey() { if (empty($this->args[0])) { - echo 'MISP apikey command line tool.' . PHP_EOL . 'To assign a new random API key for a user: ' . APP . 'Console/cake Password [email]' . PHP_EOL . 'To assign a fixed API key: ' . APP . 'Console/cake Password [email] [authkey]'; + echo 'MISP apikey command line tool.' . PHP_EOL . 'To assign a new random API key for a user: ' . APP . 'Console/cake Password [email]' . PHP_EOL . 'To assign a fixed API key: ' . APP . 'Console/cake Password [email] [authkey]' . PHP_EOL; die(); } if (!empty($this->args[1])) { @@ -393,7 +393,7 @@ class AdminShell extends AppShell 'fields' => array('User.id', 'User.email', 'User.authkey') )); if (empty($user)) { - echo 'Invalid e-mail, user not found.'; + echo 'Invalid e-mail, user not found.' . PHP_EOL; die(); } $user['User']['authkey'] = $authKey; From 58999e2b72678f66f8a552ba8fb0e0d18b8a2084 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 1 May 2019 22:53:25 +0900 Subject: [PATCH 20/26] fix: [doc] misp-modules failed to install because of a Permission issue. --- docs/generic/globalVariables.md | 2 +- docs/generic/misp-modules-debian.md | 2 -- docs/generic/supportFunctions.md | 9 +++++++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/generic/globalVariables.md b/docs/generic/globalVariables.md index 7da28eda1..9669af13c 100644 --- a/docs/generic/globalVariables.md +++ b/docs/generic/globalVariables.md @@ -65,7 +65,7 @@ MISPvars () { CAKE="$PATH_TO_MISP/app/Console/cake" # sudo config to run $LUSER commands - if [[ "$(groups |grep -o 'staff')" == "staff" ]]; then + if [[ "$(groups ${MISP_USER} |grep -o 'staff')" == "staff" ]]; then SUDO_USER="sudo -H -u ${MISP_USER} -g staff" else SUDO_USER="sudo -H -u ${MISP_USER}" diff --git a/docs/generic/misp-modules-debian.md b/docs/generic/misp-modules-debian.md index fc28ab6ed..1fa75c512 100644 --- a/docs/generic/misp-modules-debian.md +++ b/docs/generic/misp-modules-debian.md @@ -4,8 +4,6 @@ # # Main MISP Modules install function mispmodules () { - # FIXME: this is broken, ${PATH_TO_MISP} is litteral -##sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local cd /usr/local/src/ ## TODO: checkUsrLocalSrc in main doc debug "Cloning misp-modules" diff --git a/docs/generic/supportFunctions.md b/docs/generic/supportFunctions.md index 17f712454..0bbab6eda 100644 --- a/docs/generic/supportFunctions.md +++ b/docs/generic/supportFunctions.md @@ -212,6 +212,15 @@ checkID () { sudo adduser $MISP_USER staff sudo adduser $MISP_USER $WWW_USER fi + + # FIXME: the below SUDO_USER check is a duplicate from global variables, try to have just one check + # sudo config to run $LUSER commands + if [[ "$(groups ${MISP_USER} |grep -o 'staff')" == "staff" ]]; then + SUDO_USER="sudo -H -u ${MISP_USER} -g staff" + else + SUDO_USER="sudo -H -u ${MISP_USER}" + fi + } # pre-install check to make sure what we will be installing on, is ready and not a half installed system From a79ef14d04ebf6c5dd640a3eb31809a71855e805 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Wed, 1 May 2019 22:56:55 +0900 Subject: [PATCH 21/26] fix: [installer] Fixed installer misp-modules permissions. --- INSTALL/INSTALL.sh | 13 ++++++++++--- INSTALL/INSTALL.sh.sha1 | 2 +- INSTALL/INSTALL.sh.sha256 | 2 +- INSTALL/INSTALL.sh.sha384 | 2 +- INSTALL/INSTALL.sh.sha512 | 2 +- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/INSTALL/INSTALL.sh b/INSTALL/INSTALL.sh index 9f79ef596..045a3eeb8 100755 --- a/INSTALL/INSTALL.sh +++ b/INSTALL/INSTALL.sh @@ -128,7 +128,7 @@ MISPvars () { CAKE="$PATH_TO_MISP/app/Console/cake" # sudo config to run $LUSER commands - if [[ "$(groups |grep -o 'staff')" == "staff" ]]; then + if [[ "$(groups ${MISP_USER} |grep -o 'staff')" == "staff" ]]; then SUDO_USER="sudo -H -u ${MISP_USER} -g staff" else SUDO_USER="sudo -H -u ${MISP_USER}" @@ -352,6 +352,15 @@ checkID () { sudo adduser $MISP_USER staff sudo adduser $MISP_USER $WWW_USER fi + + # FIXME: the below SUDO_USER check is a duplicate from global variables, try to have just one check + # sudo config to run $LUSER commands + if [[ "$(groups ${MISP_USER} |grep -o 'staff')" == "staff" ]]; then + SUDO_USER="sudo -H -u ${MISP_USER} -g staff" + else + SUDO_USER="sudo -H -u ${MISP_USER}" + fi + } # pre-install check to make sure what we will be installing on, is ready and not a half installed system @@ -1345,8 +1354,6 @@ backgroundWorkers () { # Main MISP Modules install function mispmodules () { - # FIXME: this is broken, ${PATH_TO_MISP} is litteral -##sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local cd /usr/local/src/ ## TODO: checkUsrLocalSrc in main doc debug "Cloning misp-modules" diff --git a/INSTALL/INSTALL.sh.sha1 b/INSTALL/INSTALL.sh.sha1 index 3e95e65f5..36d0bb4b6 100644 --- a/INSTALL/INSTALL.sh.sha1 +++ b/INSTALL/INSTALL.sh.sha1 @@ -1 +1 @@ -81a19d45ae7f499cfe34e35e2c3af93af58ecc9f INSTALL.sh +eaf31965e51c8fc7a6202d7896aa0e9b0b3ecb3d INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha256 b/INSTALL/INSTALL.sh.sha256 index a6833fa19..9facf4fc4 100644 --- a/INSTALL/INSTALL.sh.sha256 +++ b/INSTALL/INSTALL.sh.sha256 @@ -1 +1 @@ -cb9ab4292ed6dac59884e4cf51f4a83a5525e499dfbc76a637f1d49811382280 INSTALL.sh +4544798f992578aaffa28ba816591ba2dbf38c18824c4df0a901f13dd7ded714 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha384 b/INSTALL/INSTALL.sh.sha384 index cd48dbbef..1c4070ec2 100644 --- a/INSTALL/INSTALL.sh.sha384 +++ b/INSTALL/INSTALL.sh.sha384 @@ -1 +1 @@ -0a807abd32d6d8f894d1466a0854b449c1bc53fde9b0fbb43582d007f48749c103c4b607f385a75c548a3422b5a9fd57 INSTALL.sh +2930fa3ddeccc2753ad91ef4869dc43357093d24c33b8e01fa5c3218922ecf3855b7cb05bb5a60da1d4f9050ad1bbeef INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha512 b/INSTALL/INSTALL.sh.sha512 index dc77c1f12..69cb9e2ff 100644 --- a/INSTALL/INSTALL.sh.sha512 +++ b/INSTALL/INSTALL.sh.sha512 @@ -1 +1 @@ -4fa55166257b1438e503323d28ffc35cab19a7503709bac699393870ff2225899204f7263b583512e15e9214b912457ab52ab9281a957720c611bbdbc3b58815 INSTALL.sh +54ba106074b191d85102b86da534a7ea5327ac208d6f3551b262b6daee6e8a289f3fe97b323a7ada4242b0a5580d8600f5caf8d2e4244b07a57a54f54cd8f426 INSTALL.sh From f28b83385688e01682c5eaf601daeec86a0f6ef9 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 1 May 2019 22:27:02 +0200 Subject: [PATCH 22/26] chg: [misp-objects] updated to the latest version --- app/files/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/misp-objects b/app/files/misp-objects index 0f6fdee7f..e76e49289 160000 --- a/app/files/misp-objects +++ b/app/files/misp-objects @@ -1 +1 @@ -Subproject commit 0f6fdee7f32ce9f57a344323564bc5e6f60bfc8f +Subproject commit e76e492894fbe260f9f67a8b72447b93f0e36196 From a4ebadecd71269d0bdb4b0611033bbb822e65970 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 2 May 2019 09:55:03 +0200 Subject: [PATCH 23/26] chg: [event:row_attribute] Added `title` to the checkboxes. This reveal to be useful if the table header is not visible. --- app/View/Elements/Events/View/row_attribute.ctp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/View/Elements/Events/View/row_attribute.ctp b/app/View/Elements/Events/View/row_attribute.ctp index f7ef3cd30..5343fcd90 100644 --- a/app/View/Elements/Events/View/row_attribute.ctp +++ b/app/View/Elements/Events/View/row_attribute.ctp @@ -193,7 +193,8 @@
- > + >
From 243108eadb06ca1b5fbdcebce69dd93465f9488e Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 2 May 2019 10:15:51 +0200 Subject: [PATCH 24/26] chg: [object:add] Changed back button text into `Back` --- app/View/Objects/add.ctp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/View/Objects/add.ctp b/app/View/Objects/add.ctp index 75a6af4dd..46940c704 100644 --- a/app/View/Objects/add.ctp +++ b/app/View/Objects/add.ctp @@ -168,7 +168,7 @@ Form->button('Submit', array('class' => 'btn btn-primary')); ?> - + Form->end(); From 3751224269b665dea5a19a8691d6eb1f1d748abf Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 2 May 2019 11:09:08 +0200 Subject: [PATCH 25/26] chg: [diagnostic] Added message if `.git` can't be read by MISP --- app/View/Elements/healthElements/diagnostics.ctp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/View/Elements/healthElements/diagnostics.ctp b/app/View/Elements/healthElements/diagnostics.ctp index 1ce017024..078403cc2 100644 --- a/app/View/Elements/healthElements/diagnostics.ctp +++ b/app/View/Elements/healthElements/diagnostics.ctp @@ -34,6 +34,12 @@ + +
+ + + +
From 8e5c6e7192dc206d063f4f35c13914bbad9c99e9 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 2 May 2019 11:21:58 +0200 Subject: [PATCH 26/26] chg: [diagnostic] Changed update button with more relevant icons --- app/View/Servers/ajax/submoduleStatus.ctp | 4 ++-- app/webroot/js/misp.js | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/app/View/Servers/ajax/submoduleStatus.ctp b/app/View/Servers/ajax/submoduleStatus.ctp index 5c7f1d0b8..c9b047ad7 100644 --- a/app/View/Servers/ajax/submoduleStatus.ctp +++ b/app/View/Servers/ajax/submoduleStatus.ctp @@ -9,7 +9,7 @@ echo $this->Form->create('Server', array('url' => array('action' => 'updateSubmodule'), 'div' => false, 'style' => 'margin: 0px; display: inline-block;')); echo $this->Form->hidden('submodule', array('value' => false)); echo $this->Form->end(); - echo ''; + echo ''; ?> @@ -56,7 +56,7 @@ '; + echo ''; } ?> diff --git a/app/webroot/js/misp.js b/app/webroot/js/misp.js index 1c8801247..e384d0a21 100644 --- a/app/webroot/js/misp.js +++ b/app/webroot/js/misp.js @@ -3753,7 +3753,8 @@ function submitSubmoduleUpdate(clicked) { var submodule_path = $clicked.data('submodule'); $.ajax({ beforeSend: function (XMLHttpRequest) { - $clicked.addClass('fa-spin'); + $clicked.removeClass('fa-download'); + $clicked.addClass('fa-spin fa-spinner'); }, dataType:"html", cache: false, @@ -3780,7 +3781,8 @@ function submitSubmoduleUpdate(clicked) { $('#submoduleGitResult').removeClass('green').addClass('red').text(data.output); }, complete:function() { - $clicked.removeClass('fa-spin'); + $clicked.removeClass('fa-spin fa-spinner'); + $clicked.addClass('fa-download'); $form.remove(); }, type:"post",