MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [updates] Implented changes requested by the PR's review #4534.

pull/4534/head
mokaddem 2019-04-29 11:09:04 +02:00
parent d6be8023e5
commit 47e13c8369
8 changed files with 136 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
app/Console/Command/AdminShell.php
View File

@ -308,11 +308,48 @@ class AdminShell extends AppShell
}
public function updateDatabase() {
echo 'Executing all updates to bring the database up to date with the current version.' . PHP_EOL;
$this->Server->runUpdates(true);
echo 'All updates completed.' . PHP_EOL;
$whoami = exec('whoami');
if ($whoami === 'httpd' || $whoami === 'www-data') {
echo 'Executing all updates to bring the database up to date with the current version.' . PHP_EOL;
$this->Server->runUpdates(true);
echo 'All updates completed.' . PHP_EOL;
} else {
die('This OS user is not allowed to run this command.'. PHP_EOL. 'Run it under `www-data` or `httpd`.' . PHP_EOL);
}
}
public function updateApp() {
$whoami = exec('whoami');
if ($whoami === 'httpd' || $whoami === 'www-data') {
$command = $this->args[0];
if (!empty($this->args[1])) {
$processId = $this->args[1];
$job = $this->Job->read(null, $processId);
} else { // create worker
$this->Job->create();
$job_data = array(
'worker' => 'prio',
'job_type' => 'update_app',
'job_input' => 'command: ' . $command,
'status' => 0,
'retries' => 0,
'org_id' => '',
'org' => '',
'message' => 'Updating.',
);
$this->Job->save($job_data);
$job = $this->Job->read(null, $this->Job->id);
}
$result = $this->Server->updateDatabase($command, false);
$job['Job']['progress'] = 100;
$job['Job']['message'] = 'Update done';
$this->Job->save($job);
} else {
die('This OS user is not allowed to run this command.'. PHP_EOL. 'Run it under `www-data` or `httpd`.' . PHP_EOL);
}
}
public function getAuthkey() {
if (empty($this->args[0])) {
echo 'Invalid parameters. Usage: ' . APP . 'Console/cake Admin getAuthkey [user_email]' . PHP_EOL;
@ -404,7 +441,7 @@ class AdminShell extends AppShell
}
echo 'Updated, new key:' . PHP_EOL . $authKey . PHP_EOL;
}
public function getOptionParser() {
$parser = parent::getOptionParser();
$parser->addSubcommand('updateJSON', array(

13
app/Console/Command/ServerShell.php
View File

@ -447,17 +447,4 @@ class ServerShell extends AppShell
$this->Task->saveField('message', count($servers) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '.');
}
public function updateApp() {
$processId = $this->args[0];
$job = $this->Job->read(null, $processId);
$command = $this->args[1];
$liveOff = $this->args[2];
$exitOnError = $this->args[3];
$useWorker = $this->args[4];
$result = $this->Server->updateDatabase($command, $liveOff, $exitOnError, $useWorker);
$job['Job']['progress'] = 100;
$job['Job']['message'] = 'Update done';
$this->Job->save($job);
}
}

4
app/Controller/AppController.php
View File

@ -824,7 +824,7 @@ class AppController extends Controller
$this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
}
public function updateDatabase($command, $liveOff=false, $exitOnError=false)
public function updateDatabase($command)
{
if (!$this->_isSiteAdmin() || !$this->request->is('post')) {
throw new MethodNotAllowedException();
@ -833,7 +833,7 @@ class AppController extends Controller
if (is_numeric($command)) {
$command = intval($command);
}
$this->Server->updateDatabase($command, $liveOff, $exitOnError);
$this->Server->updateDatabase($command);
$this->Flash->success('Done.');
if ($liveOff) {
$this->redirect(array('controller' => 'servers', 'action' => 'updateProgress'));

36
app/Controller/ServersController.php
View File

@ -1565,7 +1565,7 @@ class ServersController extends AppController
}
}
$done = $this->AdminSetting->getSetting($id);
$actions[$id]['done'] = $done !== false && $done == '1' ? true : false;
$actions[$id]['done'] = ($done == '1');
}
$this->set('actions', $actions);
$this->set('updateLocked', $this->Server->isUpdateLocked());
@ -1576,31 +1576,31 @@ class ServersController extends AppController
if (!$this->_isSiteAdmin()) {
throw new MethodNotAllowedException('You are not authorised to do that.');
}
$updateProgress = $this->Server->getUpdateProgress();
$curIndex = $updateProgress['cur'];
$curCommand = !isset($updateProgress['cmd'][$curIndex]) ? '' : $updateProgress['cmd'][$curIndex];
$lookupString = preg_replace('/\s{2,}/', '', substr($curCommand, 0, -1));
$sqlInfo = $this->Server->query("SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;");
if (empty($sqlInfo)) {
$updateProgress['process_list'] = array();
$update_progress = $this->Server->getUpdateProgress();
$current_index = $update_progress['current'];
$current_command = !isset($update_progress['commands'][$current_index]) ? '' : $update_progress['commands'][$current_index];
$lookup_string = preg_replace('/\s{2,}/', '', substr($current_command, 0, -1));
$sql_info = $this->Server->query("SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;");
if (empty($sql_info)) {
$update_progress['process_list'] = array();
} else {
// retreive current update process
foreach($sqlInfo as $row) {
if (preg_replace('/\s{2,}/', '', $row['PROCESSLIST']['INFO']) == $lookupString) {
$sqlInfo = $row['PROCESSLIST'];
foreach($sql_info as $row) {
if (preg_replace('/\s{2,}/', '', $row['PROCESSLIST']['INFO']) == $lookup_string) {
$sql_info = $row['PROCESSLIST'];
break;
}
}
$updateProgress['process_list'] = array();
$updateProgress['process_list']['STATE'] = isset($sqlInfo['STATE']) ? $sqlInfo['STATE'] : '';
$updateProgress['process_list']['PROGRESS'] = isset($sqlInfo['PROGRESS']) ? $sqlInfo['PROGRESS'] : 0;
$updateProgress['process_list']['STAGE'] = isset($sqlInfo['STAGE']) ? $sqlInfo['STAGE'] : 0;
$updateProgress['process_list']['MAX_STAGE'] = isset($sqlInfo['MAX_STAGE']) ? $sqlInfo['MAX_STAGE'] : 0;
$update_progress['process_list'] = array();
$update_progress['process_list']['STATE'] = isset($sql_info['STATE']) ? $sql_info['STATE'] : '';
$update_progress['process_list']['PROGRESS'] = isset($sql_info['PROGRESS']) ? $sql_info['PROGRESS'] : 0;
$update_progress['process_list']['STAGE'] = isset($sql_info['STAGE']) ? $sql_info['STAGE'] : 0;
$update_progress['process_list']['MAX_STAGE'] = isset($sql_info['MAX_STAGE']) ? $sql_info['MAX_STAGE'] : 0;
}
if ($this->request->is('ajax')) {
return $this->RestResponse->viewData(h($updateProgress), $this->response->type());
return $this->RestResponse->viewData(h($update_progress), $this->response->type());
} else {
$this->set('updateProgress', $updateProgress);
$this->set('updateProgress', $update_progress);
}
}

140
app/Model/AppModel.php
View File

@ -224,7 +224,7 @@ class AppModel extends Model
}
// SQL scripts for updates
public function updateDatabase($command, $liveOff=false, $exitOnError=false, $useWorker=true)
public function updateDatabase($command, $useWorker=true)
{
// Exit if updates are locked
if ($this->isUpdateLocked()) {
@ -236,27 +236,34 @@ class AppModel extends Model
$job = ClassRegistry::init('Job');
$job->create();
$data = array(
'worker' => 'prio',
'job_type' => 'update_app',
'job_input' => 'command: ' . $command,
'status' => 0,
'retries' => 0,
'org_id' => '',
'org' => '',
'message' => 'Updating.',
'worker' => 'prio',
'job_type' => 'update_app',
'job_input' => 'command: ' . $command,
'status' => 0,
'retries' => 0,
'org_id' => '',
'org' => '',
'message' => 'Updating.',
);
$job->save($data);
$jobId = $job->id;
$process_id = CakeResque::enqueue(
'prio',
'ServerShell',
array('updateApp', $jobId, $command, $liveOff, $exitOnError, false),
'AdminShell',
array('updateApp', $command, $jobId),
true
);
$job->saveField('process_id', $process_id);
return true;
}
$liveOff = false;
$exitOnError = false;
if (isset($advanced_updates_description[$command])) {
$liveOff = isset($advanced_updates_description[$command]['liveOff']) ? $advanced_updates_description[$command]['liveOff'] : $liveOff;
$exitOnError = isset($advanced_updates_description[$command]['exitOnError']) ? $advanced_updates_description[$command]['exitOnError'] : $exitOnError;
}
$dataSourceConfig = ConnectionManager::getDataSource('default')->config;
$dataSource = $dataSourceConfig['datasource'];
$sqlArray = array();
@ -1198,32 +1205,32 @@ class AppModel extends Model
}
$now = new DateTime();
$this->__changeLockState($now->format('Y-m-d H:i:s'));
$this->__changeLockState(time());
// switch MISP instance live to false
if ($liveOff) {
$this->Server = Classregistry::init('Server');
$liveSetting = 'MISP.live';
$this->Server->serverSettingsSaveValue($liveSetting, false);
}
$SqlUpdateCount = count($sqlArray);
$IndexUpdateCount = count($indexArray);
$totupdatecount = $SqlUpdateCount + $IndexUpdateCount;
$this->__setUpdateProgress(0, $totupdatecount);
$strIndexArray = array();
$sql_update_count = count($sqlArray);
$index_update_count = count($indexArray);
$total_update_count = $sql_update_count + $index_update_count;
$this->__setUpdateProgress(0, $total_update_count);
$str_index_array = array();
foreach($indexArray as $toIndex) {
$strIndexArray[] = __('Indexing ') . implode($toIndex, '->');
$str_index_array[] = __('Indexing ') . implode($toIndex, '->');
}
$this->__setUpdateCmdMessages(array_merge($sqlArray, $strIndexArray));
$flagStop = false;
$errorCount = 0;
$this->__setUpdateCmdMessages(array_merge($sqlArray, $str_index_array));
$flag_stop = false;
$error_count = 0;
foreach ($sqlArray as $i => $sql) {
try {
$this->__setUpdateProgress($i, false);
// execute test before update. Exit if it fails
if (isset($this->advanced_updates_description[$command]['preUpdate'])) {
$funName = $this->advanced_updates_description[$command]['preUpdate'];
$function_name = $this->advanced_updates_description[$command]['preUpdate'];
try {
$this->{$funName}();
$this->{$function_name}();
} catch (Exception $e) {
$exitOnError = true;
$this->__setPreUpdateTestState(false);
@ -1240,10 +1247,10 @@ class AppModel extends Model
'email' => 'SYSTEM',
'action' => 'update_database',
'user_id' => 0,
'title' => 'Successfuly executed the SQL query for ' . $command,
'change' => 'The executed SQL query was: ' . $sql
'title' => __('Successfuly executed the SQL query for ') . $command,
'change' => __('The executed SQL query was: ') . $sql
));
$this->__setUpdateResMessages($i, 'Successfuly executed the SQL query for ' . $command);
$this->__setUpdateResMessages($i, __('Successfuly executed the SQL query for ') . $command);
} catch (Exception $e) {
$this->Log->create();
$this->Log->save(array(
@ -1253,19 +1260,19 @@ class AppModel extends Model
'email' => 'SYSTEM',
'action' => 'update_database',
'user_id' => 0,
'title' => 'Issues executing the SQL query for ' . $command,
'change' => 'The executed SQL query was: ' . $sql . PHP_EOL . ' The returned error is: ' . $e->getMessage()
'title' => __('Issues executing the SQL query for ') . $command,
'change' => __('The executed SQL query was: ') . $sql . PHP_EOL . __(' The returned error is: ') . $e->getMessage()
));
$this->__setUpdateResMessages($i, 'Issues executing the SQL query for ' . $command . '. The returned error is: ' . PHP_EOL . $e->getMessage());
$this->__setUpdateResMessages($i, __('Issues executing the SQL query for ') . $command . __('. The returned error is: ') . PHP_EOL . $e->getMessage());
$this->__setUpdateError($i);
$errorCount++;
$error_count++;
if ($exitOnError) {
$flagStop = true;
$flag_stop = true;
break;
}
}
}
if (!$flagStop) {
if (!$flag_stop) {
if (!empty($indexArray)) {
if ($clean) {
$this->cleanCacheFiles();
@ -1277,7 +1284,7 @@ class AppModel extends Model
} else {
$this->__addIndex($iA[0], $iA[1]);
}
$this->__setUpdateResMessages(count($sqlArray)+$i, 'Successfuly indexed ' . implode($iA, '->'));
$this->__setUpdateResMessages(count($sqlArray)+$i, __('Successfuly indexed ') . implode($iA, '->'));
}
}
$this->__setUpdateProgress(count($sqlArray)+count($indexArray), false);
@ -1289,7 +1296,7 @@ class AppModel extends Model
$liveSetting = 'MISP.live';
$this->Server->serverSettingsSaveValue($liveSetting, true);
}
if (!$flagStop && $errorCount == 0) {
if (!$flag_stop && $error_count == 0) {
$this->__postUpdate($command);
}
$this->__changeLockState(false);
@ -1393,43 +1400,6 @@ class AppModel extends Model
}
}
/*
* Given an array, dynamically add contextual fields. An historical example is given with the *_seen fields
public function addFieldsBasedOnUpdate(&$fieldsAtt, $context = null) {
if (is_null($context)) {
$alias = '';
} else if ($context === true) {
$alias = $this->alias;
} else {
$alias = $context;
}
if ($this->additionalFeatureEnabled('seenOnAttributeAndObject')) {
// DB have *_seen columns
$fs = (strlen($alias) > 0 ? $alias . '.' : '') . 'first_seen';
$ls = (strlen($alias) > 0 ? $alias . '.' : '') . 'last_seen';
array_push($fieldsAtt, $fs, $ls);
}
}
*/
/*
* Check whether a feature is enabled (based on an update) or not. An historical example is given with the *_seen update
public function additionalFeatureEnabled($featureName) {
if (!isset($this->AdminSetting)) {
$this->AdminSetting = ClassRegistry::init('AdminSetting');
}
$value = $this->AdminSetting->getSetting('seenOnAttributeAndObject');
if($value === false) {
return false;
} else if ($value === "0") {
return false;
} else if ($value === "1") {
return true;
} else {
return $value;
}
}
*/
public function checkMISPVersion()
{
App::uses('Folder', 'Utility');
@ -1542,9 +1512,9 @@ class AppModel extends Model
private function __setUpdateProgress($current, $total=false)
{
$updateProgress = $this->getUpdateProgress();
$updateProgress['cur'] = $current;
$updateProgress['current'] = $current;
if ($total !== false) {
$updateProgress['tot'] = $total;
$updateProgress['total'] = $total;
} else {
$now = new DateTime();
$updateProgress['time']['started'][$current] = $now->format('Y-m-d H:i:s');
@ -1569,11 +1539,11 @@ class AppModel extends Model
private function __resetUpdateProgress()
{
$updateProgress = array(
'cmd' => array(),
'res' => array(),
'commands' => array(),
'results' => array(),
'time' => array('started' => array(), 'elapsed' => array()),
'cur' => '',
'tot' => '',
'current' => '',
'total' => '',
'failed_num' => array()
);
$this->__saveUpdateProgress($updateProgress);
@ -1582,14 +1552,14 @@ class AppModel extends Model
private function __setUpdateCmdMessages($messages)
{
$updateProgress = $this->getUpdateProgress();
$updateProgress['cmd'] = $messages;
$updateProgress['commands'] = $messages;
$this->__saveUpdateProgress($updateProgress);
}
private function __setUpdateResMessages($index, $message)
{
$updateProgress = $this->getUpdateProgress();
$updateProgress['res'][$index] = $message;
$updateProgress['results'][$index] = $message;
$temp = new DateTime();
$diff = $temp->diff(new DateTime($updateProgress['time']['started'][$index]));
$updateProgress['time']['elapsed'][$index] = $diff->format('%H:%I:%S');
@ -1641,20 +1611,14 @@ class AppModel extends Model
public function isUpdateLocked()
{
$lockState = $this->getUpdateLockState();
$lockState = $lockState === false ? false : $lockState;
$lockState = $lockState === '' ? false : $lockState;
if ($lockState !== false) {
if ($lockState !== false && $lockState !== '') {
// if lock is old, still allows the update
// This can be useful if the update process crashes
$lockDate = (new DateTime($lockState))->format('U');
$now = (new DateTime())->format('U');
$diffSec = $now - $lockDate;
$diffSec = time() - intval($lockState);
if (Configure::read('MISP.updateTimeThreshold')) {
$updateWaitThreshold = intval(Configure::read('MISP.updateTimeThreshold'));
} else {
if (!isset($this->Server)) {
$this->Server = ClassRegistry::init('Server');
}
$this->Server = ClassRegistry::init('Server');
$updateWaitThreshold = intval($this->Server->serverSettings['MISP']['updateTimeThreshold']['value']);
}
if ($diffSec < $updateWaitThreshold) {

2
app/Model/Server.php
View File

@ -919,7 +919,7 @@ class Server extends AppModel
'updateTimeThreshold' => array(
'level' => 1,
'description' => __('Sets the minimum time before being able to re-trigger an update if the previous one failed. (safe guard to avoid starting the same update multiple time)'),
'value' => '600',
'value' => '7200',
'test' => 'testForNumeric',
'type' => 'numeric',
'null' => true

18
app/View/Servers/update_progress.ctp
View File

@ -1,8 +1,8 @@
<?php
if (!$isSiteAdmin) exit();
if ($updateProgress['tot'] !== 0 ) {
$percentageFail = floor(count($updateProgress['failed_num']) / $updateProgress['tot']*100);
$percentage = floor(($updateProgress['cur']) / $updateProgress['tot']*100);
if ($updateProgress['total'] !== 0 ) {
$percentageFail = floor(count($updateProgress['failed_num']) / $updateProgress['total']*100);
$percentage = floor(($updateProgress['current']) / $updateProgress['total']*100);
} else {
$percentage = 100;
$percentageFail = 0;
@ -14,7 +14,7 @@ if (isset($updateProgress['preTestSuccess']) && $updateProgress['preTestSuccess'
}
?>
<div style="width: 50%;margin: 0 auto;">
<?php if (count($updateProgress['cmd']) > 0): ?>
<?php if (count($updateProgress['commands']) > 0): ?>
<h2><?php echo(__('Database Update progress'));?></h2>
<div class="" style="max-width: 1000px;">
@ -39,14 +39,14 @@ if (isset($updateProgress['preTestSuccess']) && $updateProgress['preTestSuccess'
</tr>
</thead>
<tbody>
<?php foreach($updateProgress['cmd'] as $i => $cmd):
if (isset($updateProgress['res'][$i])) {
$res = $updateProgress['res'][$i];
<?php foreach($updateProgress['commands'] as $i => $cmd):
if (isset($updateProgress['results'][$i])) {
$res = $updateProgress['results'][$i];
} else {
$res = false;
}
$rowDone = $i < $updateProgress['cur'];
$rowCurrent = $i === $updateProgress['cur'];
$rowDone = $i < $updateProgress['current'];
$rowCurrent = $i === $updateProgress['current'];
$rowFail = in_array($i, $updateProgress['failed_num']);
$rowClass = '';
$rowIcon = '<i id="icon-' . $i . '" class="fa"></i>';

26
app/webroot/js/update_progress.js
View File

@ -31,19 +31,19 @@ $(document).ready(function() {
function update_state() {
$.getJSON(urlGetProgress, function(data) {
var tot = parseInt(data['tot']);
var cur = parseInt(data['cur']);
var total = parseInt(data['total']);
var current = parseInt(data['current']);
var failArray = data['failed_num'];
for (var i=0; i<tot; i++) {
for (var i=0; i<total; i++) {
var term = $('div[data-terminalid='+i+']')
toggleVisiblity(i, true, false);
if (i < cur) {
if (i < current) {
if (failArray.indexOf(String(i)) != -1) {
update_row_state(i, 2);
} else {
update_row_state(i, 0);
}
} else if (i == cur) {
} else if (i == current) {
if (failArray.indexOf(String(i)) != -1) {
update_row_state(i, 2);
toggleVisiblity(i, true, true);
@ -57,13 +57,13 @@ function update_state() {
}
}
update_messages(data);
if (tot > 0) {
var percFail = Math.round(failArray.length/tot*100);
var perc = Math.round((cur)/tot*100);
if (total > 0) {
var percFail = Math.round(failArray.length/total*100);
var perc = Math.round(current/total*100);
update_pb(perc, percFail);
}
if ((cur+1) >= tot || failArray.indexOf(cur) != -1) {
if ((current+1) >= total || failArray.indexOf(current) != -1) {
clearInterval(pooler);
$('.single-update-progress').hide();
}
@ -72,14 +72,14 @@ function update_state() {
function update_messages(messages) {
if (messages.cmd === undefined) {
if (messages.commands === undefined) {
return;
}
messages.cmd.forEach(function(msg, i) {
messages.commands.forEach(function(msg, i) {
var div = $('#termcmd-'+i);
create_spans_from_message(div, msg);
});
messages.res.forEach(function(msg, i) {
messages.results.forEach(function(msg, i) {
var div = $('#termres-'+i);
div.css('display', '');
create_spans_from_message(div, msg);
@ -164,7 +164,7 @@ function update_single_update_progress(i, data) {
var perc = parseInt(data['process_list']['PROGRESS']);
if (data['process_list']['MAX_STAGE'] == 0) { // if MAX_STAGE == 0, progress could not be determined
perc = 5;
if (data['failed_num'].indexOf(data['cur']) >= 0) { // do not animate if failed
if (data['failed_num'].indexOf(data['current']) >= 0) { // do not animate if failed
state.text('Failed');
perc = 0;
} else {