diff --git a/INSTALL/INSTALL.sh b/INSTALL/INSTALL.sh
index ff57aa35a..045a3eeb8 100755
--- a/INSTALL/INSTALL.sh
+++ b/INSTALL/INSTALL.sh
@@ -128,7 +128,7 @@ MISPvars () {
CAKE="$PATH_TO_MISP/app/Console/cake"
# sudo config to run $LUSER commands
- if [[ "$(groups |grep -o 'staff')" == "staff" ]]; then
+ if [[ "$(groups ${MISP_USER} |grep -o 'staff')" == "staff" ]]; then
SUDO_USER="sudo -H -u ${MISP_USER} -g staff"
else
SUDO_USER="sudo -H -u ${MISP_USER}"
@@ -352,6 +352,15 @@ checkID () {
sudo adduser $MISP_USER staff
sudo adduser $MISP_USER $WWW_USER
fi
+
+ # FIXME: the below SUDO_USER check is a duplicate from global variables, try to have just one check
+ # sudo config to run $LUSER commands
+ if [[ "$(groups ${MISP_USER} |grep -o 'staff')" == "staff" ]]; then
+ SUDO_USER="sudo -H -u ${MISP_USER} -g staff"
+ else
+ SUDO_USER="sudo -H -u ${MISP_USER}"
+ fi
+
}
# pre-install check to make sure what we will be installing on, is ready and not a half installed system
@@ -1084,6 +1093,9 @@ installCore () {
# install python-magic
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install python-magic
+ # install plyara
+ $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install plyara
+
# Install Crypt_GPG and Console_CommandLine
sudo pear install ${PATH_TO_MISP}/INSTALL/dependencies/Console_CommandLine/package.xml
sudo pear install ${PATH_TO_MISP}/INSTALL/dependencies/Crypt_GPG/package.xml
@@ -1342,26 +1354,22 @@ backgroundWorkers () {
# Main MISP Modules install function
mispmodules () {
- # FIXME: this is broken, ${PATH_TO_MISP} is litteral
-##sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local
cd /usr/local/src/
## TODO: checkUsrLocalSrc in main doc
debug "Cloning misp-modules"
$SUDO_USER git clone https://github.com/MISP/misp-modules.git
cd misp-modules
# some misp-modules dependencies
- sudo apt-get install libpq5 libjpeg-dev libfuzzy-dev -y
+ sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr libpoppler-cpp-dev imagemagick virtualenv libopencv-dev zbar-tools libzbar0 libzbar-dev libfuzzy-dev -y
# If you build an egg, the user you build it as need write permissions in the CWD
sudo chgrp $WWW_USER .
sudo chmod g+w .
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I -r REQUIREMENTS
sudo chgrp staff .
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I .
- sudo apt install ruby-pygments.rb -y
+ sudo apt install ruby-pygments.rb libopencv-dev zbar-tools -y
sudo gem install asciidoctor-pdf --pre
- # install additional dependencies for extended object generation and extraction
- $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install wand yara pathlib
# Start misp-modules as a service
sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/
sudo systemctl daemon-reload
@@ -1583,11 +1591,17 @@ viper () {
$SUDO_USER sed -i "s/^misp_key\ =/misp_key\ =\ $AUTH_KEY/g" ${VIPER_HOME}/viper.conf
# Reset admin password to: admin/Password1234
echo "Fixing admin.db with default password"
+ VIPER_COUNT=0
while [ "$(sudo sqlite3 ${VIPER_HOME}/admin.db 'UPDATE auth_user SET password="pbkdf2_sha256$100000$iXgEJh8hz7Cf$vfdDAwLX8tko1t0M1TLTtGlxERkNnltUnMhbv56wK/U="'; echo $?)" -ne "0" ]; do
# FIXME This might lead to a race condition, the while loop is sub-par
sudo chown $MISP_USER:$MISP_USER ${VIPER_HOME}/admin.db
echo "Updating viper-web admin password, giving process time to start-up, sleeping 5, 4, 3,…"
sleep 6
+ VIPER_COUNT=$[$VIPER_COUNT+1]
+ if [[ "$VIPER_COUNT" > '10' ]]; then
+ echo "Something is wrong with updating viper. Continuing without db update."
+ break
+ fi
done
# Add viper-web to rc.local to be started on boot
diff --git a/INSTALL/INSTALL.sh.sha1 b/INSTALL/INSTALL.sh.sha1
index 65f6db3de..36d0bb4b6 100644
--- a/INSTALL/INSTALL.sh.sha1
+++ b/INSTALL/INSTALL.sh.sha1
@@ -1 +1 @@
-931fae16c4462a0f8aab1e7428fe4c168480d33a INSTALL.sh
+eaf31965e51c8fc7a6202d7896aa0e9b0b3ecb3d INSTALL.sh
diff --git a/INSTALL/INSTALL.sh.sha256 b/INSTALL/INSTALL.sh.sha256
index e5ccefaf7..9facf4fc4 100644
--- a/INSTALL/INSTALL.sh.sha256
+++ b/INSTALL/INSTALL.sh.sha256
@@ -1 +1 @@
-01c7ad59ac9105c2fb8e8cd86e3794d71df0091e9d6f2110a302891561b2af65 INSTALL.sh
+4544798f992578aaffa28ba816591ba2dbf38c18824c4df0a901f13dd7ded714 INSTALL.sh
diff --git a/INSTALL/INSTALL.sh.sha384 b/INSTALL/INSTALL.sh.sha384
index baf64d457..1c4070ec2 100644
--- a/INSTALL/INSTALL.sh.sha384
+++ b/INSTALL/INSTALL.sh.sha384
@@ -1 +1 @@
-0f0f2e4005833dd563ec0a71e2fcf520da798c081c6c5df3d8f5d5684c09d9eb194097a721c6c7a0687c820f0f0a8dcd INSTALL.sh
+2930fa3ddeccc2753ad91ef4869dc43357093d24c33b8e01fa5c3218922ecf3855b7cb05bb5a60da1d4f9050ad1bbeef INSTALL.sh
diff --git a/INSTALL/INSTALL.sh.sha512 b/INSTALL/INSTALL.sh.sha512
index 56dd74ed2..69cb9e2ff 100644
--- a/INSTALL/INSTALL.sh.sha512
+++ b/INSTALL/INSTALL.sh.sha512
@@ -1 +1 @@
-0ea0ea8f1de9a03a85bf475732446388a096aac0c54e8ebd7203f5119731521ef30ca99cc998637037a8f50587fa4c0bcd0f3994249898ed164729f08bdf3c7d INSTALL.sh
+54ba106074b191d85102b86da534a7ea5327ac208d6f3551b262b6daee6e8a289f3fe97b323a7ada4242b0a5580d8600f5caf8d2e4244b07a57a54f54cd8f426 INSTALL.sh
diff --git a/app/Console/Command/AdminShell.php b/app/Console/Command/AdminShell.php
index 9dc666372..c87bd1f01 100644
--- a/app/Console/Command/AdminShell.php
+++ b/app/Console/Command/AdminShell.php
@@ -65,7 +65,7 @@ class AdminShell extends AppShell
public function restartWorker()
{
if (empty($this->args[0]) || !is_numeric($this->args[0])) {
- echo 'Usage: ' . APP . '/cake ' . 'Admin restartWorker [PID]';
+ echo 'Usage: ' . APP . '/cake ' . 'Admin restartWorker [PID]' . PHP_EOL;
}
$pid = $this->args[0];
$result = $this->Server->restartWorker($pid);
@@ -85,7 +85,7 @@ class AdminShell extends AppShell
public function killWorker()
{
if (empty($this->args[0]) || !is_numeric($this->args[0])) {
- echo 'Usage: ' . APP . '/cake ' . 'Admin killWorker [PID]';
+ echo 'Usage: ' . APP . '/cake ' . 'Admin killWorker [PID]' . PHP_EOL;
die();
}
$pid = $this->args[0];
@@ -101,7 +101,7 @@ class AdminShell extends AppShell
public function startWorker()
{
if (empty($this->args[0])) {
- echo 'Usage: ' . APP . '/cake ' . 'Admin startWorker [queue]';
+ echo 'Usage: ' . APP . '/cake ' . 'Admin startWorker [queue]' . PHP_EOL;
die();
}
$queue = $this->args[0];
@@ -143,9 +143,9 @@ class AdminShell extends AppShell
$force = $value;
$result = $this->Galaxy->update($force);
if ($result) {
- echo 'Galaxies updated';
+ echo 'Galaxies updated' . PHP_EOL;
} else {
- echo 'Could not update Galaxies';
+ echo 'Could not update Galaxies' . PHP_EOL;
}
}
@@ -153,34 +153,34 @@ class AdminShell extends AppShell
public function updateTaxonomies() {
$result = $this->Taxonomy->update();
if ($result) {
- echo 'Taxonomies updated';
+ echo 'Taxonomies updated' . PHP_EOL;
} else {
- echo 'Could not update Taxonomies';
+ echo 'Could not update Taxonomies' . PHP_EOL;
}
}
public function updateWarningLists() {
$result = $this->Galaxy->update();
if ($result) {
- echo 'Warning lists updated';
+ echo 'Warning lists updated' . PHP_EOL;
} else {
- echo 'Could not update warning lists';
+ echo 'Could not update warning lists' . PHP_EOL;
}
}
public function updateNoticeLists() {
$result = $this->Noticelist->update();
if ($result) {
- echo 'Notice lists updated';
+ echo 'Notice lists updated' . PHP_EOL;
} else {
- echo 'Could not update notice lists';
+ echo 'Could not update notice lists' . PHP_EOL;
}
}
# FIXME: Debug and make it work, fails to pass userId/orgId properly
public function updateObjectTemplates() {
if (empty($this->args[0])) {
- echo 'Usage: ' . APP . '/cake ' . 'Admin updateNoticeLists [user_id]';
+ echo 'Usage: ' . APP . '/cake ' . 'Admin updateNoticeLists [user_id]' . PHP_EOL;
} else {
$userId = $this->args[0];
$user = $this->User->find('first', array(
@@ -191,13 +191,13 @@ class AdminShell extends AppShell
'fields' => array('User.id', 'User.org_id')
));
if (empty($user)) {
- echo 'User not found';
+ echo 'User not found' . PHP_EOL;
} else {
$result = $this->ObjectTemplate->update($user, false,false);
if ($result) {
- echo 'Object templates updated';
+ echo 'Object templates updated' . PHP_EOL;
} else {
- echo 'Could not update object templates';
+ echo 'Could not update object templates' . PHP_EOL;
}
}
}
@@ -275,15 +275,15 @@ class AdminShell extends AppShell
if ($value === 'true') $value = 1;
$cli_user = array('id' => 0, 'email' => 'SYSTEM', 'Organisation' => array('name' => 'SYSTEM'));
if (empty($setting_name) || $value === null) {
- echo 'Invalid parameters. Usage: ' . APP . 'Console/cake Admin setSetting [setting_name] [setting_value]';
+ echo 'Invalid parameters. Usage: ' . APP . 'Console/cake Admin setSetting [setting_name] [setting_value]' . PHP_EOL;
} else {
$setting = $this->Server->getSettingData($setting_name);
if (empty($setting)) {
- echo 'Invalid setting. Please make sure that the setting that you are attempting to change exists.';
+ echo 'Invalid setting "' . $setting_name . '". Please make sure that the setting that you are attempting to change exists and if a module parameter, the modules are running.' . PHP_EOL;
}
$result = $this->Server->serverSettingsEditValue($cli_user, $setting, $value);
if ($result === true) {
- echo 'Setting changed.';
+ echo 'Setting "' . $setting_name . '" changed to ' . $value . PHP_EOL;
} else {
echo $result;
}
@@ -308,11 +308,48 @@ class AdminShell extends AppShell
}
public function updateDatabase() {
- echo 'Executing all updates to bring the database up to date with the current version.' . PHP_EOL;
- $this->Server->runUpdates(true);
- echo 'All updates completed.' . PHP_EOL;
+ $whoami = exec('whoami');
+ if ($whoami === 'httpd' || $whoami === 'www-data') {
+ echo 'Executing all updates to bring the database up to date with the current version.' . PHP_EOL;
+ $this->Server->runUpdates(true);
+ echo 'All updates completed.' . PHP_EOL;
+ } else {
+ die('This OS user is not allowed to run this command.'. PHP_EOL. 'Run it under `www-data` or `httpd`.' . PHP_EOL);
+ }
}
+ public function updateApp() {
+ $whoami = exec('whoami');
+ if ($whoami === 'httpd' || $whoami === 'www-data') {
+ $command = $this->args[0];
+ if (!empty($this->args[1])) {
+ $processId = $this->args[1];
+ $job = $this->Job->read(null, $processId);
+ } else { // create worker
+ $this->Job->create();
+ $job_data = array(
+ 'worker' => 'prio',
+ 'job_type' => 'update_app',
+ 'job_input' => 'command: ' . $command,
+ 'status' => 0,
+ 'retries' => 0,
+ 'org_id' => '',
+ 'org' => '',
+ 'message' => 'Updating.',
+ );
+ $this->Job->save($job_data);
+ $job = $this->Job->read(null, $this->Job->id);
+ }
+ $result = $this->Server->updateDatabase($command, false);
+ $job['Job']['progress'] = 100;
+ $job['Job']['message'] = 'Update done';
+ $this->Job->save($job);
+ } else {
+ die('This OS user is not allowed to run this command.'. PHP_EOL. 'Run it under `www-data` or `httpd`.' . PHP_EOL);
+ }
+ }
+
+
public function getAuthkey() {
if (empty($this->args[0])) {
echo 'Invalid parameters. Usage: ' . APP . 'Console/cake Admin getAuthkey [user_email]' . PHP_EOL;
@@ -379,7 +416,7 @@ class AdminShell extends AppShell
public function change_authkey()
{
if (empty($this->args[0])) {
- echo 'MISP apikey command line tool.' . PHP_EOL . 'To assign a new random API key for a user: ' . APP . 'Console/cake Password [email]' . PHP_EOL . 'To assign a fixed API key: ' . APP . 'Console/cake Password [email] [authkey]';
+ echo 'MISP apikey command line tool.' . PHP_EOL . 'To assign a new random API key for a user: ' . APP . 'Console/cake Password [email]' . PHP_EOL . 'To assign a fixed API key: ' . APP . 'Console/cake Password [email] [authkey]' . PHP_EOL;
die();
}
if (!empty($this->args[1])) {
@@ -393,7 +430,7 @@ class AdminShell extends AppShell
'fields' => array('User.id', 'User.email', 'User.authkey')
));
if (empty($user)) {
- echo 'Invalid e-mail, user not found.';
+ echo 'Invalid e-mail, user not found.' . PHP_EOL;
die();
}
$user['User']['authkey'] = $authKey;
@@ -404,7 +441,7 @@ class AdminShell extends AppShell
}
echo 'Updated, new key:' . PHP_EOL . $authKey . PHP_EOL;
}
-
+
public function getOptionParser() {
$parser = parent::getOptionParser();
$parser->addSubcommand('updateJSON', array(
diff --git a/app/Console/Command/ServerShell.php b/app/Console/Command/ServerShell.php
index 67cf048dc..015d5e2b8 100644
--- a/app/Console/Command/ServerShell.php
+++ b/app/Console/Command/ServerShell.php
@@ -446,4 +446,5 @@ class ServerShell extends AppShell
$this->Task->id = $task['Task']['id'];
$this->Task->saveField('message', count($servers) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '.');
}
+
}
diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
index 66dd72d1e..0f3147f4a 100755
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@@ -372,7 +372,7 @@ class AppController extends Controller
$this->Auth->logout();
throw new MethodNotAllowedException($message);//todo this should pb be removed?
} else {
- $this->Flash->error('Warning: MISP is currently disabled for all users. Enable it in Server Settings (Administration -> Server Settings -> MISP tab -> live)', array('clear' => 1));
+ $this->Flash->error('Warning: MISP is currently disabled for all users. Enable it in Server Settings (Administration -> Server Settings -> MISP tab -> live). An update might also be in progress, you can see the progress in ' , array('params' => array('url' => $baseurl . '/servers/advancedUpdate/', 'urlName' => 'Advanced Update'), 'clear' => 1));
}
}
@@ -835,7 +835,11 @@ class AppController extends Controller
}
$this->Server->updateDatabase($command);
$this->Flash->success('Done.');
- $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
+ if ($liveOff) {
+ $this->redirect(array('controller' => 'servers', 'action' => 'updateProgress'));
+ } else {
+ $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
+ }
}
public function upgrade2324()
diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php
index b6157d995..af7e048f5 100644
--- a/app/Controller/AttributesController.php
+++ b/app/Controller/AttributesController.php
@@ -2246,7 +2246,7 @@ class AttributesController extends AppController
$data = $this->request->data;
}
if (empty($data)) {
- throw new BadRequestException(__('Either specify the search terms in the url, or POST a json array / xml (with the root element being "request" and specify the correct headers based on content type.'));
+ throw new BadRequestException(__('Either specify the search terms in the url, or POST a json array / xml (with the root element being "request" and specify the correct headers based on content type).'));
}
$paramArray = array('eventId', 'tags', 'from', 'to', 'policy', 'walled_garden', 'ns', 'email', 'serial', 'refresh', 'retry', 'expiry', 'minimum_ttl', 'ttl', 'enforceWarninglist', 'ns_alt');
foreach ($paramArray as $p) {
diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php
index e492ad3b4..b7094bad4 100644
--- a/app/Controller/Component/ACLComponent.php
+++ b/app/Controller/Component/ACLComponent.php
@@ -340,6 +340,7 @@ class ACLComponent extends Component
'getSubmoduleQuickUpdateForm' => array('perm_site_admin'),
'getVersion' => array('*'),
'index' => array('OR' => array('perm_sync', 'perm_admin')),
+ 'ondemandAction' => array(),
'postTest' => array('perm_sync'),
'previewEvent' => array(),
'previewIndex' => array(),
@@ -359,6 +360,7 @@ class ACLComponent extends Component
'testConnection' => array('perm_sync'),
'update' => array(),
'updateJSON' => array(),
+ 'updateProgress' => array(),
'updateSubmodule' => array(),
'uploadFile' => array(),
'clearWorkerQueue' => array()
diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php
index 2b5adf340..d5aa571c9 100644
--- a/app/Controller/EventsController.php
+++ b/app/Controller/EventsController.php
@@ -2621,7 +2621,7 @@ class EventsController extends AppController
{
$filesize_units = array('B', 'KB', 'MB', 'GB', 'TB');
if ($this->_isSiteAdmin()) {
- $this->Flash->info('Warning, you are logged in as a site admin, any export that you generate will contain the FULL UNRESTRICTED data-set. If you would like to generate an export for your own organisation, please log in with a different user.');
+ $this->Flash->info(__('Warning, you are logged in as a site admin, any export that you generate will contain the FULL UNRESTRICTED data-set. If you would like to generate an export for your own organisation, please log in with a different user.'));
}
// Check if the background jobs are enabled - if not, fall back to old export page.
if (Configure::read('MISP.background_jobs') && !Configure::read('MISP.disable_cached_exports')) {
@@ -3954,7 +3954,7 @@ class EventsController extends AppController
}
if ($this->request->is('post')) {
if (empty($this->request->data)) {
- throw new BadRequestException(__('Either specify the search terms in the url, or POST an xml (with the root element being "request".'));
+ throw new BadRequestException(__('Either specify the search terms in the url, or POST an xml (with the root element being "request").'));
} else {
$data = $this->request->data;
}
diff --git a/app/Controller/FeedsController.php b/app/Controller/FeedsController.php
index 0a39c85db..1242d0284 100644
--- a/app/Controller/FeedsController.php
+++ b/app/Controller/FeedsController.php
@@ -433,7 +433,7 @@ class FeedsController extends AppController
$this->redirect(array('action' => 'index'));
}
}
- $message = __('Fetching the feed has successfuly completed.');
+ $message = __('Fetching the feed has successfully completed.');
if ($this->Feed->data['Feed']['source_format'] == 'misp') {
if (isset($result['add'])) {
$message .= ' Downloaded ' . count($result['add']) . ' new event(s).';
diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php
index 9c0398dd2..4f7c33791 100644
--- a/app/Controller/ServersController.php
+++ b/app/Controller/ServersController.php
@@ -1545,6 +1545,69 @@ class ServersController extends AppController
}
}
+ public function ondemandAction()
+ {
+ if (!$this->_isSiteAdmin()) {
+ throw new MethodNotAllowedException('You are not authorised to do that.');
+ }
+ $this->AdminSetting = ClassRegistry::init('AdminSetting');
+ $actions = $this->Server->actions_description;
+ $default_fields = array(
+ 'title' => '',
+ 'description' => '',
+ 'liveOff' => false,
+ 'recommendBackup' => false,
+ 'exitOnError' => false,
+ 'requirements' => '',
+ 'url' => '/'
+ );
+ foreach($actions as $id => $action) {
+ foreach($default_fields as $field => $value) {
+ if (!isset($action[$field])) {
+ $actions[$id][$field] = $value;
+ }
+ }
+ $done = $this->AdminSetting->getSetting($id);
+ $actions[$id]['done'] = ($done == '1');
+ }
+ $this->set('actions', $actions);
+ $this->set('updateLocked', $this->Server->isUpdateLocked());
+ }
+
+ public function updateProgress()
+ {
+ if (!$this->_isSiteAdmin()) {
+ throw new MethodNotAllowedException('You are not authorised to do that.');
+ }
+ $update_progress = $this->Server->getUpdateProgress();
+ $current_index = $update_progress['current'];
+ $current_command = !isset($update_progress['commands'][$current_index]) ? '' : $update_progress['commands'][$current_index];
+ $lookup_string = preg_replace('/\s{2,}/', '', substr($current_command, 0, -1));
+ $sql_info = $this->Server->query("SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;");
+ if (empty($sql_info)) {
+ $update_progress['process_list'] = array();
+ } else {
+ // retreive current update process
+ foreach($sql_info as $row) {
+ if (preg_replace('/\s{2,}/', '', $row['PROCESSLIST']['INFO']) == $lookup_string) {
+ $sql_info = $row['PROCESSLIST'];
+ break;
+ }
+ }
+ $update_progress['process_list'] = array();
+ $update_progress['process_list']['STATE'] = isset($sql_info['STATE']) ? $sql_info['STATE'] : '';
+ $update_progress['process_list']['PROGRESS'] = isset($sql_info['PROGRESS']) ? $sql_info['PROGRESS'] : 0;
+ $update_progress['process_list']['STAGE'] = isset($sql_info['STAGE']) ? $sql_info['STAGE'] : 0;
+ $update_progress['process_list']['MAX_STAGE'] = isset($sql_info['MAX_STAGE']) ? $sql_info['MAX_STAGE'] : 0;
+ }
+ if ($this->request->is('ajax')) {
+ return $this->RestResponse->viewData(h($update_progress), $this->response->type());
+ } else {
+ $this->set('updateProgress', $update_progress);
+ }
+ }
+
+
public function getSubmoduleQuickUpdateForm($submodule_path=false) {
$this->set('submodule', base64_decode($submodule_path));
$this->render('ajax/submodule_quick_update_form');
diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php
index 4fcadca66..d44bbf8be 100644
--- a/app/Model/AppModel.php
+++ b/app/Model/AppModel.php
@@ -78,6 +78,21 @@ class AppModel extends Model
33 => false, 34 => false
);
+ public $advanced_updates_description = array(
+ );
+ public $actions_description = array(
+ 'verifyGnuPGkeys' => array(
+ 'title' => 'Verify GnuPG keys',
+ 'description' => "Run a full validation of all GnuPG keys within this instance's userbase. The script will try to identify possible issues with each key and report back on the results.",
+ 'url' => '/users/verifyGPG/'
+ ),
+ 'databaseCleanupScripts' => array(
+ 'title' => 'Database Cleanup Scripts',
+ 'description' => 'If you run into an issue with an infinite upgrade loop (when upgrading from version ~2.4.50) that ends up filling your database with upgrade script log messages, run the following script.',
+ 'url' => '/logs/pruneUpdateLogs/'
+ )
+ );
+
public function afterSave($created, $options = array())
{
if ($created) {
@@ -209,8 +224,46 @@ class AppModel extends Model
}
// SQL scripts for updates
- public function updateDatabase($command)
+ public function updateDatabase($command, $useWorker=true)
{
+ // Exit if updates are locked
+ if ($this->isUpdateLocked()) {
+ return false;
+ }
+ $this->__resetUpdateProgress();
+ // restart this function by a worker
+ if ($useWorker && Configure::read('MISP.background_jobs')) {
+ $job = ClassRegistry::init('Job');
+ $job->create();
+ $data = array(
+ 'worker' => 'prio',
+ 'job_type' => 'update_app',
+ 'job_input' => 'command: ' . $command,
+ 'status' => 0,
+ 'retries' => 0,
+ 'org_id' => '',
+ 'org' => '',
+ 'message' => 'Updating.',
+ );
+ $job->save($data);
+ $jobId = $job->id;
+ $process_id = CakeResque::enqueue(
+ 'prio',
+ 'AdminShell',
+ array('updateApp', $command, $jobId),
+ true
+ );
+ $job->saveField('process_id', $process_id);
+ return true;
+ }
+
+ $liveOff = false;
+ $exitOnError = false;
+ if (isset($advanced_updates_description[$command])) {
+ $liveOff = isset($advanced_updates_description[$command]['liveOff']) ? $advanced_updates_description[$command]['liveOff'] : $liveOff;
+ $exitOnError = isset($advanced_updates_description[$command]['exitOnError']) ? $advanced_updates_description[$command]['exitOnError'] : $exitOnError;
+ }
+
$dataSourceConfig = ConnectionManager::getDataSource('default')->config;
$dataSource = $dataSourceConfig['datasource'];
$sqlArray = array();
@@ -1150,52 +1203,123 @@ class AppModel extends Model
return false;
break;
}
- foreach ($sqlArray as $sql) {
+
+ $now = new DateTime();
+ $this->__changeLockState(time());
+ // switch MISP instance live to false
+ if ($liveOff) {
+ $this->Server = Classregistry::init('Server');
+ $liveSetting = 'MISP.live';
+ $this->Server->serverSettingsSaveValue($liveSetting, false);
+ }
+ $sql_update_count = count($sqlArray);
+ $index_update_count = count($indexArray);
+ $total_update_count = $sql_update_count + $index_update_count;
+ $this->__setUpdateProgress(0, $total_update_count);
+ $str_index_array = array();
+ foreach($indexArray as $toIndex) {
+ $str_index_array[] = __('Indexing ') . implode($toIndex, '->');
+ }
+ $this->__setUpdateCmdMessages(array_merge($sqlArray, $str_index_array));
+ $flag_stop = false;
+ $error_count = 0;
+
+ // execute test before update. Exit if it fails
+ if (isset($this->advanced_updates_description[$command]['preUpdate'])) {
+ $function_name = $this->advanced_updates_description[$command]['preUpdate'];
try {
- $this->query($sql);
- $this->Log->create();
- $this->Log->save(array(
- 'org' => 'SYSTEM',
- 'model' => 'Server',
- 'model_id' => 0,
- 'email' => 'SYSTEM',
- 'action' => 'update_database',
- 'user_id' => 0,
- 'title' => 'Successfuly executed the SQL query for ' . $command,
- 'change' => 'The executed SQL query was: ' . $sql
- ));
+ $this->{$function_name}();
} catch (Exception $e) {
- $this->Log->create();
- $this->Log->save(array(
- 'org' => 'SYSTEM',
- 'model' => 'Server',
- 'model_id' => 0,
- 'email' => 'SYSTEM',
- 'action' => 'update_database',
- 'user_id' => 0,
- 'title' => 'Issues executing the SQL query for ' . $command,
- 'change' => 'The executed SQL query was: ' . $sql . PHP_EOL . ' The returned error is: ' . $e->getMessage()
- ));
+ $this->__setPreUpdateTestState(false);
+ $this->__setUpdateProgress(0, false);
+ $this->__setUpdateResMessages(0, __('Issues executing the pre-update test `') . $function_name . __('`. The returned error is: ') . PHP_EOL . $e->getMessage());
+ $this->__setUpdateError(0);
+ $error_count++;
+ $exitOnError = true;
+ $flag_stop = true;
}
}
- if (!empty($indexArray)) {
- if ($clean) {
- $this->cleanCacheFiles();
- }
- foreach ($indexArray as $iA) {
- if (isset($iA[2])) {
- $this->__addIndex($iA[0], $iA[1], $iA[2]);
- } else {
- $this->__addIndex($iA[0], $iA[1]);
+
+ if (!$flag_stop) {
+ $this->__setPreUpdateTestState(true);
+ foreach ($sqlArray as $i => $sql) {
+ try {
+ $this->__setUpdateProgress($i, false);
+ $this->query($sql);
+ $this->Log->create();
+ $this->Log->save(array(
+ 'org' => 'SYSTEM',
+ 'model' => 'Server',
+ 'model_id' => 0,
+ 'email' => 'SYSTEM',
+ 'action' => 'update_database',
+ 'user_id' => 0,
+ 'title' => __('Successfuly executed the SQL query for ') . $command,
+ 'change' => __('The executed SQL query was: ') . $sql
+ ));
+ $this->__setUpdateResMessages($i, __('Successfuly executed the SQL query for ') . $command);
+ } catch (Exception $e) {
+ $this->Log->create();
+ $this->Log->save(array(
+ 'org' => 'SYSTEM',
+ 'model' => 'Server',
+ 'model_id' => 0,
+ 'email' => 'SYSTEM',
+ 'action' => 'update_database',
+ 'user_id' => 0,
+ 'title' => __('Issues executing the SQL query for ') . $command,
+ 'change' => __('The executed SQL query was: ') . $sql . PHP_EOL . __(' The returned error is: ') . $e->getMessage()
+ ));
+ $this->__setUpdateResMessages($i, __('Issues executing the SQL query for ') . $command . __('. The returned error is: ') . PHP_EOL . $e->getMessage());
+ $this->__setUpdateError($i);
+ $error_count++;
+ if ($exitOnError) {
+ $flag_stop = true;
+ break;
+ }
}
}
}
+ if (!$flag_stop) {
+ if (!empty($indexArray)) {
+ if ($clean) {
+ $this->cleanCacheFiles();
+ }
+ foreach ($indexArray as $i => $iA) {
+ $this->__setUpdateProgress(count($sqlArray)+$i, false);
+ if (isset($iA[2])) {
+ $this->__addIndex($iA[0], $iA[1], $iA[2]);
+ } else {
+ $this->__addIndex($iA[0], $iA[1]);
+ }
+ $this->__setUpdateResMessages(count($sqlArray)+$i, __('Successfuly indexed ') . implode($iA, '->'));
+ }
+ }
+ $this->__setUpdateProgress(count($sqlArray)+count($indexArray), false);
+ }
if ($clean) {
$this->cleanCacheFiles();
}
+ if ($liveOff) {
+ $liveSetting = 'MISP.live';
+ $this->Server->serverSettingsSaveValue($liveSetting, true);
+ }
+ if (!$flag_stop && $error_count == 0) {
+ $this->__postUpdate($command);
+ }
+ $this->__changeLockState(false);
return true;
}
+ // check whether the adminSetting should be updated after the update
+ private function __postUpdate($command) {
+ if (isset($this->advanced_updates_description[$command]['record'])) {
+ if($this->advanced_updates_description[$command]['record']) {
+ $this->AdminSetting->changeSetting($command, 1);
+ }
+ }
+ }
+
private function __dropIndex($table, $field)
{
$dataSourceConfig = ConnectionManager::getDataSource('default')->config;
@@ -1390,6 +1514,126 @@ class AppModel extends Model
if ($requiresLogout) {
$this->updateDatabase('destroyAllSessions');
}
+ return true;
+ }
+
+ private function __setUpdateProgress($current, $total=false)
+ {
+ $updateProgress = $this->getUpdateProgress();
+ $updateProgress['current'] = $current;
+ if ($total !== false) {
+ $updateProgress['total'] = $total;
+ } else {
+ $now = new DateTime();
+ $updateProgress['time']['started'][$current] = $now->format('Y-m-d H:i:s');
+ }
+ $this->__saveUpdateProgress($updateProgress);
+ }
+
+ private function __setPreUpdateTestState($state)
+ {
+ $updateProgress = $this->getUpdateProgress();
+ $updateProgress['preTestSuccess'] = $state;
+ $this->__saveUpdateProgress($updateProgress);
+ }
+
+ private function __setUpdateError($index)
+ {
+ $updateProgress = $this->getUpdateProgress();
+ $updateProgress['failed_num'][] = $index;
+ $this->__saveUpdateProgress($updateProgress);
+ }
+
+ private function __resetUpdateProgress()
+ {
+ $updateProgress = array(
+ 'commands' => array(),
+ 'results' => array(),
+ 'time' => array('started' => array(), 'elapsed' => array()),
+ 'current' => '',
+ 'total' => '',
+ 'failed_num' => array()
+ );
+ $this->__saveUpdateProgress($updateProgress);
+ }
+
+ private function __setUpdateCmdMessages($messages)
+ {
+ $updateProgress = $this->getUpdateProgress();
+ $updateProgress['commands'] = $messages;
+ $this->__saveUpdateProgress($updateProgress);
+ }
+
+ private function __setUpdateResMessages($index, $message)
+ {
+ $updateProgress = $this->getUpdateProgress();
+ $updateProgress['results'][$index] = $message;
+ $temp = new DateTime();
+ $diff = $temp->diff(new DateTime($updateProgress['time']['started'][$index]));
+ $updateProgress['time']['elapsed'][$index] = $diff->format('%H:%I:%S');
+ $this->__saveUpdateProgress($updateProgress);
+ }
+
+ public function getUpdateProgress()
+ {
+ if (!isset($this->AdminSetting)) {
+ $this->AdminSetting = ClassRegistry::init('AdminSetting');
+ }
+ $updateProgress = $this->AdminSetting->getSetting('update_progress');
+ if ($updateProgress !== false) {
+ $updateProgress = json_decode($updateProgress, true);
+ } else {
+ $this->__resetUpdateProgress();
+ $updateProgress = $this->AdminSetting->getSetting('update_progress');
+ $updateProgress = json_decode($updateProgress, true);
+ }
+ foreach($updateProgress as $setting => $value) {
+ if (!is_array($value)) {
+ $value = $value !== false && $value !== '' ? intval($value) : 0;
+ }
+ $updateProgress[$setting] = $value;
+ }
+ return $updateProgress;
+ }
+
+ private function __saveUpdateProgress($updateProgress)
+ {
+ $data = json_encode($updateProgress);
+ $this->AdminSetting->changeSetting('update_progress', $data);
+ }
+
+ private function __changeLockState($locked)
+ {
+ $this->AdminSetting->changeSetting('update_locked', $locked);
+ }
+
+ private function getUpdateLockState()
+ {
+ if (!isset($this->AdminSetting)) {
+ $this->AdminSetting = ClassRegistry::init('AdminSetting');
+ }
+ $locked = $this->AdminSetting->getSetting('update_locked');
+ return is_null($locked) ? false : $locked;
+ }
+
+ public function isUpdateLocked()
+ {
+ $lockState = $this->getUpdateLockState();
+ if ($lockState !== false && $lockState !== '') {
+ // if lock is old, still allows the update
+ // This can be useful if the update process crashes
+ $diffSec = time() - intval($lockState);
+ if (Configure::read('MISP.updateTimeThreshold')) {
+ $updateWaitThreshold = intval(Configure::read('MISP.updateTimeThreshold'));
+ } else {
+ $this->Server = ClassRegistry::init('Server');
+ $updateWaitThreshold = intval($this->Server->serverSettings['MISP']['updateTimeThreshold']['value']);
+ }
+ if ($diffSec < $updateWaitThreshold) {
+ return true;
+ }
+ }
+ return false;
}
private function __queueCleanDB()
diff --git a/app/Model/Event.php b/app/Model/Event.php
index 35476159b..77bb888aa 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -363,6 +363,118 @@ class Event extends AppModel
)
);
+ public function __construct($id = false, $table = null, $ds = null)
+ {
+ parent::__construct($id, $table, $ds);
+
+ $this->export_types = array(
+ 'json' => array(
+ 'extension' => '.json',
+ 'type' => 'JSON',
+ 'scope' => 'Event',
+ 'requiresPublished' => 0,
+ 'params' => array('includeAttachments' => 1, 'ignore' => 1, 'returnFormat' => 'json'),
+ 'description' => __('Click this to download all events and attributes that you have access to in MISP JSON format.'),
+ ),
+ 'xml' => array(
+ 'extension' => '.xml',
+ 'type' => 'XML',
+ 'scope' => 'Event',
+ 'params' => array('includeAttachments' => 1, 'ignore' => 1, 'returnFormat' => 'xml'),
+ 'requiresPublished' => 0,
+ 'description' => __('Click this to download all events and attributes that you have access to in MISP XML format.'),
+ ),
+ 'csv_sig' => array(
+ 'extension' => '.csv',
+ 'type' => 'CSV_Sig',
+ 'scope' => 'Event',
+ 'requiresPublished' => 1,
+ 'params' => array('published' => 1, 'to_ids' => 1, 'returnFormat' => 'csv'),
+ 'description' => __('Click this to download all attributes that are indicators and that you have access to (except file attachments) in CSV format.'),
+ ),
+ 'csv_all' => array(
+ 'extension' => '.csv',
+ 'type' => 'CSV_All',
+ 'scope' => 'Event',
+ 'requiresPublished' => 0,
+ 'params' => array('ignore' => 1, 'returnFormat' => 'csv'),
+ 'description' => __('Click this to download all attributes that you have access to (except file attachments) in CSV format.'),
+ ),
+ 'suricata' => array(
+ 'extension' => '.rules',
+ 'type' => 'Suricata',
+ 'scope' => 'Attribute',
+ 'requiresPublished' => 1,
+ 'params' => array('returnFormat' => 'suricata'),
+ 'description' => __('Click this to download all network related attributes that you have access to under the Suricata rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a whitelist containing host, domain name and IP numbers to exclude from the NIDS export.'),
+ ),
+ 'snort' => array(
+ 'extension' => '.rules',
+ 'type' => 'Snort',
+ 'scope' => 'Attribute',
+ 'requiresPublished' => 1,
+ 'params' => array('returnFormat' => 'snort'),
+ 'description' => __('Click this to download all network related attributes that you have access to under the Snort rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a whitelist containing host, domain name and IP numbers to exclude from the NIDS export.'),
+ ),
+ 'bro' => array(
+ 'extension' => '.intel',
+ 'type' => 'Bro',
+ 'scope' => 'Attribute',
+ 'requiresPublished' => 1,
+ 'params' => array('returnFormat' => 'bro'),
+ 'description' => __('Click this to download all network related attributes that you have access to under the Bro rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a whitelist containing host, domain name and IP numbers to exclude from the NIDS export.'),
+ ),
+ 'stix' => array(
+ 'extension' => '.xml',
+ 'type' => 'STIX',
+ 'scope' => 'Event',
+ 'requiresPublished' => 1,
+ 'params' => array('returnFormat' => 'stix', 'includeAttachments' => 1),
+ 'description' => __('Click this to download an a STIX document containing the STIX version of all events and attributes that you have access to.')
+ ),
+ 'stix2' => array(
+ 'extension' => '.json',
+ 'type' => 'STIX2',
+ 'scope' => 'Event',
+ 'requiresPublished' => 1,
+ 'params' => array('returnFormat' => 'stix2', 'includeAttachments' => 1),
+ 'description' => __('Click this to download an a STIX2 document containing the STIX2 version of all events and attributes that you have access to.')
+ ),
+ 'rpz' => array(
+ 'extension' => '.txt',
+ 'type' => 'RPZ',
+ 'scope' => 'Attribute',
+ 'requiresPublished' => 1,
+ 'params' => array('returnFormat' => 'rpz'),
+ 'description' => __('Click this to download an RPZ Zone file generated from all ip-src/ip-dst, hostname, domain attributes. This can be useful for DNS level firewalling. Only published events and attributes marked as IDS Signature are exported.')
+ ),
+ 'text' => array(
+ 'extension' => '.txt',
+ 'type' => 'TEXT',
+ 'scope' => 'Attribute',
+ 'requiresPublished' => 1,
+ 'params' => array('returnFormat' => 'text', 'includeAttachments' => 1),
+ 'description' => __('Click on one of the buttons below to download all the attributes with the matching type. This list can be used to feed forensic software when searching for susipicious files. Only published events and attributes marked as IDS Signature are exported.')
+ ),
+ 'yara' => array(
+ 'extension' => '.yara',
+ 'type' => 'Yara',
+ 'scope' => 'Event',
+ 'requiresPublished' => 1,
+ 'params' => array('returnFormat' => 'yara'),
+ 'description' => __('Click this to download Yara rules generated from all relevant attributes.')
+ ),
+ 'yara-json' => array(
+ 'extension' => '.json',
+ 'type' => 'Yara',
+ 'scope' => 'Event',
+ 'requiresPublished' => 1,
+ 'params' => array('returnFormat' => 'yara-json'),
+ 'description' => __('Click this to download Yara rules generated from all relevant attributes. Rules are returned in a JSON format with information about origin (generated or parsed) and validity.')
+ ),
+ );
+ }
+
public function beforeDelete($cascade = true)
{
// blacklist the event UUID if the feature is enabled
diff --git a/app/Model/Server.php b/app/Model/Server.php
index 25e83b7e3..f6b4f0de9 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -915,7 +915,15 @@ class Server extends AppModel
'test' => 'testBool',
'type' => 'boolean',
'null' => true
- )
+ ),
+ 'updateTimeThreshold' => array(
+ 'level' => 1,
+ 'description' => __('Sets the minimum time before being able to re-trigger an update if the previous one failed. (safe guard to avoid starting the same update multiple time)'),
+ 'value' => '7200',
+ 'test' => 'testForNumeric',
+ 'type' => 'numeric',
+ 'null' => true
+ )
),
'GnuPG' => array(
'branch' => 1,
diff --git a/app/View/Elements/Events/View/row_attribute.ctp b/app/View/Elements/Events/View/row_attribute.ctp
index f7ef3cd30..5343fcd90 100644
--- a/app/View/Elements/Events/View/row_attribute.ctp
+++ b/app/View/Elements/Events/View/row_attribute.ctp
@@ -193,7 +193,8 @@
-
diff --git a/app/View/Elements/Flash/error.ctp b/app/View/Elements/Flash/error.ctp
index 823cab6c5..50b3e7659 100644
--- a/app/View/Elements/Flash/error.ctp
+++ b/app/View/Elements/Flash/error.ctp
@@ -6,6 +6,13 @@
$message = str_replace('$flashErrorMessage', 'here ', $message);
}
echo $message;
+ if (isset($params['url'])) {
+ if (isset($params['urlName'])) {
+ echo '' . h($params['urlName']) . ' ';
+ } else {
+ echo '' . h($params['url']) . ' ';
+ }
+ }
if ($this->Session->read('flashErrorMessage')) {
echo sprintf('%s
', $this->element('flashErrorMessage', array('message' => $this->Session->read('flashErrorMessage'))));
}
diff --git a/app/View/Elements/Objects/object_similarities.ctp b/app/View/Elements/Objects/object_similarities.ctp
index 547673bdb..e06e3e399 100644
--- a/app/View/Elements/Objects/object_similarities.ctp
+++ b/app/View/Elements/Objects/object_similarities.ctp
@@ -78,7 +78,7 @@ if (!isset($simple_flattened_attribute_noval) || !isset($simple_flattened_attrib
$btn_style = 'btn-danger';
$temp_text = __('Can\'t merge due to template version');
} else {
- $temp_text = __('Merge');
+ $temp_text = __('Review merge');
$btn_style = 'btn-success';
}
?>
@@ -156,20 +156,20 @@ if (!isset($simple_flattened_attribute_noval) || !isset($simple_flattened_attrib
&& isset($multiple_attribute_allowed[$attribute['object_relation'] . ':' . $attribute['type']])
) { // Multiple allowed
$classname = 'warning';
- $title = __('This attribute is also contained by the revised object. However, as multiple instanciation is allowed by the template, the two attributes will be keept.');
+ $title = __('This attribute is also contained in the revised object. However, as multiple instantiations are allowed by the template, both attributes will be kept.');
$to_highlight = $simple_flattened_similar_attribute_noval;
} else if (
isset($simple_flattened_attribute_noval[$simple_flattened_similar_attribute_noval])
&& !isset($simple_flattened_attribute[$simple_flattened_similar_attribute])
) { // Not overridable attribute
$classname = 'error';
- $title = __('This attribute is conflicting with the one in the revised object, manual merge will be required.');
+ $title = __('This attribute is conflicting with the one in the revised object. Manual merge will be required.');
$to_highlight = $simple_flattened_similar_attribute_noval;
} else if (
!isset($simple_flattened_attribute[$simple_flattened_similar_attribute])
) { // Attribute not present in the revised object
$classname = 'info';
- $title = __('This attribute is only contained by this similar object. It will remain untouched.');
+ $title = __('This attribute is only contained in this matching object. It will remain untouched.');
} else { // Attributes are basically the same
$classname = '';
$title = __('This attribute has the same value as the one in the revised object.');
diff --git a/app/View/Elements/healthElements/diagnostics.ctp b/app/View/Elements/healthElements/diagnostics.ctp
index a475b119e..078403cc2 100644
--- a/app/View/Elements/healthElements/diagnostics.ctp
+++ b/app/View/Elements/healthElements/diagnostics.ctp
@@ -34,6 +34,12 @@
+
+
+
+
+
@@ -60,6 +66,7 @@
-
- Form->postButton(__('Prune upgrade logs'), $baseurl . '/logs/pruneUpdateLogs', $options = array('class' => 'btn btn-primary', 'style' => 'padding-top:1px;padding-bottom:1px;')); ?>
+
+
Form->submit('Enrich', array('class' => 'btn btn-primary'));
+ echo $this->Form->submit(__('Enrich'), array('class' => 'btn btn-primary'));
?>
Form->button('Submit', array('class' => 'btn btn-primary'));
+ ?>
+
Form->create('Server', array('url' => array('action' => 'updateSubmodule'), 'div' => false, 'style' => 'margin: 0px; display: inline-block;'));
echo $this->Form->hidden('submodule', array('value' => false));
echo $this->Form->end();
- echo '
@@ -56,7 +56,7 @@
';
+ echo '
diff --git a/app/View/Servers/ondemand_action.ctp b/app/View/Servers/ondemand_action.ctp
new file mode 100644
index 000000000..1e58b2b80
--- /dev/null
+++ b/app/View/Servers/ondemand_action.ctp
@@ -0,0 +1,87 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ $action): ?>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Form->create(false, array( 'url' => $baseurl . $action['url'] . $url_param ));
+ ?>
+
+
role="button" tabindex="0" aria-label="">
+
+ Form->end();
+ ?>
+
+
+
+
+
+
+
+
+ 0): ?>
+
+
+
+
+
Pre update test status:
+
+
+
+
+
+
+
+
+ Update command
+
+
+
+ $cmd):
+ if (isset($updateProgress['results'][$i])) {
+ $res = $updateProgress['results'][$i];
+ } else {
+ $res = false;
+ }
+ $rowDone = $i < $updateProgress['current'];
+ $rowCurrent = $i === $updateProgress['current'];
+ $rowFail = in_array($i, $updateProgress['failed_num']);
+ $rowClass = '';
+ $rowIcon = ' >
+
+
+
+ 60 ? '[...]' : '' );?>
+
+
+
+
+
+
+
+
+
+
+ $line) {
+ $pad = $j > 0 ? '30' : '0';
+ if ($line !== '') {
+ echo '' . h($line) . ' ';
+ }
+ }
+ ?>
+
+
+
+
+ $line) {
+ $pad = $j > 0 ? '30' : '0';
+ if ($line !== '') {
+ echo '' . h($line) . ' ';
+ }
+ }
+ }
+ ?>
+
+
+
+
+
+
+
Filling schema table
+
+
+
+
+
+
+
+
+
+
+
+ element('genericElements/assetLoader', array(
+ 'css' => array('update_progress'),
+ 'js' => array('update_progress')
+ ));
+ ?>
+
0) {
+ var percFail = Math.round(failArray.length/total*100);
+ var perc = Math.round(current/total*100);
+ update_pb(perc, percFail);
+ }
+
+ if ((current+1) >= total || failArray.indexOf(current) != -1) {
+ clearInterval(pooler);
+ $('.single-update-progress').hide();
+ }
+ });
+}
+
+
+function update_messages(messages) {
+ if (messages.commands === undefined) {
+ return;
+ }
+ messages.commands.forEach(function(msg, i) {
+ var div = $('#termcmd-'+i);
+ create_spans_from_message(div, msg);
+ });
+ messages.results.forEach(function(msg, i) {
+ var div = $('#termres-'+i);
+ div.css('display', '');
+ create_spans_from_message(div, msg);
+ });
+ messages.time.started.forEach(function(startedText, i) {
+ var elapsedText = messages.time.elapsed[i];
+ if (elapsedText === undefined) {
+ var diff = new Date((new Date()).getTime() - (new Date(startedText)).getTime());
+ elapsedText = pad(diff.getUTCHours(), 2)
+ + ':' + pad(diff.getUTCMinutes(), 2)
+ + ':' + pad(diff.getUTCSeconds(), 2);
+ }
+ update_times(i, startedText, elapsedText)
+ });
+}
+
+function create_spans_from_message(toAppendto, msg) {
+ toAppendto.empty();
+ // create span for each line of text
+ msg = msg.replace(/^\n*\s+/, '');
+ var lines = msg.split(/\s{2,}/m)
+ lines.forEach(function(line, j) {
+ var pad = j > 0 ? '30' : '0';
+ if (line !== '') {
+ var span = $('' + line + ' ');
+ toAppendto.append(span);
+ }
+ });
+}
+
+function update_row_state(i, state) {
+ var icon = $('#icon-'+i);
+ var row = $('#row-'+i);
+ switch(state) {
+ case 0: // success
+ row.removeClass('alert-danger alert-info');
+ row.addClass('alert-success');
+ icon.removeClass('fa-times-circle-o fa-cogs');
+ icon.addClass('fa-check-circle-o');
+ break;
+ case 1: // current
+ row.removeClass('alert-success alert-danger');
+ row.addClass('alert-info');
+ icon.removeClass('fa-check-circle-o', 'fa-times-circle-o');
+ icon.addClass('fa-cogs');
+ break;
+ case 2: //fail
+ row.removeClass('alert-success alert-info');
+ row.addClass('alert-danger');
+ icon.removeClass('fa-check-circle-o fa-cogs');
+ icon.addClass('fa-times-circle-o');
+ break;
+ case 3: //no state
+ default:
+ row.removeClass('alert-success alert-info alert-danger');
+ icon.removeClass('fa-check-circle-o fa-times-circle-o fa-cogs');
+ break;
+ }
+}
+
+function update_pb(perc, percFail) {
+ var pb = $('#pb-progress');
+ pb.css('width', perc+'%');
+ pb.text(perc+'%');
+ var pbF = $('#pb-fail');
+ pbF.css('width', percFail+'%');
+}
+
+function update_times(i, startedText, elapsedText) {
+ var started = $('#startedTime-'+i);
+ var elapsed = $('#elapsedTime-'+i);
+ started.text(startedText);
+ elapsed.text(elapsedText);
+}
+
+function update_single_update_progress(i, data) {
+ $('.single-update-progress').hide();
+ var div = $('#single-update-progress-'+i);
+ var pb = div.find('#single-update-pb-'+i);
+ var state = div.find('#small-state-text-'+i);
+ div.show();
+ var perc = parseInt(data['process_list']['PROGRESS']);
+ if (data['process_list']['MAX_STAGE'] == 0) { // if MAX_STAGE == 0, progress could not be determined
+ perc = 5;
+ if (data['failed_num'].indexOf(data['current']) >= 0) { // do not animate if failed
+ state.text('Failed');
+ perc = 0;
+ } else {
+ state.text('Unkown or No state');
+ pb.addClass('back-and-forth-animation');
+ }
+ } else {
+ perc = perc == 0 ? 1 : perc; // for UI, always set min progress to 1
+ pb.removeClass('back-and-forth-animation');
+ state.text(data['process_list']['STATE']);
+ }
+ pb.css('width', perc+'%');
+}
+
+function pad(num, size){ return ('000000000' + num).substr(-size); }
diff --git a/docs/INSTALL.ubuntu1804.md b/docs/INSTALL.ubuntu1804.md
index 2b8948876..32d94939a 100644
--- a/docs/INSTALL.ubuntu1804.md
+++ b/docs/INSTALL.ubuntu1804.md
@@ -175,6 +175,9 @@ installCore () {
# install python-magic
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install python-magic
+ # install plyara
+ $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install plyara
+
# Install Crypt_GPG and Console_CommandLine
sudo pear install ${PATH_TO_MISP}/INSTALL/dependencies/Console_CommandLine/package.xml
sudo pear install ${PATH_TO_MISP}/INSTALL/dependencies/Crypt_GPG/package.xml
diff --git a/docs/generic/globalVariables.md b/docs/generic/globalVariables.md
index 7da28eda1..9669af13c 100644
--- a/docs/generic/globalVariables.md
+++ b/docs/generic/globalVariables.md
@@ -65,7 +65,7 @@ MISPvars () {
CAKE="$PATH_TO_MISP/app/Console/cake"
# sudo config to run $LUSER commands
- if [[ "$(groups |grep -o 'staff')" == "staff" ]]; then
+ if [[ "$(groups ${MISP_USER} |grep -o 'staff')" == "staff" ]]; then
SUDO_USER="sudo -H -u ${MISP_USER} -g staff"
else
SUDO_USER="sudo -H -u ${MISP_USER}"
diff --git a/docs/generic/misp-modules-debian.md b/docs/generic/misp-modules-debian.md
index fc28ab6ed..1fa75c512 100644
--- a/docs/generic/misp-modules-debian.md
+++ b/docs/generic/misp-modules-debian.md
@@ -4,8 +4,6 @@
#
# Main MISP Modules install function
mispmodules () {
- # FIXME: this is broken, ${PATH_TO_MISP} is litteral
-##sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local
cd /usr/local/src/
## TODO: checkUsrLocalSrc in main doc
debug "Cloning misp-modules"
diff --git a/docs/generic/supportFunctions.md b/docs/generic/supportFunctions.md
index 17f712454..0bbab6eda 100644
--- a/docs/generic/supportFunctions.md
+++ b/docs/generic/supportFunctions.md
@@ -212,6 +212,15 @@ checkID () {
sudo adduser $MISP_USER staff
sudo adduser $MISP_USER $WWW_USER
fi
+
+ # FIXME: the below SUDO_USER check is a duplicate from global variables, try to have just one check
+ # sudo config to run $LUSER commands
+ if [[ "$(groups ${MISP_USER} |grep -o 'staff')" == "staff" ]]; then
+ SUDO_USER="sudo -H -u ${MISP_USER} -g staff"
+ else
+ SUDO_USER="sudo -H -u ${MISP_USER}"
+ fi
+
}
# pre-install check to make sure what we will be installing on, is ready and not a half installed system
diff --git a/docs/generic/viper-debian.md b/docs/generic/viper-debian.md
index 729830abc..55041670b 100644
--- a/docs/generic/viper-debian.md
+++ b/docs/generic/viper-debian.md
@@ -49,11 +49,17 @@ viper () {
$SUDO_USER sed -i "s/^misp_key\ =/misp_key\ =\ $AUTH_KEY/g" ${VIPER_HOME}/viper.conf
# Reset admin password to: admin/Password1234
echo "Fixing admin.db with default password"
+ VIPER_COUNT=0
while [ "$(sudo sqlite3 ${VIPER_HOME}/admin.db 'UPDATE auth_user SET password="pbkdf2_sha256$100000$iXgEJh8hz7Cf$vfdDAwLX8tko1t0M1TLTtGlxERkNnltUnMhbv56wK/U="'; echo $?)" -ne "0" ]; do
# FIXME This might lead to a race condition, the while loop is sub-par
sudo chown $MISP_USER:$MISP_USER ${VIPER_HOME}/admin.db
echo "Updating viper-web admin password, giving process time to start-up, sleeping 5, 4, 3,…"
sleep 6
+ VIPER_COUNT=$[$VIPER_COUNT+1]
+ if [[ "$VIPER_COUNT" > '10' ]]; then
+ echo "Something is wrong with updating viper. Continuing without db update."
+ break
+ fi
done
# Add viper-web to rc.local to be started on boot
diff --git a/docs/xINSTALL.debian9.md b/docs/xINSTALL.debian9.md
index 31b088fef..1597a409f 100644
--- a/docs/xINSTALL.debian9.md
+++ b/docs/xINSTALL.debian9.md
@@ -5,13 +5,13 @@
------------------------------------
!!! notice
- Maintained and tested by @SteveClement on 20190405
+ Maintained and tested by @SteveClement on 20190425
!!! warning
- This install document is NOT working as expected. There are Python issues as we "only" have python 3.5 but need at least python 3.6
+ This install document is **NOT** working as expected. There are Python issues as we "only" have python 3.5 but need at least python 3.6
This guide effectively converts your "stretch" install into a partial "testing" install.
Thus following the "testing" install guide is a better choice, but not for production.
- One manual work-around is to install Python >3.5 from source.
+ One manual work-around is to install Python >3.5 from source and leaving apt untouched.
### 1/ Minimal Debian install
-------------------------
@@ -80,10 +80,11 @@ sudo apt -t testing install -y \
mariadb-client \
mariadb-server
+# /!\
# This is maybe needed. If mysql does not start and you find a solution, please contribute.
# What did work for me was running mysqld interactively: sudo mysqld
-mkdir -p /var/run/mysqld
-chown mysql /var/run/mysqld
+sudo mkdir -p /var/run/mysqld
+sudo chown mysql /var/run/mysqld
sudo /etc/init.d/mysql restart
sudo apt -t testing install -y jupyter-notebook
diff --git a/docs/xINSTALL.rhel8.md b/docs/xINSTALL.rhel8.md
index c4e9e8ad0..521c94f8f 100644
--- a/docs/xINSTALL.rhel8.md
+++ b/docs/xINSTALL.rhel8.md
@@ -1,4 +1,4 @@
-# INSTALLATION INSTRUCTIONS for RHEL 8.x (beta)
+# INSTALLATION INSTRUCTIONS for RHEL 8.x (beta) and partially Fedora Server 30
-------------------------
### -1/ Installer and Manual install instructions
@@ -129,6 +129,7 @@ sudo yum install gcc git zip \
mod_ssl \
redis \
mariadb \
+ mariadb-server \
python3-devel python3-pip python3-virtualenv \
libxslt-devel zlib-devel ssdeep-devel -y
sudo alternatives --set python /usr/bin/python3
@@ -159,11 +160,6 @@ sudo yum install php php-fpm php-devel php-pear \
sudo systemctl enable --now php-fpm.service
```
-## 2.07/ Start redis service and enable to start on boot
-```bash
-sudo systemctl enable --now redis.service
-```
-
```bash
#
# GPG needs lots of entropy, haveged provides entropy
@@ -237,7 +233,8 @@ $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U zmq
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U redis
# lief needs manual compilation
-sudo yum install devtoolset-8 cmake3 cppcheck -y
+sudo yum groupinstall "Development Tools" -y
+sudo yum install cmake3 cppcheck -y
cd $PATH_TO_MISP/app/files/scripts/lief
$SUDO_WWW mkdir build
@@ -289,10 +286,10 @@ installCake_RHEL ()
sudo chown $WWW_USER:$WWW_USER /usr/share/httpd/.composer
cd $PATH_TO_MISP/app
# Update composer.phar (optional)
- #$SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
- #$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === '48e3236262b34d30969dca3c37281b3b4bbe3221bda826ac6a9a62d6444cdb0dcd0615698a5cbe587c3f0fe57a54d8f5') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
- #$SUDO_WWW php composer-setup.php
- #$SUDO_WWW php -r "unlink('composer-setup.php');"
+ $SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
+ $SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === '48e3236262b34d30969dca3c37281b3b4bbe3221bda826ac6a9a62d6444cdb0dcd0615698a5cbe587c3f0fe57a54d8f5') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
+ $SUDO_WWW php composer-setup.php
+ $SUDO_WWW php -r "unlink('composer-setup.php');"
$SUDO_WWW php composer.phar require kamisama/cake-resque:4.1.2
$SUDO_WWW php composer.phar config vendor-dir Vendor
$SUDO_WWW php composer.phar install
@@ -650,7 +647,7 @@ Make the workers' script executable and reload the systemd units :
```bash
sudo chmod +x /var/www/MISP/app/Console/worker/start.sh
sudo systemctl daemon-reload
-sudo checkmodule -M -m -o /tmp/workerstartsh.mod $PATH_TO_MISP/INSTALL/workerstartsh.te
+sudo checkmodule -M -m -o /tmp/workerstartsh.mod $PATH_TO_MISP/INSTALL/worker/startsh.te
sudo semodule_package -o /tmp/workerstartsh.pp -m /tmp/workerstartsh.mod
sudo semodule -i /tmp/workerstartsh.pp
```
@@ -670,10 +667,10 @@ sudo chown root:users /usr/local/src
cd /usr/local/src/
$SUDO_WWW git clone https://github.com/MISP/misp-modules.git
cd misp-modules
+sudo yum install rubygem-rouge rubygem-asciidoctor zbar-devel opencv-core poppler-cpp-devel -y
# pip install
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U -I -r REQUIREMENTS
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U .
-sudo yum install rubygem-rouge rubygem-asciidoctor zbar-devel opencv-core -y
echo "[Unit]
Description=MISP's modules