From 3f215743f0cae97587d01d460c222d1c84765c18 Mon Sep 17 00:00:00 2001 From: Iglocska Date: Mon, 29 Jun 2015 08:56:45 +0200 Subject: [PATCH] Complete rework of the ZeroMQ implementation - python server running in the background doing the publishing - MISP -> python script communication via redis - configurable / controllable via the admin UI --- .gitignore | 3 + INSTALL/INSTALL.ubuntu1404.txt | 10 +- INSTALL/UPDATE.txt | 8 +- VERSION.json | 2 +- app/Controller/ServersController.php | 138 ++++++++++++++---- app/Lib/Tools/PubSubTool.php | 111 ++++++++++++-- app/Model/Post.php | 3 +- app/Model/Server.php | 80 +++++++++- .../Elements/healthElements/diagnostics.ctp | 9 +- .../Elements/healthElements/settings_row.ctp | 22 +++ .../Elements/healthElements/settings_tab.ctp | 2 +- app/View/Servers/ajax/zeromqstatus.ctp | 13 ++ app/files/scripts/mispzmq/mispzmq.py | 78 ++++++++++ app/files/scripts/mispzmq/mispzmqtest.py | 5 + app/webroot/js/ajaxification.js | 38 ++++- 15 files changed, 457 insertions(+), 65 deletions(-) create mode 100644 app/View/Elements/healthElements/settings_row.ctp create mode 100644 app/View/Servers/ajax/zeromqstatus.ctp create mode 100644 app/files/scripts/mispzmq/mispzmq.py create mode 100644 app/files/scripts/mispzmq/mispzmqtest.py diff --git a/.gitignore b/.gitignore index 3d81f6932..cd4b69d49 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,9 @@ /app/files/scripts/python-cybox/ /app/files/scripts/*.pyc /app/files/scripts/*.py~ +/app/files/scripts/mispzmq/* +!/app/files/scripts/mispzmq/mispzmq.py +!/app/files/scripts/mispzmq/mispzmqtest.py /app/files/scripts/tmp/* !/app/files/scripts/tmp/empty /app/tmp/files/* diff --git a/INSTALL/INSTALL.ubuntu1404.txt b/INSTALL/INSTALL.ubuntu1404.txt index d4418a520..9e486cad3 100644 --- a/INSTALL/INSTALL.ubuntu1404.txt +++ b/INSTALL/INSTALL.ubuntu1404.txt @@ -215,11 +215,5 @@ Recommended actions Optional features ------------------- -# MISP has a new pub/sub feature, using ZeroMQ. To enable it, follow the following instructions -apt-get install libzmq-dev libtool pkg-config build-essential autoconf automake -pecl install zmq-beta -# after this is done add the following line to your php.ini -# Do this for both the apache and the CLI php.ini, for ubuntu this would be in /etc/php5/apache2/php.ini and /etc/php5/cli/php.ini -extension=zmq.so -service apache2 restart -# You can now enable the feature in the server settings \ No newline at end of file +# MISP has a new pub/sub feature, using ZeroMQ. To enable it, simply run the following command +pip install pyzmq diff --git a/INSTALL/UPDATE.txt b/INSTALL/UPDATE.txt index 722ea164b..9501b34ae 100644 --- a/INSTALL/UPDATE.txt +++ b/INSTALL/UPDATE.txt @@ -58,10 +58,4 @@ su www-data -c 'bash /var/www/MISP/app/Console/worker/start.sh' # 7. Add any new dependencies that might have been added since you've last updated (shown below) # 7.a requirements for the pubsub optional feature -apt-get install libzmq-dev libtool pkg-config build-essential autoconf automake -pecl install zmq-beta -# after this is done add the following line to your php.ini -# Do this for both the apache and the CLI php.ini, for ubuntu this would be in /etc/php5/apache2/php.ini and /etc/php5/cli/php.ini -extension=zmq.so -service apache2 restart -# You can now enable the feature in the server settings \ No newline at end of file +pip install pyzmq \ No newline at end of file diff --git a/VERSION.json b/VERSION.json index 7c59c0667..4497e6137 100644 --- a/VERSION.json +++ b/VERSION.json @@ -1 +1 @@ -{"major":2, "minor":3, "hotfix":87} \ No newline at end of file +{"major":2, "minor":3, "hotfix":88} \ No newline at end of file diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index c5776ab60..7c45c3ad9 100755 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -282,6 +282,25 @@ class ServersController extends AppController { if ($result) $this->Server->save($s); } + public function serverSettingsReloadSetting($setting, $id) { + if (!$this->_isSiteAdmin()) throw new MethodNotAllowedException(); + $pathToSetting = explode('.', $setting); + $settingObject = $this->Server->serverSettings; + foreach ($pathToSetting as $key) { + if (!isset($settingObject[$key])) throw new MethodNotAllowedException(); + $settingObject = $settingObject[$key]; + } + $result = $this->Server->serverSettingReadSingle($settingObject, $setting, $key); + $this->set('setting', $result); + $priorityErrorColours = array(0 => 'red', 1 => 'yellow', 2 => 'green'); + $this->set('priorityErrorColours', $priorityErrorColours); + $priorities = array(0 => 'Critical', 1 => 'Recommended', 2 => 'Optional', 3 => 'Deprecated'); + $this->set('priorities', $priorities); + $this->set('k', $id); + $this->layout = false; + $this->render('/Elements/healthElements/settings_row'); + } + public function serverSettings($tab=false) { if (!$this->_isSiteAdmin()) throw new MethodNotAllowedException(); if ($this->request->is('Get')) { @@ -296,7 +315,7 @@ class ServersController extends AppController { $writeableErrors = array(0 => 'OK', 1 => 'Directory doesn\'t exist', 2 => 'Directory is not writeable'); $gpgErrors = array(0 => 'OK', 1 => 'FAIL: settings not set', 2 => 'FAIL: bad GnuPG.*', 3 => 'FAIL: encrypt failed'); $proxyErrors = array(0 => 'OK', 1 => 'not configured (so not tested)', 2 => 'Getting URL via proxy failed'); - $zmqErrors = array(0 => 'OK', 1 => 'not enabled (so not tested)', 2 => 'zmq extension not installed correctly.'); + $zmqErrors = array(0 => 'OK', 1 => 'not enabled (so not tested)', 2 => 'Python ZeroMQ library not installed correctly.', 3 => 'ZeroMQ script not running.'); $stixOperational = array(0 => 'STIX or CyBox library not installed correctly', 1 => 'OK'); $stixVersion = array(0 => 'Incorrect STIX version installed, found $current, expecting $expected', 1 => 'OK'); $cyboxVersion = array(0 => 'Incorrect CyBox version installed, found $current, expecting $expected', 1 => 'OK'); @@ -337,39 +356,40 @@ class ServersController extends AppController { $diagnostic_errors = 0; App::uses('File', 'Utility'); App::uses('Folder', 'Utility'); - + $additionalViewVars = array(); // Only run this check on the diagnostics tab - if ($tab == 'diagnostics') { + if ($tab == 'diagnostics' || $tab == 'download') { // check if the current version of MISP is outdated or not $version = $this->__checkVersion(); $this->set('version', $version); if ($version && (!$version['upToDate'] || $version['upToDate'] == 'older')) $diagnostic_errors++; - } - - if ($tab == 'files') { - $files = $this->__manageFiles(); - $this->set('files', $files); - } + if ($tab == 'files') { + $files = $this->__manageFiles(); + $this->set('files', $files); + } + + // check if the STIX and Cybox libraries are working and the correct version using the test script stixtest.py + $stix = $this->Server->stixDiagnostics($diagnostic_errors, $stixVersion, $cyboxVersion); + + // if GPG is set up in the settings, try to encrypt a test message + $gpgStatus = $this->Server->gpgDiagnostics($diagnostic_errors); + + // if the message queue pub/sub is enabled, check whether the extension works + $zmqStatus = $this->Server->zmqDiagnostics($diagnostic_errors); + + // if Proxy is set up in the settings, try to connect to a test URL + $proxyStatus = $this->Server->proxyDiagnostics($diagnostic_errors); + + $additionalViewVars = array('gpgStatus', 'proxyStatus', 'zmqStatus', 'stixVersion', 'cyboxVersion','gpgErrors', 'proxyErrors', 'zmqErrors', 'stixOperational', 'stix'); + } // check whether the files are writeable $writeableDirs = $this->Server->writeableDirsDiagnostics($diagnostic_errors); - // check if the STIX and Cybox libraries are working and the correct version using the test script stixtest.py - $stix = $this->Server->stixDiagnostics($diagnostic_errors, $stixVersion, $cyboxVersion); - - // if GPG is set up in the settings, try to encrypt a test message - $gpgStatus = $this->Server->gpgDiagnostics($diagnostic_errors); - - // if the message queue pub/sub is enabled, check whether the extension works - $zmqStatus = $this->Server->zmqDiagnostics($diagnostic_errors); - - // if Proxy is set up in the settings, try to connect to a test URL - $proxyStatus = $this->Server->proxyDiagnostics($diagnostic_errors); - $viewVars = array( - 'gpgStatus', 'proxyStatus', 'zmqStatus', 'diagnostic_errors', 'tabs', 'tab', 'issues', 'finalSettings', 'writeableErrors','gpgErrors', 'proxyErrors', 'zmqErrors', 'stixOperational', 'stixVersion', 'cyboxVersion', 'stix', 'writeableDirs' + 'diagnostic_errors', 'tabs', 'tab', 'issues', 'finalSettings', 'writeableErrors', 'writeableDirs' ); - + $viewVars = array_merge($viewVars, $additionalViewVars); foreach ($viewVars as $viewVar) $this->set($viewVar, ${$viewVar}); if (Configure::read('MISP.background_jobs')) { @@ -410,10 +430,11 @@ class ServersController extends AppController { $this->response->download('MISP.report.json'); return $this->response; } + $priorities = array(0 => 'Critical', 1 => 'Recommended', 2 => 'Optional', 3 => 'Deprecated'); - $priorityErrorColours = array(0 => 'red', 1 => 'yellow', 2 => 'green'); $this->set('priorities', $priorities); $this->set('workerIssueCount', $workerIssueCount); + $priorityErrorColours = array(0 => 'red', 1 => 'yellow', 2 => 'green'); $this->set('priorityErrorColours', $priorityErrorColours); } } @@ -476,6 +497,25 @@ class ServersController extends AppController { $this->render('ajax/server_settings_edit'); } if ($this->request->is('post')) { + $this->autoRender = false; + $this->loadModel('Log'); + if (isset($found['beforeHook'])) { + $beforeResult = call_user_func_array(array($this->Server, $found['beforeHook']), array($setting, $this->request->data['Server']['value'])); + if ($afterResult !== true) { + $this->Log->create(); + $result = $this->Log->save(array( + 'org' => $this->Auth->user('org'), + 'model' => 'Server', + 'model_id' => 0, + 'email' => $this->Auth->user('email'), + 'action' => 'serverSettingsEdit', + 'user_id' => $this->Auth->user('id'), + 'title' => 'Server setting issue', + 'change' => 'There was an issue witch changing ' . $setting . ' to ' . $this->request->data['Server']['value'] . '. The error message returned is: ' . $beforeResult . 'No changes were made.', + )); + return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => $afterResult)),'status'=>200)); + } + } if ($found['type'] == 'boolean') { $this->request->data['Server']['value'] = ($this->request->data['Server']['value'] ? true : false); } @@ -490,7 +530,6 @@ class ServersController extends AppController { } else { $oldValue = Configure::read($setting); $this->Server->serverSettingsSaveValue($setting, $this->request->data['Server']['value']); - $this->loadModel('Log'); $this->Log->create(); $result = $this->Log->save(array( 'org' => $this->Auth->user('org'), @@ -502,7 +541,24 @@ class ServersController extends AppController { 'title' => 'Server setting changed', 'change' => $setting . ' (' . $oldValue . ') => (' . $this->request->data['Server']['value'] . ')', )); - $this->autoRender = false; + // execute after hook + if (isset($found['afterHook'])) { + $afterResult = call_user_func_array(array($this->Server, $found['afterHook']), array($setting, $this->request->data['Server']['value'])); + if ($afterResult !== true) { + $this->Log->create(); + $result = $this->Log->save(array( + 'org' => $this->Auth->user('org'), + 'model' => 'Server', + 'model_id' => 0, + 'email' => $this->Auth->user('email'), + 'action' => 'serverSettingsEdit', + 'user_id' => $this->Auth->user('id'), + 'title' => 'Server setting issue', + 'change' => 'There was an issue after setting a new setting. The error message returned is: ' . $afterResult, + )); + return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => $afterResult)),'status'=>200)); + } + } return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => 'Field updated.')),'status'=>200)); } } @@ -573,4 +629,34 @@ class ServersController extends AppController { } $this->redirect(array('controller' => 'servers', 'action' => 'serverSettings', 'files')); } + + public function startZeroMQServer() { + if (!$this->_isSiteAdmin()) throw new MethodNotAllowedException(); + App::uses('PubSubTool', 'Tools'); + $pubSubTool = new PubSubTool(); + $result = $pubSubTool->restartServer(); + if ($result === true) return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => 'ZeroMQ server successfully started.')),'status'=>200)); + else return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => $result)),'status'=>200)); + } + + public function stopZeroMQServer() { + if (!$this->_isSiteAdmin()) throw new MethodNotAllowedException(); + App::uses('PubSubTool', 'Tools'); + $pubSubTool = new PubSubTool(); + $result = $pubSubTool->killService(); + if ($result === true) return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => 'ZeroMQ server successfully killed.')),'status'=>200)); + else return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Could not kill the previous instance of the ZeroMQ script.')),'status'=>200)); + } + + public function statusZeroMQServer() { + App::uses('PubSubTool', 'Tools'); + $pubSubTool = new PubSubTool(); + $result = $pubSubTool->statusCheck(); + if (!empty($result)) { + $this->set('events', $result['publishCount']); + $this->set('time', date('Y/m/d H:i:s', $result['timestamp'])); + $this->set('time2', date('Y/m/d H:i:s', $result['timestampSettings'])); + } + $this->render('ajax/zeromqstatus'); + } } diff --git a/app/Lib/Tools/PubSubTool.php b/app/Lib/Tools/PubSubTool.php index 89d619e7e..280e851cc 100644 --- a/app/Lib/Tools/PubSubTool.php +++ b/app/Lib/Tools/PubSubTool.php @@ -1,29 +1,110 @@ getSocket(ZMQ::SOCKET_PUB); - $port = Configure::read('Plugin.ZeroMQ_port'); - if (empty($port)) $port = 50000; - $pub->bind("tcp://*:" . $port); - return $pub; + + private function __getSetSettings() { + $settings = array( + 'redis_host' => 'localhost', + 'redis_port' => '6379', + 'redis_password' => '', + 'redis_database' => '1', + 'redis_namespace' => 'mispq', + 'port' => '50000', + ); + foreach ($settings as $key => &$setting) { + $temp = Configure::read('Plugin.ZeroMQ_' . $key); + if ($temp) $setting = $temp; + } + $settingsFile = new File(APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'settings.json', true, 0644); + $settingsFile->write(json_encode($settings, true)); + $settingsFile->close(); + return $settings; + } + + // read the pid file, if it exists, check if the process is actually running + // if either the pid file doesn't exists or the process is not running return false + // otherwise return the pid + public function checkIfRunning() { + $pidFile = new File(APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmq.pid'); + $pid = $pidFile->read(true, 'r'); + if ($pid === false || $pid === '') return false; + if (!is_numeric($pid)) throw new Exception('Internal error (invalid PID file for the MISP zmq script)'); + $result = trim(shell_exec('ps aux | awk \'{print $2}\' | grep ' . $pid)); + if (empty($result)) return false; + return $pid; + } + + public function statusCheck() { + $redis = new Redis(); + $settings = $this->__getSetSettings(); + $redis->connect($settings['redis_host'], $settings['redis_port']); + $redis->select($settings['redis_database']); + $redis->rPush($settings['redis_namespace'] . ':command', 'status'); + sleep(1); + $response = trim($redis->lPop($settings['redis_namespace'] . ':status')); + return json_decode($response, true); + } + + public function checkIfPythonLibInstalled() { + $result = trim(shell_exec('python ' . APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmqtest.py')); + if ($result === "OK") return true; + return false; + } + + private function __setupPubServer() { + App::uses('File', 'Utility'); + $settings = $this->__getSetSettings(); + if ($this->checkIfRunning() === false) { + shell_exec('python ' . APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmq.py > ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.log 2> ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.error.log &'); + } + return $settings; } public function publishEvent($event) { - $pub = $this->__setupPub(); + $settings = $this->__setupPubServer(); App::uses('JSONConverterTool', 'Tools'); $jsonTool = new JSONConverterTool(); $json = $jsonTool->event2JSON($event); - sleep(1); - $pub->send('misp_json ' . $json); + $redis = new Redis(); + $redis->connect($settings['redis_host'], $settings['redis_port']); + $redis->select($settings['redis_database']); + $redis->rPush($settings['redis_namespace'] . ':misp_json', $json); + return true; } - public function testZMQ() { - try { - $context = new ZMQContext(); - } catch (Exception $e) { - return false; + public function killService($settings = false) { + $redis = new Redis(); + if ($settings == false) $settings = $this->__getSetSettings(); + $redis->connect($settings['redis_host'], $settings['redis_port']); + $redis->select($settings['redis_database']); + $redis->rPush($settings['redis_namespace'] . ':command', 'kill'); + $continue = true; + $counter = 0; + sleep(1); + if ($this->checkIfRunning()) return false; + return true; + } + + // reload the server if it is running, if not, start it + public function reloadServer() { + if (!$this->checkIfRunning()) { + $settings = $this->__setupPubServer(); + } else { + $settings = $this->__getSetSettings(); + $redis = new Redis(); + $redis->connect($settings['redis_host'], $settings['redis_port']); + $redis->select($settings['redis_database']); + $redis->rPush($settings['redis_namespace'] . ':command', 'reload'); } + if (!$this->checkIfRunning()) return 'Setting saved, but something is wrong with the ZeroMQ server. Please check the diagnostics page for more information.'; + return true; + } + + public function restartServer() { + if (!$this->killService()) { + return 'Could not kill the previous instance of the ZeroMQ script.'; + } + $this->__setupPubServer(); + if (!is_numeric($this->checkIfRunning())) return 'Failed starting the ZeroMQ script.'; return true; } } \ No newline at end of file diff --git a/app/Model/Post.php b/app/Model/Post.php index ac6125eb0..797c0dab2 100644 --- a/app/Model/Post.php +++ b/app/Model/Post.php @@ -109,8 +109,7 @@ class Post extends AppModel { $bodyDetail .= $message . "\n"; $subject = "[" . Configure::read('MISP.org') . " MISP] New post in discussion " . $post['Post']['thread_id'] . " - TLP Amber"; foreach ($orgMembers as &$recipient) { - $result = $this->User->sendEmail($recipient, $bodyDetail, $body, $subject); + $this->User->sendEmail($recipient, $bodyDetail, $body, $subject); } - return $result; } } diff --git a/app/Model/Server.php b/app/Model/Server.php index 8038adc02..758943157 100755 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -647,6 +647,7 @@ class Server extends AppModel { 'errorMessage' => '', 'test' => 'testBool', 'type' => 'boolean', + 'afterHook' => 'zmqAfterHook', ), 'ZeroMQ_port' => array( 'level' => 2, @@ -655,7 +656,54 @@ class Server extends AppModel { 'errorMessage' => '', 'test' => 'testForPortNumber', 'type' => 'numeric', + 'afterHook' => 'zmqAfterHook', ), + 'ZeroMQ_redis_host' => array( + 'level' => 2, + 'description' => 'Location of the Redis db used by MISP and the Python PUB script to queue data to be published.', + 'value' => 'localhost', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'afterHook' => 'zmqAfterHook', + ), + 'ZeroMQ_redis_port' => array( + 'level' => 2, + 'description' => 'The port that Redis is listening on.', + 'value' => 6379, + 'errorMessage' => '', + 'test' => 'testForPortNumber', + 'type' => 'numeric', + 'afterHook' => 'zmqAfterHook', + ), + 'ZeroMQ_redis_password' => array( + 'level' => 2, + 'description' => 'The password, if set for Redis.', + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'afterHook' => 'zmqAfterHook', + ), + 'ZeroMQ_redis_database' => array( + 'level' => 2, + 'description' => 'The database to be used for queuing messages for the pub/sub functionality.', + 'value' => '1', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'afterHook' => 'zmqAfterHook', + ), + 'ZeroMQ_redis_namespace' => array( + 'level' => 2, + 'description' => 'The namespace to be used for queuing messages for the pub/sub functionality.', + 'value' => 'mispq', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'afterHook' => 'zmqAfterHook', + ), + ), 'debug' => array( 'level' => 0, @@ -1120,6 +1168,13 @@ class Server extends AppModel { return $finalSettings; } + public function serverSettingReadSingle($settingObject, $settingName, $leafKey) { + $setting = Configure::read($settingName); + $result = $this->__evaluateLeaf($settingObject, $leafKey, $setting); + $result['setting'] = $settingName; + return $result; + } + private function __evaluateLeaf($leafValue, $leafKey, $setting) { if (isset($setting)) { $result = $this->{$leafValue['test']}($setting); @@ -1239,6 +1294,23 @@ class Server extends AppModel { return true; } + public function zmqAfterHook($setting, $value) { + App::uses('PubSubTool', 'Tools'); + $pubSubTool = new PubSubTool(); + // If we are trying to change the enable setting to false, we don't need to test anything, just kill the server and return true. + if ($setting == 'Plugin.ZeroMQ_enable') { + if ($value == false || $value == 0) { + $pubSubTool->killService(); + return true; + } + } elseif (!Configure::read('Plugin.ZeroMQ_enable')) { + // If we are changing any other ZeroMQ settings but the feature is disabled, don't reload the service + return true; + } + $pubSubTool->reloadServer(); + return true; + } + // never come here directly, always go through a secondary check like testForTermsFile in order to also pass along the expected file path private function __testForFile($value, $path) { @@ -1420,9 +1492,13 @@ class Server extends AppModel { if (!Configure::read('Plugin.ZeroMQ_enable')) return 1; App::uses('PubSubTool', 'Tools'); $pubSubTool = new PubSubTool(); - if ($pubSubTool->testZMQ()) return 0; + if (!$pubSubTool->checkIfPythonLibInstalled()) { + $diagnostic_errors++; + return 2; + } + if ($pubSubTool->checkIfRunning()) return 0; $diagnostic_errors++; - return 2; + return 3; } public function proxyDiagnostics(&$diagnostic_errors) { diff --git a/app/View/Elements/healthElements/diagnostics.ctp b/app/View/Elements/healthElements/diagnostics.ctp index 9a49895fe..2d97a562b 100644 --- a/app/View/Elements/healthElements/diagnostics.ctp +++ b/app/View/Elements/healthElements/diagnostics.ctp @@ -64,7 +64,7 @@

Mitre's STIX and Cybox python libraries have to be installed in order for MISP's STIX export to work. Make sure that you install them (as described in the MISP installation instructions) if you receive an error below.
If you run into any issues here, make sure that both STIX and CyBox are installed as described in the INSTALL.txt file. The required versions are:
STIX:
CyBox:
Other versions might work but are not tested / recommended.

-
+
1) { $colour = 'red'; } - echo 'Proxy settings....' . $message . ''; + echo 'ZeroMQ settings....' . $message . ''; ?>
+
+ Start / Restart + Stop + Status +

Proxy

diff --git a/app/View/Elements/healthElements/settings_row.ctp b/app/View/Elements/healthElements/settings_row.ctp new file mode 100644 index 000000000..93d578da8 --- /dev/null +++ b/app/View/Elements/healthElements/settings_row.ctp @@ -0,0 +1,22 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/app/View/Elements/healthElements/settings_tab.ctp b/app/View/Elements/healthElements/settings_tab.ctp index 303d7de9a..e698c8488 100644 --- a/app/View/Elements/healthElements/settings_tab.ctp +++ b/app/View/Elements/healthElements/settings_tab.ctp @@ -17,7 +17,7 @@ if ($setting['type'] == 'boolean') $setting['value'] = ($setting['value'] === true ? 'true' : 'false'); if (isset($setting['options'])) $setting['value'] = ($setting['options'][$setting['value']]); ?> - + diff --git a/app/View/Servers/ajax/zeromqstatus.ctp b/app/View/Servers/ajax/zeromqstatus.ctp new file mode 100644 index 000000000..811aea231 --- /dev/null +++ b/app/View/Servers/ajax/zeromqstatus.ctp @@ -0,0 +1,13 @@ +
+ZeroMQ Server Status +
+ +

Start time:
+ Settings read at:
+ Events processed:

+ +

The ZeroMQ server is unreachable.

+ + OK +
+
\ No newline at end of file diff --git a/app/files/scripts/mispzmq/mispzmq.py b/app/files/scripts/mispzmq/mispzmq.py new file mode 100644 index 000000000..38e79018f --- /dev/null +++ b/app/files/scripts/mispzmq/mispzmq.py @@ -0,0 +1,78 @@ +import zmq +import sys +import redis +import json +import os +import time + +socket = None +r = None +namespace = None +settings = None +current_location = os.path.dirname(os.path.realpath(__file__)) +pidfile = current_location + "/mispzmq.pid" +timestamp = time.time() +timestampSettings = timestamp +publishCount = 0 + +def setup(): + global namespace + global socket + global r + global settings + global timestampSettings + with open(current_location + '/settings.json') as settings_file: + settings = json.load(settings_file) + namespace = settings["redis_namespace"] + r = redis.StrictRedis(host=settings["redis_host"], db=settings["redis_database"], password=settings["redis_password"], port=settings["redis_port"]) + timestampSettings = time.time() + +def handleCommand(command): + if command == "kill": + print "Kill command received, shutting down.\n" + removePidFile() + sys.exit() + if command == "reload": + print "Reload command received, reloading settings from file.\n" + setup() + if command == "status": + print "Status command received, responding with latest stats.\n" + r.delete(namespace + ":status") + r.lpush(namespace + ":status", json.dumps({"timestamp": timestamp, "timestampSettings": timestampSettings, "publishCount": publishCount})) + return + +def removePidFile(): + os.unlink(pidfile) + +def createPidFile(): + pid = str(os.getpid()) + file(pidfile, 'w').write(pid) + +def pubMessage(data): + context = zmq.Context() + socket = context.socket(zmq.PUB) + socket.bind("tcp://*:%s" % settings["port"]) + print "Sending " + data + time.sleep(1) + socket.send("%s %s" % ('misp_json', data)) + socket.close() + context.term() + global publishCount + publishCount = publishCount + 1 + +def main(args): + setup() + createPidFile() + while True: + time.sleep(1) + command = r.lpop(namespace + ":command") + if command is not None: + handleCommand(command) + topic = "misp_json" + data = r.lpop(namespace + ":misp_json") + if data is None: + continue + pubMessage(data) + +if __name__ == "__main__": + main(sys.argv) diff --git a/app/files/scripts/mispzmq/mispzmqtest.py b/app/files/scripts/mispzmq/mispzmqtest.py new file mode 100644 index 000000000..9395419fd --- /dev/null +++ b/app/files/scripts/mispzmq/mispzmqtest.py @@ -0,0 +1,5 @@ +try: + import zmq +except ImportError: + print "ZeroMQ library could not be imported." +print "OK" diff --git a/app/webroot/js/ajaxification.js b/app/webroot/js/ajaxification.js index c13dabe3b..0ce6dd364 100644 --- a/app/webroot/js/ajaxification.js +++ b/app/webroot/js/ajaxification.js @@ -1329,8 +1329,21 @@ function serverSettingSubmitForm(name, setting, id) { $.ajax({ data: formData, cache: false, + beforeSend: function (XMLHttpRequest) { + $(".loading").show(); + }, success:function (data, textStatus) { - window.location.reload(); + $.ajax({ + type:"get", + url:"/servers/serverSettingsReloadSetting/" + setting + "/" + id, + success:function (data2, textStatus2) { + $('#' + id + '_row').replaceWith(data2); + $(".loading").hide(); + }, + error:function() { + showMessage('fail', 'Could not refresh the table.'); + } + }); }, error:function() { showMessage('fail', 'Request failed for an unknown reason.'); @@ -1451,3 +1464,26 @@ function lookupPGPKey(emailFieldName) { } }); } + +function zeroMQServerAction(action) { + $.ajax({ + type: "get", + url: "/servers/" + action + "ZeroMQServer/", + beforeSend: function (XMLHttpRequest) { + $(".loading").show(); + }, + success: function (data) { + $(".loading").hide(); + if (action !== 'status') { + window.location.reload(); + } else { + $("#confirmation_box").html(data); + $("#confirmation_box").fadeIn(); + $("#gray_out").fadeIn(); + } + }, + error: function (data, textStatus, errorThrown) { + showMessage('fail', textStatus + ": " + errorThrown); + } + }); +}