From dd2e6cc715beb524e2631fdf8b8949506860d7b1 Mon Sep 17 00:00:00 2001 From: iwitz Date: Wed, 6 Feb 2019 15:08:40 +0100 Subject: [PATCH 1/9] fix: lief installation * unnecessary steps removed * irrelevant comment removed * instructions updated to compile lief for python 3.6 instead of 2.7 * install lief in the folder where the other dependencies are so that it is detected by the diagnostics page --- docs/INSTALL.rhel7.md | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index ea1aff9b9..6a47775bf 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -486,44 +486,31 @@ scl enable rh-python36 pip3 install pymisp yum install devtoolset-7 cmake3 ``` -## 11.02/ Enable devtoolset-7 +## 11.02/ Create the directory and download the source code ```bash -scl enable devtoolset-7 bash +cd /var/www/MISP/app/files/scripts +git clone --branch master --single-branch https://github.com/lief-project/LIEF.git lief ``` -## 11.03/ Set env variable, create directories and download source code +## 11.03/ Compile lief and install it ```bash -mkdir -p /tmp/LIEF -mkdir -p /tmp/LIEF_INSTALL -export LIEF_TMP=/tmp/LIEF -export LIEF_INSTALL=/tmp/LIEF_INSTALL -export LIEF_BRANCH=master -cd $LIEF_TMP -git clone --branch $LIEF_BRANCH --single-branch https://github.com/lief-project/LIEF.git LIEF -``` - -## 11.04/ Compile lief and install -```bash -cd $LIEF_TMP/LIEF -mkdir -p build +cd /var/www/MISP/app/files/scripts/lief +mkdir build cd build -scl enable devtoolset-7 'bash -c "cmake3 \ +scl enable devtoolset-7 rh-python36 'bash -c "cmake3 \ -DLIEF_PYTHON_API=on \ -DLIEF_DOC=off \ -DCMAKE_INSTALL_PREFIX=$LIEF_INSTALL \ -DCMAKE_BUILD_TYPE=Release \ --DPYTHON_VERSION=2.7 \ +-DPYTHON_VERSION=3.6 \ .."' make -j3 cd api/python -scl enable rh-python36 python3 setup.py install || : -# you can ignore the error about finding suitable distribution -cd $LIEF_TMP/LIEF/build -make install -make package +# before you run setup.py it may be a good idea to switch off your Internet connection ; otherwise pip will try to fetch lief from Github which may cause issues later +scl enable rh-python36 'python3 setup.py install || :' ``` -## 11.05/ Test lief installation, if no error, package installed +## 11.04/ Test lief installation, if no error, package installed ```bash python >> import lief From 39e7dbc2962d016f68bfc54c6b6c18981b01ba9b Mon Sep 17 00:00:00 2001 From: iwitz Date: Wed, 6 Feb 2019 15:58:04 +0100 Subject: [PATCH 2/9] chg: improved setup bug workaround --- docs/INSTALL.rhel7.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index 6a47775bf..2031dfdd6 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -506,13 +506,14 @@ scl enable devtoolset-7 rh-python36 'bash -c "cmake3 \ .."' make -j3 cd api/python -# before you run setup.py it may be a good idea to switch off your Internet connection ; otherwise pip will try to fetch lief from Github which may cause issues later scl enable rh-python36 'python3 setup.py install || :' +# when running setup.py, pip will download and install remote LIEF packages that will prevent MISP from detecting the packages that you compiled ; remove them +find /opt/rh/rh-python36/root/ -name "*lief*" -exec rm -rf {} \; ``` ## 11.04/ Test lief installation, if no error, package installed ```bash -python +scl enable rh-python36 python3 >> import lief ``` From cbf3262bcac4037fa2b25f407749c21aaab09fbb Mon Sep 17 00:00:00 2001 From: iwitz Date: Thu, 7 Feb 2019 08:58:01 +0100 Subject: [PATCH 3/9] chg: remove centos step The instruction to enable the SCL repos is already provided in step 1.4, and the package centos-release-scl does not exist on RHEL7. The fact that step 1.4 may not work on a trial subscription is not relevant because assumptions 0.1 and 0.2 specifically state that a support agreement is required to be able to complete the installation. --- docs/INSTALL.rhel7.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index ea1aff9b9..549768777 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -38,8 +38,7 @@ sudo subscription-manager register --auto-attach # register your system to an ac sudo subscription-manager refresh sudo subscription-manager repos --enable rhel-7-server-optional-rpms sudo subscription-manager repos --enable rhel-7-server-extras-rpms -# This fails on a Trial subscription, it seems. -##sudo subscription-manager repos --enable rhel-server-rhscl-7-rpms +sudo subscription-manager repos --enable rhel-server-rhscl-7-rpms ``` ### 1.5a/ OPTIONAL: Install the deltarpm package to help reduce download size when installing updates @@ -64,11 +63,6 @@ yum update -y yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm -y ``` -## 1.7/ Install the SCL repo -```bash -yum install centos-release-scl -``` - # 2/ Install Dependencies Once the system is installed and updated, the following steps can be performed as root From 6914d29e3c436360db350aff98753c26a49b470a Mon Sep 17 00:00:00 2001 From: iwitz Date: Thu, 7 Feb 2019 09:31:47 +0100 Subject: [PATCH 4/9] chg: add index to all commands this is useful when only some steps apply to a particular environment --- docs/INSTALL.rhel7.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index ea1aff9b9..676067eeb 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -332,6 +332,7 @@ firewall-cmd --reload Check out the apache.misp.ssl file for an example # 8/ Log Rotation +## 8.01/ Enable log rotation MISP saves the stdout and stderr of it's workers in /var/www/MISP/app/tmp/logs To rotate these logs install the supplied logrotate script: ``` @@ -339,13 +340,13 @@ cp INSTALL/misp.logrotate /etc/logrotate.d/misp chmod 0640 /etc/logrotate.d/misp ``` -## 8.01/ Allow logrotate to work under SELinux and modify the log files +## 8.02/ Allow logrotate to work under SELinux and modify the log files ```bash semanage fcontext -a -t httpd_log_t "/var/www/MISP/app/tmp/logs(/.*)?" chcon -R -t httpd_log_t /var/www/MISP/app/tmp/logs ``` -## 8.02/ Allow logrotate to read /var/www +## 8.03/ Allow logrotate to read /var/www ```bash checkmodule -M -m -o /tmp/misplogrotate.mod INSTALL/misplogrotate.te semodule_package -o /tmp/misplogrotate.pp -m /tmp/misplogrotate.mod From 6a45e03b81566cf8f77bf30470d92f08f2dae2f5 Mon Sep 17 00:00:00 2001 From: iwitz Date: Thu, 7 Feb 2019 10:02:39 +0100 Subject: [PATCH 5/9] chg: no more assumption that only 1 shell is used scl enable was used in step 3 to make php, mysql and redis available in a new shell and during the rest of the installation process it was assumed that the same shell was used. The steps are a bit less interdependent now. --- docs/INSTALL.rhel7.md | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index 676067eeb..0dabfd82b 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -84,8 +84,7 @@ yum install rh-mariadb102 ## 2.03/ Start the MariaDB service and enable it to start on boot ```bash -systemctl start rh-mariadb102-mariadb.service -systemctl enable rh-mariadb102-mariadb.service +systemctl enable --now rh-mariadb102-mariadb.service ``` !!! note @@ -105,8 +104,7 @@ yum install rh-php71 rh-php71-php-fpm rh-php71-php-devel rh-php71-php-mysqlnd rh ## 2.05/ Start the PHP FPM service and enable to start on boot ```bash -systemctl start rh-php71-php-fpm.service -systemctl enable rh-php71-php-fpm.service +systemctl enable --now rh-php71-php-fpm.service ``` ## 2.06/ Install redis 3.2 from SCL @@ -116,22 +114,17 @@ yum install rh-redis32 ## 2.07/ Start redis service and enable to start on boot ```bash -systemctl start rh-redis32-redis.service -systemctl enable rh-redis32-redis.service -``` - -## 2.08/ Start a SCL shell with rh-mariadb102 rh-php71 and rh-redis32 enabled -```bash -scl enable rh-mariadb102 rh-php71 rh-redis32 bash +systemctl enable --now rh-redis32-redis.service ``` ## 2.08/ Secure the MariaDB installation, run the following command and follow the prompts ```bash -mysql_secure_installation +scl enable rh-mariadb102 'mysql_secure_installation' ``` ## 2.10/ Update the PHP extension repository and install required package ```bash +scl enable rh-php71 rh-redis32 bash pear channel-update pear.php.net pear install Crypt_GPG ``` @@ -139,8 +132,7 @@ pear install Crypt_GPG ## 2.11/ Install haveged and enable to start on boot to provide entropy for GPG ```bash yum install haveged -systemctl start haveged -systemctl enable haveged +systemctl enable --now haveged ``` ## 2.12/ Install Python 3.6 from SCL @@ -228,7 +220,7 @@ php composer.phar install ## 4.03/ Install and configure php redis connector through pecl ```bash -pecl install redis +scl enable rh-php71 'pecl install redis' echo "extension=redis.so" > /etc/opt/rh/rh-php71/php-fpm.d/redis.ini ln -s ../php-fpm.d/redis.ini /etc/opt/rh/rh-php71/php.d/99-redis.ini systemctl restart rh-php71-php-fpm.service @@ -276,7 +268,7 @@ systemctl restart rh-mariadb102-mariadb ## 6.02/ Start MariaDB shell and create database ```bash -mysql -u root -p +scl enable rh-mariadb102 'mysql -u root -p' ``` ``` @@ -316,8 +308,7 @@ setsebool -P httpd_can_network_connect on ## 7.03/ Enable and start the httpd service ```bash -systemctl enable httpd.service -systemctl start httpd.service +systemctl enable --now httpd.service ``` ## 7.04/ Open a hole in the firewalld service From 8a213b36aca7db3f8f0313afed99dac1140be3f6 Mon Sep 17 00:00:00 2001 From: iwitz Date: Thu, 7 Feb 2019 10:03:36 +0100 Subject: [PATCH 6/9] fix: grammar --- docs/INSTALL.rhel7.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index 0dabfd82b..f1d02e28d 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -25,7 +25,7 @@ The following assumptions with regard to this installation have been made. ## 1.2/ Configure system hostname ```bash -sudo hostnamectl set-hostname misp # You're choice, in a production environment, it's best to use a FQDN +sudo hostnamectl set-hostname misp # Your choice, in a production environment, it's best to use a FQDN ``` ## 1.3/ Register the system for updates with Red Hat Subscription Manager From 49ca9ade679b057756a0bc253da90f462a0d3b37 Mon Sep 17 00:00:00 2001 From: iwitz Date: Thu, 7 Feb 2019 10:05:49 +0100 Subject: [PATCH 7/9] chg: update php version warning --- docs/INSTALL.rhel7.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index f1d02e28d..a383d389d 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -92,7 +92,7 @@ systemctl enable --now rh-mariadb102-mariadb.service This guide installs PHP 7.1 from SCL !!! warning - [PHP 5.6 will be EOL in December 2018](https://secure.php.net/supported-versions.php). Please update accordingly. In future only PHP7 will be supported. + [PHP 5.6 and 7.0 aren't supported since December 2018](https://secure.php.net/supported-versions.php). Please update accordingly. In the future only PHP7 will be supported. ## 2.04/ Install PHP 7.1 from SCL ```bash From abd10e5f51be518184cc4889e98ecdbda253528c Mon Sep 17 00:00:00 2001 From: Martin Kulhavy Date: Thu, 7 Feb 2019 13:02:22 +0200 Subject: [PATCH 8/9] Add LDAP network timeout --- app/Config/config.default.php | 1 + app/Controller/Component/Auth/ApacheAuthenticate.php | 2 ++ 2 files changed, 3 insertions(+) diff --git a/app/Config/config.default.php b/app/Config/config.default.php index 026f9ad74..542617230 100644 --- a/app/Config/config.default.php +++ b/app/Config/config.default.php @@ -134,6 +134,7 @@ $config = array( 'apacheEnv' => 'REMOTE_USER', // If proxy variable = HTTP_REMOTE_USER 'ldapServer' => 'ldap://example.com', // FQDN or IP 'ldapProtocol' => 3, + 'ldapNetworkTimeout' => -1, // use -1 for unlimited network timeout 'ldapReaderUser' => 'cn=userWithReadAccess,ou=users,dc=example,dc=com', // DN ou RDN LDAP with reader user right 'ldapReaderPassword' => 'UserPassword', // the LDAP reader user password 'ldapDN' => 'dc=example,dc=com', diff --git a/app/Controller/Component/Auth/ApacheAuthenticate.php b/app/Controller/Component/Auth/ApacheAuthenticate.php index 38c3868ab..abedd0c99 100644 --- a/app/Controller/Component/Auth/ApacheAuthenticate.php +++ b/app/Controller/Component/Auth/ApacheAuthenticate.php @@ -51,7 +51,9 @@ class ApacheAuthenticate extends BaseAuthenticate $ldaprdn = Configure::read('ApacheSecureAuth.ldapReaderUser'); // DN ou RDN LDAP $ldappass = Configure::read('ApacheSecureAuth.ldapReaderPassword'); $ldapSearchFilter = Configure::read('ApacheSecureAuth.ldapSearchFilter'); + // LDAP connection + ldap_set_option(NULL, LDAP_OPT_NETWORK_TIMEOUT, Configure::read('ApacheSecureAuth.ldapNetworkTimeout', -1)); $ldapconn = ldap_connect(Configure::read('ApacheSecureAuth.ldapServer')) or die('LDAP server connection failed'); From aef4e51cdb610c77d8aa42ad106757594ad0a57e Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Fri, 8 Feb 2019 11:26:42 +0100 Subject: [PATCH 9/9] add: [diagnostic] Added check for STIX2 python library --- app/Controller/ServersController.php | 7 ++++--- app/Model/Server.php | 6 +++--- app/View/Elements/healthElements/diagnostics.ctp | 3 ++- app/files/scripts/stixtest.py | 10 +++++++++- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index 79d093ee2..8e7130b38 100644 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -868,10 +868,11 @@ class ServersController extends AppController $zmqErrors = array(0 => 'OK', 1 => 'not enabled (so not tested)', 2 => 'Python ZeroMQ library not installed correctly.', 3 => 'ZeroMQ script not running.'); $stixOperational = array(0 => 'Some of the libraries related to STIX are not installed. Make sure that all libraries listed below are correctly installed.', 1 => 'OK'); $stixVersion = array(0 => 'Incorrect STIX version installed, found $current, expecting $expected', 1 => 'OK'); + $stix2Version = array(0 => 'Incorrect STIX2 version installed, found $current, expecting $expected', 1 => 'OK'); $cyboxVersion = array(0 => 'Incorrect CyBox version installed, found $current, expecting $expected', 1 => 'OK'); $mixboxVersion = array(0 => 'Incorrect mixbox version installed, found $current, expecting $expected', 1 => 'OK'); $maecVersion = array(0 => 'Incorrect maec version installed, found $current, expecting $expected', 1 => 'OK'); - $pymispVersion = array(0 => 'Incorrect pymisp version installed, found $current, expecting $expected', 1 => 'OK'); + $pymispVersion = array(0 => 'Incorrect PyMISP version installed, found $current, expecting $expected', 1 => 'OK'); $sessionErrors = array(0 => 'OK', 1 => 'High', 2 => 'Alternative setting used', 3 => 'Test failed'); $moduleErrors = array(0 => 'OK', 1 => 'System not enabled', 2 => 'No modules found'); @@ -991,7 +992,7 @@ class ServersController extends AppController } // check if the STIX and Cybox libraries are working and the correct version using the test script stixtest.py - $stix = $this->Server->stixDiagnostics($diagnostic_errors, $stixVersion, $cyboxVersion, $mixboxVersion, $maecVersion, $pymispVersion); + $stix = $this->Server->stixDiagnostics($diagnostic_errors, $stixVersion, $cyboxVersion, $mixboxVersion, $maecVersion, $stix2Version, $pymispVersion); // if GnuPG is set up in the settings, try to encrypt a test message $gpgStatus = $this->Server->gpgDiagnostics($diagnostic_errors); @@ -1012,7 +1013,7 @@ class ServersController extends AppController $sessionStatus = $this->Server->sessionDiagnostics($diagnostic_errors, $sessionCount); $this->set('sessionCount', $sessionCount); - $additionalViewVars = array('gpgStatus', 'sessionErrors', 'proxyStatus', 'sessionStatus', 'zmqStatus', 'stixVersion', 'cyboxVersion', 'mixboxVersion', 'maecVersion', 'pymispVersion', 'moduleStatus', 'gpgErrors', 'proxyErrors', 'zmqErrors', 'stixOperational', 'stix', 'moduleErrors', 'moduleTypes'); + $additionalViewVars = array('gpgStatus', 'sessionErrors', 'proxyStatus', 'sessionStatus', 'zmqStatus', 'stixVersion', 'cyboxVersion', 'mixboxVersion', 'maecVersion', 'stix2Version', 'pymispVersion', 'moduleStatus', 'gpgErrors', 'proxyErrors', 'zmqErrors', 'stixOperational', 'stix', 'moduleErrors', 'moduleTypes'); } // check whether the files are writeable $writeableDirs = $this->Server->writeableDirsDiagnostics($diagnostic_errors); diff --git a/app/Model/Server.php b/app/Model/Server.php index a30091d47..35627b35a 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -3777,15 +3777,15 @@ class Server extends AppModel return $readableFiles; } - public function stixDiagnostics(&$diagnostic_errors, &$stixVersion, &$cyboxVersion, &$mixboxVersion, &$maecVersion, &$pymispVersion) + public function stixDiagnostics(&$diagnostic_errors, &$stixVersion, &$cyboxVersion, &$mixboxVersion, &$maecVersion, &$stix2Version, &$pymispVersion) { $result = array(); - $expected = array('stix' => '1.2.0.6', 'cybox' => '2.1.0.18.dev0', 'mixbox' => '1.0.3', 'maec' => '4.1.0.14', 'pymisp' => '>2.4.93'); + $expected = array('stix' => '1.2.0.6', 'cybox' => '2.1.0.18.dev0', 'mixbox' => '1.0.3', 'maec' => '4.1.0.14', 'stix2' => '1.1.1', 'pymisp' => '>2.4.93'); // check if the STIX and Cybox libraries are working using the test script stixtest.py $scriptResult = shell_exec($this->getPythonVersion() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'stixtest.py'); $scriptResult = json_decode($scriptResult, true); if ($scriptResult == null) { - return array('operational' => 0, 'stix' => array('expected' => $expected['stix']), 'cybox' => array('expected' => $expected['cybox']), 'mixbox' => array('expected' => $expected['mixbox']), 'maec' => array('expected' => $expected['maec']), 'pymisp' => array('expected' => $expected['pymisp'])); + return array('operational' => 0, 'stix' => array('expected' => $expected['stix']), 'cybox' => array('expected' => $expected['cybox']), 'mixbox' => array('expected' => $expected['mixbox']), 'maec' => array('expected' => $expected['maec']), 'stix2' => array('expected' => $expected['stix2']), 'pymisp' => array('expected' => $expected['pymisp'])); } $scriptResult['operational'] = $scriptResult['success']; if ($scriptResult['operational'] == 0) { diff --git a/app/View/Elements/healthElements/diagnostics.ctp b/app/View/Elements/healthElements/diagnostics.ctp index ea5b47fbc..f22cfadc4 100644 --- a/app/View/Elements/healthElements/diagnostics.ctp +++ b/app/View/Elements/healthElements/diagnostics.ctp @@ -203,6 +203,7 @@ CyBox:
mixbox:
maec:
+ STIX2:
PyMISP:

@@ -221,7 +222,7 @@ if (!$testReadError) { $error_count = 0; $libraries = ''; - foreach (array('stix', 'cybox', 'mixbox', 'maec', 'pymisp') as $package) { + foreach (array('stix', 'cybox', 'mixbox', 'maec', 'stix2', 'pymisp') as $package) { $lib_colour = 'green'; if ($stix[$package]['status'] == 0) { $lib_colour = 'red'; diff --git a/app/files/scripts/stixtest.py b/app/files/scripts/stixtest.py index 5888bb577..dea465d33 100644 --- a/app/files/scripts/stixtest.py +++ b/app/files/scripts/stixtest.py @@ -8,6 +8,7 @@ results = { 'cybox': 0, 'mixbox': 0, 'maec': 0, + 'stix2': 0, 'pymisp': 0 } try: @@ -40,12 +41,19 @@ try: except Exception: results['success'] = 0 +try: + import stix2 + results['stix2'] = stix2.__version__ +except Exception: + results['success'] = 0 + print(json.dumps({ 'success': results['success'], 'stix': results['stix'], 'cybox': results['cybox'], 'mixbox': results['mixbox'], 'maec': results['maec'], + 'stix2': results['stix2'], 'pymisp': results['pymisp'] })) -sys.exit(1) +sys.exit(0)