From dd2e6cc715beb524e2631fdf8b8949506860d7b1 Mon Sep 17 00:00:00 2001
From: iwitz
Date: Wed, 6 Feb 2019 15:08:40 +0100
Subject: [PATCH 1/9] fix: lief installation
* unnecessary steps removed
* irrelevant comment removed
* instructions updated to compile lief for python 3.6 instead of 2.7
* install lief in the folder where the other dependencies are so that it is detected by the diagnostics page
---
docs/INSTALL.rhel7.md | 35 +++++++++++------------------------
1 file changed, 11 insertions(+), 24 deletions(-)
diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md
index ea1aff9b9..6a47775bf 100644
--- a/docs/INSTALL.rhel7.md
+++ b/docs/INSTALL.rhel7.md
@@ -486,44 +486,31 @@ scl enable rh-python36 pip3 install pymisp
yum install devtoolset-7 cmake3
```
-## 11.02/ Enable devtoolset-7
+## 11.02/ Create the directory and download the source code
```bash
-scl enable devtoolset-7 bash
+cd /var/www/MISP/app/files/scripts
+git clone --branch master --single-branch https://github.com/lief-project/LIEF.git lief
```
-## 11.03/ Set env variable, create directories and download source code
+## 11.03/ Compile lief and install it
```bash
-mkdir -p /tmp/LIEF
-mkdir -p /tmp/LIEF_INSTALL
-export LIEF_TMP=/tmp/LIEF
-export LIEF_INSTALL=/tmp/LIEF_INSTALL
-export LIEF_BRANCH=master
-cd $LIEF_TMP
-git clone --branch $LIEF_BRANCH --single-branch https://github.com/lief-project/LIEF.git LIEF
-```
-
-## 11.04/ Compile lief and install
-```bash
-cd $LIEF_TMP/LIEF
-mkdir -p build
+cd /var/www/MISP/app/files/scripts/lief
+mkdir build
cd build
-scl enable devtoolset-7 'bash -c "cmake3 \
+scl enable devtoolset-7 rh-python36 'bash -c "cmake3 \
-DLIEF_PYTHON_API=on \
-DLIEF_DOC=off \
-DCMAKE_INSTALL_PREFIX=$LIEF_INSTALL \
-DCMAKE_BUILD_TYPE=Release \
--DPYTHON_VERSION=2.7 \
+-DPYTHON_VERSION=3.6 \
.."'
make -j3
cd api/python
-scl enable rh-python36 python3 setup.py install || :
-# you can ignore the error about finding suitable distribution
-cd $LIEF_TMP/LIEF/build
-make install
-make package
+# before you run setup.py it may be a good idea to switch off your Internet connection ; otherwise pip will try to fetch lief from Github which may cause issues later
+scl enable rh-python36 'python3 setup.py install || :'
```
-## 11.05/ Test lief installation, if no error, package installed
+## 11.04/ Test lief installation, if no error, package installed
```bash
python
>> import lief
From 39e7dbc2962d016f68bfc54c6b6c18981b01ba9b Mon Sep 17 00:00:00 2001
From: iwitz
Date: Wed, 6 Feb 2019 15:58:04 +0100
Subject: [PATCH 2/9] chg: improved setup bug workaround
---
docs/INSTALL.rhel7.md | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md
index 6a47775bf..2031dfdd6 100644
--- a/docs/INSTALL.rhel7.md
+++ b/docs/INSTALL.rhel7.md
@@ -506,13 +506,14 @@ scl enable devtoolset-7 rh-python36 'bash -c "cmake3 \
.."'
make -j3
cd api/python
-# before you run setup.py it may be a good idea to switch off your Internet connection ; otherwise pip will try to fetch lief from Github which may cause issues later
scl enable rh-python36 'python3 setup.py install || :'
+# when running setup.py, pip will download and install remote LIEF packages that will prevent MISP from detecting the packages that you compiled ; remove them
+find /opt/rh/rh-python36/root/ -name "*lief*" -exec rm -rf {} \;
```
## 11.04/ Test lief installation, if no error, package installed
```bash
-python
+scl enable rh-python36 python3
>> import lief
```
From cbf3262bcac4037fa2b25f407749c21aaab09fbb Mon Sep 17 00:00:00 2001
From: iwitz
Date: Thu, 7 Feb 2019 08:58:01 +0100
Subject: [PATCH 3/9] chg: remove centos step
The instruction to enable the SCL repos is already provided in step 1.4,
and the package centos-release-scl does not exist on RHEL7. The fact
that step 1.4 may not work on a trial subscription is not relevant
because assumptions 0.1 and 0.2 specifically state that a support
agreement is required to be able to complete the installation.
---
docs/INSTALL.rhel7.md | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md
index ea1aff9b9..549768777 100644
--- a/docs/INSTALL.rhel7.md
+++ b/docs/INSTALL.rhel7.md
@@ -38,8 +38,7 @@ sudo subscription-manager register --auto-attach # register your system to an ac
sudo subscription-manager refresh
sudo subscription-manager repos --enable rhel-7-server-optional-rpms
sudo subscription-manager repos --enable rhel-7-server-extras-rpms
-# This fails on a Trial subscription, it seems.
-##sudo subscription-manager repos --enable rhel-server-rhscl-7-rpms
+sudo subscription-manager repos --enable rhel-server-rhscl-7-rpms
```
### 1.5a/ OPTIONAL: Install the deltarpm package to help reduce download size when installing updates
@@ -64,11 +63,6 @@ yum update -y
yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm -y
```
-## 1.7/ Install the SCL repo
-```bash
-yum install centos-release-scl
-```
-
# 2/ Install Dependencies
Once the system is installed and updated, the following steps can be performed as root
From 6914d29e3c436360db350aff98753c26a49b470a Mon Sep 17 00:00:00 2001
From: iwitz
Date: Thu, 7 Feb 2019 09:31:47 +0100
Subject: [PATCH 4/9] chg: add index to all commands
this is useful when only some steps apply to a particular environment
---
docs/INSTALL.rhel7.md | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md
index ea1aff9b9..676067eeb 100644
--- a/docs/INSTALL.rhel7.md
+++ b/docs/INSTALL.rhel7.md
@@ -332,6 +332,7 @@ firewall-cmd --reload
Check out the apache.misp.ssl file for an example
# 8/ Log Rotation
+## 8.01/ Enable log rotation
MISP saves the stdout and stderr of it's workers in /var/www/MISP/app/tmp/logs
To rotate these logs install the supplied logrotate script:
```
@@ -339,13 +340,13 @@ cp INSTALL/misp.logrotate /etc/logrotate.d/misp
chmod 0640 /etc/logrotate.d/misp
```
-## 8.01/ Allow logrotate to work under SELinux and modify the log files
+## 8.02/ Allow logrotate to work under SELinux and modify the log files
```bash
semanage fcontext -a -t httpd_log_t "/var/www/MISP/app/tmp/logs(/.*)?"
chcon -R -t httpd_log_t /var/www/MISP/app/tmp/logs
```
-## 8.02/ Allow logrotate to read /var/www
+## 8.03/ Allow logrotate to read /var/www
```bash
checkmodule -M -m -o /tmp/misplogrotate.mod INSTALL/misplogrotate.te
semodule_package -o /tmp/misplogrotate.pp -m /tmp/misplogrotate.mod
From 6a45e03b81566cf8f77bf30470d92f08f2dae2f5 Mon Sep 17 00:00:00 2001
From: iwitz
Date: Thu, 7 Feb 2019 10:02:39 +0100
Subject: [PATCH 5/9] chg: no more assumption that only 1 shell is used
scl enable was used in step 3 to make php, mysql and redis available in a new shell and during the rest of the installation process it was assumed that the same shell was used. The steps are a bit less interdependent now.
---
docs/INSTALL.rhel7.md | 27 +++++++++------------------
1 file changed, 9 insertions(+), 18 deletions(-)
diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md
index 676067eeb..0dabfd82b 100644
--- a/docs/INSTALL.rhel7.md
+++ b/docs/INSTALL.rhel7.md
@@ -84,8 +84,7 @@ yum install rh-mariadb102
## 2.03/ Start the MariaDB service and enable it to start on boot
```bash
-systemctl start rh-mariadb102-mariadb.service
-systemctl enable rh-mariadb102-mariadb.service
+systemctl enable --now rh-mariadb102-mariadb.service
```
!!! note
@@ -105,8 +104,7 @@ yum install rh-php71 rh-php71-php-fpm rh-php71-php-devel rh-php71-php-mysqlnd rh
## 2.05/ Start the PHP FPM service and enable to start on boot
```bash
-systemctl start rh-php71-php-fpm.service
-systemctl enable rh-php71-php-fpm.service
+systemctl enable --now rh-php71-php-fpm.service
```
## 2.06/ Install redis 3.2 from SCL
@@ -116,22 +114,17 @@ yum install rh-redis32
## 2.07/ Start redis service and enable to start on boot
```bash
-systemctl start rh-redis32-redis.service
-systemctl enable rh-redis32-redis.service
-```
-
-## 2.08/ Start a SCL shell with rh-mariadb102 rh-php71 and rh-redis32 enabled
-```bash
-scl enable rh-mariadb102 rh-php71 rh-redis32 bash
+systemctl enable --now rh-redis32-redis.service
```
## 2.08/ Secure the MariaDB installation, run the following command and follow the prompts
```bash
-mysql_secure_installation
+scl enable rh-mariadb102 'mysql_secure_installation'
```
## 2.10/ Update the PHP extension repository and install required package
```bash
+scl enable rh-php71 rh-redis32 bash
pear channel-update pear.php.net
pear install Crypt_GPG
```
@@ -139,8 +132,7 @@ pear install Crypt_GPG
## 2.11/ Install haveged and enable to start on boot to provide entropy for GPG
```bash
yum install haveged
-systemctl start haveged
-systemctl enable haveged
+systemctl enable --now haveged
```
## 2.12/ Install Python 3.6 from SCL
@@ -228,7 +220,7 @@ php composer.phar install
## 4.03/ Install and configure php redis connector through pecl
```bash
-pecl install redis
+scl enable rh-php71 'pecl install redis'
echo "extension=redis.so" > /etc/opt/rh/rh-php71/php-fpm.d/redis.ini
ln -s ../php-fpm.d/redis.ini /etc/opt/rh/rh-php71/php.d/99-redis.ini
systemctl restart rh-php71-php-fpm.service
@@ -276,7 +268,7 @@ systemctl restart rh-mariadb102-mariadb
## 6.02/ Start MariaDB shell and create database
```bash
-mysql -u root -p
+scl enable rh-mariadb102 'mysql -u root -p'
```
```
@@ -316,8 +308,7 @@ setsebool -P httpd_can_network_connect on
## 7.03/ Enable and start the httpd service
```bash
-systemctl enable httpd.service
-systemctl start httpd.service
+systemctl enable --now httpd.service
```
## 7.04/ Open a hole in the firewalld service
From 8a213b36aca7db3f8f0313afed99dac1140be3f6 Mon Sep 17 00:00:00 2001
From: iwitz
Date: Thu, 7 Feb 2019 10:03:36 +0100
Subject: [PATCH 6/9] fix: grammar
---
docs/INSTALL.rhel7.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md
index 0dabfd82b..f1d02e28d 100644
--- a/docs/INSTALL.rhel7.md
+++ b/docs/INSTALL.rhel7.md
@@ -25,7 +25,7 @@ The following assumptions with regard to this installation have been made.
## 1.2/ Configure system hostname
```bash
-sudo hostnamectl set-hostname misp # You're choice, in a production environment, it's best to use a FQDN
+sudo hostnamectl set-hostname misp # Your choice, in a production environment, it's best to use a FQDN
```
## 1.3/ Register the system for updates with Red Hat Subscription Manager
From 49ca9ade679b057756a0bc253da90f462a0d3b37 Mon Sep 17 00:00:00 2001
From: iwitz
Date: Thu, 7 Feb 2019 10:05:49 +0100
Subject: [PATCH 7/9] chg: update php version warning
---
docs/INSTALL.rhel7.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md
index f1d02e28d..a383d389d 100644
--- a/docs/INSTALL.rhel7.md
+++ b/docs/INSTALL.rhel7.md
@@ -92,7 +92,7 @@ systemctl enable --now rh-mariadb102-mariadb.service
This guide installs PHP 7.1 from SCL
!!! warning
- [PHP 5.6 will be EOL in December 2018](https://secure.php.net/supported-versions.php). Please update accordingly. In future only PHP7 will be supported.
+ [PHP 5.6 and 7.0 aren't supported since December 2018](https://secure.php.net/supported-versions.php). Please update accordingly. In the future only PHP7 will be supported.
## 2.04/ Install PHP 7.1 from SCL
```bash
From abd10e5f51be518184cc4889e98ecdbda253528c Mon Sep 17 00:00:00 2001
From: Martin Kulhavy
Date: Thu, 7 Feb 2019 13:02:22 +0200
Subject: [PATCH 8/9] Add LDAP network timeout
---
app/Config/config.default.php | 1 +
app/Controller/Component/Auth/ApacheAuthenticate.php | 2 ++
2 files changed, 3 insertions(+)
diff --git a/app/Config/config.default.php b/app/Config/config.default.php
index 026f9ad74..542617230 100644
--- a/app/Config/config.default.php
+++ b/app/Config/config.default.php
@@ -134,6 +134,7 @@ $config = array(
'apacheEnv' => 'REMOTE_USER', // If proxy variable = HTTP_REMOTE_USER
'ldapServer' => 'ldap://example.com', // FQDN or IP
'ldapProtocol' => 3,
+ 'ldapNetworkTimeout' => -1, // use -1 for unlimited network timeout
'ldapReaderUser' => 'cn=userWithReadAccess,ou=users,dc=example,dc=com', // DN ou RDN LDAP with reader user right
'ldapReaderPassword' => 'UserPassword', // the LDAP reader user password
'ldapDN' => 'dc=example,dc=com',
diff --git a/app/Controller/Component/Auth/ApacheAuthenticate.php b/app/Controller/Component/Auth/ApacheAuthenticate.php
index 38c3868ab..abedd0c99 100644
--- a/app/Controller/Component/Auth/ApacheAuthenticate.php
+++ b/app/Controller/Component/Auth/ApacheAuthenticate.php
@@ -51,7 +51,9 @@ class ApacheAuthenticate extends BaseAuthenticate
$ldaprdn = Configure::read('ApacheSecureAuth.ldapReaderUser'); // DN ou RDN LDAP
$ldappass = Configure::read('ApacheSecureAuth.ldapReaderPassword');
$ldapSearchFilter = Configure::read('ApacheSecureAuth.ldapSearchFilter');
+
// LDAP connection
+ ldap_set_option(NULL, LDAP_OPT_NETWORK_TIMEOUT, Configure::read('ApacheSecureAuth.ldapNetworkTimeout', -1));
$ldapconn = ldap_connect(Configure::read('ApacheSecureAuth.ldapServer'))
or die('LDAP server connection failed');
From aef4e51cdb610c77d8aa42ad106757594ad0a57e Mon Sep 17 00:00:00 2001
From: chrisr3d
Date: Fri, 8 Feb 2019 11:26:42 +0100
Subject: [PATCH 9/9] add: [diagnostic] Added check for STIX2 python library
---
app/Controller/ServersController.php | 7 ++++---
app/Model/Server.php | 6 +++---
app/View/Elements/healthElements/diagnostics.ctp | 3 ++-
app/files/scripts/stixtest.py | 10 +++++++++-
4 files changed, 18 insertions(+), 8 deletions(-)
diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php
index 79d093ee2..8e7130b38 100644
--- a/app/Controller/ServersController.php
+++ b/app/Controller/ServersController.php
@@ -868,10 +868,11 @@ class ServersController extends AppController
$zmqErrors = array(0 => 'OK', 1 => 'not enabled (so not tested)', 2 => 'Python ZeroMQ library not installed correctly.', 3 => 'ZeroMQ script not running.');
$stixOperational = array(0 => 'Some of the libraries related to STIX are not installed. Make sure that all libraries listed below are correctly installed.', 1 => 'OK');
$stixVersion = array(0 => 'Incorrect STIX version installed, found $current, expecting $expected', 1 => 'OK');
+ $stix2Version = array(0 => 'Incorrect STIX2 version installed, found $current, expecting $expected', 1 => 'OK');
$cyboxVersion = array(0 => 'Incorrect CyBox version installed, found $current, expecting $expected', 1 => 'OK');
$mixboxVersion = array(0 => 'Incorrect mixbox version installed, found $current, expecting $expected', 1 => 'OK');
$maecVersion = array(0 => 'Incorrect maec version installed, found $current, expecting $expected', 1 => 'OK');
- $pymispVersion = array(0 => 'Incorrect pymisp version installed, found $current, expecting $expected', 1 => 'OK');
+ $pymispVersion = array(0 => 'Incorrect PyMISP version installed, found $current, expecting $expected', 1 => 'OK');
$sessionErrors = array(0 => 'OK', 1 => 'High', 2 => 'Alternative setting used', 3 => 'Test failed');
$moduleErrors = array(0 => 'OK', 1 => 'System not enabled', 2 => 'No modules found');
@@ -991,7 +992,7 @@ class ServersController extends AppController
}
// check if the STIX and Cybox libraries are working and the correct version using the test script stixtest.py
- $stix = $this->Server->stixDiagnostics($diagnostic_errors, $stixVersion, $cyboxVersion, $mixboxVersion, $maecVersion, $pymispVersion);
+ $stix = $this->Server->stixDiagnostics($diagnostic_errors, $stixVersion, $cyboxVersion, $mixboxVersion, $maecVersion, $stix2Version, $pymispVersion);
// if GnuPG is set up in the settings, try to encrypt a test message
$gpgStatus = $this->Server->gpgDiagnostics($diagnostic_errors);
@@ -1012,7 +1013,7 @@ class ServersController extends AppController
$sessionStatus = $this->Server->sessionDiagnostics($diagnostic_errors, $sessionCount);
$this->set('sessionCount', $sessionCount);
- $additionalViewVars = array('gpgStatus', 'sessionErrors', 'proxyStatus', 'sessionStatus', 'zmqStatus', 'stixVersion', 'cyboxVersion', 'mixboxVersion', 'maecVersion', 'pymispVersion', 'moduleStatus', 'gpgErrors', 'proxyErrors', 'zmqErrors', 'stixOperational', 'stix', 'moduleErrors', 'moduleTypes');
+ $additionalViewVars = array('gpgStatus', 'sessionErrors', 'proxyStatus', 'sessionStatus', 'zmqStatus', 'stixVersion', 'cyboxVersion', 'mixboxVersion', 'maecVersion', 'stix2Version', 'pymispVersion', 'moduleStatus', 'gpgErrors', 'proxyErrors', 'zmqErrors', 'stixOperational', 'stix', 'moduleErrors', 'moduleTypes');
}
// check whether the files are writeable
$writeableDirs = $this->Server->writeableDirsDiagnostics($diagnostic_errors);
diff --git a/app/Model/Server.php b/app/Model/Server.php
index a30091d47..35627b35a 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -3777,15 +3777,15 @@ class Server extends AppModel
return $readableFiles;
}
- public function stixDiagnostics(&$diagnostic_errors, &$stixVersion, &$cyboxVersion, &$mixboxVersion, &$maecVersion, &$pymispVersion)
+ public function stixDiagnostics(&$diagnostic_errors, &$stixVersion, &$cyboxVersion, &$mixboxVersion, &$maecVersion, &$stix2Version, &$pymispVersion)
{
$result = array();
- $expected = array('stix' => '1.2.0.6', 'cybox' => '2.1.0.18.dev0', 'mixbox' => '1.0.3', 'maec' => '4.1.0.14', 'pymisp' => '>2.4.93');
+ $expected = array('stix' => '1.2.0.6', 'cybox' => '2.1.0.18.dev0', 'mixbox' => '1.0.3', 'maec' => '4.1.0.14', 'stix2' => '1.1.1', 'pymisp' => '>2.4.93');
// check if the STIX and Cybox libraries are working using the test script stixtest.py
$scriptResult = shell_exec($this->getPythonVersion() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'stixtest.py');
$scriptResult = json_decode($scriptResult, true);
if ($scriptResult == null) {
- return array('operational' => 0, 'stix' => array('expected' => $expected['stix']), 'cybox' => array('expected' => $expected['cybox']), 'mixbox' => array('expected' => $expected['mixbox']), 'maec' => array('expected' => $expected['maec']), 'pymisp' => array('expected' => $expected['pymisp']));
+ return array('operational' => 0, 'stix' => array('expected' => $expected['stix']), 'cybox' => array('expected' => $expected['cybox']), 'mixbox' => array('expected' => $expected['mixbox']), 'maec' => array('expected' => $expected['maec']), 'stix2' => array('expected' => $expected['stix2']), 'pymisp' => array('expected' => $expected['pymisp']));
}
$scriptResult['operational'] = $scriptResult['success'];
if ($scriptResult['operational'] == 0) {
diff --git a/app/View/Elements/healthElements/diagnostics.ctp b/app/View/Elements/healthElements/diagnostics.ctp
index ea5b47fbc..f22cfadc4 100644
--- a/app/View/Elements/healthElements/diagnostics.ctp
+++ b/app/View/Elements/healthElements/diagnostics.ctp
@@ -203,6 +203,7 @@
CyBox:
mixbox:
maec:
+ STIX2:
PyMISP:
@@ -221,7 +222,7 @@
if (!$testReadError) {
$error_count = 0;
$libraries = '';
- foreach (array('stix', 'cybox', 'mixbox', 'maec', 'pymisp') as $package) {
+ foreach (array('stix', 'cybox', 'mixbox', 'maec', 'stix2', 'pymisp') as $package) {
$lib_colour = 'green';
if ($stix[$package]['status'] == 0) {
$lib_colour = 'red';
diff --git a/app/files/scripts/stixtest.py b/app/files/scripts/stixtest.py
index 5888bb577..dea465d33 100644
--- a/app/files/scripts/stixtest.py
+++ b/app/files/scripts/stixtest.py
@@ -8,6 +8,7 @@ results = {
'cybox': 0,
'mixbox': 0,
'maec': 0,
+ 'stix2': 0,
'pymisp': 0
}
try:
@@ -40,12 +41,19 @@ try:
except Exception:
results['success'] = 0
+try:
+ import stix2
+ results['stix2'] = stix2.__version__
+except Exception:
+ results['success'] = 0
+
print(json.dumps({
'success': results['success'],
'stix': results['stix'],
'cybox': results['cybox'],
'mixbox': results['mixbox'],
'maec': results['maec'],
+ 'stix2': results['stix2'],
'pymisp': results['pymisp']
}))
-sys.exit(1)
+sys.exit(0)