From ed2368fe86407e7511c7ee08becb723fe27b1112 Mon Sep 17 00:00:00 2001 From: Kortho Date: Tue, 18 Feb 2020 14:31:35 +0100 Subject: [PATCH 1/7] fixed python venv creation command The command for creating virtual environment in RHEL was wrong, fixed it :) --- docs/INSTALL.rhel7.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index 096fc3420..8eb95ecf6 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -218,7 +218,7 @@ installCoreRHEL () { # Create a python3 virtualenv sudo pip3 install virtualenv - $SUDO_WWW python3 -- virtualenv -p python3 $PATH_TO_MISP/venv + $SUDO_WWW python3 -m venv $PATH_TO_MISP/venv sudo mkdir /usr/share/httpd/.cache sudo chown $WWW_USER:$WWW_USER /usr/share/httpd/.cache $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U pip setuptools From 16c6d43d200c2db6f3062911fe353d3f920a0c28 Mon Sep 17 00:00:00 2001 From: Golbark Date: Wed, 18 Mar 2020 16:16:16 +0100 Subject: [PATCH 2/7] new: usr: add support for RHEL in the install script --- INSTALL/INSTALL.sh | 725 +++++++++++++++++++++++++++++-- INSTALL/INSTALL.sh.sha1 | 2 +- INSTALL/INSTALL.sh.sha256 | 2 +- INSTALL/INSTALL.sh.sha384 | 2 +- INSTALL/INSTALL.sh.sha512 | 2 +- INSTALL/INSTALL.tpl.sh | 130 +++++- docs/INSTALL.rhel7.md | 28 +- docs/generic/supportFunctions.md | 58 ++- 8 files changed, 879 insertions(+), 70 deletions(-) mode change 100644 => 100755 INSTALL/INSTALL.sh diff --git a/INSTALL/INSTALL.sh b/INSTALL/INSTALL.sh old mode 100644 new mode 100755 index 86b3437e5..1798a2846 --- a/INSTALL/INSTALL.sh +++ b/INSTALL/INSTALL.sh @@ -296,16 +296,17 @@ checkFlavour () { centos) if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then dist_version="$(. /etc/os-release && echo "$VERSION_ID")" + dist_version=${dist_version:0:1} fi - echo "$FLAVOUR not supported at the moment" - exit 1 + echo "$FLAVOUR support is experimental at the moment" ;; rhel|ol|sles) if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then dist_version="$(. /etc/os-release && echo "$VERSION_ID")" + dist_version=${dist_version:0:1} # Only interested about major version fi - echo "$FLAVOUR not supported at the moment" - exit 1 + # Only tested for RHEL 7 so far + echo "$FLAVOUR support is experimental at the moment" ;; *) if command_exists lsb_release; then @@ -373,25 +374,38 @@ EOF } checkInstaller () { - # TODO: Implement $FLAVOUR checks and install depending on the platform we are on - if [[ $(which shasum > /dev/null 2>&1 ; echo $?) != 0 ]]; then - sudo apt install libdigest-sha-perl -qyy - fi - # SHAsums to be computed, not the -- notatiation is for ease of use with rhash - SHA_SUMS="--sha1 --sha256 --sha384 --sha512" - for sum in $(echo ${SHA_SUMS} |sed 's/--sha//g'); do - /usr/bin/wget --no-cache -q -O /tmp/INSTALL.sh.sha${sum} https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/INSTALL.sh.sha${sum} - INSTsum=$(shasum -a ${sum} ${0} | cut -f1 -d\ ) - chsum=$(cat /tmp/INSTALL.sh.sha${sum} | cut -f1 -d\ ) - - if [[ "${chsum}" == "${INSTsum}" ]]; then - echo "sha${sum} matches" - else - echo "sha${sum}: ${chsum} does not match the installer sum of: ${INSTsum}" - echo "Delete installer, re-download and please run again." - exit 1 + # Workaround: shasum is not available on RHEL, only checking sha512 + if [[ $FLAVOUR == "rhel" ]] || [[ $FLAVOUR == "centos" ]]; then + INSTsum=$(sha512sum ${0} | cut -f1 -d\ ) + /usr/bin/wget --no-cache -q -O /tmp/INSTALL.sh.sha512 https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/INSTALL.sh.sha512 + chsum=$(cat /tmp/INSTALL.sh.sha512) + if [[ "${chsum}" == "${INSTsum}" ]]; then + echo "SHA512 matches" + else + echo "SHA512: ${chsum} does not match the installer sum of: ${INSTsum}" + # exit 1 # uncomment when/if PR is merged + fi + else + # TODO: Implement $FLAVOUR checks and install depending on the platform we are on + if [[ $(which shasum > /dev/null 2>&1 ; echo $?) != 0 ]]; then + sudo apt install libdigest-sha-perl -qyy fi - done + # SHAsums to be computed, not the -- notatiation is for ease of use with rhash + SHA_SUMS="--sha1 --sha256 --sha384 --sha512" + for sum in $(echo ${SHA_SUMS} |sed 's/--sha//g'); do + /usr/bin/wget --no-cache -q -O /tmp/INSTALL.sh.sha${sum} https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/INSTALL.sh.sha${sum} + INSTsum=$(shasum -a ${sum} ${0} | cut -f1 -d\ ) + chsum=$(cat /tmp/INSTALL.sh.sha${sum} | cut -f1 -d\ ) + + if [[ "${chsum}" == "${INSTsum}" ]]; then + echo "sha${sum} matches" + else + echo "sha${sum}: ${chsum} does not match the installer sum of: ${INSTsum}" + echo "Delete installer, re-download and please run again." + exit 1 + fi + done + fi } # Extract manufacturer @@ -1879,6 +1893,558 @@ viper () { } +enableReposRHEL () { + sudo subscription-manager refresh + sudo subscription-manager repos --enable rhel-7-server-optional-rpms + sudo subscription-manager repos --enable rhel-7-server-extras-rpms + sudo subscription-manager repos --enable rhel-server-rhscl-7-rpms +} + +centosEPEL () { + # We need some packages from the Extra Packages for Enterprise Linux repository + sudo yum install epel-release -y + + # Since MISP 2.4 PHP 5.5 is a minimal requirement, so we need a newer version than CentOS base provides + # Software Collections is a way do to this, see https://wiki.centos.org/AdditionalResources/Repositories/SCL + sudo yum install centos-release-scl -y +} + +enableEPEL () { + sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm -y +} + +yumInstallCoreDeps () { + # Install the dependencies: + sudo yum install gcc git zip rh-git218 \ + httpd24 \ + mod_ssl \ + rh-redis32 \ + rh-mariadb102 \ + libxslt-devel zlib-devel ssdeep-devel -y + + # Enable and start redis + sudo systemctl enable --now rh-redis32-redis.service + + WWW_USER="apache" + SUDO_WWW="sudo -H -u $WWW_USER" + RUN_PHP="/usr/bin/scl enable rh-php72" + PHP_INI="/etc/opt/rh/rh-php72/php.ini" + # Install PHP 7.2 from SCL, see https://www.softwarecollections.org/en/scls/rhscl/rh-php72/ + sudo yum install rh-php72 rh-php72-php-fpm rh-php72-php-devel \ + rh-php72-php-mysqlnd \ + rh-php72-php-mbstring \ + rh-php72-php-xml \ + rh-php72-php-bcmath \ + rh-php72-php-opcache \ + rh-php72-php-gd -y + + # Python 3.6 is now available in RHEL 7.7 base + sudo yum install python3 python3-devel -y + + sudo systemctl enable --now rh-php72-php-fpm.service +} + +installCoreRHEL () { + # Download MISP using git in the $PATH_TO_MISP directory. + sudo mkdir -p $(dirname $PATH_TO_MISP) + sudo chown $WWW_USER:$WWW_USER $(dirname $PATH_TO_MISP) + cd $(dirname $PATH_TO_MISP) + $SUDO_WWW git clone https://github.com/MISP/MISP.git + cd $PATH_TO_MISP + ##$SUDO_WWW git checkout tags/$(git describe --tags `git rev-list --tags --max-count=1`) + # if the last shortcut doesn't work, specify the latest version manually + # example: git checkout tags/v2.4.XY + # the message regarding a "detached HEAD state" is expected behaviour + # (you only have to create a new branch, if you want to change stuff and do a pull request for example) + + # Fetch submodules + $SUDO_WWW git submodule update --init --recursive + # Make git ignore filesystem permission differences for submodules + $SUDO_WWW git submodule foreach --recursive git config core.filemode false + # Make git ignore filesystem permission differences + $SUDO_WWW git config core.filemode false + + # Create a python3 virtualenv + sudo pip3 install virtualenv + $SUDO_WWW python3 -m venv $PATH_TO_MISP/venv + sudo mkdir /usr/share/httpd/.cache + sudo chown $WWW_USER:$WWW_USER /usr/share/httpd/.cache + $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U pip setuptools + + cd $PATH_TO_MISP/app/files/scripts + $SUDO_WWW git clone https://github.com/CybOXProject/python-cybox.git + $SUDO_WWW git clone https://github.com/STIXProject/python-stix.git + $SUDO_WWW git clone --branch master --single-branch https://github.com/lief-project/LIEF.git lief + $SUDO_WWW git clone https://github.com/CybOXProject/mixbox.git + + cd $PATH_TO_MISP/app/files/scripts/python-cybox + # If you umask is has been changed from the default, it is a good idea to reset it to 0022 before installing python modules + UMASK=$(umask) + umask 0022 + cd $PATH_TO_MISP/app/files/scripts/python-stix + $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install . + + # install mixbox to accommodate the new STIX dependencies: + cd $PATH_TO_MISP/app/files/scripts/mixbox + $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install . + + # FIXME: Remove once stix-fixed + $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2 + + # install STIX2.0 library to support STIX 2.0 export: + cd $PATH_TO_MISP/cti-python-stix2 + $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install . + + # install maec + $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U maec + + # install zmq + $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U zmq + + # install redis + $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U redis + + # lief needs manual compilation + sudo yum install devtoolset-7 cmake3 cppcheck -y + + cd $PATH_TO_MISP/app/files/scripts/lief + $SUDO_WWW mkdir build + cd build + $SUDO_WWW scl enable devtoolset-7 "bash -c 'cmake3 \ + -DLIEF_PYTHON_API=on \ + -DPYTHON_VERSION=3.6 \ + -DPYTHON_EXECUTABLE=$PATH_TO_MISP/venv/bin/python \ + -DLIEF_DOC=off \ + -DCMAKE_BUILD_TYPE=Release \ + ..'" + $SUDO_WWW make -j3 pyLIEF + + if [ $? == 2 ]; then + # In case you get "internal compiler error: Killed (program cc1plus)" + # You ran out of memory. + # Create some swap + sudo dd if=/dev/zero of=/var/swap.img bs=1024k count=4000 + sudo mkswap /var/swap.img + sudo swapon /var/swap.img + # And compile again + $SUDO_WWW make -j3 pyLIEF + sudo swapoff /var/swap.img + sudo rm /var/swap.img + fi + + # The following adds a PYTHONPATH to where the pyLIEF module has been compiled + echo $PATH_TO_MISP/app/files/scripts/lief/build/api/python |$SUDO_WWW tee $PATH_TO_MISP/venv/lib/python3.6/site-packages/lief.pth + + # install magic, pydeep + $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U python-magic git+https://github.com/kbandla/pydeep.git plyara + + # install PyMISP + cd $PATH_TO_MISP/PyMISP + $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U . + + # Enable dependencies detection in the diagnostics page + # This allows MISP to detect GnuPG, the Python modules' versions and to read the PHP settings. + # The LD_LIBRARY_PATH setting is needed for rh-git218 to work + echo "env[PATH] = /opt/rh/rh-git218/root/usr/bin:/opt/rh/rh-redis32/root/usr/bin:/opt/rh/rh-php72/root/usr/bin:/usr/local/bin:/usr/bin:/bin" |sudo tee -a /etc/opt/rh/rh-php72/php-fpm.d/www.conf + sudo sed -i.org -e 's/^;\(clear_env = no\)/\1/' /etc/opt/rh/rh-php72/php-fpm.d/www.conf + sudo systemctl restart rh-php72-php-fpm.service + umask $UMASK +} + +installCake_RHEL () +{ + sudo chown -R $WWW_USER:$WWW_USER $PATH_TO_MISP + sudo mkdir /usr/share/httpd/.composer + sudo chown $WWW_USER:$WWW_USER /usr/share/httpd/.composer + cd $PATH_TO_MISP/app + # Update composer.phar (optional) + #EXPECTED_SIGNATURE="$(wget -q -O - https://composer.github.io/installer.sig)" + #$SUDO_WWW $RUN_PHP -- php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" + #$SUDO_WWW $RUN_PHP -- php -r "if (hash_file('SHA384', 'composer-setup.php') === '$EXPECTED_SIGNATURE') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;" + #$SUDO_WWW $RUN_PHP "php composer-setup.php" + #$SUDO_WWW $RUN_PHP -- php -r "unlink('composer-setup.php');" + $SUDO_WWW $RUN_PHP "php composer.phar install" + + ## sudo yum install php-redis -y + sudo scl enable rh-php72 'pecl channel-update pecl.php.net' + sudo scl enable rh-php72 'yes no|pecl install redis' + echo "extension=redis.so" |sudo tee /etc/opt/rh/rh-php72/php.d/99-redis.ini + + # Install gnupg extension + sudo yum install gpgme-devel -y + sudo scl enable rh-php72 'pecl install gnupg' + echo "extension=gnupg.so" |sudo tee /etc/opt/rh/rh-php72/php.d/99-gnupg.ini + sudo systemctl restart rh-php72-php-fpm.service + + # If you have not yet set a timezone in php.ini + echo 'date.timezone = "Asia/Tokyo"' |sudo tee /etc/opt/rh/rh-php72/php.d/timezone.ini + + # Recommended: Change some PHP settings in /etc/opt/rh/rh-php72/php.ini + # max_execution_time = 300 + # memory_limit = 2048M + # upload_max_filesize = 50M + # post_max_size = 50M + for key in upload_max_filesize post_max_size max_execution_time max_input_time memory_limit + do + sudo sed -i "s/^\($key\).*/\1 = $(eval echo \${$key})/" $PHP_INI + done + sudo systemctl restart rh-php72-php-fpm.service + + # To use the scheduler worker for scheduled tasks, do the following: + sudo cp -fa $PATH_TO_MISP/INSTALL/setup/config.php $PATH_TO_MISP/app/Plugin/CakeResque/Config/config.php +} + +prepareDB_RHEL () { + RUN_MYSQL="/usr/bin/scl enable rh-mariadb102" + # Enable, start and secure your mysql database server + sudo systemctl enable --now rh-mariadb102-mariadb.service + echo [mysqld] |sudo tee /etc/opt/rh/rh-mariadb102/my.cnf.d/bind-address.cnf + echo bind-address=127.0.0.1 |sudo tee -a /etc/opt/rh/rh-mariadb102/my.cnf.d/bind-address.cnf + sudo systemctl restart rh-mariadb102-mariadb + + sudo yum install expect -y + + ## The following needs some thoughts about scl enable foo + #if [[ ! -e /var/opt/rh/rh-mariadb102/lib/mysql/misp/users.ibd ]]; then + + # We ask interactively your password if not run as root + pw="" + if [[ "$EUID" -ne 0 ]]; then + read -s -p "Enter sudo password: " pw + fi + + expect -f - <<-EOF + set timeout 10 + + spawn sudo scl enable rh-mariadb102 mysql_secure_installation + expect { + "*sudo*" { + send "$pw\r" + exp_continue + } + "Enter current password for root (enter for none):" { + send -- "\r" + } + } + expect "Set root password?" + send -- "y\r" + expect "New password:" + send -- "${DBPASSWORD_ADMIN}\r" + expect "Re-enter new password:" + send -- "${DBPASSWORD_ADMIN}\r" + expect "Remove anonymous users?" + send -- "y\r" + expect "Disallow root login remotely?" + send -- "y\r" + expect "Remove test database and access to it?" + send -- "y\r" + expect "Reload privilege tables now?" + send -- "y\r" + expect eof +EOF + + sudo yum remove tcl expect -y + + sudo systemctl restart rh-mariadb102-mariadb + + scl enable rh-mariadb102 "mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e 'CREATE DATABASE $DBNAME;'" + scl enable rh-mariadb102 "mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e \"GRANT USAGE on *.* to $DBUSER_MISP@localhost IDENTIFIED by '$DBPASSWORD_MISP';\"" + scl enable rh-mariadb102 "mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e \"GRANT ALL PRIVILEGES on $DBNAME.* to '$DBUSER_MISP'@'localhost';\"" + scl enable rh-mariadb102 "mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e 'FLUSH PRIVILEGES;'" + + $SUDO_WWW cat $PATH_TO_MISP/INSTALL/MYSQL.sql | sudo scl enable rh-mariadb102 "mysql -u $DBUSER_MISP -p$DBPASSWORD_MISP $DBNAME" +} + +apacheConfig_RHEL () { + # Now configure your apache server with the DocumentRoot $PATH_TO_MISP/app/webroot/ + # A sample vhost can be found in $PATH_TO_MISP/INSTALL/apache.misp.centos7 + + sudo cp $PATH_TO_MISP/INSTALL/apache.misp.centos7.ssl /etc/httpd/conf.d/misp.ssl.conf + #sudo sed -i "s/SetHandler/\#SetHandler/g" /etc/httpd/conf.d/misp.ssl.conf + sudo rm /etc/httpd/conf.d/ssl.conf + sudo chmod 644 /etc/httpd/conf.d/misp.ssl.conf + sudo sed -i '/Listen 80/a Listen 443' /etc/httpd/conf/httpd.conf + + # If a valid SSL certificate is not already created for the server, create a self-signed certificate: + echo "The Common Name used below will be: ${OPENSSL_CN}" + # This will take a rather long time, be ready. (13min on a VM, 8GB Ram, 1 core) + if [[ ! -e "/etc/pki/tls/certs/dhparam.pem" ]]; then + sudo openssl dhparam -out /etc/pki/tls/certs/dhparam.pem 4096 + fi + sudo openssl genrsa -des3 -passout pass:xxxx -out /tmp/misp.local.key 4096 + sudo openssl rsa -passin pass:xxxx -in /tmp/misp.local.key -out /etc/pki/tls/private/misp.local.key + sudo rm /tmp/misp.local.key + sudo openssl req -new -subj "/C=${OPENSSL_C}/ST=${OPENSSL_ST}/L=${OPENSSL_L}/O=${OPENSSL_O}/OU=${OPENSSL_OU}/CN=${OPENSSL_CN}/emailAddress=${OPENSSL_EMAILADDRESS}" -key /etc/pki/tls/private/misp.local.key -out /etc/pki/tls/certs/misp.local.csr + sudo openssl x509 -req -days 365 -in /etc/pki/tls/certs/misp.local.csr -signkey /etc/pki/tls/private/misp.local.key -out /etc/pki/tls/certs/misp.local.crt + sudo ln -s /etc/pki/tls/certs/misp.local.csr /etc/pki/tls/certs/misp-chain.crt + cat /etc/pki/tls/certs/dhparam.pem |sudo tee -a /etc/pki/tls/certs/misp.local.crt + + sudo systemctl restart httpd.service + + # Since SELinux is enabled, we need to allow httpd to write to certain directories + sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/files + sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/files/terms + sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/files/scripts/tmp + sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/Plugin/CakeResque/tmp + sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/Console/cake + sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/Console/worker/*.sh + sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/*.py + sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/*/*.py + sudo chcon -t httpd_sys_script_exec_t $PATH_TO_MISP/app/files/scripts/lief/build/api/python/lief.so + sudo chcon -R -t bin_t $PATH_TO_MISP/venv/bin/* + find $PATH_TO_MISP/venv -type f -name "*.so*" -or -name "*.so.*" | xargs sudo chcon -t lib_t + # Only run these if you want to be able to update MISP from the web interface + sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/.git + sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/tmp + sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/Lib + sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/Config + sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/webroot/img/orgs + sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/webroot/img/custom + sudo chcon -R -t httpd_sys_rw_content_t $PATH_TO_MISP/app/files/scripts/mispzmq +} + +firewall_RHEL () { + # Allow httpd to connect to the redis server and php-fpm over tcp/ip + sudo setsebool -P httpd_can_network_connect on + + # Allow httpd to send emails from php + sudo setsebool -P httpd_can_sendmail on + + # Enable and start the httpd service + sudo systemctl enable --now httpd.service + + # Open a hole in the iptables firewall + sudo firewall-cmd --zone=public --add-port=80/tcp --permanent + sudo firewall-cmd --zone=public --add-port=443/tcp --permanent + sudo firewall-cmd --reload +} + +# Main function to fix permissions to something sane +permissions_RHEL () { + sudo chown -R $WWW_USER:$WWW_USER $PATH_TO_MISP + ## ? chown -R root:$WWW_USER $PATH_TO_MISP + sudo find $PATH_TO_MISP -type d -exec chmod g=rx {} \; + sudo chmod -R g+r,o= $PATH_TO_MISP + ## **Note :** For updates through the web interface to work, apache must own the $PATH_TO_MISP folder and its subfolders as shown above, which can lead to security issues. If you do not require updates through the web interface to work, you can use the following more restrictive permissions : + sudo chmod -R 750 $PATH_TO_MISP + sudo chmod -R g+xws $PATH_TO_MISP/app/tmp + sudo chmod -R g+ws $PATH_TO_MISP/app/files + sudo chmod -R g+ws $PATH_TO_MISP/app/files/scripts/tmp + sudo chmod -R g+rw $PATH_TO_MISP/venv + sudo chmod -R g+rw $PATH_TO_MISP/.git + sudo chown $WWW_USER:$WWW_USER $PATH_TO_MISP/app/files + sudo chown $WWW_USER:$WWW_USER $PATH_TO_MISP/app/files/terms + sudo chown $WWW_USER:$WWW_USER $PATH_TO_MISP/app/files/scripts/tmp + sudo chown $WWW_USER:$WWW_USER $PATH_TO_MISP/app/Plugin/CakeResque/tmp + sudo chown -R $WWW_USER:$WWW_USER $PATH_TO_MISP/app/Config + sudo chown -R $WWW_USER:$WWW_USER $PATH_TO_MISP/app/tmp + sudo chown -R $WWW_USER:$WWW_USER $PATH_TO_MISP/app/webroot/img/orgs + sudo chown -R $WWW_USER:$WWW_USER $PATH_TO_MISP/app/webroot/img/custom +} + +logRotation_RHEL () { + # MISP saves the stdout and stderr of its workers in $PATH_TO_MISP/app/tmp/logs + # To rotate these logs install the supplied logrotate script: + + sudo cp $PATH_TO_MISP/INSTALL/misp.logrotate /etc/logrotate.d/misp + sudo chmod 0640 /etc/logrotate.d/misp + + # Now make logrotate work under SELinux as well + # Allow logrotate to modify the log files + sudo semanage fcontext -a -t httpd_sys_rw_content_t "$PATH_TO_MISP(/.*)?" + sudo semanage fcontext -a -t httpd_log_t "$PATH_TO_MISP/app/tmp/logs(/.*)?" + sudo chcon -R -t httpd_log_t $PATH_TO_MISP/app/tmp/logs + # Impact of the following: ?!?!?!!?111 + ##sudo restorecon -R $PATH_TO_MISP + + # Allow logrotate to read /var/www + sudo checkmodule -M -m -o /tmp/misplogrotate.mod $PATH_TO_MISP/INSTALL/misplogrotate.te + sudo semodule_package -o /tmp/misplogrotate.pp -m /tmp/misplogrotate.mod + sudo semodule -i /tmp/misplogrotate.pp +} + +configMISP_RHEL () { + # There are 4 sample configuration files in $PATH_TO_MISP/app/Config that need to be copied + $SUDO_WWW cp -a $PATH_TO_MISP/app/Config/bootstrap.default.php $PATH_TO_MISP/app/Config/bootstrap.php + $SUDO_WWW cp -a $PATH_TO_MISP/app/Config/database.default.php $PATH_TO_MISP/app/Config/database.php + $SUDO_WWW cp -a $PATH_TO_MISP/app/Config/core.default.php $PATH_TO_MISP/app/Config/core.php + $SUDO_WWW cp -a $PATH_TO_MISP/app/Config/config.default.php $PATH_TO_MISP/app/Config/config.php + + echo " 'Database/Mysql', + //'datasource' => 'Database/Postgres', + 'persistent' => false, + 'host' => '$DBHOST', + 'login' => '$DBUSER_MISP', + 'port' => 3306, // MySQL & MariaDB + //'port' => 5432, // PostgreSQL + 'password' => '$DBPASSWORD_MISP', + 'database' => '$DBNAME', + 'prefix' => '', + 'encoding' => 'utf8', + ); + }" | $SUDO_WWW tee $PATH_TO_MISP/app/Config/database.php + + # Configure the fields in the newly created files: + # config.php : baseurl (example: 'baseurl' => 'http://misp',) - don't use "localhost" it causes issues when browsing externally + # core.php : Uncomment and set the timezone: `// date_default_timezone_set('UTC');` + # database.php : login, port, password, database + # DATABASE_CONFIG has to be filled + # With the default values provided in section 6, this would look like: + # class DATABASE_CONFIG { + # public $default = array( + # 'datasource' => 'Database/Mysql', + # 'persistent' => false, + # 'host' => 'localhost', + # 'login' => 'misp', // grant usage on *.* to misp@localhost + # 'port' => 3306, + # 'password' => 'XXXXdbpasswordhereXXXXX', // identified by 'XXXXdbpasswordhereXXXXX'; + # 'database' => 'misp', // create database misp; + # 'prefix' => '', + # 'encoding' => 'utf8', + # ); + #} + + # Important! Change the salt key in $PATH_TO_MISP/app/Config/config.php + # The admin user account will be generated on the first login, make sure that the salt is changed before you create that user + # If you forget to do this step, and you are still dealing with a fresh installation, just alter the salt, + # delete the user from mysql and log in again using the default admin credentials (admin@admin.test / admin) + + # If you want to be able to change configuration parameters from the webinterface: + sudo chown $WWW_USER:$WWW_USER $PATH_TO_MISP/app/Config/config.php + sudo chcon -t httpd_sys_rw_content_t $PATH_TO_MISP/app/Config/config.php + + # Generate a GPG encryption key. + cat >/tmp/gen-key-script < /dev/null + if [ $? -eq 1 ]; then + debug "Creating MISP user" + sudo useradd -r "$MISP_USER" + fi + + debug "Enabling Extras Repos (SCL)" + if [[ $FLAVOUR == "rhel" ]]; then + sudo subscription-manager register --auto-attach + enableReposRHEL + enableEPEL + else # CentOS + centosEPEL + fi + + debug "Installing System Dependencies" + yumInstallCoreDeps + + debug "Enabling Haveged for additional entropy" + sudo yum install haveged -y + sudo systemctl enable --now haveged.service + + debug "Installing MISP code" + installCoreRHEL + + debug "Install Cake PHP" + installCake_RHEL + + debug "Setting File permissions" + permissions_RHEL + + debug "Preparing Database" + prepareDB_RHEL + + debug "Configuring Apache" + apacheConfig_RHEL + + debug "Setting up firewall" + firewall_RHEL + + debug "Enabling log rotation" + logRotation_RHEL + + debug "Configuring MISP" + configMISP_RHEL + + debug "Setting up background workers" + configWorkersRHEL + + debug "Optimizing Cake Installation" + coreCAKE + + debug "Updating tables" + updateGOWNT + + echo "Core Intallation finished, check on port 443 to see the Web UI" + fi + + if [[ -n $MODULES ]] || [[ -n $ALL ]]; then + space + echo "Installing MISP Modules" + space + + mispmodulesRHEL + + echo "MISP modules installation finished." + fi +} +# End installMISPRHEL () + ## End Function Section ## colors @@ -2417,14 +3080,14 @@ if [[ "$0" == "./INSTALL.tpl.sh" || "$(echo $0 |grep -o -e 'INSTALL.tpl.sh')" == generateInstaller fi +debug "Checking Linux distribution and flavour..." +checkFlavour debug "Checking if we are uptodate and checksums match" checkInstaller space debug "Setting MISP variables" MISPvars -debug "Checking Linux distribution and flavour..." -checkFlavour debug "Checking for parameters or Unattended Kali Install" if [[ $# == 0 && $0 != "/tmp/misp-kali.sh" ]]; then @@ -2465,9 +3128,8 @@ fi # TODO: Move support map to top SUPPORT_MAP=" -x86_64-centos-8 +x86_64-centos-7 x86_64-rhel-7 -x86_64-rhel-8 x86_64-fedora-30 x86_64-debian-stretch x86_64-debian-buster @@ -2570,3 +3232,10 @@ if [ "${FLAVOUR}" == "kali" ]; then echo "Installation done!" exit fi + +# If RHEL/CentOS is detected, run appropriate script +if [ "${FLAVOUR}" == "rhel" ] || [ "${FLAVOUR}" == "centos" ]; then + installMISPRHEL + echo "Installation done !" + exit +fi diff --git a/INSTALL/INSTALL.sh.sha1 b/INSTALL/INSTALL.sh.sha1 index ec7c207fa..12592b1b9 100644 --- a/INSTALL/INSTALL.sh.sha1 +++ b/INSTALL/INSTALL.sh.sha1 @@ -1 +1 @@ -04a834fcd3bc9da5282ede8a3d2c459fbc625e46 INSTALL.sh +c85dfcbc3da0b6b6331a9ef057e04d1921205d22 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha256 b/INSTALL/INSTALL.sh.sha256 index 1f4397bb7..0a4eab175 100644 --- a/INSTALL/INSTALL.sh.sha256 +++ b/INSTALL/INSTALL.sh.sha256 @@ -1 +1 @@ -fbca1473fec26ad5a6c1ae6ae3d9af11e47e7758f30b160bc047abe9978f4476 INSTALL.sh +4665b8df8c1baecda9bc5ef729033c7b2d64e72b6395e3fc260927cce394b16a INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha384 b/INSTALL/INSTALL.sh.sha384 index 36c9ec1e1..77e8684de 100644 --- a/INSTALL/INSTALL.sh.sha384 +++ b/INSTALL/INSTALL.sh.sha384 @@ -1 +1 @@ -7281b13aa7d6b016152096d35619c2cecc7ec49f8f41cf8a3b8284335d950d35f273fa56fea63ec5adb3669038239c61 INSTALL.sh +6bc8ba92099f339152211bee3603553a8647dcd67ec9b1929cf0c489c70b4db8772d8bb31508782d82343770dac06426 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha512 b/INSTALL/INSTALL.sh.sha512 index 0c79ff282..924c2a7f9 100644 --- a/INSTALL/INSTALL.sh.sha512 +++ b/INSTALL/INSTALL.sh.sha512 @@ -1 +1 @@ -fa17df3aa0cbc54d2b48ae14fb296c91c12fc6cf8e3704b8af1b2cb2cde9c6faf591a2e42a38c01c299c62390868e7766ef682a6b3b556bafc469688e8aed6e7 INSTALL.sh +32793e6dcf99b32fc8271a8cdda862e60bbbd256eb24a1f6711770a931ddc552ab4dc50a7d841d5030cea43717275d8ebbc6c7c641dc8f02d11f5ea20eeef35c INSTALL.sh diff --git a/INSTALL/INSTALL.tpl.sh b/INSTALL/INSTALL.tpl.sh index 774936e01..97eb57dab 100755 --- a/INSTALL/INSTALL.tpl.sh +++ b/INSTALL/INSTALL.tpl.sh @@ -90,6 +90,21 @@ ## 6_ssdeep.sh ## ## 6_viper.sh ## +## 0_RHEL_SCL.sh ## +## 0_CentOS_EPEL.sh ## +## 0_RHEL_EPEL.sh ## +## 0_yumInstallCoreDeps.sh ## +## 1_mispCoreInstall_RHEL.sh ## +## 1_installCake_RHEL.sh ## +## 1_prepareDB_RHEL.sh ## +## 1_apacheConfig_RHEL.sh ## +## 1_firewall_RHEL.sh ## +## 2_permissions_RHEL.sh ## +## 2_logRotation_RHEL.sh ## +## 2_configMISP_RHEL.sh ## +## 3_configWorkers_RHEL.sh ## +## 3_misp-modules_RHEL.sh ## + # No functions scripts: ## apt-upgrade.sh ## ## postfix.sh ## @@ -128,12 +143,12 @@ generateInstaller () { cp ../INSTALL.tpl.sh . # Pull code snippets out of Main Install Documents - for f in `echo INSTALL.ubuntu1804.md xINSTALL.debian9.md INSTALL.kali.md xINSTALL.debian10.md xINSTALL.tsurugi.md xINSTALL.debian9-postgresql.md xINSTALL.ubuntu1804.with.webmin.md`; do + for f in `echo INSTALL.ubuntu1804.md xINSTALL.debian9.md INSTALL.kali.md xINSTALL.debian10.md xINSTALL.tsurugi.md xINSTALL.debian9-postgresql.md xINSTALL.ubuntu1804.with.webmin.md INSTALL.rhel7.md`; do xsnippet . ../../docs/${f} done # Pull out code snippets from generic Install Documents - for f in `echo globalVariables.md mail_to_misp-debian.md MISP_CAKE_init.md misp-dashboard-debian.md misp-modules-debian.md gnupg.md ssdeep-debian.md sudo_etckeeper.md supportFunctions.md viper-debian.md`; do + for f in `echo globalVariables.md mail_to_misp-debian.md MISP_CAKE_init.md misp-dashboard-debian.md misp-modules-debian.md gnupg.md ssdeep-debian.md sudo_etckeeper.md supportFunctions.md viper-debian.md misp-modules-centos.md`; do xsnippet . ../../docs/generic/${f} done @@ -168,6 +183,21 @@ generateInstaller () { perl -pe 's/^## 6_viper.sh ##/`cat 6_viper.sh`/ge' -i INSTALL.tpl.sh perl -pe 's/^## 6_ssdeep.sh ##/`cat 6_ssdeep.sh`/ge' -i INSTALL.tpl.sh + perl -pe 's/^## 0_RHEL_SCL.sh ##/`cat 0_RHEL_SCL.sh`/ge' -i INSTALL.tpl.sh + perl -pe 's/^## 0_CentOS_EPEL.sh ##/`cat 0_CentOS_EPEL.sh`/ge' -i INSTALL.tpl.sh + perl -pe 's/^## 0_RHEL_EPEL.sh ##/`cat 0_RHEL_EPEL.sh`/ge' -i INSTALL.tpl.sh + perl -pe 's/^## 0_yumInstallCoreDeps.sh ##/`cat 0_yumInstallCoreDeps.sh`/ge' -i INSTALL.tpl.sh + perl -pe 's/^## 1_mispCoreInstall_RHEL.sh ##/`cat 1_mispCoreInstall_RHEL.sh`/ge' -i INSTALL.tpl.sh + perl -pe 's/^## 1_installCake_RHEL.sh ##/`cat 1_installCake_RHEL.sh`/ge' -i INSTALL.tpl.sh + perl -pe 's/^## 2_permissions_RHEL.sh ##/`cat 2_permissions_RHEL.sh`/ge' -i INSTALL.tpl.sh + perl -pe 's/^## 1_prepareDB_RHEL.sh ##/`cat 1_prepareDB_RHEL.sh`/ge' -i INSTALL.tpl.sh + perl -pe 's/^## 1_apacheConfig_RHEL.sh ##/`cat 1_apacheConfig_RHEL.sh`/ge' -i INSTALL.tpl.sh + perl -pe 's/^## 1_firewall_RHEL.sh ##/`cat 1_firewall_RHEL.sh`/ge' -i INSTALL.tpl.sh + perl -pe 's/^## 2_logRotation_RHEL.sh ##/`cat 2_logRotation_RHEL.sh`/ge' -i INSTALL.tpl.sh + perl -pe 's/^## 2_configMISP_RHEL.sh ##/`cat 2_configMISP_RHEL.sh`/ge' -i INSTALL.tpl.sh + perl -pe 's/^## 3_configWorkers_RHEL.sh ##/`cat 3_configWorkers_RHEL.sh`/ge' -i INSTALL.tpl.sh + perl -pe 's/^## 3_misp-modules_RHEL.sh ##/`cat 3_misp-modules_RHEL.sh`/ge' -i INSTALL.tpl.sh + cp INSTALL.tpl.sh ../INSTALL.sh cd .. for ALGO in $(echo "1 256 384 512"); do @@ -620,6 +650,88 @@ installMISPonKali () { } # End installMISPonKali () +# Main Install on RHEL function +installMISPRHEL () { + if [[ -n $SSDEEP ]] || [[ -n $MAIL2 ]]; then + echo "RHEL installation currently only supports Core and Modules" + echo "Ignoring other options..." + fi + + if [[ -n $CORE ]] || [[ -n $ALL ]]; then + space + echo "Proceeding with MISP core installation on RHEL $dist_version" + space + + id -u "$MISP_USER" > /dev/null + if [ $? -eq 1 ]; then + debug "Creating MISP user" + sudo useradd -r "$MISP_USER" + fi + + debug "Enabling Extras Repos (SCL)" + if [[ $FLAVOUR == "rhel" ]]; then + sudo subscription-manager register --auto-attach + enableReposRHEL + enableEPEL + else # CentOS + centosEPEL + fi + + debug "Installing System Dependencies" + yumInstallCoreDeps + + debug "Enabling Haveged for additional entropy" + sudo yum install haveged -y + sudo systemctl enable --now haveged.service + + debug "Installing MISP code" + installCoreRHEL + + debug "Install Cake PHP" + installCake_RHEL + + debug "Setting File permissions" + permissions_RHEL + + debug "Preparing Database" + prepareDB_RHEL + + debug "Configuring Apache" + apacheConfig_RHEL + + debug "Setting up firewall" + firewall_RHEL + + debug "Enabling log rotation" + logRotation_RHEL + + debug "Configuring MISP" + configMISP_RHEL + + debug "Setting up background workers" + configWorkersRHEL + + debug "Optimizing Cake Installation" + coreCAKE + + debug "Updating tables" + updateGOWNT + + echo "Core Intallation finished, check on port 443 to see the Web UI" + fi + + if [[ -n $MODULES ]] || [[ -n $ALL ]]; then + space + echo "Installing MISP Modules" + space + + mispmodulesRHEL + + echo "MISP modules installation finished." + fi +} +# End installMISPRHEL () + ## End Function Section ## colors @@ -628,14 +740,14 @@ if [[ "$0" == "./INSTALL.tpl.sh" || "$(echo $0 |grep -o -e 'INSTALL.tpl.sh')" == generateInstaller fi +debug "Checking Linux distribution and flavour..." +checkFlavour debug "Checking if we are uptodate and checksums match" checkInstaller space debug "Setting MISP variables" MISPvars -debug "Checking Linux distribution and flavour..." -checkFlavour debug "Checking for parameters or Unattended Kali Install" if [[ $# == 0 && $0 != "/tmp/misp-kali.sh" ]]; then @@ -676,9 +788,8 @@ fi # TODO: Move support map to top SUPPORT_MAP=" -x86_64-centos-8 +x86_64-centos-7 x86_64-rhel-7 -x86_64-rhel-8 x86_64-fedora-30 x86_64-debian-stretch x86_64-debian-buster @@ -781,3 +892,10 @@ if [ "${FLAVOUR}" == "kali" ]; then echo "Installation done!" exit fi + +# If RHEL/CentOS is detected, run appropriate script +if [ "${FLAVOUR}" == "rhel" ] || [ "${FLAVOUR}" == "centos" ]; then + installMISPRHEL + echo "Installation done !" + exit +fi diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index 096fc3420..fd9612c59 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -218,7 +218,7 @@ installCoreRHEL () { # Create a python3 virtualenv sudo pip3 install virtualenv - $SUDO_WWW python3 -- virtualenv -p python3 $PATH_TO_MISP/venv + $SUDO_WWW python3 -m venv $PATH_TO_MISP/venv sudo mkdir /usr/share/httpd/.cache sudo chown $WWW_USER:$WWW_USER /usr/share/httpd/.cache $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U pip setuptools @@ -355,7 +355,7 @@ installCake_RHEL () # To use the scheduler worker for scheduled tasks, do the following: sudo cp -fa $PATH_TO_MISP/INSTALL/setup/config.php $PATH_TO_MISP/app/Plugin/CakeResque/Config/config.php } -# +# ``` ### 5/ Set file permissions @@ -404,17 +404,25 @@ prepareDB_RHEL () { ## The following needs some thoughts about scl enable foo #if [[ ! -e /var/opt/rh/rh-mariadb102/lib/mysql/misp/users.ibd ]]; then - # Add your credentials if needed, if sudo has NOPASS, comment out the relevant lines - pw="Password1234" + # We ask interactively your password if not run as root + pw="" + if [[ "$EUID" -ne 0 ]]; then + read -s -p "Enter sudo password: " pw + fi expect -f - <<-EOF set timeout 10 spawn sudo scl enable rh-mariadb102 mysql_secure_installation - expect "*?assword*" - send -- "$pw\r" - expect "Enter current password for root (enter for none):" - send -- "\r" + expect { + "*sudo*" { + send "$pw\r" + exp_continue + } + "Enter current password for root (enter for none):" { + send -- "\r" + } + } expect "Set root password?" send -- "y\r" expect "New password:" @@ -480,7 +488,7 @@ apacheConfig_RHEL () { sudo openssl req -new -subj "/C=${OPENSSL_C}/ST=${OPENSSL_ST}/L=${OPENSSL_L}/O=${OPENSSL_O}/OU=${OPENSSL_OU}/CN=${OPENSSL_CN}/emailAddress=${OPENSSL_EMAILADDRESS}" -key /etc/pki/tls/private/misp.local.key -out /etc/pki/tls/certs/misp.local.csr sudo openssl x509 -req -days 365 -in /etc/pki/tls/certs/misp.local.csr -signkey /etc/pki/tls/private/misp.local.key -out /etc/pki/tls/certs/misp.local.crt sudo ln -s /etc/pki/tls/certs/misp.local.csr /etc/pki/tls/certs/misp-chain.crt - cat /etc/pki/tls/certs/dhparam.pem |sudo tee -a /etc/pki/tls/certs/misp.local.crt + cat /etc/pki/tls/certs/dhparam.pem |sudo tee -a /etc/pki/tls/certs/misp.local.crt sudo systemctl restart httpd.service @@ -709,7 +717,7 @@ Possible also due to package being installed via SCL, attempting to start worker systemctl restart misp-workers.service ``` -!!! note +!!! note No other functions were tested after the conclusion of this install. There may be issue that aren't addressed
via this guide and will need additional investigation. diff --git a/docs/generic/supportFunctions.md b/docs/generic/supportFunctions.md index 5b480f757..2d8766fe9 100644 --- a/docs/generic/supportFunctions.md +++ b/docs/generic/supportFunctions.md @@ -137,16 +137,17 @@ checkFlavour () { centos) if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then dist_version="$(. /etc/os-release && echo "$VERSION_ID")" + dist_version=${dist_version:0:1} fi - echo "$FLAVOUR not supported at the moment" - exit 1 + echo "$FLAVOUR support is experimental at the moment" ;; rhel|ol|sles) if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then dist_version="$(. /etc/os-release && echo "$VERSION_ID")" + dist_version=${dist_version:0:1} # Only interested about major version fi - echo "$FLAVOUR not supported at the moment" - exit 1 + # Only tested for RHEL 7 so far + echo "$FLAVOUR support is experimental at the moment" ;; *) if command_exists lsb_release; then @@ -214,25 +215,38 @@ EOF } checkInstaller () { - # TODO: Implement $FLAVOUR checks and install depending on the platform we are on - if [[ $(which shasum > /dev/null 2>&1 ; echo $?) != 0 ]]; then - sudo apt install libdigest-sha-perl -qyy - fi - # SHAsums to be computed, not the -- notatiation is for ease of use with rhash - SHA_SUMS="--sha1 --sha256 --sha384 --sha512" - for sum in $(echo ${SHA_SUMS} |sed 's/--sha//g'); do - /usr/bin/wget --no-cache -q -O /tmp/INSTALL.sh.sha${sum} https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/INSTALL.sh.sha${sum} - INSTsum=$(shasum -a ${sum} ${0} | cut -f1 -d\ ) - chsum=$(cat /tmp/INSTALL.sh.sha${sum} | cut -f1 -d\ ) - - if [[ "${chsum}" == "${INSTsum}" ]]; then - echo "sha${sum} matches" - else - echo "sha${sum}: ${chsum} does not match the installer sum of: ${INSTsum}" - echo "Delete installer, re-download and please run again." - exit 1 + # Workaround: shasum is not available on RHEL, only checking sha512 + if [[ $FLAVOUR == "rhel" ]] || [[ $FLAVOUR == "centos" ]]; then + INSTsum=$(sha512sum ${0} | cut -f1 -d\ ) + /usr/bin/wget --no-cache -q -O /tmp/INSTALL.sh.sha512 https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/INSTALL.sh.sha512 + chsum=$(cat /tmp/INSTALL.sh.sha512) + if [[ "${chsum}" == "${INSTsum}" ]]; then + echo "SHA512 matches" + else + echo "SHA512: ${chsum} does not match the installer sum of: ${INSTsum}" + # exit 1 # uncomment when/if PR is merged + fi + else + # TODO: Implement $FLAVOUR checks and install depending on the platform we are on + if [[ $(which shasum > /dev/null 2>&1 ; echo $?) != 0 ]]; then + sudo apt install libdigest-sha-perl -qyy fi - done + # SHAsums to be computed, not the -- notatiation is for ease of use with rhash + SHA_SUMS="--sha1 --sha256 --sha384 --sha512" + for sum in $(echo ${SHA_SUMS} |sed 's/--sha//g'); do + /usr/bin/wget --no-cache -q -O /tmp/INSTALL.sh.sha${sum} https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/INSTALL.sh.sha${sum} + INSTsum=$(shasum -a ${sum} ${0} | cut -f1 -d\ ) + chsum=$(cat /tmp/INSTALL.sh.sha${sum} | cut -f1 -d\ ) + + if [[ "${chsum}" == "${INSTsum}" ]]; then + echo "sha${sum} matches" + else + echo "sha${sum}: ${chsum} does not match the installer sum of: ${INSTsum}" + echo "Delete installer, re-download and please run again." + exit 1 + fi + done + fi } # Extract manufacturer From 80c78a38dcd62a9c66ca240d1394f2a0c34ed18d Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Thu, 19 Mar 2020 17:46:34 +0100 Subject: [PATCH 3/7] Fix virtualenv creation on RHEL 7 --- docs/INSTALL.rhel7.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index 096fc3420..3b729f610 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -218,7 +218,7 @@ installCoreRHEL () { # Create a python3 virtualenv sudo pip3 install virtualenv - $SUDO_WWW python3 -- virtualenv -p python3 $PATH_TO_MISP/venv + $SUDO_WWW python3 /usr/local/bin/virtualenv -p python3 $PATH_TO_MISP/venv sudo mkdir /usr/share/httpd/.cache sudo chown $WWW_USER:$WWW_USER /usr/share/httpd/.cache $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U pip setuptools From 22159887d798849af845a8e56b0c4cdf0b448c1a Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Thu, 19 Mar 2020 17:51:50 +0100 Subject: [PATCH 4/7] Install ssdeep PHP module on RHEL 7 --- docs/INSTALL.rhel7.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index 3b729f610..d215fb492 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -332,6 +332,10 @@ installCake_RHEL () sudo scl enable rh-php72 'yes no|pecl install redis' echo "extension=redis.so" |sudo tee /etc/opt/rh/rh-php72/php.d/99-redis.ini + sudo ln -s /usr/lib64/libfuzzy.so /usr/lib/libfuzzy.so + sudo scl enable rh-php72 'pecl install ssdeep' + echo "extension=ssdeep.so" |sudo tee /etc/opt/rh/rh-php72/php.d/99-ssdeep.ini + # Install gnupg extension sudo yum install gpgme-devel -y sudo scl enable rh-php72 'pecl install gnupg' From 708d76a8014906656f996e00abdd04c29c8f6717 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 26 Mar 2020 09:31:58 +0100 Subject: [PATCH 5/7] chg: [widgets:multiline] Added possibility to pick datapoint and see the deltas --- .../dashboard/Widgets/MultiLineChart.ctp | 187 +++++++++++++++++- 1 file changed, 179 insertions(+), 8 deletions(-) diff --git a/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp b/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp index f98795ec1..774ab227e 100644 --- a/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp +++ b/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp @@ -59,10 +59,13 @@ function init() { // variables and functions have their own scope (no override) var svg; var width, height, svg_width, svg_height; var xAxis, yAxis, cursorX, cursorY; - var x, y, xGrid, yGrid, value_line - var series, line_guides, points, pointsGroup, labels + var x, y, xGrid, yGrid, value_line; + var overlayLeft, overlayRight, tooltipPickedNodes; + var series, line_guides, points, pointsGroup, labels; var colors = d3.scale.category10(); + var pickedNodes = {start: null, end: null}; + var options = ; var options = $.extend(true, {}, default_options, options); options = _validateOptions(options); @@ -106,6 +109,10 @@ function init() { // variables and functions have their own scope (no override) }) } + function getX(datum) { + return options.abscissa_linear ? datum.index : datum.date; + } + function _init() { $loadingContainer = $('
').css({ position: 'absolute', @@ -126,6 +133,16 @@ function init() { // variables and functions have their own scope (no override) .style('color', 'white') .style('border-radius', '5px') .style('display', 'none'); + tooltipPickedNodes = d3.select('body').append('div') + .attr('class', 'tooltip tooltipPickedNodes') + .style('opacity', 0) + .style('min-width', '120px') + .style('padding', '3px') + .style('background-color', '#fff') + .style('color', 'black') + .style('border', '1px solid black') + .style('border-radius', '5px') + .style('display', 'none'); $container.append($loadingContainer); timeFormatter = d3.time.format(options.time_format).parse; } @@ -139,8 +156,8 @@ function init() { // variables and functions have their own scope (no override) if (options.abscissa_linear) { x = d3.scale.linear() - .domain(d3.extent(data, function(d) { return d.index; })) - .range([ 0, width ]); + .domain(d3.extent(data, function(d) { return d.index; })) + .range([ 0, width ]); } else { x = d3.time.scale() .domain(d3.extent(data, function(d) { return d.date; })) @@ -159,7 +176,7 @@ function init() { // variables and functions have their own scope (no override) .tickFormat(""); value_line = d3.svg.line() - .x(function(d) { return x(options.abscissa_linear ? d.index : d.date); }) + .x(function(d) { return x(getX(d)); }) .y(function(d) { return y(d.count); }); svg = d3.select(container_id) @@ -262,6 +279,23 @@ function init() { // variables and functions have their own scope (no override) svg.append('g') .classed('point-group', true); + overlayLeft = svg.append('rect') + .attr('fill', 'black') + .attr('opacity', 0.6) + .attr('class', 'overlay-left') + .attr('width', 0) + .attr('height', height) + .attr('x', 0) + .on('click', clearPickedNodes); + overlayRight = svg.append('rect') + .attr('fill', 'black') + .attr('opacity', 0.6) + .attr('class', 'overlay-right') + .attr('width', 0) + .attr('height', height) + .attr('x', 0) + .on('click', clearPickedNodes); + window.addEventListener("resize", function() { if (resize_timeout !== undefined) { clearTimeout(resize_timeout); @@ -350,7 +384,7 @@ function init() { // variables and functions have their own scope (no override) data_nodes_active = data_nodes.filter(function(d) { return !d.disabled; }) - x.domain(d3.extent(chart_data, function(d) { return options.abscissa_linear ? d.index : d.date; })) + x.domain(d3.extent(chart_data, function(d) { return getX(d); })) y.domain([ d3.min(data_nodes_active, function(c) { return d3.min(c.values, function(v) { return v.count; }); }), d3.max(data_nodes_active, function(c) { return d3.max(c.values, function(v) { return v.count; }); }) @@ -401,10 +435,10 @@ function init() { // variables and functions have their own scope (no override) points .enter() .append('circle') - .attr('class', 'datapoint d3-line-circle') + .attr('class', 'datapoint d3-line-circle useCursorPointer') .attr('r', 5) points // Update - .attr('cx', function (d) { return x(options.abscissa_linear ? d.index : d.date); }) + .attr('cx', function (d) { return x(getX(d)); }) .attr('cy', function (d) { return y(d.count); }) .style("fill", function(d) { return colors(d.name); }) .on('mouseover', function(d) { @@ -413,6 +447,9 @@ function init() { // variables and functions have their own scope (no override) .on('mouseout', function() { tooltipDate(false); }) + .on('click', function(d) { + handleMarkerClick(d); + }) pointsGroup.exit().remove(); } @@ -511,6 +548,136 @@ function init() { // variables and functions have their own scope (no override) var html = $('

').text(datum.name).html() + ' (' + formated_date + ', ' + $('

').text(datum.count).html() + ') '; return html; } + + function handleMarkerClick(datum) { + var xVal = getX(datum); + if (pickedNodes.start === null) { + pickedNodes.start = datum; + } else { + if (getX(pickedNodes.start) < xVal) { + pickedNodes.end = datum; + } else { + pickedNodes.end = pickedNodes.start; + pickedNodes.start = datum; + } + } + updatePickedNodesOverlays(); + } + + function clearPickedNodes() { + pickedNodes.start = null; + pickedNodes.end = null; + updatePickedNodesOverlays(); + } + + function updatePickedNodesOverlays() { + if (pickedNodes.start === null) { + overlayLeft.attr('width', 0); + overlayRight.attr('x', 0) + .attr('width', 0); + togglePickedNodeTooltip(false); + } else { + overlayLeft.attr('width', x(getX(pickedNodes.start))); + if (pickedNodes.end !== null) { + overlayRight.attr('x', x(getX(pickedNodes.end))) + .attr('width', width - x(getX(pickedNodes.end))); + togglePickedNodeTooltip(true); + } + } + } + + function togglePickedNodeTooltip(show) { + if (show) { + tooltipPickedNodes.html(genTooltipPickedNodeHtml()); + tooltipPickedNodes + .style('display', 'block') + .style('opacity', '0.8'); + + var overlayLeftBCR = overlayLeft.node().getBoundingClientRect(); + var overlayRightBCR = overlayRight.node().getBoundingClientRect(); + var tooltipBCR = tooltipPickedNodes.node().getBoundingClientRect(); + var left = (overlayLeftBCR.width - overlayRightBCR.width > 0 ? + overlayLeftBCR.left + overlayLeftBCR.width/2 : + overlayRightBCR.left + overlayRightBCR.width/2) - tooltipBCR.width / 2; + var top = overlayLeftBCR.top + 30; + + tooltipPickedNodes + .style('left', left + 'px') + .style('top', top + 'px') + } else { + tooltipPickedNodes.style('display', 'none'); + } + return tooltipPickedNodes; + } + + function genTooltipPickedNodeHtml() { + var xValueStart = getX(pickedNodes.start) + var xValueEnd = getX(pickedNodes.end) + var yValues = [] + data_nodes_active.forEach(function(serie) { + var startPoint = serie.values.find(function(point) { + return getX(point) == xValueStart; + }) + var endPoint = serie.values.find(function(point) { + return getX(point) == xValueEnd; + }) + if (startPoint !== undefined && endPoint !== undefined) + var deltaY = endPoint.count - startPoint.count; + var deltaYPerc = startPoint.count != 0 ? Math.abs(100*deltaY / startPoint.count).toFixed(2) : '-'; + yValues.push({ + name: serie.name, + nameColor: colors(serie.name), + deltaY: deltaY, + deltaYPerc: deltaYPerc + '%', + yColor: deltaY == 0 ? '' : (deltaY > 0 ? 'success' : 'error') + }) + }) + if (!options.abscissa_linear) { + xValueStart = d3.time.format(options.time_format)(xValueStart); + xValueEnd = d3.time.format(options.time_format)(xValueEnd); + } + var $content = $('
').append( + $('
').append( + $('').text(xValueStart), + $(''), + $('').text(xValueEnd) + ) + ); + var $table = $('
').append( + $('').append($('').append( + $('').text('Name'), + $('').text('Delta'), + $('').text('Delta %') + ) + ) + ); + yValues.forEach(function(serie) { + $table.append( + $('').append($('').append( + $('').append( + $('').append($('') + .attr('cx', 5) + .attr('cy', 5) + .attr('r', 5) + .css('fill', serie.nameColor) + ), + $('').text(serie.name) + ), + $('') + .addClass('text-'+serie.yColor) + .text(serie.deltaY) + .append($('').addClass(serie.deltaY > 0 ? 'fas fa-caret-up' : 'fas fa-caret-down')), + $('') + .addClass('text-'+serie.yColor) + .text(serie.deltaYPerc) + .append($('').addClass(serie.deltaY > 0 ? 'fas fa-caret-up' : 'fas fa-caret-down')), + ) + ) + ); + }); + $content.append($table); + return $content[0].outerHTML; + } }; @@ -578,4 +745,8 @@ function init() { // variables and functions have their own scope (no override) .axis.grid path { stroke-width: 0; } + +.overlay-right, .overlay-left { + cursor: pointer; +} From 4074e9b1cad5b8c5c9ed187c0b9121eb434cb588 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 26 Mar 2020 21:46:04 +0900 Subject: [PATCH 6/7] Update INSTALL.rhel7.md --- docs/INSTALL.rhel7.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md index d215fb492..7af631c81 100644 --- a/docs/INSTALL.rhel7.md +++ b/docs/INSTALL.rhel7.md @@ -218,7 +218,7 @@ installCoreRHEL () { # Create a python3 virtualenv sudo pip3 install virtualenv - $SUDO_WWW python3 /usr/local/bin/virtualenv -p python3 $PATH_TO_MISP/venv + $SUDO_WWW python3 -m venv $PATH_TO_MISP/venv sudo mkdir /usr/share/httpd/.cache sudo chown $WWW_USER:$WWW_USER /usr/share/httpd/.cache $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U pip setuptools From 8eff432fc5ee3e3353a379d7c89c83723cf630c3 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 26 Mar 2020 14:13:01 +0100 Subject: [PATCH 7/7] fix: [widgets:multiline] Racecondition executing `init` and fetching d3.js twice --- .../dashboard/Widgets/MultiLineChart.ctp | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp b/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp index 774ab227e..23d7e2cbb 100644 --- a/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp +++ b/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp @@ -10,14 +10,25 @@