MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch '2.4' of github.com:MISP/MISP into 2.4

pull/7276/head
iglocska 2021-03-30 09:39:29 +02:00
parent 8f13e5f41a 309b54c331
commit 75de46a6a7
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
6 changed files with 2313 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/Model/Server.php
View File

@ -2234,7 +2234,7 @@ class Server extends AppModel
'debug', 'MISP', 'GnuPG', 'SMIME', 'Proxy', 'SecureAuth',
'Security', 'Session.defaults', 'Session.timeout', 'Session.cookieTimeout',
'Session.autoRegenerate', 'Session.checkAgent', 'site_admin_debug',
'Plugin', 'CertAuth', 'ApacheShibbAuth', 'ApacheSecureAuth', 'OidcAuth',
'Plugin', 'CertAuth', 'ApacheShibbAuth', 'ApacheSecureAuth', 'OidcAuth', 'AadAuth'
);
$settingsArray = array();
foreach ($settingsToSave as $setting) {

2211
docs/Changelog.md
View File

File diff suppressed because it is too large Load Diff

55
docs/INSTALL.rhel7.md
View File

@ -112,6 +112,9 @@ centosEPEL () {
# Since MISP 2.4 PHP 5.5 is a minimal requirement, so we need a newer version than CentOS base provides
# Software Collections is a way do to this, see https://wiki.centos.org/AdditionalResources/Repositories/SCL
sudo yum install centos-release-scl -y
sudo yum install yum-utils -y
sudo yum install http://rpms.remirepo.net/enterprise/remi-release-7.rpm -y
sudo yum-config-manager --enable remi-php74
}
# <snippet-end 0_CentOS_EPEL.sh>
```
@ -169,6 +172,8 @@ enableEPEL () {
# <snippet-begin 0_yumInstallCoreDeps.sh>
yumInstallCoreDeps () {
# Install the dependencies:
PHP_BASE="/etc/"
PHP_INI="/etc/php.ini"
sudo yum install gcc git zip \
mod_ssl \
redis \
@ -185,27 +190,26 @@ yumInstallCoreDeps () {
rm mariadb_repo_setup
sudo yum install MariaDB-server -y
PHP_INI="/etc/opt/remi/php74/php.ini"
# Install PHP 7.4 from Remi's repo, see https://rpms.remirepo.net/enterprise/7/php74/x86_64/repoview/
sudo yum install php74 php74-php-fpm php74-php-devel \
php74-php-mysqlnd \
php74-php-mbstring \
php74-php-xml \
php74-php-bcmath \
php74-php-opcache \
php74-php-zip \
php74-php-pear \
php74-php-brotli \
php74-php-intl \
php74-php-gd -y
sudo yum install php php-fpm php-devel \
php-mysqlnd \
php-mbstring \
php-xml \
php-bcmath \
php-opcache \
php-zip \
php-pear \
php-brotli \
php-intl \
php-gd -y
# cake has php baked in, thus we link to it
sudo ln -s /usr/bin/php74 /usr/bin/php
# cake has php baked in, thus we link to it if necessary.
[[ ! -e "/usr/bin/php" ]] && sudo ln -s /usr/bin/php74 /usr/bin/php
# Python 3.6 is now available in RHEL 7.7 base
sudo yum install python3 python3-devel -y
sudo systemctl enable --now php74-php-fpm.service
sudo systemctl enable --now php-fpm.service
}
# <snippet-end 0_yumInstallCoreDeps.sh>
```
@ -315,9 +319,11 @@ installCoreRHEL () {
# Enable dependencies detection in the diagnostics page
# This allows MISP to detect GnuPG, the Python modules' versions and to read the PHP settings.
echo "env[PATH] = /usr/local/bin:/usr/bin:/bin" |sudo tee -a /etc/opt/remi/php74/php-fpm.d/www.conf
sudo sed -i.org -e 's/^;\(clear_env = no\)/\1/' /etc/opt/remi/php74/php-fpm.d/www.conf
sudo systemctl restart php74-php-fpm.service
echo "env[PATH] = /usr/local/bin:/usr/bin:/bin" |sudo tee -a ${PHP_BASE}/php-fpm.d/www.conf
sudo sed -i.org -e 's/^;\(clear_env = no\)/\1/' ${PHP_BASE}/php-fpm.d/www.conf
sudo sed -i.org -e 's/^\(listen =\) \/run\/php-fpm\/www\.sock/\1 127.0.0.1:9000/' ${PHP_BASE}/php-fpm.d/www.conf
sudo systemctl restart php-fpm.service
umask $UMASK
}
# <snippet-end 1_mispCoreInstall_RHEL.sh>
@ -345,14 +351,15 @@ installCake_RHEL ()
#$SUDO_WWW php -r "unlink('composer-setup.php');"
$SUDO_WWW php composer.phar install
sudo yum install php74-php-pecl-redis php74-php-pecl-ssdeep php74-php-pecl-gnupg -y
sudo yum install php-pecl-redis php-pecl-ssdeep php-pecl-gnupg -y
sudo systemctl restart php74-php-fpm.service
sudo systemctl restart php-fpm.service
# If you have not yet set a timezone in php.ini
echo 'date.timezone = "Asia/Tokyo"' |sudo tee /etc/opt/remi/php74/php.d/timezone.ini
echo 'date.timezone = "Asia/Tokyo"' |sudo tee /etc/php-fpm.d/timezone.ini
sudo ln -s ../php-fpm.d/timezone.ini /etc/php.d/99-timezone.ini
# Recommended: Change some PHP settings in /etc/opt/remi/php74/php.ini
# Recommended: Change some PHP settings in /etc/php.ini
# max_execution_time = 300
# memory_limit = 2048M
# upload_max_filesize = 50M
@ -363,7 +370,7 @@ installCake_RHEL ()
done
sudo sed -i "s/^\(session.sid_length\).*/\1 = $(eval echo \${session0sid_length})/" $PHP_INI
sudo sed -i "s/^\(session.use_strict_mode\).*/\1 = $(eval echo \${session0use_strict_mode})/" $PHP_INI
sudo systemctl restart php74-php-fpm.service
sudo systemctl restart php-fpm.service
# To use the scheduler worker for scheduled tasks, do the following:
sudo cp -fa $PATH_TO_MISP/INSTALL/setup/config.php $PATH_TO_MISP/app/Plugin/CakeResque/Config/config.php
@ -655,7 +662,7 @@ EOF
configWorkersRHEL () {
echo "[Unit]
Description=MISP background workers
After=mariadb.service redis.service php74-php-fpm.service
After=mariadb.service redis.service php-fpm.service
[Service]
Type=forking

14
docs/generic/globalVariables.md
View File

@ -1,8 +1,19 @@
#### MISP configuration variables
If you are doing a manual install, copy and pasting from this document, please do the following before starting:
```bash
eval "$(curl -fsSL https://raw.githubusercontent.com/MISP/MISP/2.4/docs/generic/globalVariables.md | awk '/^# <snippet-begin/,0' | grep -v \`\`\`)"
MISPvars
```
!!! notice
Do NOT leave your session after this mid-install as some initial passwords have been generated and thus would be lost.
Consider using a mux like [screen or tmux](https://superuser.com/questions/423310/byobu-vs-gnu-screen-vs-tmux-usefulness-and-transferability-of-skills).
```bash
# <snippet-begin 0_global-vars.sh>
# $ eval "$(curl -fsSL https://raw.githubusercontent.com/MISP/MISP/2.4/docs/generic/globalVariables.md | grep -v \`\`\`)"
# $ eval "$(curl -fsSL https://raw.githubusercontent.com/MISP/MISP/2.4/docs/generic/globalVariables.md | awk '/^# <snippet-begin/,0' | grep -v \`\`\`)"
# $ MISPvars
MISPvars () {
debug "Setting generic ${LBLUE}MISP${NC} variables shared by all flavours" 2> /dev/null
@ -19,6 +30,7 @@ MISPvars () {
# The web server user
# RHEL/CentOS
if [[ -f "/etc/redhat-release" ]]; then
SE_LINUX=$(sestatus -v -b |grep "^SELinux status"| grep enabled ; echo $?)
WWW_USER="apache"
SUDO_WWW="sudo -H -u ${WWW_USER} "
# Debian flavoured

89
docs/generic/misp-modules-centos.md
View File

@ -4,25 +4,24 @@
# <snippet-begin 3_misp-modules_RHEL.sh>
mispmodulesRHEL () {
# some misp-modules dependencies for RHEL<8
sudo yum install openjpeg-devel gcc-c++ poppler-cpp-devel pkgconfig python-devel redhat-rpm-config -y
[[ "${DIST_VER}" =~ ^[7].* ]] && sudo yum install openjpeg-devel gcc-c++ poppler-cpp-devel pkgconfig python-devel redhat-rpm-config -y
# some misp-modules dependencies for RHEL8
sudo yum install openjpeg2-devel gcc-c++ poppler-cpp-devel pkgconfig python2-devel redhat-rpm-config -y
[[ "${DIST_VER}" =~ ^[8].* ]] && sudo yum install openjpeg2-devel gcc-c++ poppler-cpp-devel pkgconfig python2-devel redhat-rpm-config -y
sudo chmod 2777 /usr/local/src
sudo chown root:users /usr/local/src
cd /usr/local/src/
false; while [[ $? -ne 0 ]]; do $SUDO_WWW git clone https://github.com/MISP/misp-modules.git; done
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git clone https://github.com/MISP/misp-modules.git; done
cd misp-modules
# pip install
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U -I -r REQUIREMENTS
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U .
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install pyfaup censys
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U -I -r REQUIREMENTS
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U .
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install pyfaup censys
# some misp-modules dependencies for RHEL<8
sudo yum install rubygem-rouge rubygem-asciidoctor zbar-devel opencv-devel -y
[[ "${DIST_VER}" =~ ^[7].* ]] && sudo yum install rubygem-rouge rubygem-asciidoctor zbar-devel opencv-devel -y
# some misp-modules dependencies for RHEL8
sudo dnf install https://packages.endpoint.com/rhel/8/main/x86_64/endpoint-repo-8-1.ep8.noarch.rpm -y
sudo yum install zbar-devel opencv-devel -y
[[ "${DIST_VER}" =~ ^[8].* ]] && sudo dnf install https://packages.endpoint.com/rhel/8/main/x86_64/endpoint-repo-8-1.ep8.noarch.rpm -y && sudo yum install zbar-devel opencv-devel -y
echo "[Unit]
Description=MISP modules
@ -30,8 +29,8 @@ mispmodulesRHEL () {
[Service]
Type=simple
User=$WWW_USER
Group=$WWW_USER
User=${WWW_USER}
Group=${WWW_USER}
WorkingDirectory=/usr/local/src/misp-modules
Environment="PATH=/var/www/MISP/venv/bin"
ExecStart=\"${PATH_TO_MISP}/venv/bin/misp-modules -l 127.0.0.1 -s\"
@ -43,48 +42,48 @@ mispmodulesRHEL () {
sudo systemctl daemon-reload
# Test misp-modules
$SUDO_WWW $PATH_TO_MISP/venv/bin/misp-modules -l 127.0.0.1 -s &
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/misp-modules -l 127.0.0.1 -s &
sudo systemctl enable --now misp-modules
# Enable Enrichment, set better timeouts
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Enrichment_services_enable" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Enrichment_hover_enable" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Enrichment_timeout" 300
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Enrichment_hover_timeout" 150
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_services_enable" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_hover_enable" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_timeout" 300
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_hover_timeout" 150
# TODO:"Investigate why the next one fails"
#$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Enrichment_asn_history_enabled" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Enrichment_cve_enabled" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Enrichment_dns_enabled" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Enrichment_btc_steroids_enabled" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Enrichment_ipasn_enabled" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Enrichment_yara_syntax_validator_enabled" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Enrichment_yara_query_enabled" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Enrichment_pdf_enabled" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Enrichment_docx_enabled" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Enrichment_xlsx_enabled" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Enrichment_pptx_enabled" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Enrichment_ods_enabled" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Enrichment_odt_enabled" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Enrichment_services_url" "http://127.0.0.1"
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Enrichment_services_port" 6666
#${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_asn_history_enabled" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_cve_enabled" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_dns_enabled" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_btc_steroids_enabled" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_ipasn_enabled" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_yara_syntax_validator_enabled" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_yara_query_enabled" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_pdf_enabled" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_docx_enabled" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_xlsx_enabled" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_pptx_enabled" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_ods_enabled" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_odt_enabled" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_services_url" "http://127.0.0.1"
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_services_port" 6666
# Enable Import modules, set better timeout
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Import_services_enable" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Import_services_url" "http://127.0.0.1"
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Import_services_port" 6666
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Import_timeout" 300
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Import_ocr_enabled" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Import_mispjson_enabled" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Import_openiocimport_enabled" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Import_threatanalyzer_import_enabled" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Import_csvimport_enabled" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Import_services_enable" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Import_services_url" "http://127.0.0.1"
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Import_services_port" 6666
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Import_timeout" 300
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Import_ocr_enabled" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Import_mispjson_enabled" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Import_openiocimport_enabled" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Import_threatanalyzer_import_enabled" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Import_csvimport_enabled" true
# Enable Export modules, set better timeout
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Export_services_enable" true
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Export_services_url" "http://127.0.0.1"
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Export_services_port" 6666
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Export_timeout" 300
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Export_pdfexport_enabled" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Export_services_enable" true
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Export_services_url" "http://127.0.0.1"
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Export_services_port" 6666
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Export_timeout" 300
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Export_pdfexport_enabled" true
}
# <snippet-end 3_misp-modules_RHEL.sh>
```

24
tools/gen_misp_install_docs.sh
View File

@ -3,7 +3,7 @@
set -e
# set -x
if [ -e "/usr/bin/virtualenv" ]; then
if [ -e "$(which virtualenv)" ]; then
echo "Python virtualenv exists, continuing with mkdocs build"
else
echo "NO virtualenv present, bye."
@ -16,6 +16,8 @@ if [ -z "$VIRTUAL_ENV" ]; then
${PWD}/mkdocs/bin/pip install mkdocs==1.0.4 mkdocs-material==4.6.3 markdown-include python-markdown-comments gitchangelog git+https://github.com/ryneeverett/python-markdown-comments.git
fi
[[ -e "$(which gsed)" ]] && xSED="gsed" || xSED="sed"
# Fixing ASCII aborration introduced in: https://github.com/MISP/MISP/commit/1b028ee15a3bd2f209102cd6204e6c4bb519be97
${PWD}/mkdocs/bin/gitchangelog |grep -v -e " ,," -e "\.\.," > ../docs/Changelog.md
# Removing consecutive dupe lines
@ -28,18 +30,18 @@ ${PWD}/gen_misp_changelog.py
# This search and replace is sub-optimal. It replaces 3 "~"s beginning of the line
# and then just replaces the remaining 2 following tildes in the document.
# This might change the sense of some commit messages...
sed -i "s/^\~\~\~/---/" ../docs/Changelog.md
sed -i "s/^- \#/- \\\#/" ../docs/Changelog.md
sed -i "s/\~\~/--/g" ../docs/Changelog.md
sed -i "s/%%version%%/v2.4 aka 2.4 for ever/g" ../docs/Changelog.md
sed -i "s/\(unreleased\)/current changelog/g" ../docs/Changelog.md
${xSED} -i "s/^\~\~\~/---/" ../docs/Changelog.md
${xSED} -i "s/^- \#/- \\\#/" ../docs/Changelog.md
${xSED} -i "s/\~\~/--/g" ../docs/Changelog.md
${xSED} -i "s/%%version%%/v2.4 aka 2.4 for ever/g" ../docs/Changelog.md
${xSED} -i "s/\(unreleased\)/current changelog/g" ../docs/Changelog.md
# Emojifying things
sed -i "s/\/\!\\\/:warning:/g" ../docs/Changelog.md
sed -i "s/WiP/:construction:/g" ../docs/Changelog.md
sed -i "s/WIP/:construction:/g" ../docs/Changelog.md
sed -i "s/Wip:/:construction:/g" ../docs/Changelog.md
sed -i "s/\[security\]/:lock:/g" ../docs/Changelog.md
${xSED} -i "s/\/\!\\\/:warning:/g" ../docs/Changelog.md
${xSED} -i "s/WiP/:construction:/g" ../docs/Changelog.md
${xSED} -i "s/WIP/:construction:/g" ../docs/Changelog.md
${xSED} -i "s/Wip:/:construction:/g" ../docs/Changelog.md
${xSED} -i "s/\[security\]/:lock:/g" ../docs/Changelog.md
## Other creative ways in sprinkling emoji goodness:
### Source: https://gist.github.com/pocotan001/68f96bf86891db316f20