Merge branch '2.4' of github.com:MISP/MISP into 2.4

pull/4769/head
iglocska 2019-06-16 19:15:48 +02:00
commit 98aca016e0
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
14 changed files with 107 additions and 38 deletions

View File

@ -120,9 +120,11 @@ MISPvars () {
# GPG configuration
GPG_REAL_NAME='Autogenerated Key'
# On a REAL install, please do not set a comment, see here for why: https://www.debian-administration.org/users/dkg/weblog/97
GPG_COMMENT='WARNING: MISP AutoGenerated Key consider this Key VOID!'
GPG_EMAIL_ADDRESS='admin@admin.test'
GPG_KEY_LENGTH='2048'
# 3072 bits used as per suggestions here: https://riseup.net/en/security/message-security/openpgp/best-practices
GPG_KEY_LENGTH='3072'
GPG_PASSPHRASE="$(openssl rand -hex 32)"
# debug alias to make sure people are not confused when blindly copy pasting blobs of code
@ -247,7 +249,7 @@ checkCoreOS () {
if [[ -f "/etc/redhat-release" ]]; then
echo "This is some redhat flavour"
REDHAT=1
RHfla=$(cat /etc/redhat-release | cut -f 1 -d\ | tr [A-Z] [a-z])
RHfla=$(cat /etc/redhat-release | cut -f 1 -d\ | tr '[:upper:]' '[:lower:]')
fi
}
@ -259,7 +261,7 @@ checkFlavour () {
sudo apt install lsb-release dialog -y
fi
FLAVOUR=$(lsb_release -s -i |tr [A-Z] [a-z])
FLAVOUR=$(lsb_release -s -i |tr '[:upper:]' '[:lower:]')
if [ FLAVOUR == "ubuntu" ]; then
RELEASE=$(lsb_release -s -r)
debug "We detected the following Linux flavour: ${YELLOW}$(tr '[:lower:]' '[:upper:]' <<< ${FLAVOUR:0:1})${FLAVOUR:1} ${RELEASE}${NC}"
@ -377,6 +379,27 @@ checkFail () {
fi
}
ask_o () {
ANSWER=""
if [ -z "${1}" ]; then
echo "This function needs at least 1 parameter."
exit 1
fi
[ -z "${2}" ] && OPT1="y" || OPT1="${2}"
[ -z "${3}" ] && OPT2="n" || OPT2="${3}"
while true; do
case "${ANSWER}" in "${OPT1}" | "${OPT2}") break ;; esac
echo -n "${1} (${OPT1}/${OPT2}) "
read ANSWER
ANSWER=$(echo "${ANSWER}" | tr '[:upper:]' '[:lower:]')
done
}
# Check if misp user is present and if run as root
checkID () {
debug "Checking if run as root and $MISP_USER is present"
@ -387,7 +410,7 @@ checkID () {
if [[ "$UNATTENDED" != "1" ]]; then
echo "There is NO user called '$MISP_USER' create a user '$MISP_USER' (y) or continue as $USER (n)? (y/n) "
read ANSWER
ANSWER=$(echo $ANSWER |tr [A-Z] [a-z])
ANSWER=$(echo $ANSWER |tr '[:upper:]' '[:lower:]')
else
ANSWER="y"
fi
@ -530,7 +553,7 @@ setBaseURL () {
echo "You can now enter your own MISP_BASEURL, if you wish to NOT do that, the MISP_BASEURL will be empty, which will work, but ideally you configure it afterwards."
echo "Do you want to change it now? (y/n) "
read ANSWER
ANSWER=$(echo $ANSWER |tr [A-Z] [a-z])
ANSWER=$(echo $ANSWER |tr '[:upper:]' '[:lower:]')
if [[ "$ANSWER" == "y" ]]; then
if [[ ! -z $IP ]]; then
echo "It seems you have an interface called $IFACE UP with the following IP: $IP - FYI"
@ -854,6 +877,7 @@ genRCLOCAL () {
sed -i -e '$i \echo never > /sys/kernel/mm/transparent_hugepage/enabled\n' /etc/rc.local
sed -i -e '$i \echo 1024 > /proc/sys/net/core/somaxconn\n' /etc/rc.local
sed -i -e '$i \sysctl vm.overcommit_memory=1\n' /etc/rc.local
sed -i -e '$i \[ -f /etc/init.d/firstBoot ] && bash /etc/init.d/firstBoot\n' /etc/rc.local
}
# Run PyMISP tests
@ -2318,7 +2342,7 @@ fi
# If Ubuntu is detected, figure out which release it is and run the according scripts
if [ "${FLAVOUR}" == "ubuntu" ]; then
RELEASE=$(lsb_release -s -r| tr [A-Z] [a-z])
RELEASE=$(lsb_release -s -r| tr '[:upper:]' '[:lower:]')
if [ "${RELEASE}" == "18.04" ]; then
echo "Install on Ubuntu 18.04 LTS fully supported."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
@ -2344,7 +2368,7 @@ fi
# If Debian is detected, figure out which release it is and run the according scripts
if [ "${FLAVOUR}" == "debian" ]; then
CODE=$(lsb_release -s -c| tr [A-Z] [a-z])
CODE=$(lsb_release -s -c| tr '[:upper:]' '[:lower:]')
if [ "${CODE}" == "buster" ]; then
echo "Install on Debian testing fully supported."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
@ -2366,7 +2390,7 @@ fi
# If Tsurugi is detected, figure out which release it is and run the according scripts
if [ "${FLAVOUR}" == "tsurugi" ]; then
CODE=$(lsb_release -s -c| tr [A-Z] [a-z])
CODE=$(lsb_release -s -c| tr '[:upper:]' '[:lower:]')
if [ "${CODE}" == "bamboo" ]; then
echo "Install on Tsurugi Lab partially supported."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"

View File

@ -1,5 +1,5 @@
; Generated by RHash v1.3.8 on 2019-06-13 at 09:09.32
; Generated by RHash v1.3.8 on 2019-06-15 at 11:59.41
; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
;
; 94127 09:09.32 2019-06-13 INSTALL.sh
INSTALL.sh BE7DE7EC5AA24D277F5F752E59356BAD26DC6EF7 E3ACE0081A95A94771EFE51AB8EDEE10CE5687F07A4C36DFD791B9B5F7E128DA 715AE909E454D2A74EEAF8A83D38F82534CB17B9C2BBDF5927251F08BD34651CF46D3260506ACC2BD3F26370E3A89696 24C56E075B7D4415517614C35A255A1D8217495A6A658F6B344A5D98AC24BD9A903E85EC3DC0C25CA89070EF0C0FB82D114ED7FA96D8120179DF4AEF0B1914ED
; 94958 11:59.41 2019-06-15 INSTALL.sh
INSTALL.sh AD8916A355616615634DD364BB78945301039BDD E12BB967B309DB78E723476B79AC6F1001F02248A0001593B5EFA5C18D0BEBB6 37BC162A792BDBC86AEC04F629E5C410D754C92CA76D796F8FA9F11272A35064D6D3371165A330F747E5C71BF69311F0 71C1FF6F5FB0C2A61A9C0AACE2D017C7EA4A7094B49F22CD22E5723702CFAC27F66D05C52368F169FD8F40166AFC493F5512E078EDF5C279939AB85110A27EBD

View File

@ -1 +1 @@
be7de7ec5aa24d277f5f752e59356bad26dc6ef7 INSTALL.sh
ad8916a355616615634dd364bb78945301039bdd INSTALL.sh

View File

@ -1 +1 @@
e3ace0081a95a94771efe51ab8edee10ce5687f07a4c36dfd791b9b5f7e128da INSTALL.sh
e12bb967b309db78e723476b79ac6f1001f02248a0001593b5efa5c18d0bebb6 INSTALL.sh

View File

@ -1 +1 @@
715ae909e454d2a74eeaf8a83d38f82534cb17b9c2bbdf5927251f08bd34651cf46d3260506acc2bd3f26370e3a89696 INSTALL.sh
37bc162a792bdbc86aec04f629e5c410d754c92ca76d796f8fa9f11272a35064d6d3371165a330f747e5c71bf69311f0 INSTALL.sh

View File

@ -1 +1 @@
24c56e075b7d4415517614c35a255a1d8217495a6a658f6b344a5d98ac24bd9a903e85ec3dc0c25ca89070ef0c0fb82d114ed7fa96d8120179df4aef0b1914ed INSTALL.sh
71c1ff6f5fb0c2a61a9c0aace2d017c7ea4a7094b49f22cd22e5723702cfac27f66d05c52368f169fd8f40166afc493f5512e078edf5c279939ab85110a27ebd INSTALL.sh

View File

@ -666,7 +666,7 @@ fi
# If Ubuntu is detected, figure out which release it is and run the according scripts
if [ "${FLAVOUR}" == "ubuntu" ]; then
RELEASE=$(lsb_release -s -r| tr [A-Z] [a-z])
RELEASE=$(lsb_release -s -r| tr '[:upper:]' '[:lower:]')
if [ "${RELEASE}" == "18.04" ]; then
echo "Install on Ubuntu 18.04 LTS fully supported."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
@ -692,7 +692,7 @@ fi
# If Debian is detected, figure out which release it is and run the according scripts
if [ "${FLAVOUR}" == "debian" ]; then
CODE=$(lsb_release -s -c| tr [A-Z] [a-z])
CODE=$(lsb_release -s -c| tr '[:upper:]' '[:lower:]')
if [ "${CODE}" == "buster" ]; then
echo "Install on Debian testing fully supported."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
@ -714,7 +714,7 @@ fi
# If Tsurugi is detected, figure out which release it is and run the according scripts
if [ "${FLAVOUR}" == "tsurugi" ]; then
CODE=$(lsb_release -s -c| tr [A-Z] [a-z])
CODE=$(lsb_release -s -c| tr '[:upper:]' '[:lower:]')
if [ "${CODE}" == "bamboo" ]; then
echo "Install on Tsurugi Lab partially supported."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"

View File

@ -22,12 +22,15 @@ import time
import uuid
import base64
import stix2misp_mapping
import stix_edh
import stix.extensions.marking.ais
from mixbox.namespaces import NamespaceNotFoundError
from operator import attrgetter
from stix.core import STIXPackage
from collections import defaultdict
try:
import stix_edh
except ModuleNotFoundError:
pass
_MISP_dir = "/".join([p for p in os.path.dirname(os.path.realpath(__file__)).split('/')[:-3]])
_PyMISP_dir = '{_MISP_dir}/PyMISP'.format(_MISP_dir=_MISP_dir)
@ -1225,13 +1228,16 @@ def generate_event(filename, tries=0):
sys.exit()
_update_namespaces()
return generate_event(filename, 1)
except NotImplementedError:
print('ERROR - Missing python library: stix_edh', file=sys.stderr)
except Exception:
try:
import maec
print(2)
except ImportError:
print('ERROR - Missing python library: maec', file=sys.stderr)
print(3)
sys.exit(0)
sys.exit(0)
def main(args):
filename = '{}/tmp/{}'.format(os.path.dirname(args[0]), args[1])

View File

@ -57,9 +57,11 @@ MISPvars () {
# GPG configuration
GPG_REAL_NAME='Autogenerated Key'
# On a REAL install, please do not set a comment, see here for why: https://www.debian-administration.org/users/dkg/weblog/97
GPG_COMMENT='WARNING: MISP AutoGenerated Key consider this Key VOID!'
GPG_EMAIL_ADDRESS='admin@admin.test'
GPG_KEY_LENGTH='2048'
# 3072 bits used as per suggestions here: https://riseup.net/en/security/message-security/openpgp/best-practices
GPG_KEY_LENGTH='3072'
GPG_PASSPHRASE="$(openssl rand -hex 32)"
# debug alias to make sure people are not confused when blindly copy pasting blobs of code

View File

@ -95,7 +95,7 @@ checkCoreOS () {
if [[ -f "/etc/redhat-release" ]]; then
echo "This is some redhat flavour"
REDHAT=1
RHfla=$(cat /etc/redhat-release | cut -f 1 -d\ | tr [A-Z] [a-z])
RHfla=$(cat /etc/redhat-release | cut -f 1 -d\ | tr '[:upper:]' '[:lower:]')
fi
}
@ -107,7 +107,7 @@ checkFlavour () {
sudo apt install lsb-release dialog -y
fi
FLAVOUR=$(lsb_release -s -i |tr [A-Z] [a-z])
FLAVOUR=$(lsb_release -s -i |tr '[:upper:]' '[:lower:]')
if [ FLAVOUR == "ubuntu" ]; then
RELEASE=$(lsb_release -s -r)
debug "We detected the following Linux flavour: ${YELLOW}$(tr '[:lower:]' '[:upper:]' <<< ${FLAVOUR:0:1})${FLAVOUR:1} ${RELEASE}${NC}"
@ -225,6 +225,27 @@ checkFail () {
fi
}
ask_o () {
ANSWER=""
if [ -z "${1}" ]; then
echo "This function needs at least 1 parameter."
exit 1
fi
[ -z "${2}" ] && OPT1="y" || OPT1="${2}"
[ -z "${3}" ] && OPT2="n" || OPT2="${3}"
while true; do
case "${ANSWER}" in "${OPT1}" | "${OPT2}") break ;; esac
echo -n "${1} (${OPT1}/${OPT2}) "
read ANSWER
ANSWER=$(echo "${ANSWER}" | tr '[:upper:]' '[:lower:]')
done
}
# Check if misp user is present and if run as root
checkID () {
debug "Checking if run as root and $MISP_USER is present"
@ -235,7 +256,7 @@ checkID () {
if [[ "$UNATTENDED" != "1" ]]; then
echo "There is NO user called '$MISP_USER' create a user '$MISP_USER' (y) or continue as $USER (n)? (y/n) "
read ANSWER
ANSWER=$(echo $ANSWER |tr [A-Z] [a-z])
ANSWER=$(echo $ANSWER |tr '[:upper:]' '[:lower:]')
else
ANSWER="y"
fi
@ -378,7 +399,7 @@ setBaseURL () {
echo "You can now enter your own MISP_BASEURL, if you wish to NOT do that, the MISP_BASEURL will be empty, which will work, but ideally you configure it afterwards."
echo "Do you want to change it now? (y/n) "
read ANSWER
ANSWER=$(echo $ANSWER |tr [A-Z] [a-z])
ANSWER=$(echo $ANSWER |tr '[:upper:]' '[:lower:]')
if [[ "$ANSWER" == "y" ]]; then
if [[ ! -z $IP ]]; then
echo "It seems you have an interface called $IFACE UP with the following IP: $IP - FYI"
@ -706,6 +727,7 @@ genRCLOCAL () {
sed -i -e '$i \echo never > /sys/kernel/mm/transparent_hugepage/enabled\n' /etc/rc.local
sed -i -e '$i \echo 1024 > /proc/sys/net/core/somaxconn\n' /etc/rc.local
sed -i -e '$i \sysctl vm.overcommit_memory=1\n' /etc/rc.local
sed -i -e '$i \[ -f /etc/init.d/firstBoot ] && bash /etc/init.d/firstBoot\n' /etc/rc.local
}
# Run PyMISP tests

View File

@ -26,11 +26,14 @@
## vi misp-backup.conf # adjust values
## sudo bash misp-backup.sh 2>&1 | tee misp-backup.log
##
## TODO: Target directory, rudimentary free space check: stat -f --format="%a" OutputDirName
## TODO: Make sure no directories are blank
## TODO: Make sure no directories are blank, $OutputDirName==Done
## TODO: Review how much sense it makes to ask fo MySQL credentials when most of the script does auto detection anyway.
##
# This makes use of the standard variables used by the installer
eval "$(curl -fsSL https://raw.githubusercontent.com/MISP/MISP/2.4/docs/generic/globalVariables.md | grep -v \`\`\`)"
MISPvars > /dev/null 2>&1
# Leave empty for NO debug messages, if run with set -x or bash -x it will enable DEBUG by default
DEBUG=
@ -59,6 +62,10 @@ checkDiskFree () {
if [[ ! -e $1 ]]; then
echo "$1 does not exist, creating"
mkdir -p $1
if [[ $? != 0 ]]; then
echo "Could not create $1, please fix permissions."
exit 126
fi
fi
threshhold=90
free=$(df -l --sync --output=pcent $1 |tail -1|cut -f 1 -d% | tr -d \ )

View File

@ -27,6 +27,10 @@
# TODO: Move DB, check DB?
# TODO: Check db user exists.
# This makes use of the standard variables used by the installer
eval "$(curl -fsSL https://raw.githubusercontent.com/MISP/MISP/2.4/docs/generic/globalVariables.md | grep -v \`\`\`)"
MISPvars > /dev/null 2>&1
# Leave empty for NO debug messages, if run with set -x or bash -x it will enable DEBUG by default
DEBUG=

View File

@ -1 +1 @@
MISPPath=/var/www/MISP
PATH_TO_MISP=/var/www/MISP

View File

@ -25,6 +25,10 @@
## Time to set some variables
##
# This makes use of the standard variables used by the installer
eval "$(curl -fsSL https://raw.githubusercontent.com/MISP/MISP/2.4/docs/generic/globalVariables.md | grep -v \`\`\`)"
MISPvars > /dev/null 2>&1
LUSER_ID="$(id -u)"
if [[ "${LUSER_ID}" > "0" ]]; then
@ -54,17 +58,17 @@ fi
# Fill in any missing values with defaults
# MISP path
MISPPath=${MISPPath:-$(locate MISP/app/webroot/index.php|sed 's/\/app\/webroot\/index\.php//')}
PATH_TO_MISP=${PATH_TO_MISP:-$(locate MISP/app/webroot/index.php|sed 's/\/app\/webroot\/index\.php//')}
# database.php
MySQLUUser=$(grep -o -P "(?<='login' => ').*(?=')" $MISPPath/app/Config/database.php)
MySQLUPass=$(grep -o -P "(?<='password' => ').*(?=')" $MISPPath/app/Config/database.php)
MISPDB=$(grep -o -P "(?<='database' => ').*(?=')" $MISPPath/app/Config/database.php)
DB_Port=$(grep -o -P "(?<='port' => ).*(?=,)" $MISPPath/app/Config/database.php)
MISPDBHost=$(grep -o -P "(?<='host' => ').*(?=')" $MISPPath/app/Config/database.php)
MySQLUUser=$(grep -o -P "(?<='login' => ').*(?=')" $PATH_TO_MISP/app/Config/database.php)
MySQLUPass=$(grep -o -P "(?<='password' => ').*(?=')" $PATH_TO_MISP/app/Config/database.php)
MISPDB=$(grep -o -P "(?<='database' => ').*(?=')" $PATH_TO_MISP/app/Config/database.php)
DB_Port=$(grep -o -P "(?<='port' => ).*(?=,)" $PATH_TO_MISP/app/Config/database.php)
MISPDBHost=$(grep -o -P "(?<='host' => ').*(?=')" $PATH_TO_MISP/app/Config/database.php)
echo "Clearing data model cache files"
rm -f $MISPPath/app/tmp/cache/models/myapp_*
rm -f $MISPPath/app/tmp/cache/persistent/myapp_*
rm -f $PATH_TO_MISP/app/tmp/cache/models/myapp_*
rm -f $PATH_TO_MISP/app/tmp/cache/persistent/myapp_*
echo "Wiping MySQL tables"
echo "Removes all users and organizations, except default (id=1)"
@ -76,7 +80,7 @@ mysql --host $MISPDBHost -u $MySQLRUser -p$MySQLRPass $MISPDB < $SQL
echo "Inserting default values to MySQL tables"
TMP=/tmp/misp-wipe-$$.sql
cd $MISPPath
cd $PATH_TO_MISP
sed -n '/Default values for initial installation/ { s///; :a; n; p; ba; }' INSTALL/MYSQL.sql | egrep -v '(admin_settings|db_version)' > $TMP
mysql --host $MISPDBHost -u $MySQLRUser -p$MySQLRPass $MISPDB < $TMP
rm -f $TMP
@ -88,7 +92,7 @@ git clean -f -d -x app/tmp
git clean -f -d -x app/files
echo "Updating taxonomies"
baseurl=$(grep -o -P "(?<='baseurl' => ').*(?=')" $MISPPath/app/Config/config.php)
baseurl=$(grep -o -P "(?<='baseurl' => ').*(?=')" $PATH_TO_MISP/app/Config/config.php)
AuthKey=$(echo 'select authkey from users where role_id = 1 order by id limit 1;' | mysql -u $MySQLRUser -p$MySQLRPass $MISPDB 2>/dev/null | tail -1)
curl --header "Authorization: $AuthKey" --header "Accept: application/json" --header "Content-Type: application/json" -o /dev/null -s -X POST ${baseurl}/taxonomies/update