Merge branch '2.4' of github.com:MISP/MISP into 2.4

pull/4769/head
chrisr3d 2019-06-17 09:16:47 +02:00
commit 139eab9fe9
14 changed files with 138 additions and 56 deletions

View File

@ -120,9 +120,11 @@ MISPvars () {
# GPG configuration
GPG_REAL_NAME='Autogenerated Key'
# On a REAL install, please do not set a comment, see here for why: https://www.debian-administration.org/users/dkg/weblog/97
GPG_COMMENT='WARNING: MISP AutoGenerated Key consider this Key VOID!'
GPG_EMAIL_ADDRESS='admin@admin.test'
GPG_KEY_LENGTH='2048'
# 3072 bits used as per suggestions here: https://riseup.net/en/security/message-security/openpgp/best-practices
GPG_KEY_LENGTH='3072'
GPG_PASSPHRASE="$(openssl rand -hex 32)"
# debug alias to make sure people are not confused when blindly copy pasting blobs of code
@ -247,7 +249,7 @@ checkCoreOS () {
if [[ -f "/etc/redhat-release" ]]; then
echo "This is some redhat flavour"
REDHAT=1
RHfla=$(cat /etc/redhat-release | cut -f 1 -d\ | tr [A-Z] [a-z])
RHfla=$(cat /etc/redhat-release | cut -f 1 -d\ | tr '[:upper:]' '[:lower:]')
fi
}
@ -259,7 +261,7 @@ checkFlavour () {
sudo apt install lsb-release dialog -y
fi
FLAVOUR=$(lsb_release -s -i |tr [A-Z] [a-z])
FLAVOUR=$(lsb_release -s -i |tr '[:upper:]' '[:lower:]')
if [ FLAVOUR == "ubuntu" ]; then
RELEASE=$(lsb_release -s -r)
debug "We detected the following Linux flavour: ${YELLOW}$(tr '[:lower:]' '[:upper:]' <<< ${FLAVOUR:0:1})${FLAVOUR:1} ${RELEASE}${NC}"
@ -377,6 +379,27 @@ checkFail () {
fi
}
ask_o () {
ANSWER=""
if [ -z "${1}" ]; then
echo "This function needs at least 1 parameter."
exit 1
fi
[ -z "${2}" ] && OPT1="y" || OPT1="${2}"
[ -z "${3}" ] && OPT2="n" || OPT2="${3}"
while true; do
case "${ANSWER}" in "${OPT1}" | "${OPT2}") break ;; esac
echo -n "${1} (${OPT1}/${OPT2}) "
read ANSWER
ANSWER=$(echo "${ANSWER}" | tr '[:upper:]' '[:lower:]')
done
}
# Check if misp user is present and if run as root
checkID () {
debug "Checking if run as root and $MISP_USER is present"
@ -387,7 +410,7 @@ checkID () {
if [[ "$UNATTENDED" != "1" ]]; then
echo "There is NO user called '$MISP_USER' create a user '$MISP_USER' (y) or continue as $USER (n)? (y/n) "
read ANSWER
ANSWER=$(echo $ANSWER |tr [A-Z] [a-z])
ANSWER=$(echo $ANSWER |tr '[:upper:]' '[:lower:]')
else
ANSWER="y"
fi
@ -530,7 +553,7 @@ setBaseURL () {
echo "You can now enter your own MISP_BASEURL, if you wish to NOT do that, the MISP_BASEURL will be empty, which will work, but ideally you configure it afterwards."
echo "Do you want to change it now? (y/n) "
read ANSWER
ANSWER=$(echo $ANSWER |tr [A-Z] [a-z])
ANSWER=$(echo $ANSWER |tr '[:upper:]' '[:lower:]')
if [[ "$ANSWER" == "y" ]]; then
if [[ ! -z $IP ]]; then
echo "It seems you have an interface called $IFACE UP with the following IP: $IP - FYI"
@ -854,6 +877,7 @@ genRCLOCAL () {
sed -i -e '$i \echo never > /sys/kernel/mm/transparent_hugepage/enabled\n' /etc/rc.local
sed -i -e '$i \echo 1024 > /proc/sys/net/core/somaxconn\n' /etc/rc.local
sed -i -e '$i \sysctl vm.overcommit_memory=1\n' /etc/rc.local
sed -i -e '$i \[ -f /etc/init.d/firstBoot ] && bash /etc/init.d/firstBoot\n' /etc/rc.local
}
# Run PyMISP tests
@ -2318,7 +2342,7 @@ fi
# If Ubuntu is detected, figure out which release it is and run the according scripts
if [ "${FLAVOUR}" == "ubuntu" ]; then
RELEASE=$(lsb_release -s -r| tr [A-Z] [a-z])
RELEASE=$(lsb_release -s -r| tr '[:upper:]' '[:lower:]')
if [ "${RELEASE}" == "18.04" ]; then
echo "Install on Ubuntu 18.04 LTS fully supported."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
@ -2344,7 +2368,7 @@ fi
# If Debian is detected, figure out which release it is and run the according scripts
if [ "${FLAVOUR}" == "debian" ]; then
CODE=$(lsb_release -s -c| tr [A-Z] [a-z])
CODE=$(lsb_release -s -c| tr '[:upper:]' '[:lower:]')
if [ "${CODE}" == "buster" ]; then
echo "Install on Debian testing fully supported."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
@ -2366,7 +2390,7 @@ fi
# If Tsurugi is detected, figure out which release it is and run the according scripts
if [ "${FLAVOUR}" == "tsurugi" ]; then
CODE=$(lsb_release -s -c| tr [A-Z] [a-z])
CODE=$(lsb_release -s -c| tr '[:upper:]' '[:lower:]')
if [ "${CODE}" == "bamboo" ]; then
echo "Install on Tsurugi Lab partially supported."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"

View File

@ -1,5 +1,5 @@
; Generated by RHash v1.3.8 on 2019-06-13 at 09:09.32
; Generated by RHash v1.3.8 on 2019-06-15 at 11:59.41
; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
;
; 94127 09:09.32 2019-06-13 INSTALL.sh
INSTALL.sh BE7DE7EC5AA24D277F5F752E59356BAD26DC6EF7 E3ACE0081A95A94771EFE51AB8EDEE10CE5687F07A4C36DFD791B9B5F7E128DA 715AE909E454D2A74EEAF8A83D38F82534CB17B9C2BBDF5927251F08BD34651CF46D3260506ACC2BD3F26370E3A89696 24C56E075B7D4415517614C35A255A1D8217495A6A658F6B344A5D98AC24BD9A903E85EC3DC0C25CA89070EF0C0FB82D114ED7FA96D8120179DF4AEF0B1914ED
; 94958 11:59.41 2019-06-15 INSTALL.sh
INSTALL.sh AD8916A355616615634DD364BB78945301039BDD E12BB967B309DB78E723476B79AC6F1001F02248A0001593B5EFA5C18D0BEBB6 37BC162A792BDBC86AEC04F629E5C410D754C92CA76D796F8FA9F11272A35064D6D3371165A330F747E5C71BF69311F0 71C1FF6F5FB0C2A61A9C0AACE2D017C7EA4A7094B49F22CD22E5723702CFAC27F66D05C52368F169FD8F40166AFC493F5512E078EDF5C279939AB85110A27EBD

View File

@ -1 +1 @@
be7de7ec5aa24d277f5f752e59356bad26dc6ef7 INSTALL.sh
ad8916a355616615634dd364bb78945301039bdd INSTALL.sh

View File

@ -1 +1 @@
e3ace0081a95a94771efe51ab8edee10ce5687f07a4c36dfd791b9b5f7e128da INSTALL.sh
e12bb967b309db78e723476b79ac6f1001f02248a0001593b5efa5c18d0bebb6 INSTALL.sh

View File

@ -1 +1 @@
715ae909e454d2a74eeaf8a83d38f82534cb17b9c2bbdf5927251f08bd34651cf46d3260506acc2bd3f26370e3a89696 INSTALL.sh
37bc162a792bdbc86aec04f629e5c410d754c92ca76d796f8fa9f11272a35064d6d3371165a330f747e5c71bf69311f0 INSTALL.sh

View File

@ -1 +1 @@
24c56e075b7d4415517614c35a255a1d8217495a6a658f6b344a5d98ac24bd9a903e85ec3dc0c25ca89070ef0c0fb82d114ed7fa96d8120179df4aef0b1914ed INSTALL.sh
71c1ff6f5fb0c2a61a9c0aace2d017c7ea4a7094b49f22cd22e5723702cfac27f66d05c52368f169fd8f40166afc493f5512e078edf5c279939ab85110a27ebd INSTALL.sh

View File

@ -666,7 +666,7 @@ fi
# If Ubuntu is detected, figure out which release it is and run the according scripts
if [ "${FLAVOUR}" == "ubuntu" ]; then
RELEASE=$(lsb_release -s -r| tr [A-Z] [a-z])
RELEASE=$(lsb_release -s -r| tr '[:upper:]' '[:lower:]')
if [ "${RELEASE}" == "18.04" ]; then
echo "Install on Ubuntu 18.04 LTS fully supported."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
@ -692,7 +692,7 @@ fi
# If Debian is detected, figure out which release it is and run the according scripts
if [ "${FLAVOUR}" == "debian" ]; then
CODE=$(lsb_release -s -c| tr [A-Z] [a-z])
CODE=$(lsb_release -s -c| tr '[:upper:]' '[:lower:]')
if [ "${CODE}" == "buster" ]; then
echo "Install on Debian testing fully supported."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
@ -714,7 +714,7 @@ fi
# If Tsurugi is detected, figure out which release it is and run the according scripts
if [ "${FLAVOUR}" == "tsurugi" ]; then
CODE=$(lsb_release -s -c| tr [A-Z] [a-z])
CODE=$(lsb_release -s -c| tr '[:upper:]' '[:lower:]')
if [ "${CODE}" == "bamboo" ]; then
echo "Install on Tsurugi Lab partially supported."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"

View File

@ -192,6 +192,15 @@ class Server extends AppModel
'type' => 'boolean',
'null' => true
),
'server_settings_skip_backup_rotate' => array(
'level' => 1,
'description' => __('Enable this setting to directly save the config.php file without first creating a temporary file and moving it to avoid concurency issues. Generally not recommended, but useful when for example other tools modify/maintain the config.php file.'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean',
'null' => true
),
'python_bin' => array(
'level' => 1,
'description' => __('It is highly recommended to install all the python dependencies in a virtualenv. The recommended location is: %s/venv', ROOT),
@ -3076,6 +3085,9 @@ class Server extends AppModel
public function testForBinExec($value)
{
if (substr($value, 0, 7) === "phar://") {
return 'Phar protocol not allowed.';
}
$finfo = finfo_open(FILEINFO_MIME_TYPE);
if ($value === '') {
return true;
@ -3094,6 +3106,9 @@ class Server extends AppModel
public function testForWritableDir($value)
{
if (substr($value, 0, 7) === "phar://") {
return 'Phar protocol not allowed.';
}
if (!is_dir($value)) {
return 'Not a valid directory.';
}
@ -3724,26 +3739,30 @@ class Server extends AppModel
if (function_exists('opcache_reset')) {
opcache_reset();
}
$randomFilename = $this->generateRandomFileName();
// To protect us from 2 admin users having a concurent file write to the config file, solar flares and the bogeyman
file_put_contents(APP . 'Config' . DS . $randomFilename, $settingsString);
rename(APP . 'Config' . DS . $randomFilename, APP . 'Config' . DS . 'config.php');
$config_saved = file_get_contents(APP . 'Config' . DS . 'config.php');
// if the saved config file is empty, restore the backup.
if (strlen($config_saved) < 20) {
copy(APP . 'Config' . DS . 'config.php.bk', APP . 'Config' . DS . 'config.php');
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => $id,
'email' => 'SYSTEM',
'action' => 'error',
'user_id' => 0,
'title' => 'Error: Something went wrong saving the config file, reverted to backup file.',
));
return false;
if (empty(Configure::read('MISP.server_settings_skip_backup_rotate'))) {
$randomFilename = $this->generateRandomFileName();
// To protect us from 2 admin users having a concurent file write to the config file, solar flares and the bogeyman
file_put_contents(APP . 'Config' . DS . $randomFilename, $settingsString);
rename(APP . 'Config' . DS . $randomFilename, APP . 'Config' . DS . 'config.php');
$config_saved = file_get_contents(APP . 'Config' . DS . 'config.php');
// if the saved config file is empty, restore the backup.
if (strlen($config_saved) < 20) {
copy(APP . 'Config' . DS . 'config.php.bk', APP . 'Config' . DS . 'config.php');
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => $id,
'email' => 'SYSTEM',
'action' => 'error',
'user_id' => 0,
'title' => 'Error: Something went wrong saving the config file, reverted to backup file.',
));
return false;
}
} else {
file_put_contents(APP . 'Config' . DS . 'config.php', $settingsString);
}
return true;
}

View File

@ -57,9 +57,11 @@ MISPvars () {
# GPG configuration
GPG_REAL_NAME='Autogenerated Key'
# On a REAL install, please do not set a comment, see here for why: https://www.debian-administration.org/users/dkg/weblog/97
GPG_COMMENT='WARNING: MISP AutoGenerated Key consider this Key VOID!'
GPG_EMAIL_ADDRESS='admin@admin.test'
GPG_KEY_LENGTH='2048'
# 3072 bits used as per suggestions here: https://riseup.net/en/security/message-security/openpgp/best-practices
GPG_KEY_LENGTH='3072'
GPG_PASSPHRASE="$(openssl rand -hex 32)"
# debug alias to make sure people are not confused when blindly copy pasting blobs of code

View File

@ -95,7 +95,7 @@ checkCoreOS () {
if [[ -f "/etc/redhat-release" ]]; then
echo "This is some redhat flavour"
REDHAT=1
RHfla=$(cat /etc/redhat-release | cut -f 1 -d\ | tr [A-Z] [a-z])
RHfla=$(cat /etc/redhat-release | cut -f 1 -d\ | tr '[:upper:]' '[:lower:]')
fi
}
@ -107,7 +107,7 @@ checkFlavour () {
sudo apt install lsb-release dialog -y
fi
FLAVOUR=$(lsb_release -s -i |tr [A-Z] [a-z])
FLAVOUR=$(lsb_release -s -i |tr '[:upper:]' '[:lower:]')
if [ FLAVOUR == "ubuntu" ]; then
RELEASE=$(lsb_release -s -r)
debug "We detected the following Linux flavour: ${YELLOW}$(tr '[:lower:]' '[:upper:]' <<< ${FLAVOUR:0:1})${FLAVOUR:1} ${RELEASE}${NC}"
@ -225,6 +225,27 @@ checkFail () {
fi
}
ask_o () {
ANSWER=""
if [ -z "${1}" ]; then
echo "This function needs at least 1 parameter."
exit 1
fi
[ -z "${2}" ] && OPT1="y" || OPT1="${2}"
[ -z "${3}" ] && OPT2="n" || OPT2="${3}"
while true; do
case "${ANSWER}" in "${OPT1}" | "${OPT2}") break ;; esac
echo -n "${1} (${OPT1}/${OPT2}) "
read ANSWER
ANSWER=$(echo "${ANSWER}" | tr '[:upper:]' '[:lower:]')
done
}
# Check if misp user is present and if run as root
checkID () {
debug "Checking if run as root and $MISP_USER is present"
@ -235,7 +256,7 @@ checkID () {
if [[ "$UNATTENDED" != "1" ]]; then
echo "There is NO user called '$MISP_USER' create a user '$MISP_USER' (y) or continue as $USER (n)? (y/n) "
read ANSWER
ANSWER=$(echo $ANSWER |tr [A-Z] [a-z])
ANSWER=$(echo $ANSWER |tr '[:upper:]' '[:lower:]')
else
ANSWER="y"
fi
@ -378,7 +399,7 @@ setBaseURL () {
echo "You can now enter your own MISP_BASEURL, if you wish to NOT do that, the MISP_BASEURL will be empty, which will work, but ideally you configure it afterwards."
echo "Do you want to change it now? (y/n) "
read ANSWER
ANSWER=$(echo $ANSWER |tr [A-Z] [a-z])
ANSWER=$(echo $ANSWER |tr '[:upper:]' '[:lower:]')
if [[ "$ANSWER" == "y" ]]; then
if [[ ! -z $IP ]]; then
echo "It seems you have an interface called $IFACE UP with the following IP: $IP - FYI"
@ -706,6 +727,7 @@ genRCLOCAL () {
sed -i -e '$i \echo never > /sys/kernel/mm/transparent_hugepage/enabled\n' /etc/rc.local
sed -i -e '$i \echo 1024 > /proc/sys/net/core/somaxconn\n' /etc/rc.local
sed -i -e '$i \sysctl vm.overcommit_memory=1\n' /etc/rc.local
sed -i -e '$i \[ -f /etc/init.d/firstBoot ] && bash /etc/init.d/firstBoot\n' /etc/rc.local
}
# Run PyMISP tests

View File

@ -26,11 +26,14 @@
## vi misp-backup.conf # adjust values
## sudo bash misp-backup.sh 2>&1 | tee misp-backup.log
##
## TODO: Target directory, rudimentary free space check: stat -f --format="%a" OutputDirName
## TODO: Make sure no directories are blank
## TODO: Make sure no directories are blank, $OutputDirName==Done
## TODO: Review how much sense it makes to ask fo MySQL credentials when most of the script does auto detection anyway.
##
# This makes use of the standard variables used by the installer
eval "$(curl -fsSL https://raw.githubusercontent.com/MISP/MISP/2.4/docs/generic/globalVariables.md | grep -v \`\`\`)"
MISPvars > /dev/null 2>&1
# Leave empty for NO debug messages, if run with set -x or bash -x it will enable DEBUG by default
DEBUG=
@ -59,6 +62,10 @@ checkDiskFree () {
if [[ ! -e $1 ]]; then
echo "$1 does not exist, creating"
mkdir -p $1
if [[ $? != 0 ]]; then
echo "Could not create $1, please fix permissions."
exit 126
fi
fi
threshhold=90
free=$(df -l --sync --output=pcent $1 |tail -1|cut -f 1 -d% | tr -d \ )

View File

@ -27,6 +27,10 @@
# TODO: Move DB, check DB?
# TODO: Check db user exists.
# This makes use of the standard variables used by the installer
eval "$(curl -fsSL https://raw.githubusercontent.com/MISP/MISP/2.4/docs/generic/globalVariables.md | grep -v \`\`\`)"
MISPvars > /dev/null 2>&1
# Leave empty for NO debug messages, if run with set -x or bash -x it will enable DEBUG by default
DEBUG=

View File

@ -1 +1 @@
MISPPath=/var/www/MISP
PATH_TO_MISP=/var/www/MISP

View File

@ -25,6 +25,10 @@
## Time to set some variables
##
# This makes use of the standard variables used by the installer
eval "$(curl -fsSL https://raw.githubusercontent.com/MISP/MISP/2.4/docs/generic/globalVariables.md | grep -v \`\`\`)"
MISPvars > /dev/null 2>&1
LUSER_ID="$(id -u)"
if [[ "${LUSER_ID}" > "0" ]]; then
@ -54,17 +58,17 @@ fi
# Fill in any missing values with defaults
# MISP path
MISPPath=${MISPPath:-$(locate MISP/app/webroot/index.php|sed 's/\/app\/webroot\/index\.php//')}
PATH_TO_MISP=${PATH_TO_MISP:-$(locate MISP/app/webroot/index.php|sed 's/\/app\/webroot\/index\.php//')}
# database.php
MySQLUUser=$(grep -o -P "(?<='login' => ').*(?=')" $MISPPath/app/Config/database.php)
MySQLUPass=$(grep -o -P "(?<='password' => ').*(?=')" $MISPPath/app/Config/database.php)
MISPDB=$(grep -o -P "(?<='database' => ').*(?=')" $MISPPath/app/Config/database.php)
DB_Port=$(grep -o -P "(?<='port' => ).*(?=,)" $MISPPath/app/Config/database.php)
MISPDBHost=$(grep -o -P "(?<='host' => ').*(?=')" $MISPPath/app/Config/database.php)
MySQLUUser=$(grep -o -P "(?<='login' => ').*(?=')" $PATH_TO_MISP/app/Config/database.php)
MySQLUPass=$(grep -o -P "(?<='password' => ').*(?=')" $PATH_TO_MISP/app/Config/database.php)
MISPDB=$(grep -o -P "(?<='database' => ').*(?=')" $PATH_TO_MISP/app/Config/database.php)
DB_Port=$(grep -o -P "(?<='port' => ).*(?=,)" $PATH_TO_MISP/app/Config/database.php)
MISPDBHost=$(grep -o -P "(?<='host' => ').*(?=')" $PATH_TO_MISP/app/Config/database.php)
echo "Clearing data model cache files"
rm -f $MISPPath/app/tmp/cache/models/myapp_*
rm -f $MISPPath/app/tmp/cache/persistent/myapp_*
rm -f $PATH_TO_MISP/app/tmp/cache/models/myapp_*
rm -f $PATH_TO_MISP/app/tmp/cache/persistent/myapp_*
echo "Wiping MySQL tables"
echo "Removes all users and organizations, except default (id=1)"
@ -76,7 +80,7 @@ mysql --host $MISPDBHost -u $MySQLRUser -p$MySQLRPass $MISPDB < $SQL
echo "Inserting default values to MySQL tables"
TMP=/tmp/misp-wipe-$$.sql
cd $MISPPath
cd $PATH_TO_MISP
sed -n '/Default values for initial installation/ { s///; :a; n; p; ba; }' INSTALL/MYSQL.sql | egrep -v '(admin_settings|db_version)' > $TMP
mysql --host $MISPDBHost -u $MySQLRUser -p$MySQLRPass $MISPDB < $TMP
rm -f $TMP
@ -88,7 +92,7 @@ git clean -f -d -x app/tmp
git clean -f -d -x app/files
echo "Updating taxonomies"
baseurl=$(grep -o -P "(?<='baseurl' => ').*(?=')" $MISPPath/app/Config/config.php)
baseurl=$(grep -o -P "(?<='baseurl' => ').*(?=')" $PATH_TO_MISP/app/Config/config.php)
AuthKey=$(echo 'select authkey from users where role_id = 1 order by id limit 1;' | mysql -u $MySQLRUser -p$MySQLRPass $MISPDB 2>/dev/null | tail -1)
curl --header "Authorization: $AuthKey" --header "Accept: application/json" --header "Content-Type: application/json" -o /dev/null -s -X POST ${baseurl}/taxonomies/update