MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch '2.4' into cerebrate

pull/6703/head
iglocska 2020-11-30 23:49:40 +01:00
parent 776405c5ec f202128af1
commit a332e1379c
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
46 changed files with 2366 additions and 1282 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.github/workflows/main.yml vendored
View File

@ -212,9 +212,15 @@ jobs:
./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
./app/Vendor/bin/phpunit app/Test/ComplexTypeToolTest.php
./app/Vendor/bin/phpunit app/Test/JSONConverterToolTest.php
# Ensure the perms
# Ensure the perms of config files
sudo chown -R $USER:www-data `pwd`/app/Config
sudo chmod -R 777 `pwd`/app/Config
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.server_settings_skip_backup_rotate" 1'
sudo chown -R $USER:www-data `pwd`/app/Config
sudo chmod -R 777 `pwd`/app/Config
python3 tests/testlive_security.py
pushd tests
./curl_tests_GH.sh $AUTH $HOST
@ -229,3 +235,8 @@ jobs:
poetry run python ./create_massive_dummy_events.py -l 5 -a 30
popd
python3 tools/misp-feed/validate.py
- name: Logs
if: ${{ always() }}
run: |
tail -n +1 `pwd`/app/tmp/logs/*

252
INSTALL/INSTALL.sh
View File

@ -206,6 +206,19 @@ usage () {
space
echo -e "Recommended is either a barebone MISP install (ideal for syncing from other instances) or"
echo -e "MISP + modules - ${SCRIPT_NAME} -c -M"
echo -e ""
echo -e ""
echo -e "Interesting environment variables that get considered are:"
echo -e ""
echo -e "MISP_USER/MISP_PASSWORD # Local username on machine, default: misp/opensslGeneratedPassword"
echo -e ""
echo -e "PATH_TO_MISP # Where MISP will be installed, default: /var/www/MISP (recommended)"
echo -e ""
echo -e "DBHOST/DBNAME # database hostname, MISP database name, default: localhost/misp"
echo -e "DBUSER_ADMIN/DBPASSWORD_ADMIN # MySQL admin user, default: root/opensslGeneratedPassword"
echo -e "DBUSER_MISP/DBPASSWORD_MISP # MISP database user, default: misp/opensslGeneratedPassword"
echo -e ""
echo -e "You need to export the variable(s) to be taken into account. (or specified in-line when invoking INSTALL.sh)"
space
}
@ -1266,59 +1279,39 @@ installDepsPhp70 () {
}
prepareDB () {
if [[ ! -e /var/lib/mysql/misp/users.ibd ]]; then
if sudo test ! -e "/var/lib/mysql/mysql/"; then
#Make sure initial tables are created in MySQL
debug "Install mysql tables"
sudo mysql_install_db --user=mysql --basedir=/usr --datadir=/var/lib/mysql
sudo service mysql start
fi
if sudo test ! -e "/var/lib/mysql/misp/"; then
debug "Start mysql"
sudo service mysql start
debug "Setting up database"
# Kill the anonymous users
sudo mysql -e "DROP USER IF EXISTS ''@'localhost'"
# Because our hostname varies we'll use some Bash magic here.
sudo mysql -e "DROP USER IF EXISTS ''@'$(hostname)'"
# Kill off the demo database
sudo mysql -e "DROP DATABASE IF EXISTS test"
# No root remote logins
sudo mysql -e "DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1')"
# Make sure that NOBODY can access the server without a password
sudo mysqladmin -u "${DBUSER_ADMIN}" password "${DBPASSWORD_ADMIN}"
# Make our changes take effect
sudo mysql -e "FLUSH PRIVILEGES"
# FIXME: If user 'misp' exists, and has a different password, the below WILL fail. Partially fixed with the Env-Var check in the beginning. (Need to implement pre-flight checks to exit gracefully if not set)
# Add your credentials if needed, if sudo has NOPASS, comment out the relevant lines
if [[ "${PACKER}" == "1" ]]; then
pw="Password1234"
else
pw=${MISP_PASSWORD}
fi
if [[ ! -z ${INSTALL_USER} ]]; then
SUDO_EXPECT="sudo mysql_secure_installation"
echo "Making sure sudo session is buffered"
sudo ls -la /tmp > /dev/null 2> /dev/null
else
SUDO_EXPECT="sudo -k mysql_secure_installation"
fi
expect -f - <<-EOF
set timeout 10
spawn ${SUDO_EXPECT}
expect "*?assword*"
send -- "${pw}\r"
expect "Enter current password for root (enter for none):"
send -- "\r"
expect "Set root password?"
send -- "y\r"
expect "New password:"
send -- "${DBPASSWORD_ADMIN}\r"
expect "Re-enter new password:"
send -- "${DBPASSWORD_ADMIN}\r"
expect "Remove anonymous users?"
send -- "y\r"
expect "Disallow root login remotely?"
send -- "y\r"
expect "Remove test database and access to it?"
send -- "y\r"
expect "Reload privilege tables now?"
send -- "y\r"
expect eof
EOF
sudo apt-get purge -y expect ; sudo apt autoremove -qy
fi
sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "CREATE DATABASE ${DBNAME};"
sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "CREATE USER '${DBUSER_MISP}'@'localhost' IDENTIFIED BY '${DBPASSWORD_MISP}';"
sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "GRANT USAGE ON *.* to ${DBUSER_MISP}@localhost;"
sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "GRANT ALL PRIVILEGES on ${DBNAME}.* to '${DBUSER_MISP}'@'localhost';"
sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "FLUSH PRIVILEGES;"
# Import the empty MISP database from MYSQL.sql
${SUDO_WWW} cat ${PATH_TO_MISP}/INSTALL/MYSQL.sql | mysql -u ${DBUSER_MISP} -p${DBPASSWORD_MISP} ${DBNAME}
sudo mysql -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "CREATE DATABASE ${DBNAME};"
sudo mysql -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "CREATE USER '${DBUSER_MISP}'@'localhost' IDENTIFIED BY '${DBPASSWORD_MISP}';"
sudo mysql -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "GRANT USAGE ON *.* to '${DBUSER_MISP}'@'localhost';"
sudo mysql -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "GRANT ALL PRIVILEGES on ${DBNAME}.* to '${DBUSER_MISP}'@'localhost';"
sudo mysql -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "FLUSH PRIVILEGES;"
# Import the empty MISP database from MYSQL.sql
${SUDO_WWW} cat ${PATH_TO_MISP}/INSTALL/MYSQL.sql | mysql -u "${DBUSER_MISP}" -p"${DBPASSWORD_MISP}" ${DBNAME}
fi
}
apacheConfig () {
@ -1358,78 +1351,89 @@ apacheConfig () {
installCore () {
debug "Installing ${LBLUE}MISP${NC} core"
# Download MISP using git in the /var/www/ directory.
sudo mkdir ${PATH_TO_MISP}
sudo chown ${WWW_USER}:${WWW_USER} ${PATH_TO_MISP}
cd ${PATH_TO_MISP}
${SUDO_WWW} git clone https://github.com/MISP/MISP.git ${PATH_TO_MISP}
${SUDO_WWW} git submodule update --progress --init --recursive
# Make git ignore filesystem permission differences for submodules
$SUDO_WWW git submodule foreach --recursive git config core.filemode false
if [[ ! -d ${PATH_TO_MISP} ]]; then
sudo mkdir ${PATH_TO_MISP}
sudo chown ${WWW_USER}:${WWW_USER} ${PATH_TO_MISP}
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git clone https://github.com/MISP/MISP.git ${PATH_TO_MISP}; done
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git -C ${PATH_TO_MISP} submodule update --progress --init --recursive; done
# Make git ignore filesystem permission differences for submodules
${SUDO_WWW} git -C ${PATH_TO_MISP} submodule foreach --recursive git config core.filemode false
# Make git ignore filesystem permission differences
$SUDO_WWW git config core.filemode false
# Make git ignore filesystem permission differences
${SUDO_WWW} git -C ${PATH_TO_MISP} config core.filemode false
# Create a python3 virtualenv
$SUDO_WWW virtualenv -p python3 ${PATH_TO_MISP}/venv
# Create a python3 virtualenv
${SUDO_WWW} virtualenv -p python3 ${PATH_TO_MISP}/venv
# make pip happy
sudo mkdir /var/www/.cache/
sudo chown $WWW_USER:$WWW_USER /var/www/.cache
# make pip happy
sudo mkdir /var/www/.cache/
sudo chown ${WWW_USER}:${WWW_USER} /var/www/.cache
cd ${PATH_TO_MISP}/app/files/scripts
$SUDO_WWW git clone https://github.com/CybOXProject/python-cybox.git
$SUDO_WWW git clone https://github.com/STIXProject/python-stix.git
$SUDO_WWW git clone https://github.com/MAECProject/python-maec.git
PATH_TO_MISP_SCRIPTS=${PATH_TO_MISP}/app/files/scripts
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git clone https://github.com/CybOXProject/python-cybox.git ${PATH_TO_MISP_SCRIPTS}/python-cybox; done
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git clone https://github.com/STIXProject/python-stix.git ${PATH_TO_MISP_SCRIPTS}/python-stix; done
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git clone https://github.com/MAECProject/python-maec.git ${PATH_TO_MISP_SCRIPTS}/python-maec; done
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git clone https://github.com/CybOXProject/mixbox.git ${PATH_TO_MISP_SCRIPTS}/mixbox; done
# install mixbox to accommodate the new STIX dependencies:
$SUDO_WWW git clone https://github.com/CybOXProject/mixbox.git
cd ${PATH_TO_MISP}/app/files/scripts/mixbox
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
cd ${PATH_TO_MISP}/app/files/scripts/python-cybox
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
cd ${PATH_TO_MISP}/app/files/scripts/python-stix
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
cd $PATH_TO_MISP/app/files/scripts/python-maec
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
# install STIX2.0 library to support STIX 2.0 export:
cd ${PATH_TO_MISP}/cti-python-stix2
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/mixbox
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/python-cybox
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/python-maec
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/python-stix
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/cti-python-stix2
# install PyMISP
cd ${PATH_TO_MISP}/PyMISP
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
# FIXME: Remove libfaup etc once the egg has the library baked-in
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
cd /tmp
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
cd gtcaca
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
debug "Install PyMISP"
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/PyMISP
# FIXME: Remove libfaup etc once the egg has the library baked-in
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
cd /tmp
false; while [[ $? -ne 0 ]]; do [[ ! -d "faup" ]] && ${SUDO_CMD} git clone git://github.com/stricaud/faup.git faup; done
false; while [[ $? -ne 0 ]]; do [[ ! -d "gtcaca" ]] && ${SUDO_CMD} git clone git://github.com/stricaud/gtcaca.git gtcaca; done
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
cd gtcaca
${SUDO_CMD} mkdir -p build
cd build
${SUDO_CMD} cmake .. && ${SUDO_CMD} make
sudo make install
cd ../../faup
${SUDO_CMD} mkdir -p build
cd build
${SUDO_CMD} cmake .. && ${SUDO_CMD} make
sudo make install
sudo ldconfig
# install pydeep
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/pydeep.git
# install pydeep
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/pydeep.git; done
# install lief
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install lief
# install lief
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install lief
# install zmq needed by mispzmq
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install zmq redis
# install zmq needed by mispzmq
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install zmq redis
# install python-magic
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install python-magic
# install python-magic
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install python-magic
# install plyara
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install plyara
# install plyara
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install plyara
else
debug "Trying to git pull existing install"
${SUDO_WWW} git pull -C ${PATH_TO_MISP}
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git -C ${PATH_TO_MISP} submodule update --progress --init --recursive; done
PATH_TO_MISP_SCRIPTS=${PATH_TO_MISP}/app/files/scripts
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git -C ${PATH_TO_MISP_SCRIPTS}/python-cybox pull; done
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git -C ${PATH_TO_MISP_SCRIPTS}/python-stix pull; done
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git -C ${PATH_TO_MISP_SCRIPTS}/python-maec pull; done
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git -C ${PATH_TO_MISP_SCRIPTS}/mixbox pull; done
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U setuptools pip lief zmq redis python-magic plyara
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP}/app/files/scripts/mixbox
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP}/app/files/scripts/python-cybox
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP}/app/files/scripts/python-maec
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP}/app/files/scripts/python-stix
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP}/cti-python-stix2
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP}/PyMISP
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U git+https://github.com/kbandla/pydeep.git; done
fi
}
installCake () {
@ -1439,15 +1443,15 @@ installCake () {
cd ${PATH_TO_MISP}/app
# Make composer cache happy
# /!\ composer on Ubuntu when invoked with sudo -u doesn't set $HOME to /var/www but keeps it /home/misp \!/
sudo mkdir /var/www/.composer ; sudo chown $WWW_USER:$WWW_USER /var/www/.composer
$SUDO_WWW php composer.phar install
sudo mkdir /var/www/.composer ; sudo chown ${WWW_USER}:${WWW_USER} /var/www/.composer
${SUDO_WWW} php composer.phar install
# Enable CakeResque with php-redis
sudo phpenmod redis
sudo phpenmod gnupg
# To use the scheduler worker for scheduled tasks, do the following:
$SUDO_WWW cp -fa ${PATH_TO_MISP}/INSTALL/setup/config.php ${PATH_TO_MISP}/app/Plugin/CakeResque/Config/config.php
${SUDO_WWW} cp -fa ${PATH_TO_MISP}/INSTALL/setup/config.php ${PATH_TO_MISP}/app/Plugin/CakeResque/Config/config.php
# If you have multiple MISP instances on the same system, don't forget to have a different Redis per MISP instance for the CakeResque workers
# The default Redis port can be updated in Plugin/CakeResque/Config/config.php
@ -1728,8 +1732,14 @@ mispmodules () {
cd /usr/local/src/
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
## TODO: checkUsrLocalSrc in main doc
debug "Cloning misp-modules"
false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone https://github.com/MISP/misp-modules.git; done
if [[ ! -d /usr/local/src/misp-modules ]]; then
debug "Cloning misp-modules"
false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone https://github.com/MISP/misp-modules.git; done
else
false; while [[ $? -ne 0 ]]; do $SUDO_CMD git -C /usr/local/src/misp-modules pull; done
fi
# Install faup/gtcaca
[[ ! -d "faup" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone git://github.com/stricaud/faup.git faup; done
[[ ! -d "gtcaca" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca; done
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
@ -1739,14 +1749,15 @@ mispmodules () {
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
cd /usr/loca/src/faup
# Install faup
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
cd ../../misp-modules
cd /usr/local/src/misp-modules
# some misp-modules dependencies
sudo apt install libpq5 libjpeg-dev tesseract-ocr libpoppler-cpp-dev imagemagick libopencv-dev zbar-tools libzbar0 libzbar-dev libfuzzy-dev -y
# If you build an egg, the user you build it as need write permissions in the CWD
@ -1755,10 +1766,9 @@ mispmodules () {
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I -r REQUIREMENTS
sudo chgrp staff .
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I .
## sudo gem install asciidoctor-pdf --pre
# Start misp-modules as a service
sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/
sudo cp /usr/local/src/misp-modules/etc/systemd/system/misp-modules.service /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable --now misp-modules

6
INSTALL/INSTALL.sh.sfv
View File

@ -1,5 +1,5 @@
; Generated by RHash v1.3.8 on 2020-07-15 at 04:14.11
; Generated by RHash v1.3.9 on 2020-11-30 at 18:25.58
; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
;
; 133908 04:14.11 2020-07-15 INSTALL.sh
INSTALL.sh 1048857D0C71A2FB6029A448090BC88E008AA499 2E3E878D35568521B5DEC1E7F6DA3193FE3C51049E4BCC127068659E5375939E ED7092DC612C51D7B81969418B4EEA90CE5E990DDE693A3CE83566DEC11E1CF456452DCF103689F05B7E6CCB63F9BC45 2D4CC5D6E02135B337541CE00CDDB205E10EDE924B89B1EEFE069DD1FCE7CE552970AB65A7342838A3BB41E0DCDAB6460E7F96AB7F63EA554D9A1DB61116AE2A
; 137439 18:25.58 2020-11-30 INSTALL.sh
INSTALL.sh FBE9B0213997DAB872BF388909509F1711705C06 5571EFC696328757C2A5556C2A991BF25F448BC1D0F77482086EC70437485C15 4DAF3F805A4EDAD6B556F8870BD0D2F57EE50B667BE4F2581465DC43E56191DF23F1930A983C458124FBB69383CFC821 41543621CB64B670058429D2D9080C7535E1CF3E9892AC2B3620CD099C2E9455DA6FA8A292C908FBF9C74F8CF6B65BD61B552F0EB36E5A7BFDCF51F24D8080C2

2
INSTALL/INSTALL.sh.sha1
View File

@ -1 +1 @@
d1da7c4045eb88f05236acb72df96cf5671c9eb8 INSTALL.sh
fbe9b0213997dab872bf388909509f1711705c06 INSTALL.sh

2
INSTALL/INSTALL.sh.sha256
View File

@ -1 +1 @@
156e96e2aa2f3f4d7ef6342b2355eef445a7020f6e826d8c167b85965847a540 INSTALL.sh
5571efc696328757c2a5556c2a991bf25f448bc1d0f77482086ec70437485c15 INSTALL.sh

2
INSTALL/INSTALL.sh.sha384
View File

@ -1 +1 @@
cea44e962b4162f2388170cb304ff6c37dee5189fa70ae9af6ab9911a8a39eb74cd67a0827f90a72ac1c94ec5a291748 INSTALL.sh
4daf3f805a4edad6b556f8870bd0d2f57ee50b667be4f2581465dc43e56191df23f1930a983c458124fbb69383cfc821 INSTALL.sh

2
INSTALL/INSTALL.sh.sha512
View File

@ -1 +1 @@
0cf6d3ceee4ce78a85c617a8993a49d8966367ef29966b70048c7c172f51684ba1128b9578367ed05b8312e397fceb65dd0395dccb5c1cdbbb5e131f06e42232 INSTALL.sh
41543621cb64b670058429d2d9080c7535e1cf3e9892ac2b3620cd099c2e9455da6fa8a292c908fbf9c74f8cf6b65bd61b552f0eb36e5a7bfdcf51f24d8080c2 INSTALL.sh

55
app/Controller/AppController.php
View File

@ -73,6 +73,9 @@ class AppController extends Controller
protected $_legacyParams = array();
/** @var User */
public $User;
public function __construct($id = false, $table = null, $ds = null)
{
parent::__construct($id, $table, $ds);
@ -154,8 +157,8 @@ class AppController extends Controller
$this->set('ajax', $this->request->is('ajax'));
$this->set('queryVersion', $this->__queryVersion);
$this->loadModel('User');
$auth_user_fields = $this->User->describeAuthFields();
$this->User = ClassRegistry::init('User');
$language = Configure::read('MISP.language');
if (!empty($language) && $language !== 'eng') {
Configure::write('Config.language', $language);
@ -174,18 +177,19 @@ class AppController extends Controller
$this->Server->serverSettingsSaveValue('MISP.uuid', CakeText::uuid());
}
// check if Apache provides kerberos authentication data
$authUserFields = $this->User->describeAuthFields();
$envvar = Configure::read('ApacheSecureAuth.apacheEnv');
if (isset($_SERVER[$envvar])) {
if ($envvar && isset($_SERVER[$envvar])) {
$this->Auth->className = 'ApacheSecureAuth';
$this->Auth->authenticate = array(
'Apache' => array(
// envvar = field returned by Apache if user is authenticated
'fields' => array('username' => 'email', 'envvar' => $envvar),
'userFields' => $auth_user_fields
'userFields' => $authUserFields,
)
);
} else {
$this->Auth->authenticate['Form']['userFields'] = $auth_user_fields;
$this->Auth->authenticate[AuthComponent::ALL]['userFields'] = $authUserFields;
}
if (!empty($this->params['named']['disable_background_processing'])) {
Configure::write('MISP.background_jobs', 0);
@ -681,23 +685,6 @@ class AppController extends Controller
public $userRole = null;
protected function _isJson($data=false)
{
if ($data) {
return (json_decode($data) != null) ? true : false;
}
return $this->request->header('Accept') === 'application/json' || $this->RequestHandler->prefers() === 'json';
}
protected function _isCsv($data=false)
{
if ($this->params['ext'] === 'csv' || $this->request->header('Accept') === 'application/csv' || $this->RequestHandler->prefers() === 'csv') {
return true;
} else {
return false;
}
}
protected function _isRest()
{
return $this->IndexFilter->isRest();
@ -740,11 +727,6 @@ class AppController extends Controller
return $this->userRole['perm_site_admin'];
}
protected function _checkOrg()
{
return $this->Auth->user('org_id');
}
protected function _getApiAuthUser(&$key, &$exception)
{
if (strlen($key) == 40) {
@ -1174,13 +1156,24 @@ class AppController extends Controller
$this->redirect($targetRoute);
}
protected function _loadAuthenticationPlugins() {
/**
* @throws Exception
*/
protected function _loadAuthenticationPlugins()
{
// load authentication plugins from Configure::read('Security.auth')
$auth = Configure::read('Security.auth');
if (!$auth) return;
if (!$auth) {
return;
}
if (!is_array($auth)) {
throw new Exception("`Security.auth` config value must be array.");
}
$this->Auth->authenticate = array_merge($auth, $this->Auth->authenticate);
// Disable Form authentication
if (Configure::read('Security.auth_enforced')) {
unset($this->Auth->authenticate['Form']);
}
if ($this->Auth->startup($this)) {
$user = $this->Auth->user();
if ($user) {

9
app/Controller/AttributesController.php
View File

@ -1653,7 +1653,6 @@ class AttributesController extends AppController
return [[], []];
}
$sightingsData = array();
$this->Feed = ClassRegistry::init('Feed');
$this->loadModel('Sighting');
@ -1684,11 +1683,6 @@ class AttributesController extends AppController
$attributes[$k]['Attribute']['AttributeTag'] = $attributes[$k]['AttributeTag'];
$attributes[$k]['Attribute'] = $this->Attribute->Event->massageTags($this->Auth->user(), $attributes[$k]['Attribute'], 'Attribute', $excludeGalaxy = false, $cullGalaxyTags = true);
unset($attributes[$k]['AttributeTag']);
$sightingsData = array_merge(
$sightingsData,
$this->Sighting->attachToEvent($attribute, $user, $attribute, $extraConditions = false)
);
}
// Fetch correlations in one query
@ -1708,8 +1702,7 @@ class AttributesController extends AppController
$attributes[$k]['Attribute']['RelatedAttribute'] = $correlations[$attribute['Attribute']['id']];
}
}
$sightingsData = $this->Attribute->Event->getSightingData(array('Sighting' => $sightingsData));
$sightingsData = $this->Sighting->attributesStatistics($attributes, $user);
return array($attributes, $sightingsData);
}

25
app/Controller/AuthKeysController.php
View File

@ -1,5 +1,4 @@
<?php
App::uses('AppController', 'Controller');
class AuthKeysController extends AppController
@ -11,10 +10,10 @@ class AuthKeysController extends AppController
);
public $paginate = array(
'limit' => 60,
'order' => array(
'AuthKey.name' => 'ASC'
)
'limit' => 60,
'order' => array(
'AuthKey.name' => 'ASC',
)
);
public function index($id = false)
@ -28,8 +27,13 @@ class AuthKeysController extends AppController
'filters' => ['User.username', 'authkey', 'comment', 'User.id'],
'quickFilters' => ['authkey', 'comment'],
'contain' => ['User'],
'exclude_fields' => ['authkey'],
'conditions' => $conditions,
'afterFind' => function (array $authKeys) {
foreach ($authKeys as &$authKey) {
unset($authKey['AuthKey']['authkey']);
}
return $authKeys;
}
]);
if ($this->IndexFilter->isRest()) {
return $this->restResponsePayload;
@ -86,14 +90,21 @@ class AuthKeysController extends AppController
public function view($id = false)
{
$this->set('menuData', array('menuList' => $this->_isSiteAdmin() ? 'admin' : 'globalActions', 'menuItem' => 'authKeyView'));
$this->CRUD->view($id, [
'contain' => ['User.id', 'User.email'],
'conditions' => $this->__prepareConditions(),
'afterFind' => function (array $authKey) {
unset($authKey['AuthKey']['authkey']);
return $authKey;
}
]);
if ($this->IndexFilter->isRest()) {
return $this->restResponsePayload;
}
$this->set('menuData', [
'menuList' => $this->_isSiteAdmin() ? 'admin' : 'globalActions',
'menuItem' => 'authKeyView',
]);
}
/**

4
app/Controller/Component/CRUDComponent.php
View File

@ -194,6 +194,9 @@ class CRUDComponent extends Component
if (empty($data)) {
throw new NotFoundException(__('Invalid %s.', $modelName));
}
if (isset($params['afterFind'])) {
$data = $params['afterFind']($data);
}
if ($this->Controller->IndexFilter->isRest()) {
$this->Controller->restResponsePayload = $this->Controller->RestResponse->viewData($data, 'json');
} else {
@ -259,7 +262,6 @@ class CRUDComponent extends Component
protected function setFilters($params, $query)
{
$params = $this->massageFilters($params);
$conditions = array();
if (!empty($params['simpleFilters'])) {
foreach ($params['simpleFilters'] as $filter => $filterValue) {
if ($filter === 'quickFilter') {

9
app/Controller/DecayingModelController.php
View File

@ -638,8 +638,6 @@ class DecayingModelController extends AppController
);
$attributes = $this->paginate($this->User->Event->Attribute);
// attach sightings and massage tags
$sightingsData = array();
if (!empty($options['overrideLimit'])) {
$overrideLimit = true;
} else {
@ -658,10 +656,6 @@ class DecayingModelController extends AppController
unset($attributes[$k]['Attribute']['AttributeTag'][$k2]);
}
}
$sightingsData = array_merge(
$sightingsData,
$this->Sighting->attachToEvent($attribute, $this->Auth->user(), $attributes[$k]['Attribute']['id'], $extraConditions = false)
);
if (!empty($params['includeEventTags'])) {
$tagConditions = array('EventTag.event_id' => $attribute['Event']['id']);
if (empty($params['includeAllTags'])) {
@ -694,8 +688,7 @@ class DecayingModelController extends AppController
}
}
}
$sightingsData = $this->User->Event->getSightingData(array('Sighting' => $sightingsData));
$this->set('sightingsData', $sightingsData);
$this->set('sightingsData', $this->Sighting->attributesStatistics($attributes, $this->Auth->user()));
$this->set('attributes', $attributes);
$this->set('attrDescriptions', $this->User->Event->Attribute->fieldDescriptions);
$this->set('typeDefinitions', $this->User->Event->Attribute->typeDefinitions);

27
app/Controller/EventGraphController.php
View File

@ -1,6 +1,9 @@
<?php
App::uses('AppController', 'Controller');
/**
* @property EventGraph $EventGraph
*/
class EventGraphController extends AppController
{
public $components = array(
@ -19,30 +22,16 @@ class EventGraphController extends AppController
throw new MethodNotAllowedException(__('No event ID set.'));
}
// retrieve current org_id
$org_id = $this->_checkOrg();
// validate event
$this->loadModel('Event');
if (Validation::uuid($event_id)) {
$temp = $this->Event->find('first', array('recursive' => -1, 'fields' => array('Event.id'), 'conditions' => array('Event.uuid' => $eventId)));
if (empty($temp)) {
throw new NotFoundException('Invalid event');
}
$event_id = $temp['Event']['id'];
} elseif (!is_numeric($event_id)) {
throw new NotFoundException(__('Invalid event'));
}
$event = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $event_id));
$event = $this->Event->fetchSimpleEvent($this->Auth->user(), $event_id);
if (empty($event)) {
throw new NotFoundException('Invalid event');
}
// fetch eventGraphs
$conditions = [
'EventGraph.event_id' => $event_id,
'EventGraph.org_id' => $org_id
'EventGraph.event_id' => $event['Event']['id'],
'EventGraph.org_id' => $this->Auth->user('org_id'),
];
if (!is_null($graph_id)) {
$conditions['EventGraph.id'] = $graph_id;
@ -82,7 +71,7 @@ class EventGraphController extends AppController
}
$this->loadModel('Event');
$event = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $event_id));
$event = $this->Event->fetchSimpleEvent($this->Auth->user(), $event_id);
if (empty($event)) {
throw new NotFoundException('Invalid event');
}
@ -91,7 +80,7 @@ class EventGraphController extends AppController
if (!$this->_isSiteAdmin() && ($event['Event']['orgc_id'] != $this->Auth->user('org_id') && !$this->userRole['perm_modify'])) {
throw new UnauthorizedException(__('You do not have permission to do that.'));
} else {
$eventGraph['EventGraph']['event_id'] = $event_id;
$eventGraph['EventGraph']['event_id'] = $event['Event']['id'];
}
if (!isset($this->request->data['EventGraph']['network_json'])) {

10
app/Controller/EventsController.php
View File

@ -1093,6 +1093,7 @@ class EventsController extends AppController
$conditions['includeGranularCorrelations'] = 1;
$conditions['includeEventCorrelations'] = false;
$conditions['noEventReports'] = true; // event reports for view are loaded dynamically
$conditions['noSightings'] = true;
if (!empty($filters['includeRelatedTags'])) {
$this->set('includeRelatedTags', 1);
$conditions['includeRelatedTags'] = 1;
@ -1174,7 +1175,8 @@ class EventsController extends AppController
$filters['sort'] = 'timestamp';
$filters['direction'] = 'desc';
}
$sightingsData = $this->Event->getSightingData($event);
$this->loadModel('Sighting');
$sightingsData = $this->Sighting->eventsStatistic([$event], $this->Auth->user());
$this->set('sightingsData', $sightingsData);
$params = $this->Event->rearrangeEventForView($event, $filters, $all, $sightingsData);
if (!empty($filters['includeSightingdb']) && Configure::read('Plugin.Sightings_sighting_db_enable')) {
@ -1430,7 +1432,8 @@ class EventsController extends AppController
}
}
unset($modificationMap);
$sightingsData = $this->Event->getSightingData($event);
$this->loadModel('Sighting');
$sightingsData = $this->Sighting->eventsStatistic([$event], $this->Auth->user());
$this->set('sightingsData', $sightingsData);
$params = $this->Event->rearrangeEventForView($event, $filters, false, $sightingsData);
if (!empty($filters['includeSightingdb']) && Configure::read('Plugin.Sightings_sighting_db_enable')) {
@ -1535,10 +1538,11 @@ class EventsController extends AppController
}
if ($this->_isRest()) {
$conditions['includeAttachments'] = true;
$conditions['includeAttachments'] = isset($this->params['named']['includeAttachments']) ? $this->params['named']['includeAttachments'] : true;
} else {
$conditions['includeAllTags'] = true;
$conditions['noEventReports'] = true; // event reports for view are loaded dynamically
$conditions['noSightings'] = true;
}
$deleted = 0;
if (isset($this->params['named']['deleted'])) {

72
app/Controller/GalaxyClustersController.php
View File

@ -1,6 +1,9 @@
<?php
App::uses('AppController', 'Controller');
/**
* @property GalaxyCluster $GalaxyCluster
*/
class GalaxyClustersController extends AppController
{
public $components = array('Session', 'RequestHandler');
@ -99,67 +102,39 @@ class GalaxyClustersController extends AppController
$this->paginate['conditions']['AND'][] = $aclConditions;
$this->paginate['contain'] = array_merge($this->paginate['contain'], array('Org', 'Orgc', 'SharingGroup', 'GalaxyClusterRelation', 'TargetingClusterRelation'));
$clusters = $this->paginate();
$sgs = $this->GalaxyCluster->Tag->EventTag->Event->SharingGroup->fetchAllAuthorised($this->Auth->user());
$tagIds = array();
foreach ($clusters as $k => $cluster) {
$clusters[$k] = $this->GalaxyCluster->attachExtendByInfo($this->Auth->user(), $clusters[$k]);
$clusters[$k] = $this->GalaxyCluster->attachExtendFromInfo($this->Auth->user(), $clusters[$k]);
$tag = $this->GalaxyCluster->Tag->find('first', array(
'conditions' => array(
'LOWER(name)' => strtolower($cluster['GalaxyCluster']['tag_name']),
),
'fields' => array('id'),
'recursive' => -1,
'contain' => array('EventTag.event_id')
));
if (!empty($tag['Tag']['id'])) {
$clusters[$k]['GalaxyCluster']['event_count'] = $this->GalaxyCluster->Tag->EventTag->countForTag($tag['Tag']['id'], $this->Auth->user());
} else {
$clusters[$k]['GalaxyCluster']['event_count'] = 0;
}
$clusters[$k]['GalaxyCluster']['relation_counts'] = array(
'out' => count($clusters[$k]['GalaxyClusterRelation']),
'in' => count($clusters[$k]['TargetingClusterRelation']),
);
}
$tagIds = array();
$sightings = array();
if (!empty($clusters)) {
$galaxyType = $clusters[0]['GalaxyCluster']['type'];
foreach ($clusters as $k => $v) {
$clusters[$k]['event_ids'] = array();
if (!empty($v['Tag'])) {
$tagIds[] = $v['Tag']['id'];
$clusters[$k]['GalaxyCluster']['tag_id'] = $v['Tag']['id'];
}
$clusters[$k]['GalaxyCluster']['synonyms'] = array();
foreach ($v['GalaxyElement'] as $element) {
$clusters[$k]['GalaxyCluster']['synonyms'][] = $element['value'];
}
if (isset($cluster['Tag']['id'])) {
$tagIds[] = $cluster['Tag']['id'];
$clusters[$k]['GalaxyCluster']['tag_id'] = $cluster['Tag']['id'];
}
$clusters[$k]['GalaxyCluster']['synonyms'] = array();
foreach ($cluster['GalaxyElement'] as $element) {
$clusters[$k]['GalaxyCluster']['synonyms'][] = $element['value'];
}
$clusters[$k]['GalaxyCluster']['event_count'] = 0; // real number is assigned later
}
$eventCountsForTags = $this->GalaxyCluster->Tag->EventTag->countForTags($tagIds, $this->Auth->user());
$this->loadModel('Sighting');
$sightings['tags'] = array();
$csvForTags = $this->Sighting->tagsSparkline($tagIds, $this->Auth->user(), '0');
foreach ($clusters as $k => $cluster) {
if (!empty($cluster['GalaxyCluster']['tag_id'])) {
$temp = $this->Sighting->getSightingsForTag($this->Auth->user(), $cluster['GalaxyCluster']['tag_id']);
$clusters[$k]['sightings'] = $temp;
}
}
$csv = array();
foreach ($clusters as $k => $cluster) {
$startDate = !empty($cluster['sightings']) ? min(array_keys($cluster['sightings'])) : date('Y-m-d');
$startDate = date('Y-m-d', strtotime("-3 days", strtotime($startDate)));
$to = date('Y-m-d', time());
for ($date = $startDate; strtotime($date) <= strtotime($to); $date = date('Y-m-d', strtotime("+1 day", strtotime($date)))) {
if (!isset($csv[$k])) {
$csv[$k] = 'Date,Close\n';
if (isset($cluster['GalaxyCluster']['tag_id'])) {
if (isset($csvForTags[$cluster['GalaxyCluster']['tag_id']])) {
$clusters[$k]['csv'] = $csvForTags[$cluster['GalaxyCluster']['tag_id']];
}
if (isset($cluster['sightings'][$date])) {
$csv[$k] .= $date . ',' . $cluster['sightings'][$date] . '\n';
} else {
$csv[$k] .= $date . ',0\n';
if (isset($eventCountsForTags[$cluster['GalaxyCluster']['tag_id']])) {
$clusters[$k]['GalaxyCluster']['event_count'] = $eventCountsForTags[$cluster['GalaxyCluster']['tag_id']];
}
$clusters[$k]['csv'] = $csv[$k];
}
}
$customClusterCount = $this->GalaxyCluster->fetchGalaxyClusters($this->Auth->user(), [
@ -173,7 +148,6 @@ class GalaxyClustersController extends AppController
$distributionLevels = $this->Attribute->distributionLevels;
unset($distributionLevels[5]);
$this->set('distributionLevels', $distributionLevels);
$this->set('csv', $csv);
$this->set('list', $clusters);
$this->set('galaxy_id', $galaxyId);
$this->set('custom_cluster_count', $customClusterCount);

81
app/Controller/SightingsController.php
View File

@ -294,24 +294,15 @@ class SightingsController extends AppController
$org_id = $this->Toolbox->findIdByUuid($this->Organisation, $org_id);
}
$sightings = $this->Sighting->listSightings($this->Auth->user(), $id, $context, $org_id);
if ($this->_isRest()) {
return $this->RestResponse->viewData($sightings, $this->response->type());
}
$this->set('org_id', $org_id);
$this->set('rawId', $rawId);
$this->set('context', $context);
$this->set('types', array('Sighting', 'False-positive', 'Expiration'));
if (Configure::read('Plugin.Sightings_anonymise') && !$this->_isSiteAdmin()) {
if (!empty($sightings)) {
foreach ($sightings as $k => $v) {
if ($v['Sighting']['org_id'] != $this->Auth->user('org_id')) {
$sightings[$k]['Organisation']['name'] = '';
$sightings[$k]['Sighting']['org_id'] = 0;
}
}
}
}
if ($this->_isRest()) {
return $this->RestResponse->viewData($sightings, $this->response->type());
}
$this->set('sightings', empty($sightings) ? array() : $sightings);
$this->set('sightings', $sightings);
$this->layout = false;
$this->render('ajax/list_sightings');
}
@ -321,65 +312,21 @@ class SightingsController extends AppController
$this->loadModel('Event');
$id = $this->Sighting->explodeIdList($id);
if ($context === 'attribute') {
$attribute_id = $id;
$object = $this->Event->Attribute->fetchAttributes($this->Auth->user(), array('conditions' => array('Attribute.id' => $id, 'Attribute.deleted' => 0), 'flatten' => 1));
if (empty($object)) {
$objects = $this->Event->Attribute->fetchAttributes($this->Auth->user(), array('conditions' => array('Attribute.id' => $id, 'Attribute.deleted' => 0), 'flatten' => 1));
if (empty($objects)) {
throw new MethodNotAllowedException('Invalid object.');
}
$eventIds = array();
foreach ($object as $v) {
$eventIds[] = $v['Attribute']['event_id'];
}
$events = $this->Event->fetchSimpleEvents($this->Auth->user(), ['conditions' => ['id' => $eventIds]]);
} else {
$attribute_id = false;
$statistics = $this->Sighting->attributesStatistics($objects, $this->Auth->user(), true);
} elseif ($context === 'event') {
// let's set the context to event here, since we reuse the variable later on for some additional lookups.
// Passing $context = 'org' could have interesting results otherwise...
$events = $this->Event->fetchSimpleEvents($this->Auth->user(), ['conditions' => ['id' => $id]]);
$statistics = $this->Sighting->eventsStatistic($events, $this->Auth->user(), true);
} else {
throw new MethodNotAllowedException('Invalid context');
}
if (empty($events)) {
throw new MethodNotAllowedException('Invalid object.');
}
$raw = array();
foreach ($events as $event) {
$raw = array_merge($raw, $this->Sighting->attachToEvent($event, $this->Auth->user(), $attribute_id));
}
$results = array();
foreach ($raw as $sighting) {
$results[$sighting['type']][date('Ymd', $sighting['date_sighting'])][] = $sighting;
}
unset($raw);
$dataPoints = array();
$startDate = date('Ymd');
$range = date('Ymd', $this->Sighting->getMaximumRange());
foreach ($results as $type => $data) {
foreach ($data as $date => $sighting) {
if ($date < $startDate) {
if ($date >= $range) {
$startDate = $date;
}
}
$temp = array();
foreach ($sighting as $sightingInstance) {
if (!isset($sightingInstance['Organisation']['name'])) {
$org = 'Anonymised';
} else {
$org = $sightingInstance['Organisation']['name'];
}
$temp[$org] = isset($temp[$org]) ? $temp[$org] + 1 : 1;
}
$dataPoints[$date][$type] = array('count' => count($sighting), 'details' => $temp);
}
}
$startDate = date('Ymd', strtotime("-3 days", strtotime($startDate)));
$tsv = 'date\tSighting\tFalse-positive\n';
for ($i = $startDate; $i < date('Ymd') + 1; $i++) {
if (checkdate(substr($i, 4, 2), substr($i, 6, 2), substr($i, 0, 4))) {
$tsv .= $i . '\t' . (isset($dataPoints[$i][0]['count']) ? $dataPoints[$i][0]['count'] : 0) . '\t' . (isset($dataPoints[$i][1]['count']) ? $dataPoints[$i][1]['count'] : 0) . '\n';
}
}
$this->set('tsv', $tsv);
$this->set('results', $results);
$this->set('csv', $statistics['csv']['all']);
$this->layout = 'ajax';
$this->render('ajax/view_sightings');
}

131
app/Controller/TagsController.php
View File

@ -1,7 +1,9 @@
<?php
App::uses('AppController', 'Controller');
/**
* @property Tag $Tag
*/
class TagsController extends AppController
{
public $components = array('Security' ,'RequestHandler');
@ -12,12 +14,6 @@ class TagsController extends AppController
'Tag.name' => 'asc'
),
'contain' => array(
'EventTag' => array(
'fields' => array('EventTag.event_id')
),
'AttributeTag' => array(
'fields' => array('AttributeTag.event_id', 'AttributeTag.attribute_id')
),
'FavouriteTag',
'Organisation' => array(
'fields' => array('id', 'name')
@ -33,7 +29,7 @@ class TagsController extends AppController
$this->Security->unlockedActions[] = 'search';
}
public function index($favouritesOnly = false)
public function index()
{
$this->loadModel('Attribute');
$this->loadModel('Event');
@ -49,14 +45,7 @@ class TagsController extends AppController
);
$exception = false;
$passedArgsArray = $this->_harvestParameters($filterData, $exception);
$taxonomies = $this->Taxonomy->listTaxonomies(array('full' => false, 'enabled' => true));
$taxonomyNamespaces = array();
if (!empty($taxonomies)) {
foreach ($taxonomies as $taxonomy) {
$taxonomyNamespaces[$taxonomy['namespace']] = $taxonomy;
}
}
$taxonomyTags = array();
$this->Event->recursive = -1;
if (!empty($passedArgsArray['favouritesOnly'])) {
$tag_id_list = $this->Tag->FavouriteTag->find('list', array(
@ -78,101 +67,62 @@ class TagsController extends AppController
}
if ($this->_isRest()) {
unset($this->paginate['limit']);
unset($this->paginate['contain']['EventTag']);
unset($this->paginate['contain']['AttributeTag']);
$paginated = $this->Tag->find('all', $this->paginate);
} else {
if (!empty($passedArgsArray['exclude_statistics'])) {
unset($this->paginate['contain']['EventTag']);
unset($this->paginate['contain']['AttributeTag']);
$this->set('exclude_statistics', true);
}
$paginated = $this->paginate();
}
$tagList = array();
$csv = array();
$sgs = $this->Tag->EventTag->Event->SharingGroup->fetchAllAuthorised($this->Auth->user());
$taxonomyTags = array();
$taxonomyNamespaces = $this->Taxonomy->listTaxonomies(array('full' => false, 'enabled' => true));
foreach ($paginated as $k => $tag) {
$tagList[] = $tag['Tag']['id'];
if (empty($passedArgsArray['exclude_statistics'])) {
$paginated[$k]['Tag']['count'] = $this->Tag->EventTag->countForTag($tag['Tag']['id'], $this->Auth->user(), $sgs);
if (!$this->_isRest()) {
$paginated[$k]['event_ids'] = array();
$paginated[$k]['attribute_ids'] = array();
foreach ($paginated[$k]['EventTag'] as $et) {
$paginated[$k]['event_ids'][] = $et['event_id'];
}
unset($paginated[$k]['EventTag']);
foreach ($paginated[$k]['AttributeTag'] as $at) {
$paginated[$k]['attribute_ids'][] = $at['attribute_id'];
}
unset($paginated[$k]['AttributeTag']);
}
$paginated[$k]['Tag']['attribute_count'] = $this->Tag->AttributeTag->countForTag($tag['Tag']['id'], $this->Auth->user(), $sgs);
}
$favourite = false;
if (!empty($tag['FavouriteTag'])) {
foreach ($tag['FavouriteTag'] as $ft) {
if ($ft['user_id'] == $this->Auth->user('id')) {
$paginated[$k]['Tag']['favourite'] = true;
$favourite = true;
break;
}
}
if (!isset($paginated[$k]['Tag']['favourite'])) {
$paginated[$k]['Tag']['favourite'] = false;
}
} else {
$paginated[$k]['Tag']['favourite'] = false;
}
$paginated[$k]['Tag']['favourite'] = $favourite;
unset($paginated[$k]['FavouriteTag']);
if (!empty($taxonomyNamespaces)) {
$taxonomyNamespaceArrayKeys = array_keys($taxonomyNamespaces);
foreach ($taxonomyNamespaceArrayKeys as $tns) {
if (substr(strtoupper($tag['Tag']['name']), 0, strlen($tns)) === strtoupper($tns)) {
$paginated[$k]['Tag']['Taxonomy'] = $taxonomyNamespaces[$tns];
if (!isset($taxonomyTags[$tns])) {
$taxonomyTags[$tns] = $this->Taxonomy->getTaxonomyTags($taxonomyNamespaces[$tns]['id'], true);
}
$paginated[$k]['Tag']['Taxonomy']['expanded'] = isset($taxonomyTags[$tns][strtoupper($tag['Tag']['name'])]) ? $taxonomyTags[$tns][strtoupper($tag['Tag']['name'])] : $tag['Tag']['name'];
foreach ($taxonomyNamespaces as $namespace => $taxonomy) {
if (substr(strtoupper($tag['Tag']['name']), 0, strlen($namespace)) === strtoupper($namespace)) {
$paginated[$k]['Tag']['Taxonomy'] = $taxonomy;
if (!isset($taxonomyTags[$namespace])) {
$taxonomyTags[$namespace] = $this->Taxonomy->getTaxonomyTags($taxonomy['id'], true);
}
$paginated[$k]['Tag']['Taxonomy']['expanded'] = isset($taxonomyTags[$namespace][strtoupper($tag['Tag']['name'])]) ? $taxonomyTags[$namespace][strtoupper($tag['Tag']['name'])] : $tag['Tag']['name'];
break;
}
}
}
if (!$this->_isRest() && empty($passedArgsArray['exclude_statistics'])) {
$this->loadModel('Sighting');
$sightings['event'] = $this->Sighting->getSightingsForObjectIds($this->Auth->user(), $tagList);
$sightings['attribute'] = $this->Sighting->getSightingsForObjectIds($this->Auth->user(), $tagList, 'attribute');
if (empty($passedArgsArray['exclude_statistics'])) {
$attributeCount = $this->Tag->AttributeTag->countForTags($tagList, $this->Auth->user());
// TODO: this must be called before `tagsSparkline`!
$eventCount = $this->Tag->EventTag->countForTags($tagList, $this->Auth->user());
if ($this->_isRest()) {
$csvForTags = []; // Sightings sparkline doesn't make sense for REST requests
} else {
$this->loadModel('Sighting');
$csvForTags = $this->Sighting->tagsSparkline($tagList, $this->Auth->user(), '0');
}
foreach ($paginated as $k => $tag) {
$objects = array('event', 'attribute');
foreach ($objects as $object) {
foreach ($tag[$object . '_ids'] as $objectid) {
if (isset($sightings[$object][$objectid])) {
foreach ($sightings[$object][$objectid] as $date => $sightingCount) {
if (!isset($tag['sightings'][$date])) {
$tag['sightings'][$date] = $sightingCount;
} else {
$tag['sightings'][$date] += $sightingCount;
}
}
}
}
$tagId = $tag['Tag']['id'];
if (isset($csvForTags[$tagId])) {
$paginated[$k]['Tag']['csv'] = $csvForTags[$tagId];
}
if (!empty($tag['sightings'])) {
$startDate = !empty($tag['sightings']) ? min(array_keys($tag['sightings'])) : date('Y-m-d');
$startDate = date('Y-m-d', strtotime("-3 days", strtotime($startDate)));
$to = date('Y-m-d', time());
for ($date = $startDate; strtotime($date) <= strtotime($to); $date = date('Y-m-d', strtotime("+1 day", strtotime($date)))) {
if (!isset($paginated[$k]['Tag']['csv'])) {
$paginated[$k]['Tag']['csv'] = 'Date,Close\n';
}
if (isset($tag['sightings'][$date])) {
$paginated[$k]['Tag']['csv'] .= $date . ',' . $tag['sightings'][$date] . '\n';
} else {
$paginated[$k]['Tag']['csv'] .= $date . ',0\n';
}
}
}
unset($paginated[$k]['event_ids']);
$paginated[$k]['Tag']['count'] = isset($eventCount[$tagId]) ? (int)$eventCount[$tagId] : 0;
$paginated[$k]['Tag']['attribute_count'] = isset($attributeCount[$tagId]) ? (int)$attributeCount[$tagId] : 0;
}
} else {
$this->set('exclude_statistics', true);
}
if ($this->_isRest()) {
foreach ($paginated as $key => $tag) {
$paginated[$key] = $tag['Tag'];
@ -182,9 +132,8 @@ class TagsController extends AppController
} else {
$this->set('passedArgs', json_encode($this->passedArgs));
$this->set('passedArgsArray', $passedArgsArray);
$this->set('csv', $csv);
$this->set('list', $paginated);
$this->set('favouritesOnly', $favouritesOnly);
$this->set('favouritesOnly', !empty($passedArgsArray['favouritesOnly']));
}
// send perm_tagger to view for action buttons
}

39
app/Controller/UsersController.php
View File

@ -843,17 +843,17 @@ class UsersController extends AppController
{
$this->set('currentOrg', $this->Auth->user('org_id'));
$this->User->id = $id;
if (!$this->User->exists()) {
throw new NotFoundException(__('Invalid user'));
}
$params = array();
$allowedRole = '';
$userToEdit = $this->User->find('first', array(
'conditions' => array('User.id' => $id),
'recursive' => -1,
'fields' => array('User.id', 'User.role_id', 'User.email', 'User.org_id', 'Role.perm_site_admin'),
'contain' => array('Role')
'conditions' => array('User.id' => $id),
'recursive' => -1,
'fields' => array('User.id', 'User.role_id', 'User.email', 'User.org_id', 'Role.perm_site_admin'),
'contain' => array('Role')
));
if (empty($userToEdit)) {
throw new NotFoundException(__('Invalid user'));
}
if (!$this->_isSiteAdmin()) {
// Org admins should be able to select the role that is already assigned to an org user when editing them.
// What happened previously:
@ -971,7 +971,7 @@ class UsersController extends AppController
if (!$this->_isRest()) {
$fields[] = 'role_id';
}
if (!$this->_isSiteAdmin()) {
if (!$this->_isSiteAdmin() && isset($this->request->data['User']['role_id'])) {
$this->loadModel('Role');
$this->Role->recursive = -1;
$chosenRole = $this->Role->findById($this->request->data['User']['role_id']);
@ -979,7 +979,7 @@ class UsersController extends AppController
throw new Exception('You are not authorised to assign that role to a user.');
}
}
if ($this->User->save($this->request->data, true, $fields)) {
if (!empty($fields) && $this->User->save($this->request->data, true, $fields)) {
// newValues to array
$fieldsNewValues = array();
foreach ($fields as $field) {
@ -1139,25 +1139,9 @@ class UsersController extends AppController
if (!empty($this->request->data['User']['email'])) {
if ($this->Bruteforce->isBlocklisted($_SERVER['REMOTE_ADDR'], $this->request->data['User']['email'])) {
$expire = Configure::check('SecureAuth.expire') ? Configure::read('SecureAuth.expire') : 300;
throw new ForbiddenException('You have reached the maximum number of login attempts. Please wait ' . Configure::read('SecureAuth.expire') . ' seconds and try again.');
throw new ForbiddenException('You have reached the maximum number of login attempts. Please wait ' . $expire . ' seconds and try again.');
}
}
// Check the length of the user's authkey
$userPass = $this->User->find('first', array(
'conditions' => array('User.email' => $this->request->data['User']['email']),
'fields' => array('User.password'),
'recursive' => -1
));
if (!empty($userPass) && strlen($userPass['User']['password']) == 40) {
$this->AdminSetting = ClassRegistry::init('AdminSetting');
$db_version = $this->AdminSetting->find('all', array('conditions' => array('setting' => 'db_version')));
$versionRequirementMet = $this->User->checkVersionRequirements($db_version[0]['AdminSetting']['value'], '2.4.77');
if ($versionRequirementMet) {
$passwordToSave = $this->request->data['User']['password'];
}
unset($this->Auth->authenticate['Form']['passwordHasher']);
$this->Auth->constructAuthenticate();
}
}
if ($this->request->is('post') && Configure::read('Security.email_otp_enabled')) {
$user = $this->Auth->identify($this->request, $this->response);
@ -1166,6 +1150,9 @@ class UsersController extends AppController
return $this->redirect('email_otp');
}
}
$formLoginEnabled = isset($this->Auth->authenticate['Form']);
$this->set('formLoginEnabled', $formLoginEnabled);
if ($this->Auth->login()) {
$this->_postlogin();
} else {

5
app/Lib/Export/CsvExport.php
View File

@ -32,6 +32,7 @@ class CsvExport
));
unset($params['fields']);
$params['withAttachments'] = 0;
$params['includeContext'] = 0; // Needed as fetchAttributes override the Event entry
return $params;
}
@ -140,9 +141,9 @@ class CsvExport
$attribute['event_analysis'] = $attribute_raw['Event']['analysis'];
$attribute['event_date'] = $attribute_raw['Event']['date'];
$attribute['event_timestamp'] = $attribute_raw['Event']['timestamp'];
if (!empty($attribute_raw['EventTag'])) {
if (!empty($attribute_raw['Event']['EventTag'])) {
$tags = array();
foreach ($attribute_raw['EventTag'] as $et) {
foreach ($attribute_raw['Event']['EventTag'] as $et) {
$tags[] = $et['Tag']['name'];
}
$tags = implode(',', $tags);

4
app/Lib/Tools/ClusterRelationsTreeTool.php
View File

@ -20,6 +20,7 @@
*/
public function getTree(array $cluster)
{
$relationCache = []; // Needed as some default clusters have the same UUID
$treeRight = array(array(
'GalaxyCluster' => $cluster['GalaxyCluster'],
'children' => array()
@ -48,7 +49,8 @@
));
if (!empty($cluster['GalaxyCluster']['TargetingClusterRelation'])) {
foreach ($cluster['GalaxyCluster']['TargetingClusterRelation'] as $relation) {
if (isset($relation['GalaxyCluster'])) { // not set if cluster is unkown
if (isset($relation['GalaxyCluster']) && !isset($relationCache[$relation['GalaxyCluster']['uuid']])) { // not set if cluster is unkown
$relationCache[$relation['GalaxyCluster']['uuid']] = true;
$tmp = array(
'Relation' => array_diff_key($relation, array_flip(array('GalaxyCluster'))),
'children' => array(

13
app/Model/AppModel.php
View File

@ -2563,19 +2563,6 @@ class AppModel extends Model
$this->elasticSearchClient = $client;
}
public function checkVersionRequirements($versionString, $minVersion)
{
$version = explode('.', $versionString);
$minVersion = explode('.', $minVersion);
if (count($version) > $minVersion) {
return true;
}
if (count($version) == 1) {
return $minVersion <= $version;
}
return ($version[0] >= $minVersion[0] && $version[1] >= $minVersion[1] && $version[2] >= $minVersion[2]);
}
// generate a generic subquery - options needs to include conditions
public function subQueryGenerator($model, $options, $lookupKey, $negation = false)
{

4
app/Model/Attribute.php
View File

@ -1199,7 +1199,7 @@ class Attribute extends AppModel
}
break;
case 'filename|vhash':
if (preg_match('#^.+\|[a-zA-Z0-9&!=\"]+$#', $value)) {
if (preg_match('#^.+\|.+$#', $value)) {
$returnValue = true;
} else {
$returnValue = __('Checksum has an invalid length or format (expected: filename|string characters). Please double check the value or select type "other".');
@ -1461,7 +1461,7 @@ class Attribute extends AppModel
}
break;
case 'vhash':
if (preg_match('/^[a-zA-Z0-9&!="]+$/', $value)) {
if (preg_match('/^.+$/', $value)) {
$returnValue = true;
}
break;

21
app/Model/AttributeTag.php
View File

@ -180,12 +180,25 @@ class AttributeTag extends AppModel
}
}
public function countForTag($tag_id, $user)
/**
* @param array $tagIds
* @param array $user - Currently ignored for performance reasons
* @return array
*/
public function countForTags(array $tagIds, array $user)
{
return $this->find('count', array(
if (empty($tagIds)) {
return [];
}
$this->virtualFields['attribute_count'] = 'COUNT(AttributeTag.id)';
$counts = $this->find('list', [
'recursive' => -1,
'conditions' => array('AttributeTag.tag_id' => $tag_id)
));
'fields' => ['AttributeTag.tag_id', 'attribute_count'],
'conditions' => ['AttributeTag.tag_id' => $tagIds],
'group' => ['AttributeTag.tag_id'],
]);
unset($this->virtualFields['attribute_count']);
return $counts;
}
// Fetch all tags attached to attribute belonging to supplied event. No ACL if user not provided

82
app/Model/Event.php
View File

@ -5772,88 +5772,6 @@ class Event extends AppModel
);
}
public function getSightingData(array $event)
{
if (empty($event['Sighting'])) {
return ['data' => [], 'csv' => []];
}
$this->Sighting = ClassRegistry::init('Sighting');
$sightingsData = array();
$sparklineData = array();
$startDates = array();
$range = $this->Sighting->getMaximumRange();
foreach ($event['Sighting'] as $sighting) {
$type = $this->Sighting->type[$sighting['type']];
if (!isset($sightingsData[$sighting['attribute_id']][$type])) {
$sightingsData[$sighting['attribute_id']][$type] = array('count' => 0);
}
$sightingsData[$sighting['attribute_id']][$type]['count']++;
$orgName = isset($sighting['Organisation']['name']) ? $sighting['Organisation']['name'] : 'Others';
if (!isset($sightingsData[$sighting['attribute_id']][$type]['orgs'][$orgName])) {
$sightingsData[$sighting['attribute_id']][$type]['orgs'][$orgName] = array('count' => 1, 'date' => $sighting['date_sighting']);
} else {
$sightingsData[$sighting['attribute_id']][$type]['orgs'][$orgName]['count']++;
if ($sightingsData[$sighting['attribute_id']][$type]['orgs'][$orgName]['date'] < $sighting['date_sighting']) {
$sightingsData[$sighting['attribute_id']][$type]['orgs'][$orgName]['date'] = $sighting['date_sighting'];
}
}
if ($sighting['type'] !== '0') {
continue;
}
if (!isset($startDates[$sighting['attribute_id']]) || $startDates[$sighting['attribute_id']] > $sighting['date_sighting']) {
if ($sighting['date_sighting'] >= $range) {
$startDates[$sighting['attribute_id']] = $sighting['date_sighting'];
}
}
if (!isset($startDates['event']) || $startDates['event'] > $sighting['date_sighting']) {
if ($sighting['date_sighting'] >= $range) {
$startDates['event'] = $sighting['date_sighting'];
}
}
$date = date("Y-m-d", $sighting['date_sighting']);
if (!isset($sparklineData[$sighting['attribute_id']][$date])) {
$sparklineData[$sighting['attribute_id']][$date] = 1;
} else {
$sparklineData[$sighting['attribute_id']][$date]++;
}
if (!isset($sparklineData['event'][$date])) {
$sparklineData['event'][$date] = 1;
} else {
$sparklineData['event'][$date]++;
}
}
$csv = array();
$today = strtotime(date('Y-m-d', time()));
foreach ($startDates as $k => $v) {
$startDates[$k] = date('Y-m-d', $v);
}
foreach ($sparklineData as $aid => $data) {
if (!isset($startDates[$aid])) {
continue;
}
$startDate = $startDates[$aid];
if (strtotime($startDate) < $range) {
$startDate = date('Y-m-d');
}
$startDate = date('Y-m-d', strtotime("-3 days", strtotime($startDate)));
$sighting = $data;
$csv[$aid] = 'Date,Close\n';
for ($date = $startDate; strtotime($date) <= $today; $date = date('Y-m-d', strtotime("+1 day", strtotime($date)))) {
if (isset($sighting[$date])) {
$csv[$aid] .= $date . ',' . $sighting[$date] . '\n';
} else {
$csv[$aid] .= $date . ',0\n';
}
}
}
return array(
'data' => $sightingsData,
'csv' => $csv
);
}
public function cacheSgids($user, $useCache = false)
{
if ($useCache && isset($this->assetCache['sgids'])) {

38
app/Model/EventTag.php
View File

@ -1,6 +1,9 @@
<?php
App::uses('AppModel', 'Model');
/**
* @property Event $Event
*/
class EventTag extends AppModel
{
public $actsAs = array('Containable');
@ -157,12 +160,39 @@ class EventTag extends AppModel
return $tags;
}
public function countForTag($tag_id, $user)
/**
* @param int $tagId
* @param array $user
* @return int
*/
public function countForTag($tagId, array $user)
{
return $this->find('count', array(
$count = $this->countForTags([$tagId], $user);
return isset($count[$tagId]) ? (int)$count[$tagId] : 0;
}
/**
* @param array $tagIds
* @param array $user
* @return array Key is tag ID, value is event count that user can see
*/
public function countForTags(array $tagIds, array $user)
{
if (empty($tagIds)) {
return [];
}
$conditions = $this->Event->createEventConditions($user);
$conditions['AND']['EventTag.tag_id'] = $tagIds;
$this->virtualFields['event_count'] = 'COUNT(EventTag.id)';
$counts = $this->find('list', [
'recursive' => -1,
'conditions' => array('EventTag.tag_id' => $tag_id)
));
'contain' => ['Event'],
'fields' => ['EventTag.tag_id', 'event_count'],
'conditions' => $conditions,
'group' => ['EventTag.tag_id'],
]);
unset($this->virtualFields['event_count']);
return $counts;
}
public function getTagScores($eventId=0, $allowedTags=array(), $propagateToAttribute=false)

8
app/Model/Server.php
View File

@ -1343,6 +1343,14 @@ class Server extends AppModel
'test' => 'testBool',
'type' => 'boolean',
),
'auth_enforced' => [
'level' => self::SETTING_OPTIONAL,
'description' => __('This optional can be enabled if external auth provider is used and when set to true, it will disable default form authentication.'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean',
],
'rest_client_enable_arbitrary_urls' => array(
'level' => 0,
'description' => __('Enable this setting if you wish for users to be able to query any arbitrary URL via the rest client. Keep in mind that queries are executed by the MISP server, so internal IPs in your MISP\'s network may be reachable.'),

161
app/Model/SharingGroup.php
View File

@ -1,6 +1,11 @@
<?php
App::uses('AppModel', 'Model');
/**
* @property SharingGroupOrg $SharingGroupOrg
* @property SharingGroupServer $SharingGroupServer
* @property Organisation $Organisation
*/
class SharingGroup extends AppModel
{
public $actsAs = array(
@ -53,7 +58,6 @@ class SharingGroup extends AppModel
)
);
private $__sgoCache = array();
private $__sgAuthorisationCache = array(
'save' => array(),
'access' => array()
@ -114,12 +118,22 @@ class SharingGroup extends AppModel
return $sgs;
}
// returns a list of all sharing groups that the user is allowed to see
// scope can be:
// full: Entire SG object with all organisations and servers attached
// name: array in ID => name key => value format
// false: array with all IDs
public function fetchAllAuthorised($user, $scope = false, $active = false, $id = false)
/**
* Returns a list of all sharing groups that the user is allowed to see.
* Scope can be:
* - full: Entire SG object with all organisations and servers attached
* - simplified: Just imporant fields from SG, organisations and servers
* - name: array in ID => name key => value format
* - uuid
* - false: array with all IDs
*
* @param array $user
* @param string|false $scope
* @param bool $active
* @param int|false $id
* @return array
*/
public function fetchAllAuthorised(array $user, $scope = false, $active = false, $id = false)
{
$conditions = array();
if ($id) {
@ -128,24 +142,25 @@ class SharingGroup extends AppModel
if ($active !== false) {
$conditions['AND']['SharingGroup.active'] = $active;
}
if ($user['Role']['perm_site_admin']) {
$sgs = $this->find('all', array(
$ids = array_values($this->find('list', array(
'recursive' => -1,
'fields' => array('id'),
'conditions' => $conditions
));
$ids = array();
foreach ($sgs as $sg) {
$ids[] = $sg['SharingGroup']['id'];
}
)));
} else {
$ids = array_unique(array_merge($this->SharingGroupServer->fetchAllAuthorised(), $this->SharingGroupOrg->fetchAllAuthorised($user['Organisation']['id'])));
$ids = array_unique(array_merge(
$this->SharingGroupServer->fetchAllAuthorised(),
$this->SharingGroupOrg->fetchAllAuthorised($user['Organisation']['id'])
));
}
if (!empty($ids)) {
$conditions['AND'][] = array('SharingGroup.id' => $ids);
} else {
return array();
}
if ($scope === 'full') {
$sgs = $this->find('all', array(
'contain' => array('SharingGroupServer' => array('Server'), 'SharingGroupOrg' => array('Organisation'), 'Organisation'),
@ -153,10 +168,9 @@ class SharingGroup extends AppModel
'order' => 'SharingGroup.name ASC'
));
return $sgs;
} elseif ($scope == 'simplified') {
} elseif ($scope === 'simplified') {
$fieldsOrg = array('id', 'name', 'uuid');
$fieldsServer = array('id', 'url', 'name');
$fields = array();
$permissionTree = ($user['Role']['perm_site_admin'] || $user['Role']['perm_sync']) ? 1 : 0;
$fieldsSharingGroup = array(
array(
@ -175,9 +189,7 @@ class SharingGroup extends AppModel
'fields' => array('SharingGroup.*'),
'contain' => array(
'SharingGroupOrg',
'SharingGroupServer' => array(
'Server' => array('fields' => $fieldsServer),
)
'SharingGroupServer',
)
)
);
@ -187,32 +199,8 @@ class SharingGroup extends AppModel
'fields' => $fieldsSharingGroup[$permissionTree]['fields'],
'order' => 'SharingGroup.name ASC'
));
if (empty($this->__sgoCache)) {
$temp = $this->Organisation->find('all', array(
'recursive' => -1,
'fields' => $fieldsOrg
));
$this->__sgoCache = array();
foreach ($temp as $o) {
$this->__sgoCache[$o['Organisation']['id']] = $o;
}
}
foreach ($sgs as &$sg) {
if(isset($this->__sgoCache[$sg['SharingGroup']['org_id']]['Organisation'])) {
$sg['Organisation'] = $this->__sgoCache[$sg['SharingGroup']['org_id']]['Organisation'];
} else {
$sg['Organisation'] = '';
}
if (!empty($sg['SharingGroupOrg'])) {
foreach ($sg['SharingGroupOrg'] as &$sgo) {
if (!empty($this->__sgoCache[$sgo['org_id']]['Organisation'])) {
$sgo['Organisation'] = $this->__sgoCache[$sgo['org_id']]['Organisation'];
}
}
}
}
return $sgs;
} elseif ($scope == 'name') {
return $this->appendOrgsAndServers($sgs, $fieldsOrg, $fieldsServer);
} elseif ($scope === 'name') {
$sgs = $this->find('list', array(
'recursive' => -1,
'fields' => array('SharingGroup.id', 'SharingGroup.name'),
@ -220,11 +208,11 @@ class SharingGroup extends AppModel
'conditions' => $conditions,
));
return $sgs;
} elseif ($scope == 'uuid') {
} elseif ($scope === 'uuid') {
$sgs = $this->find('list', array(
'recursive' => -1,
'fields' => array('SharingGroup.id', 'SharingGroup.uuid'),
'conditions' => $conditions,
'recursive' => -1,
'fields' => array('SharingGroup.id', 'SharingGroup.uuid'),
'conditions' => $conditions,
));
return $sgs;
} else {
@ -232,6 +220,81 @@ class SharingGroup extends AppModel
}
}
/**
* @param array $sharingGroups
* @param array|null $orgFields
* @param array|null $serverFields
* @return array
*/
private function appendOrgsAndServers(array $sharingGroups, $orgFields = null, $serverFields = null)
{
$orgsToFetch = [];
$serverToFetch = [];
foreach($sharingGroups as $sg) {
$orgsToFetch[$sg['SharingGroup']['org_id']] = true;
if (isset($sg['SharingGroupOrg'])) {
foreach ($sg['SharingGroupOrg'] as $sgo) {
$orgsToFetch[$sgo['org_id']] = true;
}
}
if (isset($sg['SharingGroupServer'])) {
foreach ($sg['SharingGroupServer'] as $sgs) {
if ($sgs['server_id'] == 0) { // local server
continue;
}
$serverToFetch[$sgs['server_id']] = true;
}
}
}
$orgsById = [];
if (!empty($orgsToFetch)) {
$orgs = $this->Organisation->find('all', [
'recursive' => -1,
'fields' => $orgFields,
'conditions' => ['id' => array_keys($orgsToFetch)],
]);
foreach ($orgs as $org) {
$orgsById[$org['Organisation']['id']] = $org['Organisation'];
}
}
$serversById = [];
if (!empty($serverToFetch)) {
$servers = $this->SharingGroupServer->Server->find('all', [
'recursive' => -1,
'fields' => $serverFields,
'conditions' => ['id' => array_keys($serverToFetch)],
]);
foreach ($servers as $server) {
$serversById[$server['Server']['id']] = $server['Server'];
}
}
foreach ($sharingGroups as &$sg) {
if (isset($orgsById[$sg['SharingGroup']['org_id']])) {
$sg['Organisation'] = $orgsById[$sg['SharingGroup']['org_id']];
}
if (isset($sg['SharingGroupOrg'])) {
foreach($sg['SharingGroupOrg'] as &$sgo) {
if (isset($orgsById[$sgo['org_id']])) {
$sgo['Organisation'] = $orgsById[$sgo['org_id']];
}
}
}
if (isset($sg['SharingGroupServer'])) {
foreach($sg['SharingGroupServer'] as &$sgs) {
if (isset($serversById[$sgs['server_id']])) {
$sgs['Server'] = $serversById[$sgs['server_id']];
}
}
}
}
return $sharingGroups;
}
// Who can create a new sharing group with the elements pre-defined (via REST for example)?
// 1. site admins
// 2. Sharing group enabled users

638
app/Model/Sighting.php
View File

@ -1,14 +1,23 @@
<?php
App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
/**
* @property Attribute $Attribute
* @property Event $Event
* @property Organisation $Organisation
*/
class Sighting extends AppModel
{
const ONE_DAY = 86400; // in seconds
// Possible values of `Plugin.Sightings_policy` setting
const SIGHTING_POLICY_EVENT_OWNER = 0,
SIGHTING_POLICY_SIGHTING_REPORTER = 1,
SIGHTING_POLICY_EVERYONE = 2;
private $orgCache = [];
public $useTable = 'sightings';
public $recursive = -1;
@ -156,30 +165,18 @@ class Sighting extends AppModel
$event = array('Event' => $sighting['Event']);
}
$ownEvent = false;
if ($user['Role']['perm_site_admin'] || $event['Event']['org_id'] == $user['org_id']) {
$ownEvent = true;
}
$ownEvent = $user['Role']['perm_site_admin'] || $event['Event']['org_id'] == $user['org_id'];
if (!$ownEvent) {
$sightingPolicy = $this->sightingsPolicy();
// if sighting policy == 0 then return false if the sighting doesn't belong to the user
if (!Configure::read('Plugin.Sightings_policy') || Configure::read('Plugin.Sightings_policy') == 0) {
if ($sightingPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
if ($sighting['Sighting']['org_id'] != $user['org_id']) {
return array();
}
}
// if sighting policy == 1, the user can only see the sighting if they've sighted something in the event once
if (Configure::read('Plugin.Sightings_policy') == 1) {
$temp = $this->find(
'first',
array(
'recursive' => -1,
'conditions' => array(
'Sighting.event_id' => $sighting['Sighting']['event_id'],
'Sighting.org_id' => $user['org_id']
)
)
);
if (empty($temp)) {
else if ($sightingPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) {
if (!$this->isReporter($sighting['Sighting']['event_id'], $user['org_id'])) {
return array();
}
}
@ -207,6 +204,292 @@ class Sighting extends AppModel
return $result;
}
/**
* @param array $tagIds
* @param array $user
* @param null|string $type
* @return array
*/
public function tagsSparkline(array $tagIds, array $user, $type = null)
{
if (empty($tagIds)) {
return [];
}
$conditions = ['Sighting.date_sighting >' => $this->getMaximumRange()];
if ($type !== null) {
$conditions['Sighting.type'] = $type;
}
$sightingsPolicy = $this->sightingsPolicy();
if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
$conditions['Sighting.org_id'] = $user['org_id'];
}
// TODO: Currently, we dont support `SIGHTING_POLICY_SIGHTING_REPORTER` for tags
$sparklineData = [];
foreach (['event', 'attribute'] as $context) {
$sightings = $this->fetchGroupedSightingsForTags($tagIds, $conditions, $context);
$objectElement = ucfirst($context) . 'Tag';
foreach ($sightings as $sighting) {
$tagId = $sighting[$objectElement]['tag_id'];
$date = $sighting['Sighting']['date_sighting'];
$count = (int)$sighting['Sighting']['sighting_count'];
if (isset($sparklineData[$tagId][$date]['sighting'])) {
$sparklineData[$tagId][$date]['sighting'] += $count;
} else {
$sparklineData[$tagId][$date]['sighting'] = $count;
}
}
}
return $this->generateSparkline($sparklineData, false);
}
/**
* @param array $attributes Attribute must contain Event
* @param array $user
* @param bool $csvWithFalsePositive
* @return array[]
*/
public function attributesStatistics(array $attributes, array $user, $csvWithFalsePositive = false)
{
if (empty($attributes)) {
return ['data' => [], 'csv' => []];
}
$sightingsPolicy = $this->sightingsPolicy();
$conditions = [];
foreach ($attributes as $attribute) {
$attributeConditions = ['Sighting.attribute_id' => $attribute['Attribute']['id']];
$ownEvent = $user['Role']['perm_site_admin'] || $attribute['Event']['org_id'] == $user['org_id'];
if (!$ownEvent) {
if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
$attributeConditions['Sighting.org_id'] = $user['org_id'];
} else if ($sightingsPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) {
if (!$this->isReporter($attribute['Event']['id'], $user['org_id'])) {
continue; // skip attribute
}
}
}
$conditions['OR'][] = $attributeConditions;
}
$groupedSightings = $this->fetchGroupedSightings($conditions, $user);
return $this->generateStatistics($groupedSightings, $csvWithFalsePositive);
}
/**
* @param array $events
* @param array $user
* @param bool $csvWithFalsePositive
* @return array
*/
public function eventsStatistic(array $events, array $user, $csvWithFalsePositive = false)
{
if (empty($events)) {
return ['data' => [], 'csv' => []];
}
$sightingPolicy = $this->sightingsPolicy();
$conditions = [];
foreach ($events as $event) {
$eventCondition = ['Sighting.event_id' => $event['Event']['id']];
$ownEvent = $user['Role']['perm_site_admin'] || $event['Event']['org_id'] == $user['org_id'];
if (!$ownEvent) {
if ($sightingPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
$eventCondition['Sighting.org_id'] = $user['org_id'];
} else if ($sightingPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) {
if (!$this->isReporter($event['Event']['id'], $user['org_id'])) {
continue;
}
}
}
$conditions['OR'][] = $eventCondition;
}
$groupedSightings = $this->fetchGroupedSightings($conditions, $user);
return $this->generateStatistics($groupedSightings, $csvWithFalsePositive);
}
/**
* @param array $conditions
* @param array $user
* @return array
*/
private function fetchGroupedSightings(array $conditions, array $user)
{
if (empty($conditions)) {
return [];
}
// Returns date in `Y-m-d` format
$this->virtualFields['date_sighting'] = $this->dateVirtualColumn();
$this->virtualFields['sighting_count'] = 'COUNT(id)';
$this->virtualFields['last_timestamp'] = 'MAX(date_sighting)';
$groupedSightings = $this->find('all', array(
'conditions' => $conditions,
'fields' => ['org_id', 'attribute_id', 'type', 'date_sighting', 'last_timestamp', 'sighting_count'],
'recursive' => -1,
'group' => ['org_id', 'attribute_id', 'type', 'date_sighting'],
'order' => ['date_sighting'], // from oldest
));
unset(
$this->virtualFields['date_sighting'],
$this->virtualFields['sighting_count'],
$this->virtualFields['last_timestamp']
);
return $this->attachOrgToSightings($groupedSightings, $user, false);
}
/**
* @param array $tagIds
* @param array $conditions
* @param string $context
* @return array
*/
private function fetchGroupedSightingsForTags(array $tagIds, array $conditions, $context)
{
$conditions[ucfirst($context) . 'Tag.tag_id'] = $tagIds;
// Temporary bind EventTag or AttributeTag model
$this->bindModel([
'hasOne' => [
ucfirst($context) . 'Tag' => [
'foreignKey' => false,
'conditions' => ucfirst($context) . 'Tag.' . $context . '_id = Sighting.' . $context . '_id',
]
]
]);
// Returns date in `Y-m-d` format
$this->virtualFields['date_sighting'] = $this->dateVirtualColumn();
$this->virtualFields['sighting_count'] = 'COUNT(Sighting.id)';
$sightings = $this->find('all', array(
'recursive' => -1,
'contain' => [ucfirst($context) . 'Tag'],
'conditions' => $conditions,
'fields' => [ucfirst($context) . 'Tag.tag_id', 'date_sighting', 'sighting_count'],
'group' => [ucfirst($context) . 'Tag.id', 'date_sighting'],
'order' => ['date_sighting'], // from oldest
));
unset($this->virtualFields['date_sighting'], $this->virtualFields['sighting_count']);
return $sightings;
}
/**
* @param array $groupedSightings
* @param bool $csvWithFalsePositive
* @return array[]
*/
private function generateStatistics(array $groupedSightings, $csvWithFalsePositive = false)
{
$sightingsData = [];
$sparklineData = [];
$range = $this->getMaximumRange();
foreach ($groupedSightings as $sighting) {
$type = $this->type[$sighting['type']];
$orgName = isset($sighting['Organisation']['name']) ? $sighting['Organisation']['name'] : __('Others');
$count = (int)$sighting['sighting_count'];
$inRange = strtotime($sighting['date_sighting']) >= $range;
foreach ([$sighting['attribute_id'], 'all'] as $needle) {
if (!isset($sightingsData[$needle][$type])) {
$sightingsData[$needle][$type] = ['count' => 0, 'orgs' => []];
}
$ref = &$sightingsData[$needle][$type];
$ref['count'] += $count;
if (!isset($ref['orgs'][$orgName])) {
$ref['orgs'][$orgName] = ['count' => $count, 'date' => $sighting['last_timestamp']];
} else {
$ref['orgs'][$orgName]['count'] += $count;
$ref['orgs'][$orgName]['date'] = $sighting['last_timestamp'];
}
if ($inRange) {
if (isset($sparklineData[$needle][$sighting['date_sighting']][$type])) {
$sparklineData[$needle][$sighting['date_sighting']][$type] += $count;
} else {
$sparklineData[$needle][$sighting['date_sighting']][$type] = $count;
}
}
}
}
return ['data' => $sightingsData, 'csv' => $this->generateSparkline($sparklineData, $csvWithFalsePositive)];
}
/**
* @param array $sparklineData
* @param bool $csvWithFalsePositive
* @return array
*/
private function generateSparkline(array $sparklineData, $csvWithFalsePositive)
{
$todayString = date('Y-m-d');
$today = strtotime($todayString);
// If nothing found, generate default "empty" CSV for 'all'
if (!isset($sparklineData['all'])) {
$sparklineData['all'][$todayString] = null;
}
$csv = [];
foreach ($sparklineData as $object => $data) {
$startDate = key($data); // oldest date for sparkline
$startDate = strtotime($startDate) - (self::ONE_DAY * 3);
$csvForObject = $csvWithFalsePositive ? 'Date,Sighting,False-positive\n' : 'Date,Close\n';
for ($date = $startDate; $date <= $today; $date += self::ONE_DAY) {
$dateAsString = date('Y-m-d', $date);
$csvForObject .= $dateAsString . ',' . (isset($data[$dateAsString]['sighting']) ? $data[$dateAsString]['sighting'] : '0');
if ($csvWithFalsePositive) {
$csvForObject .= ',' . (isset($data[$dateAsString]['false-positive']) ? $data[$dateAsString]['false-positive'] : '0');
}
$csvForObject .= '\n';
}
$csv[$object] = $csvForObject;
}
return $csv;
}
/**
* @param array $sightings
* @param array $user
* @param false $forSync
* @return array
*/
private function attachOrgToSightings(array $sightings, array $user, $forSync = false)
{
$showOrg = Configure::read('MISP.showorg');
$anonymise = Configure::read('Plugin.Sightings_anonymise');
$anonymiseAs = Configure::read('Plugin.Sightings_anonymise_as');
$anonOrg = null;
if ($forSync && !empty($anonymiseAs)) {
$anonOrg = $this->getOrganisationById($anonymiseAs);
}
foreach ($sightings as $k => $sighting) {
$sighting = $sighting['Sighting'];
if ($showOrg && $sighting['org_id']) {
$sighting['Organisation'] = $this->getOrganisationById($sighting['org_id']);
}
if ($sighting['org_id'] != $user['org_id'] && ($anonymise || !empty($anonOrg))) {
if (empty($anonOrg)) {
unset($sighting['org_id']);
unset($sighting['Organisation']);
} else {
$sighting['org_id'] = $anonOrg['Organisation']['id'];
$sighting['Organisation'] = $anonOrg['Organisation'];
}
}
$sightings[$k] = $sighting;
}
$this->orgCache = []; // clear org cache
return $sightings;
}
/**
* @param array $event
* @param array $user
@ -217,8 +500,6 @@ class Sighting extends AppModel
*/
public function attachToEvent(array $event, array $user, $attribute = null, $extraConditions = false, $forSync = false)
{
$ownEvent = $user['Role']['perm_site_admin'] || $event['Event']['org_id'] == $user['org_id'];
$contain = [];
$conditions = array('Sighting.event_id' => $event['Event']['id']);
if (isset($attribute['Attribute']['id'])) {
@ -234,32 +515,20 @@ class Sighting extends AppModel
$contain['Attribute'] = ['fields' => 'Attribute.uuid'];
}
if (!$ownEvent && (!Configure::read('Plugin.Sightings_policy') || Configure::read('Plugin.Sightings_policy') == 0)) {
$conditions['Sighting.org_id'] = $user['org_id'];
$ownEvent = $user['Role']['perm_site_admin'] || $event['Event']['org_id'] == $user['org_id'];
if (!$ownEvent) {
$sightingPolicy = $this->sightingsPolicy();
if ($sightingPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
$conditions['Sighting.org_id'] = $user['org_id'];
} elseif ($sightingPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) {
if (!$this->isReporter($event['Event']['id'], $user['org_id'])) {
return array();
}
}
}
if ($extraConditions !== false) {
$conditions['AND'] = $extraConditions;
}
if (Configure::read('MISP.showorg')) {
$contain['Organisation'] = array('fields' => array('Organisation.id', 'Organisation.uuid', 'Organisation.name'));
}
// Sighting reporters setting
// If the event has any sightings for the user's org, then the user is a sighting reporter for the event too.
// This means that he /she has access to the sightings data contained within
if (!$ownEvent && Configure::read('Plugin.Sightings_policy') == 1) {
$temp = $this->find('first', array(
'recursive' => -1,
'conditions' => array(
'Sighting.event_id' => $event['Event']['id'],
'Sighting.org_id' => $user['org_id'],
)
));
if (empty($temp)) {
return array();
}
}
$sightings = $this->find('all', array(
'conditions' => $conditions,
'recursive' => -1,
@ -268,46 +537,15 @@ class Sighting extends AppModel
if (empty($sightings)) {
return array();
}
$anonymise = Configure::read('Plugin.Sightings_anonymise');
$anonymiseAs = Configure::read('Plugin.Sightings_anonymise_as');
$anonOrg = null;
if ($forSync && !empty($anonymiseAs)) {
$this->Organisation = ClassRegistry::init('Organisation');
$anonOrg = $this->Organisation->find('first', [
'recursive' => -1,
'conditions' => ['Organisation.id' => $anonymiseAs],
'fields' => ['Organisation.id', 'Organisation.uuid', 'Organisation.name']
]);
}
foreach ($sightings as $k => $sighting) {
if (
($sighting['Sighting']['org_id'] == 0 && !empty($sighting['Organisation'])) ||
$anonymise || !empty($anonOrg)
) {
if ($sighting['Sighting']['org_id'] != $user['org_id']) {
if (empty($anonOrg)) {
unset($sighting['Sighting']['org_id']);
unset($sighting['Organisation']);
} else {
$sighting['Sighting']['org_id'] = $anonOrg['Organisation']['id'];
$sighting['Organisation'] = $anonOrg['Organisation'];
}
}
}
// rearrange it to match the event format of fetchevent
if (isset($sighting['Organisation'])) {
$sighting['Sighting']['Organisation'] = $sighting['Organisation'];
}
// zeroq: add attribute UUID to sighting to make synchronization easier
if (isset($sighting['Attribute']['uuid'])) {
$sighting['Sighting']['attribute_uuid'] = $sighting['Attribute']['uuid'];
} else {
$sighting['Sighting']['attribute_uuid'] = $attribute['Attribute']['uuid'];
}
$sightings[$k] = $sighting['Sighting'] ;
$sightings[$k] = $sighting;
}
return $sightings;
return $this->attachOrgToSightings($sightings, $user, $forSync);
}
public function saveSightings($id, $values, $timestamp, $user, $type = false, $source = false, $sighting_uuid = false, $publish = false, $saveOnBehalfOf = false)
@ -445,161 +683,102 @@ class Sighting extends AppModel
return $id;
}
public function getSightingsForTag($user, $tag_id, $sgids = array(), $type = false)
{
$conditions = array(
'Sighting.date_sighting >' => $this->getMaximumRange(),
'EventTag.tag_id' => $tag_id
);
if ($type !== false) {
$conditions['Sighting.type'] = $type;
}
$this->bindModel(
array(
'hasOne' => array(
'EventTag' => array(
'className' => 'EventTag',
'foreignKey' => false,
'conditions' => 'EventTag.event_id = Sighting.event_id'
)
)
)
);
$sightings = $this->find('all', array(
'recursive' => -1,
'contain' => array('EventTag'),
'conditions' => $conditions,
'fields' => array('Sighting.id', 'Sighting.event_id', 'Sighting.date_sighting', 'EventTag.tag_id')
));
$sightingsRearranged = array();
foreach ($sightings as $sighting) {
$date = date("Y-m-d", $sighting['Sighting']['date_sighting']);
if (isset($sightingsRearranged[$date])) {
$sightingsRearranged[$date]++;
} else {
$sightingsRearranged[$date] = 1;
}
}
return $sightingsRearranged;
}
/**
* @param $user - not used
* @param array $tagIds
* @param string $context 'event' or 'attribute'
* @param string|false $type
* @return array
* @param array $user
* @param $ids
* @param string $context
* @param int|false $orgId
* @param int|false $sightingsType
* @param bool $orderDesc
* @return array|int|null
* @throws Exception
*/
public function getSightingsForObjectIds($user, array $tagIds, $context = 'event', $type = '0')
{
$conditions = array(
'Sighting.date_sighting >' => $this->getMaximumRange(),
ucfirst($context) . 'Tag.tag_id' => $tagIds
);
if ($type !== false) {
$conditions['Sighting.type'] = $type;
}
$this->bindModel(array('hasOne' => array(ucfirst($context) . 'Tag' => array('foreignKey' => false, 'conditions' => ucfirst($context) . 'Tag.' . $context . '_id = Sighting.' . $context . '_id'))));
$sightings = $this->find('all', array(
'recursive' => -1,
'contain' => array(ucfirst($context) . 'Tag'),
'conditions' => $conditions,
'fields' => array('Sighting.' . $context . '_id', 'Sighting.date_sighting')
));
$sightingsRearranged = array();
foreach ($sightings as $sighting) {
$date = date("Y-m-d", $sighting['Sighting']['date_sighting']);
$contextId = $sighting['Sighting'][$context . '_id'];
if (isset($sightingsRearranged[$contextId][$date])) {
$sightingsRearranged[$contextId][$date]++;
} else {
$sightingsRearranged[$contextId][$date] = 1;
}
}
return $sightingsRearranged;
}
public function listSightings($user, $id, $context, $org_id = false, $sightings_type = false, $order_desc = true)
public function listSightings(array $user, $ids, $context, $orgId = false, $sightingsType = false, $orderDesc = true)
{
$this->Event = ClassRegistry::init('Event');
$id = is_array($id) ? $id : $this->explodeIdList($id);
$ids = is_array($ids) ? $ids : $this->explodeIdList($ids);
$objectIds = [];
$eventOwnerOrgIdList = [];
if ($context === 'attribute') {
$object = $this->Event->Attribute->fetchAttributes($user, array('conditions' => array('Attribute.id' => $id, 'Attribute.deleted' => 0), 'flatten' => 1));
} else {
$objects = $this->Event->Attribute->fetchAttributes($user, ['conditions' => ['Attribute.id' => $ids, 'Attribute.deleted' => 0], 'flatten' => 1]);
foreach ($objects as $object) {
$objectIds[] = $object['Attribute']['id'];
$eventOwnerOrgIdList[$object['Event']['id']] = $object['Event']['orgc_id'];
}
} elseif ($context === 'event') {
// let's set the context to event here, since we reuse the variable later on for some additional lookups.
// Passing $context = 'org' could have interesting results otherwise...
$context = 'event';
$object = $this->Event->fetchEvent($user, $options = array('eventid' => $id, 'metadata' => true));
$objects = $this->Event->fetchSimpleEvents($user, ['conditions' => ['Event.id' => $ids]]);
foreach ($objects as $object) {
$objectIds[] = $object['Event']['id'];
$eventOwnerOrgIdList[$object['Event']['id']] = $object['Event']['orgc_id'];
}
} else {
throw new InvalidArgumentException("Invalid context '$context'.");
}
if (empty($object)) {
unset($objects);
if (empty($objectIds)) {
throw new MethodNotAllowedException('Invalid object.');
}
$conditions = array(
'Sighting.' . $context . '_id' => $id
'Sighting.' . $context . '_id' => $objectIds
);
if ($org_id) {
$conditions[] = array('Sighting.org_id' => $org_id);
if ($orgId) {
$conditions[] = array('Sighting.org_id' => $orgId);
}
if ($sightings_type !== false) {
$conditions[] = array('Sighting.type' => $sightings_type);
if ($sightingsType !== false) {
$conditions[] = array('Sighting.type' => $sightingsType);
}
$sightings = $this->find('all', array(
'conditions' => $conditions,
'recursive' => -1,
'contain' => array('Organisation.name'),
'order' => array(sprintf('Sighting.date_sighting %s', $order_desc ? 'DESC' : ''))
'order' => array(sprintf('Sighting.date_sighting %s', $orderDesc ? 'DESC' : ''))
));
if (!empty($sightings) && empty(Configure::read('Plugin.Sightings_policy')) && !$user['Role']['perm_site_admin']) {
$eventOwnerOrgIdList = array();
if (empty($sightings)) {
return [];
}
if ($user['Role']['perm_site_admin']) {
return $sightings; // site admin can see all sightings, do not limit him
}
$sightingsPolicy = $this->sightingsPolicy();
if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
$userOrgId = $user['org_id'];
foreach ($sightings as $k => $sighting) {
if (empty($eventOwnerOrgIdList[$sighting['Sighting']['event_id']])) {
$temp_event = $this->Event->find('first', array(
'recursive' => -1,
'conditions' => array('Event.id' => $sighting['Sighting']['event_id']),
'fields' => array('Event.id', 'Event.orgc_id')
));
$eventOwnerOrgIdList[$temp_event['Event']['id']] = $temp_event['Event']['orgc_id'];
}
if (
empty($eventOwnerOrgIdList[$sighting['Sighting']['event_id']]) ||
($eventOwnerOrgIdList[$sighting['Sighting']['event_id']] !== $user['org_id'] && $sighting['Sighting']['org_id'] !== $user['org_id'])
) {
if ($eventOwnerOrgIdList[$sighting['Sighting']['event_id']] !== $userOrgId && $sighting['Sighting']['org_id'] !== $userOrgId) {
unset($sightings[$k]);
}
}
$sightings = array_values($sightings);
} else if (!empty($sightings) && Configure::read('Plugin.Sightings_policy') == 1 && !$user['Role']['perm_site_admin']) {
} else if ($sightingsPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) {
$eventsWithOwnSightings = array();
foreach ($sightings as $k => $sighting) {
if (empty($eventsWithOwnSightings[$sighting['Sighting']['event_id']])) {
$eventsWithOwnSightings[$sighting['Sighting']['event_id']] = false;
$sighting_temp = $this->find('first', array(
'recursive' => -1,
'conditions' => array(
'Sighting.event_id' => $sighting['Sighting']['event_id'],
'Sighting.org_id' => $user['org_id']
)
));
if (empty($sighting_temp)) {
$temp_event = $this->Event->find('first', array(
'recursive' => -1,
'conditions' => array(
'Event.id' => $sighting['Sighting']['event_id'],
'Event.orgc_id' => $user['org_id']
),
'fields' => array('Event.id', 'Event.orgc_id')
));
$eventsWithOwnSightings[$sighting['Sighting']['event_id']] = !empty($temp_event);
$eventId = $sighting['Sighting']['event_id'];
if (!isset($eventsWithOwnSightings[$eventId])) {
$isReporter = $this->isReporter($eventId, $user['org_id']);
if ($isReporter) {
$eventsWithOwnSightings[$eventId] = true;
} else {
$eventsWithOwnSightings[$sighting['Sighting']['event_id']] = true;
$ownEvent = $eventOwnerOrgIdList[$eventId] == $user['org_id'];
$eventsWithOwnSightings[$eventId] = $ownEvent;
}
}
if (!$eventsWithOwnSightings[$sighting['Sighting']['event_id']]) {
if (!$eventsWithOwnSightings[$eventId]) {
unset($sightings[$k]);
}
}
$sightings = array_values($sightings);
}
if (Configure::read('Plugin.Sightings_anonymise')) {
foreach ($sightings as $k => $v) {
if ($v['Sighting']['org_id'] != $user['org_id']) {
$sightings[$k]['Organisation']['name'] = '';
$sightings[$k]['Sighting']['org_id'] = 0;
}
}
}
return $sightings;
}
@ -825,4 +1004,71 @@ class Sighting extends AppModel
$rangeInDays = (!empty($rangeInDays) && is_numeric($rangeInDays)) ? $rangeInDays : 365;
return strtotime("-$rangeInDays days");
}
/**
* Sighting reporters setting
* If the event has any sightings for the user's org, then the user is a sighting reporter for the event too.
* This means that he /she has access to the sightings data contained within.
*
* @param int $eventId
* @param int $orgId
* @return bool
*/
private function isReporter($eventId, $orgId)
{
return (bool)$this->find('first', array(
'recursive' => -1,
'callbacks' => false,
'fields' => ['Sighting.id'],
'conditions' => array(
'Sighting.event_id' => $eventId,
'Sighting.org_id' => $orgId,
)
));
}
/**
* Reduce memory usage by not fetching organisation object for every sighting but just once. Then organisation
* object will be deduplicated in memory.
*
* @param int $orgId
* @return array
*/
private function getOrganisationById($orgId)
{
if (isset($this->orgCache[$orgId])) {
return $this->orgCache[$orgId];
}
$org = $this->Organisation->find('first', [
'recursive' => -1,
'conditions' => ['Organisation.id' => $orgId],
'fields' => ['Organisation.id', 'Organisation.uuid', 'Organisation.name']
]);
if (!empty($org)) {
$org = $org['Organisation'];
}
$this->orgCache[$orgId] = $org;
return $this->orgCache[$orgId];
}
/**
* @return int
*/
private function sightingsPolicy()
{
$policy = Configure::read('Plugin.Sightings_policy');
if ($policy === null) { // default policy
return self::SIGHTING_POLICY_EVENT_OWNER;
}
return (int)$policy;
}
private function dateVirtualColumn()
{
if (in_array($this->getDataSource()->config['datasource'], ['Database/Mysql', 'Database/MysqlObserver'], true)) {
return 'DATE(FROM_UNIXTIME(Sighting.date_sighting))';
} else {
return "to_char(date(to_timestamp(Sighting.date_sighting)), 'YYYY-MM-DD')"; // PostgreSQL
}
}
}

43
app/Model/User.php
View File

@ -862,18 +862,19 @@ class User extends AppModel
return $gpgTool->fetchGpgKey($fingerprint);
}
/**
* Returns fields that should be fetched from database.
* @return array
*/
public function describeAuthFields()
{
$fields = array();
$fields = array_merge($fields, array_keys($this->getColumnTypes()));
foreach ($fields as $k => $field) {
if (in_array($field, array('gpgkey', 'certif_public'))) {
unset($fields[$k]);
}
}
$fields = array_values($fields);
$relatedModels = array_keys($this->belongsTo);
foreach ($relatedModels as $relatedModel) {
$fields = $this->schema();
// Do not include keys, because they are big and usually not necessary
unset($fields['gpgkey']);
unset($fields['certif_public']);
$fields = array_keys($fields);
foreach ($this->belongsTo as $relatedModel => $foo) {
$fields[] = $relatedModel . '.*';
}
return $fields;
@ -1363,6 +1364,28 @@ class User extends AppModel
return $this->save($user, true, array('id', 'last_login', 'current_login'));
}
/**
* Update field in user model and also set `date_modified`
*
* @param array $user
* @param string $name
* @param mixed $value
* @throws Exception
*/
public function updateField(array $user, $name, $value)
{
if (!isset($user['id'])) {
throw new InvalidArgumentException("Invalid user object provided.");
}
$success = $this->save([
'id' => $user['id'],
$name => $value,
], true, ['id', $name, 'date_modified']);
if (!$success) {
throw new RuntimeException("Could not save field `$name` with value `$value` for user `{$user['id']}`.");
}
}
/**
* Initialize GPG. Returns `null` if initialization failed.
*

408
app/Plugin/ShibbAuth/Controller/Component/Auth/ApacheShibbAuthenticate.php
View File

@ -1,12 +1,11 @@
<?php
App::uses('BaseAuthenticate', 'Controller/Component/Auth');
App::uses('RandomTool', 'Tools');
if (session_status() == PHP_SESSION_NONE) {
session_start();
if (session_status() === PHP_SESSION_NONE) {
session_start();
}
session_regenerate_id();
/*
* custom class for Apache-based authentication
*
@ -21,228 +20,223 @@ session_regenerate_id();
* @see ApacheAuthComponent::$authenticate
*/
class ApacheShibbAuthenticate extends BaseAuthenticate {
class ApacheShibbAuthenticate extends BaseAuthenticate
{
/**
* Authentication class
*
* Configuration in app/Config/Config.php is:
*
* 'ApacheShibbAuth' => // Configuration for shibboleth authentication
* array(
* 'MailTag' => 'EMAIL_TAG',
* 'OrgTag' => 'FEDERATION_TAG',
* 'GroupTag' => 'GROUP_TAG',
* 'GroupSeparator' => ';',
* 'GroupRoleMatching' => array( // 3:User, 1:admin. May be good to set "1" for the first user
* 'group_three' => '3',
* 'group_two' => 2,
* 'group_one' => 1,
* ),
* 'DefaultOrg' => 'MY_ORG',
* ),
* @param CakeRequest $request The request that contains login information.
* @param CakeResponse $response Unused response object.
* @return mixed False on login failure. An array of User data on success.
* @throws Exception
*/
public function authenticate(CakeRequest $request, CakeResponse $response)
{
return self::getUser($request);
}
/**
* @param CakeRequest $request
* @return array|bool
* @throws Exception
*/
public function getUser(CakeRequest $request)
{
// If the url contains sso=disable we return false so the main misp authentication form is used to log in
if (array_key_exists('sso', $request->query) && $request->query['sso'] == 'disable' || (isset($_SESSION["sso_disable"]) && $_SESSION["sso_disable"] === true)) {
$_SESSION["sso_disable"] = true;
return false;
}
/**
* Authentication class
*
* Configuration in app/Config/Config.php is:
*
* 'ApacheShibbAuth' => // Configuration for shibboleth authentication
* array(
* 'MailTag' => 'EMAIL_TAG',
* 'OrgTag' => 'FEDERATION_TAG',
* 'GroupTag' => 'GROUP_TAG',
* 'GroupSeparator' => ';',
* 'GroupRoleMatching' => array( // 3:User, 1:admin. May be good to set "1" for the first user
* 'group_three' => '3',
* 'group_two' => 2,
* 'group_one' => 1,
* ),
* 'DefaultOrg' => 'MY_ORG',
* ),
* @param CakeRequest $request The request that contains login information.
* @param CakeResponse $response Unused response object.
* @return mixed False on login failure. An array of User data on success.
*/
// Get Default parameters
$roleId = -1;
$org = Configure::read('ApacheShibbAuth.DefaultOrg');
$useDefaultOrg = Configure::read('ApacheShibbAuth.UseDefaultOrg');
// Get tags from SSO config
$mailTag = Configure::read('ApacheShibbAuth.MailTag');
$orgTag = Configure::read('ApacheShibbAuth.OrgTag');
$groupTag = Configure::read('ApacheShibbAuth.GroupTag');
$groupRoleMatching = Configure::read('ApacheShibbAuth.GroupRoleMatching');
public function authenticate(CakeRequest $request, CakeResponse $response)
{
return self::getUser($request);
}
// Get user values
if (!isset($_SERVER[$mailTag])) {
CakeLog::error('Mail tag is not given by the SSO SP. Not processing login.');
return false;
}
$mispUsername = $_SERVER[$mailTag];
/**
* @return array|bool
*/
public function getUser(CakeRequest $request)
{
if (filter_var($mispUsername, FILTER_VALIDATE_EMAIL) === false) {
CakeLog::error( "Mail tag `$mispUsername` given by the SSO SP, but it is not valid email address.");
return false;
}
//If the url contains sso=disable we return false so the main misp authentication form is used to log in
if (array_key_exists('sso', $request->query) && $request->query['sso'] == 'disable' || (isset($_SESSION["sso_disable"]) && $_SESSION["sso_disable"] === True)) {
$_SESSION["sso_disable"]=True;
return false;
}
CakeLog::info("Trying login of user: `$mispUsername`.");
// Get Default parameters
$roleId = -1;
$org = Configure::read('ApacheShibbAuth.DefaultOrg');
$useDefaultOrg = Configure::read('ApacheShibbAuth.UseDefaultOrg');
// Get tags from SSO config
$mailTag = Configure::read('ApacheShibbAuth.MailTag');
$OrgTag = Configure::read('ApacheShibbAuth.OrgTag');
$groupTag = Configure::read('ApacheShibbAuth.GroupTag');
$groupRoleMatching = Configure::read('ApacheShibbAuth.GroupRoleMatching');
// Change username column for email (username in shibboleth attributes corresponds to the email in MISPs DB)
$this->settings['fields'] = array('username' => 'email');
// Get user values
if (!isset($_SERVER[$mailTag]) || filter_var($_SERVER[$mailTag], FILTER_VALIDATE_EMAIL) === FALSE) {
CakeLog::write('error', 'Mail tag is not given by the SSO SP. Not processing login.');
return false;
}
// Find user with real username (mail)
$user = $this->_findUser($mispUsername);
$mispUsername = $_SERVER[$mailTag];
CakeLog::write('info', "Trying login of user: ${mispUsername}.");
// Obtain default org. If default is not enforced and it is given, org keeps the default value
if (!$useDefaultOrg && isset($_SERVER[$orgTag])) {
$org = $_SERVER[$orgTag];
}
//Change username column for email (username in shibboleth attributes corresponds to the email in MISPs DB)
$this->settings['fields'] = array('username' => 'email');
// Check if the organization exits and create it if not
$org = $this->checkOrganization($org, $user);
if (!$org) {
return false;
}
// Find user with real username (mail)
$user = $this->_findUser($mispUsername);
// Get user role from its list of groups
list($roleChanged, $roleId) = $this->getUserRoleFromGroup($groupTag, $groupRoleMatching, $roleId);
if ($roleId < 0) {
CakeLog::error('No role was assigned, no egroup matched the configuration.');
return false; // Deny if the user is not in any egroup
}
//Obtain default org. If default is not enforced and it is given, org keeps the default value
if (!$useDefaultOrg && isset($_SERVER[$OrgTag])) {
$org = $_SERVER[$OrgTag];
}
/** @var User $userModel */
$userModel = ClassRegistry::init($this->settings['userModel']);
//Check if the organization exits and create it if not
$org = $this->checkOrganization($org, $user);
if ($user) { // User already exists
CakeLog::info( "User `$mispUsername` found in database.");
$user = $this->updateUserRole($roleChanged, $user, $roleId, $userModel);
$user = $this->updateUserOrg($org, $user, $userModel);
CakeLog::info("User `$mispUsername` logged in.");
return $user;
}
//Get user role from its list of groups
list($roleChanged, $roleId) = $this->getUserRoleFromGroup($groupTag, $groupRoleMatching, $roleId);
if($roleId < 0) {
CakeLog::write('error', 'No role was assigned, no egroup matched the configuration.');
return false; //Deny if the user is not in any egroup
}
CakeLog::info("User `$mispUsername` not found in database.");
// Database model object
$userModel = ClassRegistry::init($this->settings['userModel']);
// Insert user in database if not existent
$userData = array('User' => array(
'email' => $mispUsername,
'org_id' => $org,
'newsread' => time(),
'role_id' => $roleId,
'change_pw' => 0,
'date_created' => time(),
));
if ($user) { // User already exists
CakeLog::write('info', "User ${mispUsername} found in database.");
$user = $this->updateUserRole($roleChanged, $user, $roleId, $userModel);
$user = $this->updateUserOrg($org, $user, $userModel);
CakeLog::write('info', "User ${mispUsername} logged in.");
return $user;
}
// save user
$userModel->save($userData);
CakeLog::info("User `$mispUsername` saved in database.");
CakeLog::info("User `$mispUsername` logged in.");
return $this->_findUser($mispUsername);
}
CakeLog::write('info', "User ${mispUsername} not found in database.");
//Insert user in database if not existent
/**
* @param string $org
* @param array $user
* @return int
*/
private function checkOrganization($org, $user)
{
$orgIsUuid = Validation::uuid($org);
// Generate random password
$password = $userModel->generateRandomPassword();
// Generate random auth key
$authKey = $userModel->generateAuthKey();
// get maximum nids value
$nidsMax = $userModel->find('all', array(
'fields' => array('MAX(User.nids_sid) AS nidsMax'),
)
);
// create user
$userData = array('User' => array(
'email' => $mispUsername,
'org_id' => $org,
'password' => $password, //Since it is done via shibboleth the password will be a random 40 character string
'confirm_password' => $password,
'authkey' => $authKey,
'nids_sid' => ((int)$nidsMax[0][0]['nidsMax'])+1,
'newsread' => time(),
'role_id' => $roleId,
'change_pw' => 0,
'date_created' => time()
));
/** @var Organisation $orgModel */
$orgModel = ClassRegistry::init('Organisation');
$orgAux = $orgModel->find('first', [
'fields' => array('Organisation.id'),
'conditions' => $orgIsUuid ? ['uuid' => strtolower($org)] : ['name' => $org],
]);
if (empty($orgAux)) {
if ($orgIsUuid) {
CakeLog::error("Could not found organisation with UUID `$org`.");
return false;
}
// save user
$userModel->save($userData, false);
CakeLog::write('info', "User ${mispUsername} saved in database.");
CakeLog::write('info', "User ${mispUsername} logged in.");
return $this->_findUser(
$mispUsername
);
}
$orgUserId = 1; // By default created by the admin
if ($user) {
$orgUserId = $user['id'];
}
$orgId = $orgModel->createOrgFromName($org, $orgUserId, 0); // Created with local set to 0 by default
CakeLog::info("User organisation `$org` created with ID $orgId.");
} else {
$orgId = $orgAux['Organisation']['id'];
CakeLog::info("User organisation `$org` found with ID $orgId.");
}
return $orgId;
}
/**
* @param $roleChanged
* @param $user
* @param $roleId
* @param $userModel
* @return mixed
*/
public function updateUserRole($roleChanged, $user, $roleId, $userModel)
{
if ($roleChanged && $user['role_id'] != $roleId) {
CakeLog::write('warning', "User role changed from ${user['role_id']} to ${roleId}.");
$user['role_id'] = $roleId; // Different role either increase or decrease permissions
$userUpdatedData = array('User' => $user);
$userModel->set(array(
'role_id' => $roleId,
'id' => $user['id'],
)); // Update the user
$userModel->save($userUpdatedData, false);
return $user;
}
return $user;
}
/**
* @param string $groupTag
* @param array $groupRoleMatching
* @param int $roleId
* @return array
*/
public function getUserRoleFromGroup($groupTag, $groupRoleMatching, $roleId)
{
// Check the role mapping to get the user's role level and update it if needed
$roleChanged = false;
if (isset($_SERVER[$groupTag])) {
$groupSeparator = Configure::read('ApacheShibbAuth.GroupSeparator');
$groupList = explode($groupSeparator, $_SERVER[$groupTag]);
// Check user roles and egroup match and update if needed
foreach ($groupList as $group) {
// TODO: Can be optimized inverting the search group and using only array_key_exists
if (array_key_exists($group, $groupRoleMatching)) { //In case there is an group not defined in the config.php file
CakeLog::write('info', "User group ${group} found.");
$roleVal = $groupRoleMatching[$group];
if ($roleVal <= $roleId || $roleId == -1) {
$roleId = $roleVal;
$roleChanged = true;
}
CakeLog::write('info', "User role ${roleId} assigned.");
}
}
return array($roleChanged, $roleId);
}
return array($roleChanged, $roleId);
}
/**
* @param $groupTag
* @param $groupRoleMatching
* @param $roleId
* @return array
*/
public function getUserRoleFromGroup($groupTag, $groupRoleMatching, $roleId)
{
//Check the role mapping to get the user's role level and update it if needed
$roleChanged = false;
if (isset($_SERVER[$groupTag])) {
$groupSeparator = Configure::read('ApacheShibbAuth.GroupSeparator');
$groupList = explode($groupSeparator, $_SERVER[$groupTag]);
//Check user roles and egroup match and update if needed
foreach ($groupList as $group) {
//TODO: Can be optimized inverting the search group and using only array_key_exists
if (array_key_exists($group, $groupRoleMatching)) { //In case there is an group not defined in the config.php file
CakeLog::write('info', "User group ${group} found.");
$roleVal = $groupRoleMatching[$group];
if ($roleVal <= $roleId || $roleId == -1) {
$roleId = $roleVal;
$roleChanged = true;
}
CakeLog::write('info', "User role ${roleId} assigned.");
}
}
return array($roleChanged, $roleId);
}
return array($roleChanged, $roleId);
}
/**
* @param bool $roleChanged
* @param array $user
* @param int $roleId
* @param User $userModel
* @return array
* @throws Exception
*/
private function updateUserRole($roleChanged, array $user, $roleId, User $userModel)
{
if ($roleChanged && $user['role_id'] != $roleId) {
CakeLog::write('warning', "User role changed from ${user['role_id']} to $roleId.");
$userModel->updateField($user, 'role_id', $roleId);
}
return $user;
}
/**
* @param $org
* @param $user
* @return array|bool|int|mixed|string
*/
public function checkOrganization($org, $user)
{
$orgModel = ClassRegistry::init('Organisation');
$orgAux = $orgModel->find('first', array(
'fields' => array('Organisation.id'),
'conditions' => array('name' => $org),
)
);
if ($orgAux == null) {
$organisations = new Organisation();
$orgUserId = 1; //By default created by the admin
if ($user) $orgUserId = $user['id'];
$orgId = $organisations->createOrgFromName($org, $orgUserId, 0); //Created with local set to 0 by default
CakeLog::write('info', "User organisation ${org} created with id ${orgId}.");
} else {
$orgId = $orgAux['Organisation']['id'];
CakeLog::write('info', "User organisation ${org} found with id ${orgId}.");
}
return $orgId;
}
private function updateUserOrg($org, $user, $userModel)
{
if ($user['org_id'] != $org) {
CakeLog::write('warning', "User organisation ${org} changed.");
$user['org_id'] = $org; // Different role either increase or decrease permissions
$userUpdatedData = array('User' => $user);
$userModel->set(array(
'org_id' => $org,
'id' => $user['id'],
)); // Update the user
$userModel->save($userUpdatedData, false);
return $user;
}
return $user;
}
/**
* @param int $orgId
* @param array $user
* @param User $userModel
* @return array
* @throws Exception
*/
private function updateUserOrg($orgId, array $user, User $userModel)
{
if ($user['org_id'] != $orgId) {
CakeLog::write('warning', "User organisation $orgId changed.");
$user['org_id'] = $orgId; // Different role either increase or decrease permissions
$userModel->updateField($user, 'org_id', $orgId);
}
return $user;
}
}

19
app/View/Elements/Events/View/eventSightingValue.ctp
View File

@ -1,23 +1,22 @@
<?php
$ownOrgSightingsCount = 0;
if (isset($event['Sighting'])) {
$meOrgId = $this->get('me')['org_id'];
foreach ($event['Sighting'] as $sighting) {
if (isset($sighting['org_id']) && $sighting['org_id'] == $meOrgId) {
++$ownOrgSightingsCount;
}
}
$userOrgName = $this->get('me')['Organisation']['name'];
$totalCount = 0;
$ownCount = 0;
foreach ($sightingsData as $data) {
$totalCount += $data['count'];
$ownCount += isset($data['orgs'][$userOrgName]['count']) ? $data['orgs'][$userOrgName]['count'] : 0;
}
echo sprintf(
'%s (%s) %s %s',
sprintf(
'<span id="eventSightingCount" class="bold sightingsCounter">%s</span>',
count($event['Sighting'])
$totalCount
),
sprintf(
'<span id="eventOwnSightingCount" class="green bold sightingsCounter">%s</span>',
$ownOrgSightingsCount
$ownCount
),
(Configure::read('Plugin.Sightings_policy')) ? '' : __('- restricted to own organisation only.'),
sprintf(

2
app/View/Elements/genericElements/SideMenu/side_menu.ctp
View File

@ -1040,7 +1040,7 @@ $divider = $this->element('/genericElements/SideMenu/side_menu_divider');
case 'tags':
echo $this->element('/genericElements/SideMenu/side_menu_link', array(
'element_id' => 'indexfav',
'url' => $baseurl . '/tags/index/1',
'url' => $baseurl . '/tags/index/favouritesOnly:1',
'text' => __('List Favourite Tags')
));
echo $this->element('/genericElements/SideMenu/side_menu_link', array(

5
app/View/Events/view.ctp
View File

@ -225,13 +225,14 @@
'element' => '/Events/View/eventSightingValue',
'element_params' => array(
'event' => $event,
'sightingsData' => isset($sightingsData['data']['all']) ? $sightingsData['data']['all'] : [],
)
);
if (!empty($sightingsData['csv']['event'])) {
if (isset($sightingsData['data']['all'])) {
$table_data[] = array(
'key' => __('Activity'),
'element' => 'sparkline',
'element_params' => array('scope' => 'event', 'id' => $event['Event']['id'], 'csv' => $sightingsData['csv']['event'])
'element_params' => array('scope' => 'event', 'id' => $event['Event']['id'], 'csv' => $sightingsData['csv']['all'])
);
}
if (!empty($delegationRequest)) {

4
app/View/News/index.ctp
View File

@ -9,7 +9,7 @@
<div class="templateGlass"></div>
<div class ="templateElementHeaderText" style="width:100%;">
<div style="float:left;width:83%;"><?php echo $newsItem['User']['email'] ? h($newsItem['User']['email']) : 'Administrator'; ?></div>
<div style="float:left;width:17%;"><?php echo date('Y/m/d H:i:s', $newsItem['News']['date_created']); ?></div>
<div style="float:left;width:17%;"><?php echo date('Y-m-d H:i:s', $newsItem['News']['date_created']); ?></div>
</div>
</div>
<div style="padding:6px;">
@ -21,7 +21,7 @@
?>
<br /><a href="<?php echo $baseurl; ?>/news/edit/<?php echo h($newsItem['News']['id']);?>" class="fa fa-edit" title="<?php echo __('Edit news message');?>" aria-label="<?php echo __('Edit');?>"></a>
<?php
echo $this->Form->postLink('', array('action' => 'delete', $newsItem['News']['id']), array('class' => 'fa fa-trash', 'title' => __('Delete'), 'aria-label' => __('Delete')), __('Are you sure you want to delete news item # %s?', $newsItem['News']['id']));
echo $this->Form->postLink('', array('action' => 'delete', $newsItem['News']['id']), array('class' => 'fa fa-trash', 'title' => __('Delete'), 'aria-label' => __('Delete')), __('Are you sure you want to delete news item #%s?', $newsItem['News']['id']));
endif;
?>
</div>

14
app/View/Sightings/ajax/list_sightings.ctp
View File

@ -1,3 +1,7 @@
<?php
// Calling `__` method for every sighting can be surprisingly quite slow, so better to call just once
$deleteSightingTitle = __('Delete sighting');
?>
<div>
<div id="org_id" class="hidden"><?php echo h($org_id); ?></div>
<table class="table table-striped table-hover table-condensed" style="display:block; overflow-y:auto;max-height:500px;">
@ -21,15 +25,15 @@
}
?>
</td>
<td class="short"><?= $types[$item['Sighting']['type']]; ?></td>
<td class="short"><?php echo h($item['Sighting']['source']);?></td>
<td class="short"><?php echo h($item['Sighting']['event_id']);?></td>
<td class="short"><?php echo h($item['Sighting']['attribute_id']);?></td>
<td class="short"><?= $types[$item['Sighting']['type']] ?></td>
<td class="short"><?= h($item['Sighting']['source']) ?></td>
<td class="short"><?= h($item['Sighting']['event_id']) ?></td>
<td class="short"><?= h($item['Sighting']['attribute_id']) ?></td>
<td class="short action-links">
<?php
if ($isSiteAdmin || ($item['Sighting']['org_id'] == $me['org_id'] && $isAclAdd)):
?>
<span class="fa fa-trash useCursorPointer" title="<?= __('Delete sighting');?>" role="button" tabindex="0" aria-label="<?php echo __('Delete sighting');?>" onClick="quickDeleteSighting('<?php echo h($item['Sighting']['id']); ?>', '<?php echo h($rawId); ?>', '<?php echo h($context); ?>');"></span>
<span class="fa fa-trash useCursorPointer" title="<?= $deleteSightingTitle ?>" role="button" tabindex="0" aria-label="<?= $deleteSightingTitle ?>" onClick="quickDeleteSighting('<?= h($item['Sighting']['id']) ?>', '<?= h($rawId) ?>', '<?= h($context) ?>');"></span>
<?php
endif;
?>

14
app/View/Sightings/ajax/view_sightings.ctp
View File

@ -4,7 +4,7 @@
?>
<div id="graphContent" class="graphContent"></div>
<script>
var myData = "<?php echo $tsv; ?>";
var myData = "<?php echo $csv; ?>";
var colours = {
'Sighting': 'blue',
@ -20,7 +20,7 @@
width = 980 - margin.left - margin.right,
height = 300 - margin.top - margin.bottom;
var parseDate = d3.time.format("%Y%m%d").parse;
var parseDate = d3.time.format("%Y-%m-%d").parse;
var x = d3.time.scale()
.range([0, width]);
@ -53,14 +53,14 @@
.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
var data = d3.tsv.parse(myData);
var data = d3.csv.parse(myData);
color.domain(d3.keys(data[0]).filter(function(key) {
return key !== "date";
return key !== "Date";
}));
data.forEach(function(d) {
d.date = parseDate(d.date);
d.Date = parseDate(d.Date);
});
var sightings = color.domain().map(function(name) {
@ -68,7 +68,7 @@
name: name,
values: data.map(function(d) {
return {
date: d.date,
date: d.Date,
count: +d[name]
};
})
@ -76,7 +76,7 @@
});
x.domain(d3.extent(data, function(d) {
return d.date;
return d.Date;
}));
y.domain([

4
app/View/Users/login.ctp
View File

@ -32,6 +32,7 @@
</div>
<?php
endif;
if ($formLoginEnabled):
echo $this->Form->create('User');
?>
<legend><?php echo __('Login');?></legend>
@ -52,6 +53,7 @@
<?= $this->Form->button(__('Login'), array('class' => 'btn btn-primary')); ?>
<?php
echo $this->Form->end();
endif;
if (Configure::read('ApacheShibbAuth') == true) {
echo '<div class="clear"></div><a class="btn btn-info" href="/Shibboleth.sso/Login">Login with SAML</a>';
}
@ -65,7 +67,7 @@
</div>
<script>
$(document).ready(function() {
$(function() {
$('#UserLoginForm').submit(function(event) {
event.preventDefault()
submitLoginForm()

2
app/files/misp-galaxy

@ -1 +1 @@
Subproject commit ac5301871577bcc792ffcf4feb3c55e598f74d92
Subproject commit 9a731470d313c636d9bd7d8d688d3cb9bf17f667

10
app/webroot/js/event-timeline.js
View File

@ -511,13 +511,13 @@ function enable_timeline() {
},
success: function( data, textStatus, jQxhr ){
if (data.items.length > hardThreshold) {
$('#eventtimeline_div').html('<div class="alert alert-danger" style="margin: 10px;">Timeline: To much data to show</div>');
$('#eventtimeline_div').html('<div class="alert alert-danger" style="margin: 10px;">Timeline: Too much data to show</div>');
timeline_disabled = true;
return;
} else if (data.items.length > softThreshold) {
var res = confirm('You are about to draw a lot ('+data.items.length+') of items in the timeline. Do you wish to continue?');
if (!res) {
$('#eventtimeline_div').html('<div class="alert alert-danger" style="margin: 10px;">Timeline: To much data to show</div>');
$('#eventtimeline_div').html('<div class="alert alert-danger" style="margin: 10px;">Timeline: Too much data to show</div>');
timeline_disabled = true;
return;
}
@ -673,8 +673,8 @@ function init_popover() {
});
menu_display_timeline.add_slider({
id: 'slider_timeline_display_max_char_num',
label: "Charater to show",
title: "Maximum number of charater to display in the label",
label: "Characters to show",
title: "Maximum number of characters to display in the label",
min: 8,
max: 2048,
value: max_displayed_char_timeline,
@ -690,7 +690,7 @@ function init_popover() {
menu_display_timeline.add_checkbox({
id: 'checkbox_timeline_display_hide_not_seen_enabled',
label: "Hide first seen not set",
title: "Hide items that does not have first seen sets",
title: "Hide items that do not have first seen set",
event: function(value) {
handle_not_seen_enabled(value)
}

18
app/webroot/js/misp.js
View File

@ -2788,6 +2788,14 @@ function freetextImportResultsSubmit(id, count) {
}
function moduleResultsSubmit(id) {
var attributeValue = function ($attributeValue) {
if ($attributeValue.find("[data-full]").length) {
return $attributeValue.find("[data-full]").data('full');
} else {
return $attributeValue.text()
}
}
var typesWithData = ['attachment', 'malware-sample'];
var data_collected = {};
var temp;
@ -2802,7 +2810,7 @@ function moduleResultsSubmit(id) {
}
if ($('.MISPObject').length) {
var objects = [];
$(".MISPObject").each(function(o) {
$(".MISPObject").each(function() {
var object_uuid = $(this).find('.ObjectUUID').text();
temp = {
uuid: object_uuid,
@ -2845,14 +2853,14 @@ function moduleResultsSubmit(id) {
}
if ($(this).find('.ObjectAttribute').length) {
var object_attributes = [];
$(this).find('.ObjectAttribute').each(function(a) {
$(this).find('.ObjectAttribute').each(function() {
var attribute_type = $(this).find('.AttributeType').text();
attribute = {
var attribute = {
import_attribute: $(this).find('.ImportMISPObjectAttribute')[0].checked,
object_relation: $(this).find('.ObjectRelation').text(),
category: $(this).find('.AttributeCategory').text(),
type: attribute_type,
value: $(this).find('.AttributeValue').text(),
value: attributeValue($(this).find('.AttributeValue')),
uuid: $(this).find('.AttributeUuid').text(),
to_ids: $(this).find('.AttributeToIds')[0].checked,
disable_correlation: $(this).find('.AttributeDisableCorrelation')[0].checked,
@ -2911,7 +2919,7 @@ function moduleResultsSubmit(id) {
import_attribute: $(this).find('.ImportMISPAttribute')[0].checked,
category: category_value,
type: type_value,
value: $(this).find('.AttributeValue').text(),
value: attributeValue($(this).find('.AttributeValue')),
uuid: $(this).find('.AttributeUuid').text(),
to_ids: $(this).find('.AttributeToIds')[0].checked,
disable_correlation: $(this).find('.AttributeDisableCorrelation')[0].checked,

221
docs/INSTALL.ubuntu1804.md
View File

@ -133,78 +133,89 @@ installDepsPhp72 () {
installCore () {
debug "Installing ${LBLUE}MISP${NC} core"
# Download MISP using git in the /var/www/ directory.
sudo mkdir ${PATH_TO_MISP}
sudo chown ${WWW_USER}:${WWW_USER} ${PATH_TO_MISP}
cd ${PATH_TO_MISP}
${SUDO_WWW} git clone https://github.com/MISP/MISP.git ${PATH_TO_MISP}
${SUDO_WWW} git submodule update --progress --init --recursive
# Make git ignore filesystem permission differences for submodules
$SUDO_WWW git submodule foreach --recursive git config core.filemode false
if [[ ! -d ${PATH_TO_MISP} ]]; then
sudo mkdir ${PATH_TO_MISP}
sudo chown ${WWW_USER}:${WWW_USER} ${PATH_TO_MISP}
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git clone https://github.com/MISP/MISP.git ${PATH_TO_MISP}; done
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git -C ${PATH_TO_MISP} submodule update --progress --init --recursive; done
# Make git ignore filesystem permission differences for submodules
${SUDO_WWW} git -C ${PATH_TO_MISP} submodule foreach --recursive git config core.filemode false
# Make git ignore filesystem permission differences
$SUDO_WWW git config core.filemode false
# Make git ignore filesystem permission differences
${SUDO_WWW} git -C ${PATH_TO_MISP} config core.filemode false
# Create a python3 virtualenv
$SUDO_WWW virtualenv -p python3 ${PATH_TO_MISP}/venv
# Create a python3 virtualenv
${SUDO_WWW} virtualenv -p python3 ${PATH_TO_MISP}/venv
# make pip happy
sudo mkdir /var/www/.cache/
sudo chown $WWW_USER:$WWW_USER /var/www/.cache
# make pip happy
sudo mkdir /var/www/.cache/
sudo chown ${WWW_USER}:${WWW_USER} /var/www/.cache
cd ${PATH_TO_MISP}/app/files/scripts
$SUDO_WWW git clone https://github.com/CybOXProject/python-cybox.git
$SUDO_WWW git clone https://github.com/STIXProject/python-stix.git
$SUDO_WWW git clone https://github.com/MAECProject/python-maec.git
PATH_TO_MISP_SCRIPTS=${PATH_TO_MISP}/app/files/scripts
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git clone https://github.com/CybOXProject/python-cybox.git ${PATH_TO_MISP_SCRIPTS}/python-cybox; done
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git clone https://github.com/STIXProject/python-stix.git ${PATH_TO_MISP_SCRIPTS}/python-stix; done
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git clone https://github.com/MAECProject/python-maec.git ${PATH_TO_MISP_SCRIPTS}/python-maec; done
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git clone https://github.com/CybOXProject/mixbox.git ${PATH_TO_MISP_SCRIPTS}/mixbox; done
# install mixbox to accommodate the new STIX dependencies:
$SUDO_WWW git clone https://github.com/CybOXProject/mixbox.git
cd ${PATH_TO_MISP}/app/files/scripts/mixbox
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
cd ${PATH_TO_MISP}/app/files/scripts/python-cybox
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
cd ${PATH_TO_MISP}/app/files/scripts/python-stix
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
cd $PATH_TO_MISP/app/files/scripts/python-maec
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
# install STIX2.0 library to support STIX 2.0 export:
cd ${PATH_TO_MISP}/cti-python-stix2
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/mixbox
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/python-cybox
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/python-maec
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/python-stix
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/cti-python-stix2
# install PyMISP
cd ${PATH_TO_MISP}/PyMISP
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
# FIXME: Remove libfaup etc once the egg has the library baked-in
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
cd /tmp
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
cd gtcaca
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
debug "Install PyMISP"
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/PyMISP
# FIXME: Remove libfaup etc once the egg has the library baked-in
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
cd /tmp
false; while [[ $? -ne 0 ]]; do [[ ! -d "faup" ]] && ${SUDO_CMD} git clone git://github.com/stricaud/faup.git faup; done
false; while [[ $? -ne 0 ]]; do [[ ! -d "gtcaca" ]] && ${SUDO_CMD} git clone git://github.com/stricaud/gtcaca.git gtcaca; done
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
cd gtcaca
${SUDO_CMD} mkdir -p build
cd build
${SUDO_CMD} cmake .. && ${SUDO_CMD} make
sudo make install
cd ../../faup
${SUDO_CMD} mkdir -p build
cd build
${SUDO_CMD} cmake .. && ${SUDO_CMD} make
sudo make install
sudo ldconfig
# install pydeep
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/pydeep.git
# install pydeep
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/pydeep.git; done
# install lief
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install lief
# install lief
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install lief
# install zmq needed by mispzmq
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install zmq redis
# install zmq needed by mispzmq
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install zmq redis
# install python-magic
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install python-magic
# install python-magic
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install python-magic
# install plyara
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install plyara
# install plyara
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install plyara
else
debug "Trying to git pull existing install"
${SUDO_WWW} git pull -C ${PATH_TO_MISP}
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git -C ${PATH_TO_MISP} submodule update --progress --init --recursive; done
PATH_TO_MISP_SCRIPTS=${PATH_TO_MISP}/app/files/scripts
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git -C ${PATH_TO_MISP_SCRIPTS}/python-cybox pull; done
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git -C ${PATH_TO_MISP_SCRIPTS}/python-stix pull; done
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git -C ${PATH_TO_MISP_SCRIPTS}/python-maec pull; done
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git -C ${PATH_TO_MISP_SCRIPTS}/mixbox pull; done
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U setuptools pip lief zmq redis python-magic plyara
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP}/app/files/scripts/mixbox
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP}/app/files/scripts/python-cybox
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP}/app/files/scripts/python-maec
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP}/app/files/scripts/python-stix
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP}/cti-python-stix2
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP}/PyMISP
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U git+https://github.com/kbandla/pydeep.git; done
fi
}
# <snippet-end 1_mispCoreInstall.sh>
```
@ -221,15 +232,15 @@ installCake () {
cd ${PATH_TO_MISP}/app
# Make composer cache happy
# /!\ composer on Ubuntu when invoked with sudo -u doesn't set $HOME to /var/www but keeps it /home/misp \!/
sudo mkdir /var/www/.composer ; sudo chown $WWW_USER:$WWW_USER /var/www/.composer
$SUDO_WWW php composer.phar install
sudo mkdir /var/www/.composer ; sudo chown ${WWW_USER}:${WWW_USER} /var/www/.composer
${SUDO_WWW} php composer.phar install
# Enable CakeResque with php-redis
sudo phpenmod redis
sudo phpenmod gnupg
# To use the scheduler worker for scheduled tasks, do the following:
$SUDO_WWW cp -fa ${PATH_TO_MISP}/INSTALL/setup/config.php ${PATH_TO_MISP}/app/Plugin/CakeResque/Config/config.php
${SUDO_WWW} cp -fa ${PATH_TO_MISP}/INSTALL/setup/config.php ${PATH_TO_MISP}/app/Plugin/CakeResque/Config/config.php
# If you have multiple MISP instances on the same system, don't forget to have a different Redis per MISP instance for the CakeResque workers
# The default Redis port can be updated in Plugin/CakeResque/Config/config.php
@ -262,59 +273,39 @@ permissions () {
```bash
# <snippet-begin 1_prepareDB.sh>
prepareDB () {
if [[ ! -e /var/lib/mysql/misp/users.ibd ]]; then
if sudo test ! -e "/var/lib/mysql/mysql/"; then
#Make sure initial tables are created in MySQL
debug "Install mysql tables"
sudo mysql_install_db --user=mysql --basedir=/usr --datadir=/var/lib/mysql
sudo service mysql start
fi
if sudo test ! -e "/var/lib/mysql/misp/"; then
debug "Start mysql"
sudo service mysql start
debug "Setting up database"
# Kill the anonymous users
sudo mysql -e "DROP USER IF EXISTS ''@'localhost'"
# Because our hostname varies we'll use some Bash magic here.
sudo mysql -e "DROP USER IF EXISTS ''@'$(hostname)'"
# Kill off the demo database
sudo mysql -e "DROP DATABASE IF EXISTS test"
# No root remote logins
sudo mysql -e "DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1')"
# Make sure that NOBODY can access the server without a password
sudo mysqladmin -u "${DBUSER_ADMIN}" password "${DBPASSWORD_ADMIN}"
# Make our changes take effect
sudo mysql -e "FLUSH PRIVILEGES"
# FIXME: If user 'misp' exists, and has a different password, the below WILL fail. Partially fixed with the Env-Var check in the beginning. (Need to implement pre-flight checks to exit gracefully if not set)
# Add your credentials if needed, if sudo has NOPASS, comment out the relevant lines
if [[ "${PACKER}" == "1" ]]; then
pw="Password1234"
else
pw=${MISP_PASSWORD}
fi
if [[ ! -z ${INSTALL_USER} ]]; then
SUDO_EXPECT="sudo mysql_secure_installation"
echo "Making sure sudo session is buffered"
sudo ls -la /tmp > /dev/null 2> /dev/null
else
SUDO_EXPECT="sudo -k mysql_secure_installation"
fi
expect -f - <<-EOF
set timeout 10
spawn ${SUDO_EXPECT}
expect "*?assword*"
send -- "${pw}\r"
expect "Enter current password for root (enter for none):"
send -- "\r"
expect "Set root password?"
send -- "y\r"
expect "New password:"
send -- "${DBPASSWORD_ADMIN}\r"
expect "Re-enter new password:"
send -- "${DBPASSWORD_ADMIN}\r"
expect "Remove anonymous users?"
send -- "y\r"
expect "Disallow root login remotely?"
send -- "y\r"
expect "Remove test database and access to it?"
send -- "y\r"
expect "Reload privilege tables now?"
send -- "y\r"
expect eof
EOF
sudo apt-get purge -y expect ; sudo apt autoremove -qy
fi
sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "CREATE DATABASE ${DBNAME};"
sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "CREATE USER '${DBUSER_MISP}'@'localhost' IDENTIFIED BY '${DBPASSWORD_MISP}';"
sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "GRANT USAGE ON *.* to ${DBUSER_MISP}@localhost;"
sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "GRANT ALL PRIVILEGES on ${DBNAME}.* to '${DBUSER_MISP}'@'localhost';"
sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "FLUSH PRIVILEGES;"
# Import the empty MISP database from MYSQL.sql
${SUDO_WWW} cat ${PATH_TO_MISP}/INSTALL/MYSQL.sql | mysql -u ${DBUSER_MISP} -p${DBPASSWORD_MISP} ${DBNAME}
sudo mysql -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "CREATE DATABASE ${DBNAME};"
sudo mysql -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "CREATE USER '${DBUSER_MISP}'@'localhost' IDENTIFIED BY '${DBPASSWORD_MISP}';"
sudo mysql -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "GRANT USAGE ON *.* to '${DBUSER_MISP}'@'localhost';"
sudo mysql -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "GRANT ALL PRIVILEGES on ${DBNAME}.* to '${DBUSER_MISP}'@'localhost';"
sudo mysql -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "FLUSH PRIVILEGES;"
# Import the empty MISP database from MYSQL.sql
${SUDO_WWW} cat ${PATH_TO_MISP}/INSTALL/MYSQL.sql | mysql -u "${DBUSER_MISP}" -p"${DBPASSWORD_MISP}" ${DBNAME}
fi
}
# <snippet-end 1_prepareDB.sh>
```

216
docs/INSTALL.ubuntu2004.md
View File

@ -98,8 +98,6 @@ installCoreDeps () {
# install Mitre's STIX and its dependencies by running the following commands:
sudo apt-get install python3-dev python3-pip libxml2-dev libxslt1-dev zlib1g-dev python-setuptools -qy
sudo apt install expect -qy
}
# <snippet-end 0_installCoreDeps.sh>
@ -133,77 +131,89 @@ installDepsPhp74 () {
installCore () {
debug "Installing ${LBLUE}MISP${NC} core"
# Download MISP using git in the /var/www/ directory.
sudo mkdir ${PATH_TO_MISP}
sudo chown ${WWW_USER}:${WWW_USER} ${PATH_TO_MISP}
cd ${PATH_TO_MISP}
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git clone https://github.com/MISP/MISP.git ${PATH_TO_MISP}; done
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git submodule update --progress --init --recursive; done
# Make git ignore filesystem permission differences for submodules
${SUDO_WWW} git submodule foreach --recursive git config core.filemode false
if [[ ! -d ${PATH_TO_MISP} ]]; then
sudo mkdir ${PATH_TO_MISP}
sudo chown ${WWW_USER}:${WWW_USER} ${PATH_TO_MISP}
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git clone https://github.com/MISP/MISP.git ${PATH_TO_MISP}; done
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git -C ${PATH_TO_MISP} submodule update --progress --init --recursive; done
# Make git ignore filesystem permission differences for submodules
${SUDO_WWW} git -C ${PATH_TO_MISP} submodule foreach --recursive git config core.filemode false
# Make git ignore filesystem permission differences
${SUDO_WWW} git config core.filemode false
# Make git ignore filesystem permission differences
${SUDO_WWW} git -C ${PATH_TO_MISP} config core.filemode false
# Create a python3 virtualenv
${SUDO_WWW} virtualenv -p python3 ${PATH_TO_MISP}/venv
# Create a python3 virtualenv
${SUDO_WWW} virtualenv -p python3 ${PATH_TO_MISP}/venv
# make pip happy
sudo mkdir /var/www/.cache/
sudo chown ${WWW_USER}:${WWW_USER} /var/www/.cache
# make pip happy
sudo mkdir /var/www/.cache/
sudo chown ${WWW_USER}:${WWW_USER} /var/www/.cache
cd ${PATH_TO_MISP}/app/files/scripts
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git clone https://github.com/CybOXProject/python-cybox.git; done
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git clone https://github.com/STIXProject/python-stix.git; done
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git clone https://github.com/MAECProject/python-maec.git; done
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git clone https://github.com/CybOXProject/mixbox.git; done
PATH_TO_MISP_SCRIPTS=${PATH_TO_MISP}/app/files/scripts
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git clone https://github.com/CybOXProject/python-cybox.git ${PATH_TO_MISP_SCRIPTS}/python-cybox; done
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git clone https://github.com/STIXProject/python-stix.git ${PATH_TO_MISP_SCRIPTS}/python-stix; done
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git clone https://github.com/MAECProject/python-maec.git ${PATH_TO_MISP_SCRIPTS}/python-maec; done
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git clone https://github.com/CybOXProject/mixbox.git ${PATH_TO_MISP_SCRIPTS}/mixbox; done
cd ${PATH_TO_MISP}/app/files/scripts/mixbox
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install .
cd ${PATH_TO_MISP}/app/files/scripts/python-cybox
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install .
cd ${PATH_TO_MISP}/app/files/scripts/python-stix
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install .
cd ${PATH_TO_MISP}/app/files/scripts/python-maec
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install .
# install STIX2.0 library to support STIX 2.0 export:
cd ${PATH_TO_MISP}/cti-python-stix2
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install .
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/mixbox
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/python-cybox
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/python-maec
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/python-stix
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/cti-python-stix2
# install PyMISP
cd ${PATH_TO_MISP}/PyMISP
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install .
# FIXME: Remove libfaup etc once the egg has the library baked-in
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
cd /tmp
false; while [[ $? -ne 0 ]]; do [[ ! -d "faup" ]] && ${SUDO_CMD} git clone git://github.com/stricaud/faup.git faup; done
false; while [[ $? -ne 0 ]]; do [[ ! -d "gtcaca" ]] && ${SUDO_CMD} git clone git://github.com/stricaud/gtcaca.git gtcaca; done
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
cd gtcaca
${SUDO_CMD} mkdir -p build
cd build
${SUDO_CMD} cmake .. && ${SUDO_CMD} make
sudo make install
cd ../../faup
${SUDO_CMD} mkdir -p build
cd build
${SUDO_CMD} cmake .. && ${SUDO_CMD} make
sudo make install
sudo ldconfig
debug "Install PyMISP"
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/PyMISP
# FIXME: Remove libfaup etc once the egg has the library baked-in
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
cd /tmp
false; while [[ $? -ne 0 ]]; do [[ ! -d "faup" ]] && ${SUDO_CMD} git clone git://github.com/stricaud/faup.git faup; done
false; while [[ $? -ne 0 ]]; do [[ ! -d "gtcaca" ]] && ${SUDO_CMD} git clone git://github.com/stricaud/gtcaca.git gtcaca; done
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
cd gtcaca
${SUDO_CMD} mkdir -p build
cd build
${SUDO_CMD} cmake .. && ${SUDO_CMD} make
sudo make install
cd ../../faup
${SUDO_CMD} mkdir -p build
cd build
${SUDO_CMD} cmake .. && ${SUDO_CMD} make
sudo make install
sudo ldconfig
# install pydeep
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/pydeep.git; done
# install pydeep
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/pydeep.git; done
# install lief
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install lief
# install lief
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install lief
# install zmq needed by mispzmq
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install zmq redis
# install zmq needed by mispzmq
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install zmq redis
# install python-magic
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install python-magic
# install python-magic
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install python-magic
# install plyara
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install plyara
# install plyara
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install plyara
else
debug "Trying to git pull existing install"
${SUDO_WWW} git pull -C ${PATH_TO_MISP}
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git -C ${PATH_TO_MISP} submodule update --progress --init --recursive; done
PATH_TO_MISP_SCRIPTS=${PATH_TO_MISP}/app/files/scripts
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git -C ${PATH_TO_MISP_SCRIPTS}/python-cybox pull; done
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git -C ${PATH_TO_MISP_SCRIPTS}/python-stix pull; done
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git -C ${PATH_TO_MISP_SCRIPTS}/python-maec pull; done
false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git -C ${PATH_TO_MISP_SCRIPTS}/mixbox pull; done
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U setuptools pip lief zmq redis python-magic plyara
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP}/app/files/scripts/mixbox
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP}/app/files/scripts/python-cybox
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP}/app/files/scripts/python-maec
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP}/app/files/scripts/python-stix
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP}/cti-python-stix2
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP}/PyMISP
false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U git+https://github.com/kbandla/pydeep.git; done
fi
}
# <snippet-end 1_mispCoreInstall.sh>
```
@ -261,59 +271,39 @@ permissions () {
```bash
# <snippet-begin 1_prepareDB.sh>
prepareDB () {
if [[ ! -e /var/lib/mysql/misp/users.ibd ]]; then
if sudo test ! -e "/var/lib/mysql/mysql/"; then
#Make sure initial tables are created in MySQL
debug "Install mysql tables"
sudo mysql_install_db --user=mysql --basedir=/usr --datadir=/var/lib/mysql
sudo service mysql start
fi
if sudo test ! -e "/var/lib/mysql/misp/"; then
debug "Start mysql"
sudo service mysql start
debug "Setting up database"
# Kill the anonymous users
sudo mysql -e "DROP USER IF EXISTS ''@'localhost'"
# Because our hostname varies we'll use some Bash magic here.
sudo mysql -e "DROP USER IF EXISTS ''@'$(hostname)'"
# Kill off the demo database
sudo mysql -e "DROP DATABASE IF EXISTS test"
# No root remote logins
sudo mysql -e "DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1')"
# Make sure that NOBODY can access the server without a password
sudo mysqladmin -u "${DBUSER_ADMIN}" password "${DBPASSWORD_ADMIN}"
# Make our changes take effect
sudo mysql -e "FLUSH PRIVILEGES"
# FIXME: If user 'misp' exists, and has a different password, the below WILL fail. Partially fixed with the Env-Var check in the beginning. (Need to implement pre-flight checks to exit gracefully if not set)
# Add your credentials if needed, if sudo has NOPASS, comment out the relevant lines
if [[ "${PACKER}" == "1" ]]; then
pw="Password1234"
else
pw=${MISP_PASSWORD}
fi
if [[ ! -z ${INSTALL_USER} ]]; then
SUDO_EXPECT="sudo mysql_secure_installation"
echo "Making sure sudo session is buffered"
sudo ls -la /tmp > /dev/null 2> /dev/null
else
SUDO_EXPECT="sudo -k mysql_secure_installation"
fi
expect -f - <<-EOF
set timeout 10
spawn ${SUDO_EXPECT}
expect "*?assword*"
send -- "${pw}\r"
expect "Enter current password for root (enter for none):"
send -- "\r"
expect "Set root password?"
send -- "y\r"
expect "New password:"
send -- "${DBPASSWORD_ADMIN}\r"
expect "Re-enter new password:"
send -- "${DBPASSWORD_ADMIN}\r"
expect "Remove anonymous users?"
send -- "y\r"
expect "Disallow root login remotely?"
send -- "y\r"
expect "Remove test database and access to it?"
send -- "y\r"
expect "Reload privilege tables now?"
send -- "y\r"
expect eof
EOF
sudo apt-get purge -y expect ; sudo apt autoremove -qy
fi
sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "CREATE DATABASE ${DBNAME};"
sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "CREATE USER '${DBUSER_MISP}'@'localhost' IDENTIFIED BY '${DBPASSWORD_MISP}';"
sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "GRANT USAGE ON *.* to ${DBUSER_MISP}@localhost;"
sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "GRANT ALL PRIVILEGES on ${DBNAME}.* to '${DBUSER_MISP}'@'localhost';"
sudo mysql -u ${DBUSER_ADMIN} -p${DBPASSWORD_ADMIN} -e "FLUSH PRIVILEGES;"
# Import the empty MISP database from MYSQL.sql
${SUDO_WWW} cat ${PATH_TO_MISP}/INSTALL/MYSQL.sql | mysql -u ${DBUSER_MISP} -p${DBPASSWORD_MISP} ${DBNAME}
sudo mysql -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "CREATE DATABASE ${DBNAME};"
sudo mysql -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "CREATE USER '${DBUSER_MISP}'@'localhost' IDENTIFIED BY '${DBPASSWORD_MISP}';"
sudo mysql -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "GRANT USAGE ON *.* to '${DBUSER_MISP}'@'localhost';"
sudo mysql -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "GRANT ALL PRIVILEGES on ${DBNAME}.* to '${DBUSER_MISP}'@'localhost';"
sudo mysql -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "FLUSH PRIVILEGES;"
# Import the empty MISP database from MYSQL.sql
${SUDO_WWW} cat ${PATH_TO_MISP}/INSTALL/MYSQL.sql | mysql -u "${DBUSER_MISP}" -p"${DBPASSWORD_MISP}" ${DBNAME}
fi
}
# <snippet-end 1_prepareDB.sh>
```

18
docs/generic/misp-modules-debian.md
View File

@ -7,8 +7,14 @@ mispmodules () {
cd /usr/local/src/
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
## TODO: checkUsrLocalSrc in main doc
debug "Cloning misp-modules"
false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone https://github.com/MISP/misp-modules.git; done
if [[ ! -d /usr/local/src/misp-modules ]]; then
debug "Cloning misp-modules"
false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone https://github.com/MISP/misp-modules.git; done
else
false; while [[ $? -ne 0 ]]; do $SUDO_CMD git -C /usr/local/src/misp-modules pull; done
fi
# Install faup/gtcaca
[[ ! -d "faup" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone git://github.com/stricaud/faup.git faup; done
[[ ! -d "gtcaca" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca; done
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
@ -18,14 +24,15 @@ mispmodules () {
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
cd /usr/loca/src/faup
# Install faup
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
cd ../../misp-modules
cd /usr/local/src/misp-modules
# some misp-modules dependencies
sudo apt install libpq5 libjpeg-dev tesseract-ocr libpoppler-cpp-dev imagemagick libopencv-dev zbar-tools libzbar0 libzbar-dev libfuzzy-dev -y
# If you build an egg, the user you build it as need write permissions in the CWD
@ -34,10 +41,9 @@ mispmodules () {
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I -r REQUIREMENTS
sudo chgrp staff .
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I .
## sudo gem install asciidoctor-pdf --pre
# Start misp-modules as a service
sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/
sudo cp /usr/local/src/misp-modules/etc/systemd/system/misp-modules.service /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable --now misp-modules

13
docs/generic/supportFunctions.md
View File

@ -43,6 +43,19 @@ usage () {
space
echo -e "Recommended is either a barebone MISP install (ideal for syncing from other instances) or"
echo -e "MISP + modules - ${SCRIPT_NAME} -c -M"
echo -e ""
echo -e ""
echo -e "Interesting environment variables that get considered are:"
echo -e ""
echo -e "MISP_USER/MISP_PASSWORD # Local username on machine, default: misp/opensslGeneratedPassword"
echo -e ""
echo -e "PATH_TO_MISP # Where MISP will be installed, default: /var/www/MISP (recommended)"
echo -e ""
echo -e "DBHOST/DBNAME # database hostname, MISP database name, default: localhost/misp"
echo -e "DBUSER_ADMIN/DBPASSWORD_ADMIN # MySQL admin user, default: root/opensslGeneratedPassword"
echo -e "DBUSER_MISP/DBPASSWORD_MISP # MISP database user, default: misp/opensslGeneratedPassword"
echo -e ""
echo -e "You need to export the variable(s) to be taken into account. (or specified in-line when invoking INSTALL.sh)"
space
}

44
tests/modify_config.php Normal file
View File

@ -0,0 +1,44 @@
<?php
function fail($code, $message) {
fwrite(STDERR, "$message\n");
exit($code);
}
if (!isset($argv[2])) {
fail(1, "Required arguments not provided.");
}
if (!in_array($argv[1], ['modify', 'replace'], true)) {
fail(1, "Invalid argument '{$argv[1]}', it must be 'modify' or 'replace'.");
}
$newConfig = json_decode($argv[2], true);
if ($newConfig === null) {
fail(2, "Could not decode new config, it is not JSON: " . json_last_error_msg());
}
if (!is_array($newConfig)) {
fail(2, "Provided new config is not array, `" . gettype($newConfig) . "` given.");
}
$configFile = realpath(__DIR__ . '/../app/Config/config.php');
if ($configFile === false) {
fail(3, "File $configFile not found.");
}
if (!is_readable($configFile)) {
fail(3, "File $configFile is not readable.");
}
if (!is_writable($configFile)) {
$owner = posix_getpwuid(fileowner($configFile))["name"] . ':' . posix_getgrgid(filegroup($configFile))["name"];
$perms = substr(sprintf('%o', fileperms($configFile)), -4);
fail(3, "File $configFile is not writeable (owner $owner, permissions $perms).");
}
require_once $configFile;
if (!isset($config)) {
fail(3, "Original config variable not found.");
}
if ($argv[1] === 'modify') {
$merged = array_merge_recursive($config, $newConfig);
} else {
$merged = $newConfig;
}
file_put_contents($configFile, "<?php\n\$config = " . var_export($merged, true) . ';');
// Returns config file before modification
echo json_encode($config);

878
tests/testlive_security.py Normal file
View File

@ -0,0 +1,878 @@
#!/usr/bin/env python3
import os
import sys
import json
import unittest
from typing import Union
import urllib3 # type: ignore
import logging
import uuid
import warnings
import requests
import subprocess
from lxml.html import fromstring
from enum import Enum
try:
from pymisp import PyMISP, MISPOrganisation, MISPUser, MISPRole
from pymisp.exceptions import PyMISPError, NoKey, MISPServerError
except ImportError:
if sys.version_info < (3, 6):
print('This test suite requires Python 3.6+, breaking.')
sys.exit(0)
else:
raise
# Load access information for env variables
url = "http://" + os.environ["HOST"]
key = os.environ["AUTH"]
# TODO?
urllib3.disable_warnings()
logging.disable(logging.CRITICAL)
logger = logging.getLogger('pymisp')
class ROLE(Enum):
USER = 3
ADMIN = 1
ORG_ADMIN = 2
def check_response(response):
if isinstance(response, dict) and "errors" in response:
raise Exception(response["errors"])
def assert_error_response(response):
if "errors" not in response:
raise Exception(response)
def login(url: str, email: str, password: str) -> bool:
session = requests.Session()
r = session.get(url)
r.raise_for_status()
parsed = fromstring(r.text)
form = parsed.forms[0]
form_fields = form.fields
login_form = {}
for name in form_fields:
login_form[name] = form_fields[name]
login_form["data[User][email]"] = email
login_form["data[User][password]"] = password
r = session.post(url + form.action, login_form, allow_redirects=False)
r.raise_for_status()
if r.status_code == 302:
r = session.get(r.headers['Location'].replace(":8080", ""), allow_redirects=False) # TODO
r.raise_for_status()
r = session.get(url + "/users/view/me.json")
try:
r.raise_for_status()
except requests.HTTPError:
return False
r = r.json()
if email != r["User"]["email"]:
raise Exception(r) # logged in as different user
return True
class MISPSetting:
def __init__(self, connection: PyMISP, setting: str, value):
self.__connection = connection
self.__setting = setting
self.__value = value
def __enter__(self):
original = self.__connection.get_server_setting(self.__setting)
if "value" not in original:
raise Exception(original)
self.__original = original["value"]
result = self.__connection.set_server_setting(self.__setting, self.__value, force=True)
if "saved" not in result or not result["saved"]:
raise Exception(result)
def __exit__(self, exc_type, exc_val, exc_tb):
result = self.__connection.set_server_setting(self.__setting, self.__original, force=True)
if "saved" not in result or not result["saved"]:
raise Exception(result)
class MISPComplexSetting:
def __init__(self, new_setting: dict):
self.new_setting = new_setting
def __enter__(self):
self.original = self.__run("modify", json.dumps(self.new_setting))
def __exit__(self, exc_type, exc_val, exc_tb):
self.__run("replace", self.original)
@staticmethod
def __run(command: str, data: str) -> str:
dir_path = os.path.dirname(os.path.realpath(__file__))
r = subprocess.run(["php", dir_path + "/modify_config.php", command, data], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if r.returncode != 0:
raise Exception([r.returncode, r.stdout, r.stderr])
return r.stdout.decode("utf-8")
def send(api: PyMISP, request_type: str, url: str, data=None, check_errors: bool = True) -> dict:
if data is None:
data = {}
response = api._prepare_request(request_type, url, data=data)
response = api._check_json_response(response)
if check_errors:
check_response(response)
return response
def random() -> str:
return str(uuid.uuid4()).split("-")[0]
class TestSecurity(unittest.TestCase):
@classmethod
def setUpClass(cls):
warnings.simplefilter("ignore", ResourceWarning)
# Connect as admin
cls.admin_misp_connector = PyMISP(url, key)
cls.admin_misp_connector.set_server_setting('debug', 1, force=True)
cls.admin_misp_connector.global_pythonify = True
# Check if admin is really site admin
assert cls.admin_misp_connector._current_role.perm_site_admin
# Create advanced authkey, so connector will work even after advanced keys are required
cls.admin_advanced_authkey = cls.__create_advanced_authkey(cls, cls.admin_misp_connector._current_user.id)
cls.admin_misp_connector.key = cls.admin_misp_connector.key + "," + cls.admin_advanced_authkey["authkey_raw"]
# Creates an org
organisation = MISPOrganisation()
organisation.name = 'Test Org ' + random() # make name always unique
cls.test_org = cls.admin_misp_connector.add_organisation(organisation)
check_response(cls.test_org)
# Creates org admin
org_admin = MISPUser()
org_admin.email = 'testorgadmin@user' + random() + '.local' # make name always unique
org_admin.org_id = cls.test_org.id
org_admin.role_id = 2 # Org admin role
cls.test_org_admin = cls.admin_misp_connector.add_user(org_admin)
check_response(cls.test_org_admin)
# Creates advanced auth key for org admin
cls.org_admin_advanced_authkey = cls.__create_advanced_authkey(cls, cls.test_org_admin.id)
cls.org_admin_misp_connector = PyMISP(url, cls.test_org_admin.authkey + "," + cls.org_admin_advanced_authkey["authkey_raw"])
cls.org_admin_misp_connector.global_pythonify = True
# Creates an user
cls.test_usr_password = str(uuid.uuid4())
user = MISPUser()
user.email = 'testusr@user' + random() + '.local' # make name always unique
user.org_id = cls.test_org.id
user.role_id = 3 # User role
user.password = cls.test_usr_password
cls.test_usr = cls.admin_misp_connector.add_user(user)
check_response(cls.test_usr)
# Try to connect as user to check if everything works
PyMISP(url, cls.test_usr.authkey)
# Check if user can login with given password
assert login(url, cls.test_usr.email, cls.test_usr_password)
@classmethod
def tearDownClass(cls):
cls.admin_misp_connector.delete_user(cls.test_usr)
cls.admin_misp_connector.delete_user(cls.test_org_admin)
cls.admin_misp_connector.delete_organisation(cls.test_org)
cls.__delete_advanced_authkey(cls, cls.admin_advanced_authkey["id"])
cls.__delete_advanced_authkey(cls, cls.org_admin_advanced_authkey["id"])
def setUp(self):
# Do not show warning about not closed resources, because that something we want
warnings.simplefilter("ignore", ResourceWarning)
def test_not_logged_in(self):
session = requests.Session()
# Should redirect to login page
for path in ("/", "/events/index", "/servers/index", "/users/checkIfLoggedIn"):
r = session.get(url + path, allow_redirects=False)
self.assertEqual(302, r.status_code, path)
self.assertEqual(url + "/users/login", r.headers['Location'], path)
# Should be accessible without login
for path in ("/users/login", ):
r = session.get(url + path, allow_redirects=False)
self.assertEqual(200, r.status_code, path)
with MISPSetting(self.admin_misp_connector, "Security.allow_self_registration", True):
r = session.get(url + "/users/register", allow_redirects=False)
self.assertEqual(200, r.status_code, path)
with MISPSetting(self.admin_misp_connector, "Security.allow_self_registration", False):
r = session.get(url + "/users/register", allow_redirects=False)
self.assertEqual(302, r.status_code)
self.assertEqual(url + "/users/login", r.headers['Location'])
def test_empty_authkey(self):
with self.assertRaises(NoKey):
PyMISP(url, "")
def test_invalid_length_authkey(self):
with self.assertRaises(PyMISPError):
PyMISP(url, "ahoj")
def test_invalid_authkey(self):
with self.assertRaises(PyMISPError):
PyMISP(url, "pCZDbBr3wYPlY0DrlQzoD8EWrcClGc0Dqu2yMYyE")
def test_invalid_authkey_start_end_correct(self):
authkey = self.test_usr.authkey[0:4] + ("a" * 32) + self.test_usr.authkey[:-4]
with self.assertRaises(PyMISPError):
PyMISP(url, authkey)
def test_no_auth_access(self):
no_access_role = MISPRole()
no_access_role.name = "No auth access"
no_access_role = send(self.admin_misp_connector, "POST", 'admin/roles/add', data=no_access_role)
self.assertFalse(no_access_role["Role"]["perm_auth"])
no_access_role_id = no_access_role["Role"]["id"]
# Change user role to no access role
updated_user = self.admin_misp_connector.update_user({'role_id': no_access_role_id}, self.test_usr)
check_response(updated_user)
self.assertEqual(no_access_role_id, updated_user.role_id)
with self.assertRaises(PyMISPError):
PyMISP(url, self.test_usr.authkey)
# Change user role back to origin one and try to connect
updated_user = self.admin_misp_connector.update_user({'role_id': self.test_usr.role_id}, self.test_usr)
check_response(updated_user)
self.assertEqual(self.test_usr.role_id, updated_user.role_id)
PyMISP(url, self.test_usr.authkey)
# Delete test role
self.admin_misp_connector._prepare_request('POST', f'admin/roles/delete/{no_access_role_id}')
def test_assign_role_by_myself(self):
logged_in = PyMISP(url, self.test_usr.authkey)
logged_in.global_pythonify = True
update_user = logged_in.update_user({'role_id': 1}, self.test_usr)
# Check if role was not changed
self.assertEqual(self.test_usr.role_id, update_user.role_id)
def test_assign_site_admin_role_by_org_admin(self):
with self.assertRaises(MISPServerError):
self.org_admin_misp_connector.update_user({'role_id': 1}, self.test_usr)
def test_user_must_change_password(self):
updated_user = self.admin_misp_connector.update_user({'change_pw': 1}, self.test_usr)
check_response(updated_user)
self.assertEqual(updated_user.change_pw, "1")
# Try to login, should still work because key is still valid
PyMISP(url, self.test_usr.authkey)
updated_user = self.admin_misp_connector.update_user({'change_pw': 0}, self.test_usr)
check_response(updated_user)
self.assertEqual(updated_user.change_pw, "0")
# Try to login, should also still works
PyMISP(url, self.test_usr.authkey)
def test_user_must_change_password_by_myself(self):
# Admin set that user must change password
updated_user = self.admin_misp_connector.update_user({'change_pw': 1}, self.test_usr)
check_response(updated_user)
self.assertEqual(updated_user.change_pw, "1")
# User try to change back trough API
logged_in = PyMISP(url, self.test_usr.authkey)
logged_in.update_user({'change_pw': 0}, self.test_usr)
updated_user = self.admin_misp_connector.get_user(self.test_usr)
# Should not be possible
self.assertEqual(updated_user.change_pw, "1")
def test_disabled_user(self):
# Disable user
updated_user = self.admin_misp_connector.update_user({'disabled': True}, self.test_usr)
check_response(updated_user)
self.assertTrue(updated_user.disabled)
# Try to login
self.assertFalse(login(url, self.test_usr.email, self.test_usr_password))
# Enable user
updated_user = self.admin_misp_connector.update_user({'disabled': False}, self.test_usr)
check_response(updated_user)
self.assertFalse(updated_user.disabled)
# Try to login
self.assertTrue(login(url, self.test_usr.email, self.test_usr_password))
def test_disabled_user_api_access(self):
# Disable user
updated_user = self.admin_misp_connector.update_user({'disabled': True}, self.test_usr)
check_response(updated_user)
self.assertTrue(updated_user.disabled)
# Try to login
with self.assertRaises(PyMISPError):
PyMISP(url, self.test_usr.authkey)
# Enable user
updated_user = self.admin_misp_connector.update_user({'disabled': False}, self.test_usr)
check_response(updated_user)
self.assertFalse(updated_user.disabled)
# Try to login
PyMISP(url, self.test_usr.authkey)
def test_disabled_misp(self):
with MISPSetting(self.admin_misp_connector, "MISP.live", False):
self.assertFalse(login(url, self.test_usr.email, self.test_usr_password))
# Check if user can login with given password
self.assertTrue(login(url, self.test_usr.email, self.test_usr_password))
def test_disabled_misp_api_access(self):
with MISPSetting(self.admin_misp_connector, "MISP.live", False):
# Try to login
with self.assertRaises(PyMISPError):
PyMISP(url, self.test_usr.authkey)
# Try to login
PyMISP(url, self.test_usr.authkey)
def test_advanced_authkeys(self):
with MISPSetting(self.admin_misp_connector, "Security.advanced_authkeys", True):
# Create advanced authkey
auth_key = self.__create_advanced_authkey(self.test_usr.id)
# Try to login
logged_in = PyMISP(url, auth_key["authkey_raw"])
self.assertEqual(logged_in._current_user.id, self.test_usr.id)
self.__delete_advanced_authkey(auth_key["id"])
def test_advanced_authkeys_expired(self):
with MISPSetting(self.admin_misp_connector, "Security.advanced_authkeys", True):
# Create expired advanced authkey
auth_key = self.__create_advanced_authkey(self.test_usr.id, {
"expiration": "1990-01-05",
})
# Try to login
with self.assertRaises(PyMISPError):
PyMISP(url, auth_key["authkey_raw"])
self.__delete_advanced_authkey(auth_key["id"])
def test_advanced_authkeys_invalid_start_end_correct(self):
with MISPSetting(self.admin_misp_connector, "Security.advanced_authkeys", True):
# Create advanced authkey
auth_key = self.__create_advanced_authkey(self.test_usr.id)
# Try to login
authkey = auth_key["authkey_raw"][0:4] + ("a" * 32) + auth_key["authkey_raw"][:-4]
with self.assertRaises(PyMISPError):
PyMISP(url, authkey)
self.__delete_advanced_authkey(auth_key["id"])
def test_advanced_authkeys_reset_own(self):
with MISPSetting(self.admin_misp_connector, "Security.advanced_authkeys", True):
# Create advanced authkey
auth_key = self.__create_advanced_authkey(self.test_usr.id)
# Try to login
logged_in = PyMISP(url, auth_key["authkey_raw"])
self.assertEqual(logged_in._current_user.id, self.test_usr.id)
# Reset auth key
new_auth_key = send(logged_in, "POST", "users/resetauthkey/me")
new_auth_key = new_auth_key["message"].replace("Authkey updated: ", "")
# Try to login with old key
with self.assertRaises(PyMISPError):
PyMISP(url, auth_key["authkey_raw"])
# Try to login with new key
logged_in = PyMISP(url, new_auth_key)
self.assertEqual(logged_in._current_user.id, self.test_usr.id)
self.__delete_advanced_authkey(auth_key["id"])
# TODO: Delete new key
def test_advanced_authkeys_reset_for_different_user(self):
with MISPSetting(self.admin_misp_connector, "Security.advanced_authkeys", True):
# Create advanced authkey
auth_key = self.__create_advanced_authkey(self.test_usr.id)
# Try to login
logged_in = PyMISP(url, auth_key["authkey_raw"])
self.assertEqual(logged_in._current_user.id, self.test_usr.id)
# Reset auth key for different user
new_auth_key = send(logged_in, "POST", "users/resetauthkey/1", check_errors=False)
assert_error_response(new_auth_key)
# Try to login again
logged_in = PyMISP(url, auth_key["authkey_raw"])
self.assertEqual(logged_in._current_user.id, self.test_usr.id)
self.__delete_advanced_authkey(auth_key["id"])
def test_advanced_authkeys_reset_org_admin(self):
with MISPSetting(self.admin_misp_connector, "Security.advanced_authkeys", True):
# Create advanced authkey
auth_key = self.__create_advanced_authkey(self.test_usr.id)
# Try to login
logged_in = PyMISP(url, auth_key["authkey_raw"])
self.assertEqual(logged_in._current_user.id, self.test_usr.id)
# Reset auth key from org admin account
new_auth_key = send(self.org_admin_misp_connector, "POST", f"users/resetauthkey/{self.test_usr.id}")
new_auth_key = new_auth_key["message"].replace("Authkey updated: ", "")
# Try to login with old key
with self.assertRaises(PyMISPError):
PyMISP(url, auth_key["authkey_raw"])
# Try to login with new key
logged_in = PyMISP(url, new_auth_key)
self.assertEqual(logged_in._current_user.id, self.test_usr.id)
self.__delete_advanced_authkey(auth_key["id"])
# TODO: Delete new key
def test_advanced_authkeys_view(self):
with MISPSetting(self.admin_misp_connector, "Security.advanced_authkeys", True):
auth_key = self.__create_advanced_authkey(self.test_usr.id)
auth_key_id = auth_key["id"]
auth_key = send(self.admin_misp_connector, "GET", f'authKeys/view/{auth_key_id}')
self.__delete_advanced_authkey(auth_key_id)
self.assertNotIn("authkey", auth_key["AuthKey"], "Response should not contain hashed authkey")
def test_advanced_authkeys_index(self):
with MISPSetting(self.admin_misp_connector, "Security.advanced_authkeys", True):
auth_key_id = self.__create_advanced_authkey(self.test_usr.id)["id"]
auth_keys = send(self.admin_misp_connector, "GET", 'authKeys/index/')
self.__delete_advanced_authkey(auth_key_id)
self.assertGreaterEqual(len(auth_keys), 1, "Response should contains at least one key")
for auth_key in auth_keys:
self.assertNotIn("authkey", auth_key["AuthKey"], "Response should not contain hashed authkey")
def test_change_login(self):
new_email = 'testusr@user' + random() + '.local'
logged_in = PyMISP(url, self.test_usr.authkey)
logged_in.global_pythonify = True
# Change email
updated_user = logged_in.update_user({'email': new_email}, self.test_usr)
check_response(updated_user)
self.assertEqual(new_email, updated_user.email)
# Change email back
updated_user = logged_in.update_user({'email': self.test_usr.email}, self.test_usr)
check_response(updated_user)
self.assertEqual(self.test_usr.email, updated_user.email)
def test_change_login_disabled(self):
with MISPSetting(self.admin_misp_connector, "MISP.disable_user_login_change", True):
new_email = 'testusr@user' + random() + '.local'
logged_in = PyMISP(url, self.test_usr.authkey)
logged_in.global_pythonify = True
# Try to change email
updated_user = logged_in.update_user({'email': new_email}, self.test_usr)
check_response(updated_user)
# Change should be not successful
self.assertEqual(self.test_usr.email, updated_user.email)
def test_change_login_org_admin(self):
# Try to change email as org admin
new_email = 'testusr@user' + random() + '.local'
updated_user = self.org_admin_misp_connector.update_user({'email': new_email}, self.test_usr)
check_response(updated_user)
# Change should be successful
self.assertEqual(new_email, updated_user.email)
# Change email back
updated_user = self.org_admin_misp_connector.update_user({'email': self.test_usr.email}, self.test_usr)
check_response(updated_user)
self.assertEqual(self.test_usr.email, updated_user.email)
def test_change_login_disabled_org_admin(self):
with MISPSetting(self.admin_misp_connector, "MISP.disable_user_login_change", True):
# Try to change email as org admin
new_email = 'testusr@user' + random() + '.local'
updated_user = self.org_admin_misp_connector.update_user({'email': new_email}, self.test_usr)
assert_error_response(updated_user)
def test_change_pw_disabled(self):
with MISPSetting(self.admin_misp_connector, "MISP.disable_user_password_change", True):
logged_in = PyMISP(url, self.test_usr.authkey)
logged_in.global_pythonify = True
logged_in.change_user_password(str(uuid.uuid4()))
# Password should be still the same
self.assertTrue(login(url, self.test_usr.email, self.test_usr_password))
def test_change_pw_disabled_different_way(self):
with MISPSetting(self.admin_misp_connector, "MISP.disable_user_password_change", True):
logged_in = PyMISP(url, self.test_usr.authkey)
logged_in.global_pythonify = True
logged_in.update_user({"password": str(uuid.uuid4())}, self.test_usr.id)
# Password should be still the same
self.assertTrue(login(url, self.test_usr.email, self.test_usr_password))
def test_change_pw_disabled_by_org_admin(self):
with MISPSetting(self.admin_misp_connector, "MISP.disable_user_password_change", True):
self.org_admin_misp_connector.update_user({"password": str(uuid.uuid4())}, self.test_usr.id)
# Password should be still the same
self.assertTrue(login(url, self.test_usr.email, self.test_usr_password))
def test_add_user_by_org_admin(self):
user = MISPUser()
user.email = 'testusr@user' + random() + '.local' # make name always unique
user.org_id = self.test_org.id
user.role_id = 3
created_user = self.org_admin_misp_connector.add_user(user)
check_response(created_user)
deleted = self.org_admin_misp_connector.delete_user(created_user)
check_response(deleted)
def test_add_user_by_org_admin_to_different_org(self):
user = MISPUser()
user.email = 'testusr@user' + random() + '.local' # make name always unique
user.org_id = 1
user.role_id = 3
created_user = self.org_admin_misp_connector.add_user(user)
check_response(created_user)
# Org should be silently changed to correct org
self.assertEqual(created_user.org_id, self.test_org_admin.org_id)
deleted = self.org_admin_misp_connector.delete_user(created_user)
check_response(deleted)
def test_add_user_by_org_admin_disabled(self):
with MISPSetting(self.admin_misp_connector, "MISP.disable_user_add", True):
user = MISPUser()
user.email = 'testusr@user' + random() + '.local' # make name always unique
user.org_id = self.test_org.id
user.role_id = 3
created_user = self.org_admin_misp_connector.add_user(user)
assert_error_response(created_user)
def test_change_user_org_by_org_admin_different_org(self):
updated_user = self.org_admin_misp_connector.update_user({'org_id': 1}, self.test_usr)
check_response(updated_user)
# Org should be silently keep to correct org
self.assertEqual(updated_user.org_id, self.test_usr.org_id)
def test_change_user_org_by_myself(self):
logged_in = PyMISP(url, self.test_usr.authkey)
logged_in.global_pythonify = True
updated_user = logged_in.update_user({'org_id': 1}, self.test_usr)
# Org should be silently keep to correct org
self.assertEqual(updated_user.org_id, self.test_usr.org_id)
def test_shibb_existing_user(self):
with MISPComplexSetting(self.__default_shibb_config()):
session = requests.Session()
session.headers["Email-Tag"] = self.test_usr.email
session.headers["Federation-Tag"] = self.test_org.name
session.headers["Group-Tag"] = "user"
session.get(url, allow_redirects=False)
r = session.get(url + "/users/view/me.json")
r.raise_for_status()
json_response = r.json()
self.assertEqual(self.test_usr.email, json_response["User"]["email"])
self.assertEqual(3, int(json_response["User"]["role_id"]))
self.assertEqual(session.headers["Federation-Tag"], json_response["Organisation"]["name"])
def test_shibb_new_user(self):
with MISPComplexSetting(self.__default_shibb_config()):
session = requests.Session()
session.headers["Email-Tag"] = "external@user" + random() + ".local"
session.headers["Federation-Tag"] = self.test_org.name
session.headers["Group-Tag"] = "user"
session.get(url, allow_redirects=False)
r = session.get(url + "/users/view/me.json")
r.raise_for_status()
json_response = r.json()
self.assertEqual(session.headers["Email-Tag"], json_response["User"]["email"])
self.assertEqual(3, int(json_response["User"]["role_id"]))
self.assertEqual(session.headers["Federation-Tag"], json_response["Organisation"]["name"])
self.admin_misp_connector.delete_user(json_response["User"]["id"])
def test_shibb_new_user_multiple_groups(self):
with MISPComplexSetting(self.__default_shibb_config()):
session = requests.Session()
session.headers["Email-Tag"] = "external@user" + random() + ".local"
session.headers["Federation-Tag"] = self.test_org.name
session.headers["Group-Tag"] = "user,invalid,admin"
session.get(url, allow_redirects=False)
r = session.get(url + "/users/view/me.json")
r.raise_for_status()
json_response = r.json()
self.assertEqual(session.headers["Email-Tag"], json_response["User"]["email"])
self.assertEqual(1, int(json_response["User"]["role_id"]))
self.assertEqual(session.headers["Federation-Tag"], json_response["Organisation"]["name"])
self.admin_misp_connector.delete_user(json_response["User"]["id"])
def test_shibb_new_user_non_exists_org(self):
with MISPComplexSetting(self.__default_shibb_config()):
session = requests.Session()
session.headers["Email-Tag"] = "external@user" + random() + ".local"
session.headers["Federation-Tag"] = "Non exists org " + random()
session.headers["Group-Tag"] = "user"
session.get(url, allow_redirects=False)
r = session.get(url + "/users/view/me.json")
r.raise_for_status()
json_response = r.json()
self.assertEqual(session.headers["Email-Tag"], json_response["User"]["email"])
self.assertEqual(3, int(json_response["User"]["role_id"]))
self.assertEqual(session.headers["Federation-Tag"], json_response["Organisation"]["name"])
self.admin_misp_connector.delete_user(json_response["User"]["id"])
self.admin_misp_connector.delete_organisation(json_response["User"]["org_id"])
def test_shibb_new_user_org_uuid(self):
with MISPComplexSetting(self.__default_shibb_config()):
r = self.__shibb_login({
"Email-Tag": "external@user" + random() + ".local",
"Federation-Tag": self.test_org.uuid,
"Group-Tag": "user",
})
r.raise_for_status()
json_response = r.json()
self.assertEqual(r.request.headers["Email-Tag"], json_response["User"]["email"])
self.assertEqual(3, int(json_response["User"]["role_id"]))
self.assertEqual(self.test_org.name, json_response["Organisation"]["name"])
self.admin_misp_connector.delete_user(json_response["User"]["id"])
self.admin_misp_connector.delete_organisation(json_response["User"]["org_id"])
def test_shibb_new_user_non_exists_org_uuid(self):
with MISPComplexSetting(self.__default_shibb_config()):
r = self.__shibb_login({
"Email-Tag": "external@user" + random() + ".local",
"Federation-Tag": str(uuid.uuid4()),
"Group-Tag": "user",
})
if r.status_code != 403:
print(r.text)
self.fail()
def test_shibb_new_user_no_org_provided(self):
with MISPComplexSetting(self.__default_shibb_config()):
session = requests.Session()
session.headers["Email-Tag"] = "external@user" + random() + ".local"
session.headers["Group-Tag"] = "user"
session.get(url, allow_redirects=False)
r = session.get(url + "/users/view/me.json")
r.raise_for_status()
json_response = r.json()
self.assertEqual(3, int(json_response["User"]["role_id"]))
# Default org is used
self.assertEqual(self.test_org.name, json_response["Organisation"]["name"])
self.admin_misp_connector.delete_user(json_response["User"]["id"])
def test_shibb_invalid_group(self):
with MISPComplexSetting(self.__default_shibb_config()):
session = requests.Session()
session.headers["Email-Tag"] = "external@user" + random() + ".local"
session.headers["Federation-Tag"] = self.test_org.name
session.headers["Group-Tag"] = "invalid"
session.get(url, allow_redirects=False)
r = session.get(url + "/users/view/me.json")
if r.status_code != 403:
print(r.text)
self.fail()
def test_shibb_invalid_email(self):
with MISPComplexSetting(self.__default_shibb_config()):
session = requests.Session()
session.headers["Email-Tag"] = "external.user" + random() + ".local"
session.headers["Federation-Tag"] = self.test_org.name
session.headers["Group-Tag"] = "user"
session.get(url, allow_redirects=False)
r = session.get(url + "/users/view/me.json")
if r.status_code != 403:
print(r.text)
self.fail()
def test_shibb_change_role(self):
org_admin = self.__create_user(self.test_org.id, ROLE.ORG_ADMIN)
with MISPComplexSetting(self.__default_shibb_config()):
session = requests.Session()
session.headers["Email-Tag"] = org_admin.email
session.headers["Federation-Tag"] = self.test_org.name
session.headers["Group-Tag"] = "user"
session.get(url, allow_redirects=False)
r = session.get(url + "/users/view/me.json")
r.raise_for_status()
json_response = r.json()
# Change role back to user
self.assertEqual(3, int(json_response["User"]["role_id"]))
self.admin_misp_connector.delete_user(org_admin)
def test_shibb_change_org(self):
user = self.__create_user(self.test_org.id, ROLE.USER)
with MISPComplexSetting(self.__default_shibb_config()):
session = requests.Session()
session.headers["Email-Tag"] = user.email
session.headers["Federation-Tag"] = "Non exists org " + random()
session.headers["Group-Tag"] = "user"
session.get(url, allow_redirects=False)
r = session.get(url + "/users/view/me.json")
r.raise_for_status()
json_response = r.json()
# Change role back to user
self.assertEqual(session.headers["Federation-Tag"], json_response["Organisation"]["name"])
self.admin_misp_connector.delete_user(user)
self.admin_misp_connector.delete_organisation(json_response["User"]["org_id"])
def test_shibb_form_login(self):
with MISPComplexSetting(self.__default_shibb_config()):
# Form login should still works when no header provided
self.assertTrue(login(url, self.test_usr.email, self.test_usr_password))
def test_shibb_api_login(self):
with MISPComplexSetting(self.__default_shibb_config()):
PyMISP(url, self.test_usr.authkey)
def test_shibb_enforced_existing_user(self):
config = self.__default_shibb_config()
config["Security"]["auth_enforced"] = True
with MISPComplexSetting(config):
r = self.__shibb_login({
"Email-Tag": self.test_usr.email,
"Federation-Tag": self.test_org.name,
"Group-Tag": "user",
})
r.raise_for_status()
json_response = r.json()
self.assertEqual(self.test_usr.email, json_response["User"]["email"])
self.assertEqual(3, int(json_response["User"]["role_id"]))
self.assertEqual(self.test_org.name, json_response["Organisation"]["name"])
def test_shibb_enforced_form_login(self):
config = self.__default_shibb_config()
config["Security"]["auth_enforced"] = True
with MISPComplexSetting(config):
# Form login should not work when shibb is enforced, because form doesn't exists
with self.assertRaises(IndexError):
login(url, self.test_usr.email, self.test_usr_password)
def test_shibb_enforced_api_login(self):
config = self.__default_shibb_config()
config["Security"]["auth_enforced"] = True
with MISPComplexSetting(config):
PyMISP(url, self.test_usr.authkey)
def __shibb_login(self, headers: dict) -> requests.Response:
session = requests.Session()
session.headers.update(headers)
r = session.get(url, allow_redirects=False)
if 500 <= r.status_code < 600:
raise Exception(r)
r = session.get(url + "/users/view/me.json")
if 500 <= r.status_code < 600:
raise Exception(r)
return r
def __create_user(self, org_id: int = None, role_id: Union[int, ROLE] = None) -> MISPUser:
if isinstance(role_id, ROLE):
role_id = role_id.value
user = MISPUser()
user.email = 'test@' + random() + '.local' # make name always unique
if org_id:
user.org_id = org_id
if role_id:
user.role_id = role_id
user = self.admin_misp_connector.add_user(user)
check_response(user)
if org_id:
self.assertEqual(int(org_id), int(user.org_id))
if role_id:
self.assertEqual(int(role_id), int(user.role_id))
return user
def __create_advanced_authkey(self, user_id: int, data=None):
return send(self.admin_misp_connector, "POST", f'authKeys/add/{user_id}', data=data)["AuthKey"]
def __delete_advanced_authkey(self, key_id: int):
return send(self.admin_misp_connector, "POST", f'authKeys/delete/{key_id}')
def __default_shibb_config(self) -> dict:
return {
"ApacheShibbAuth": {
"DefaultOrg": self.test_org.name,
"UseDefaultOrg": False,
"MailTag": "HTTP_EMAIL_TAG",
"OrgTag": "HTTP_FEDERATION_TAG",
"GroupTag": "HTTP_GROUP_TAG",
"GroupSeparator": ",",
"GroupRoleMatching": {
"admin": 1,
"user": 3,
}
},
"Security": {
"auth": ["ShibbAuth.ApacheShibb"],
}
}
if __name__ == '__main__':
unittest.main()