MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch '2.4' of github.com:MISP/MISP into pr-5210

pull/5615/head
mokaddem 2020-02-10 11:09:14 +01:00
parent fd4588154a 8839770242
commit 6e66256f7a
No known key found for this signature in database
GPG Key ID: 164C473F627A06FA
291 changed files with 50388 additions and 20123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.github/ISSUE_TEMPLATE.md → .github/ISSUE_TEMPLATE/bug_report.md vendored
View File

@ -1,6 +1,11 @@
# This template is meant for bug reports, if you have a feature request, please be as descriptive as possible and delete the template
---
name: Bug report
about: Create a report to help us improve
labels: bug, potential-bug, needs triage
---
# This template is meant for bug reports, if you have a feature request user the other template.
@ -15,9 +20,9 @@
| Questions | Answers
|---------------------------|--------------------
| Type of issue | Bug, Question, Feature Request, support...
| OS version (server) | Debian, ubuntu, CentOS, RedHat, ...
| OS version (client) | XP, Seven, 10, Ubuntu, ...
| Type of issue | Bug
| OS version (server) | Debian, Ubuntu, CentOS, RedHat, ...
| OS version (client) | MacOS, Win10, Ubuntu, ...
| PHP version | 5.4, 5.5, 5.6, 7.0, 7.1...
| MISP version / git hash | 2.4.XX, hash of the commit
| Browser | If applicable

19
.github/ISSUE_TEMPLATE/feature_request.md vendored Normal file
View File

@ -0,0 +1,19 @@
---
name: Feature request
about: Suggest an idea for this project
labels: feature request, needs triage
---
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
**Additional context**
Add any other context or screenshots about the feature request here.

34
.github/ISSUE_TEMPLATE/support_request.md vendored Normal file
View File

@ -0,0 +1,34 @@
---
name: Support
about: Support requests for MISP
labels: support, needs triage
---
## Please consider the following notes
- Critical security bugs can be reported, preferably PGP encrypted, by mail to: info@circl.lu
- Bug reports and feature requests can be filed as an issue on github: https://github.com/MISP/MISP/issues
- For interactive support please join the Gitter chat on: https://gitter.im/MISP/MISP
The official documentation of MISP can be found here: https://www.circl.lu/doc/misp/
We also periodically do user/admin/developer trainings and have our training material online: https://www.circl.lu/services/misp-training-materials/
Nevertheless you can of course file a Support request as an issue. Please be as precise as possible and fill the template as detailed as possible too.
Please remove this text until the line below. Thanks a lot.
--------8<------
### Work environment
| Questions | Answers
|---------------------------|--------------------
| Type of issue | Support
| OS version (server) | Debian, Ubuntu, CentOS, RedHat, ...
| OS version (client) | MacOS, Win10, Ubuntu, ...
| PHP version | 5.4, 5.5, 5.6, 7.0, 7.1... (if relevant)
| MISP version / git hash | 2.4.XX, hash of the commit
| Browser | If applicable
### Support Questions
### Logs, screenshots, configuration dump, ...

43
.travis.yml
View File

@ -1,9 +1,9 @@
language: php
php:
- 7.1
- 7.2
- 7.3
- 7.4
- nightly
services:
@ -11,7 +11,6 @@ services:
sudo: required
dist: bionic
group: edge
addons:
mariadb: '10.2'
@ -95,14 +94,13 @@ install:
# Get authkey
- sudo usermod -a -G www-data $USER
- sudo -E su $USER -c 'app/Console/cake userInit -q | sudo tee ./key.txt'
- sudo -E su $USER -c 'app/Console/cake Admin updateDatabase'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "/usr/bin/python3"'
- sudo -E su $USER -c 'app/Console/cake Admin runUpdates'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.autoRegenerate" 0'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.timeout" 600'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.cookieTimeout" 3600'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.host_org_id" 1'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.email" "info@admin.test"'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.disable_emailing" true'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.disable_emailing" false'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "debug" true'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1"'
@ -113,7 +111,7 @@ install:
- sudo -E su $USER -c 'app/Console/cake Admin updateTaxonomies'
- sudo -E su $USER -c 'app/Console/cake Admin updateWarningLists'
- sudo -E su $USER -c 'app/Console/cake Admin updateNoticeLists'
- sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates "1337"'
- sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates 1'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" true'
- sudo -E su $USER -c 'app/Console/cake Live 1'
- sudo chmod 777 ./key.txt
@ -123,27 +121,18 @@ install:
- app/Console/worker/start.sh &
- sleep 10
# Dirty install python stuff
- virtualenv -p python3.6 ./venv
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$TRAVIS_BUILD_DIR/venv/bin/python"'
- . ./venv/bin/activate
- pushd cti-python-stix2
- sudo pip3 install .
- popd
- pushd app/files/scripts
- sudo -H -u $USER git clone https://github.com/CybOXProject/python-cybox.git
- sudo -H -u $USER git clone https://github.com/STIXProject/python-stix.git
- sudo -H -u $USER git clone https://github.com/MAECProject/python-maec.git
- sudo -H -u $USER git clone https://github.com/CybOXProject/mixbox.git
- pushd mixbox
- sudo pip3 install .
- popd
- pushd python-cybox
- sudo pip3 install .
- popd
- pushd python-stix
- sudo pip3 install .
- popd
- pushd python-maec
- sudo pip3 install .
- pip install .
- popd
- pushd PyMISP
- pip install .[fileobjects]
- popd
- pip install stix zmq redis plyara
- pip install -I antlr4-python3-runtime==4.7.2 # antlr4-python3-runtime==4.8 causes issues
- deactivate
before_script:
- curl http://misp.local
@ -160,12 +149,10 @@ script:
- ./curl_tests.sh $AUTH
- popd
- pushd PyMISP
- git submodule init
- git submodule update
- pipenv install -d
- pushd tests
- git clone https://github.com/viper-framework/viper-test-files.git
- popd
- pipenv run python tests/testlive_comprehensive.py
- pipenv run python tests/test.py
- pipenv run python tests/test_mispevent.py
- popd
- cp PyMISP/tests/keys.py PyMISP/examples/events/

133
INSTALL/INSTALL.sh
View File

@ -148,9 +148,9 @@ MISPvars () {
# sudo config to run $LUSER commands
if [[ "$(groups ${MISP_USER} |grep -o 'staff')" == "staff" ]]; then
SUDO_USER="sudo -H -u ${MISP_USER} -g staff"
SUDO_CMD="sudo -H -u ${MISP_USER} -g staff"
else
SUDO_USER="sudo -H -u ${MISP_USER}"
SUDO_CMD="sudo -H -u ${MISP_USER}"
fi
SUDO_WWW="sudo -H -u ${WWW_USER} "
@ -185,7 +185,8 @@ usage () {
echo -e "${SCRIPT_NAME} -c | Install ONLY ${LBLUE}MISP${NC} Core" # core
echo -e " -M | ${LBLUE}MISP${NC} modules" # modules
echo -e " -D | ${LBLUE}MISP${NC} dashboard" # dashboard
echo -e " -V | Viper" # viper
## FIXME: The current stat of Viper is broken, disabling any use.
##echo -e " -V | Viper" # viper
echo -e " -m | Mail 2 ${LBLUE}MISP${NC}" # mail2
echo -e " -S | Experimental ssdeep correlations" # ssdeep
echo -e " -A | Install ${YELLOW}all${NC} of the above" # all
@ -196,7 +197,7 @@ usage () {
echo -e "${HIDDEN} -U | Attempt and upgrade of selected item${NC}" # UPGRADE
echo -e "${HIDDEN} -N | Nuke this MISP Instance${NC}" # NUKE
echo -e "${HIDDEN} -f | Force test install on current Ubuntu LTS schim, add -B for 18.04 -> 18.10, or -BB 18.10 -> 19.10)${NC}" # FORCE
echo -e "Options can be combined: ${SCRIPT_NAME} -c -V -D # Will install Core+Viper+Dashboard"
echo -e "Options can be combined: ${SCRIPT_NAME} -c -D # Will install Core+Dashboard"
space
echo -e "Recommended is either a barebone MISP install (ideal for syncing from other instances) or"
echo -e "MISP + modules - ${SCRIPT_NAME} -c -M"
@ -544,12 +545,12 @@ checkID () {
sudo adduser $MISP_USER $WWW_USER
fi
# FIXME: the below SUDO_USER check is a duplicate from global variables, try to have just one check
# FIXME: the below SUDO_CMD check is a duplicate from global variables, try to have just one check
# sudo config to run $LUSER commands
if [[ "$(groups ${MISP_USER} |grep -o 'staff')" == "staff" ]]; then
SUDO_USER="sudo -H -u ${MISP_USER} -g staff"
SUDO_CMD="sudo -H -u ${MISP_USER} -g staff"
else
SUDO_USER="sudo -H -u ${MISP_USER}"
SUDO_CMD="sudo -H -u ${MISP_USER}"
fi
}
@ -953,7 +954,8 @@ composer73 () {
# Update composer.phar
# If hash changes, check here: https://getcomposer.org/download/ and replace with the correct one
# Current Sum for: v1.8.3
SHA384_SUM='48e3236262b34d30969dca3c37281b3b4bbe3221bda826ac6a9a62d6444cdb0dcd0615698a5cbe587c3f0fe57a54d8f5'
SHA384_SUM="$(wget -q -O - https://composer.github.io/installer.sig)"
php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
$SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === '$SHA384_SUM') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); exit(137); } echo PHP_EOL;"
checkFail "composer.phar checksum failed, please investigate manually. " $?
@ -985,8 +987,8 @@ genRCLOCAL () {
# Run PyMISP tests
runTests () {
echo "url = ${MISP_BASEURL}
key = ${AUTH_KEY}" |sudo tee ${PATH_TO_MISP}/PyMISP/tests/keys.py
echo "url = '${MISP_BASEURL}'
key = '${AUTH_KEY}'" |sudo tee ${PATH_TO_MISP}/PyMISP/tests/keys.py
sudo chown -R $WWW_USER:$WWW_USER $PATH_TO_MISP/PyMISP/
sudo -H -u $WWW_USER sh -c "cd $PATH_TO_MISP/PyMISP && git submodule foreach git pull origin master"
@ -1008,11 +1010,12 @@ nuke () {
# Final function to let the user know what happened
theEnd () {
space
echo "Admin (root) DB Password: $DBPASSWORD_ADMIN" |$SUDO_USER tee /home/${MISP_USER}/mysql.txt
echo "User (misp) DB Password: $DBPASSWORD_MISP" |$SUDO_USER tee -a /home/${MISP_USER}/mysql.txt
echo "Authkey: $AUTH_KEY" |$SUDO_USER tee -a /home/${MISP_USER}/MISP-authkey.txt
echo "Admin (root) DB Password: $DBPASSWORD_ADMIN" |$SUDO_CMD tee /home/${MISP_USER}/mysql.txt
echo "User (misp) DB Password: $DBPASSWORD_MISP" |$SUDO_CMD tee -a /home/${MISP_USER}/mysql.txt
echo "Authkey: $AUTH_KEY" |$SUDO_CMD tee -a /home/${MISP_USER}/MISP-authkey.txt
clear
# Commenting out, see: https://github.com/MISP/MISP/issues/5368
# clear -x
space
echo -e "${LBLUE}MISP${NC} Installed, access here: ${MISP_BASEURL}"
echo
@ -1224,7 +1227,7 @@ EOF
}
apacheConfig () {
debug "Generating Apache config"
debug "Generating Apache config, if this hangs, make sure you have enough entropy (install: haveged or wait)"
sudo cp ${PATH_TO_MISP}/INSTALL/apache.24.misp.ssl /etc/apache2/sites-available/misp-ssl.conf
if [[ ! -z ${MISP_BASEURL} ]] && [[ "$(echo $MISP_BASEURL|cut -f 1 -d :)" == "http" || "$(echo $MISP_BASEURL|cut -f 1 -d :)" == "https" ]]; then
@ -1293,6 +1296,8 @@ installCore () {
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
cd $PATH_TO_MISP/app/files/scripts/python-maec
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
# FIXME: Remove once stix-fixed
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
# install STIX2.0 library to support STIX 2.0 export:
cd ${PATH_TO_MISP}/cti-python-stix2
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
@ -1308,7 +1313,7 @@ installCore () {
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
# install zmq needed by mispzmq
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install zmq
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install zmq redis
# install python-magic
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install python-magic
@ -1395,7 +1400,7 @@ coreCAKE () {
$SUDO_WWW $RUN_PHP -- $CAKE userInit -q
# This makes sure all Database upgrades are done, without logging in.
$SUDO_WWW $RUN_PHP -- $CAKE Admin updateDatabase
$SUDO_WWW $RUN_PHP -- $CAKE Admin runUpdates
# The default install is Python >=3.6 in a virtualenv, setting accordingly
$SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.python_bin" "${PATH_TO_MISP}/venv/bin/python"
@ -1607,7 +1612,7 @@ mispmodules () {
cd /usr/local/src/
## TODO: checkUsrLocalSrc in main doc
debug "Cloning misp-modules"
$SUDO_USER git clone https://github.com/MISP/misp-modules.git
$SUDO_CMD git clone https://github.com/MISP/misp-modules.git
cd misp-modules
# some misp-modules dependencies
sudo apt install libpq5 libjpeg-dev tesseract-ocr libpoppler-cpp-dev imagemagick libopencv-dev zbar-tools libzbar0 libzbar-dev libfuzzy-dev -y
@ -1762,39 +1767,39 @@ mail2misp () {
debug "Installing Mail2${LBLUE}MISP${NC}"
cd /usr/local/src/
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
$SUDO_USER git clone https://github.com/MISP/mail_to_misp.git
$SUDO_USER git clone git://github.com/stricaud/faup.git faup
$SUDO_USER git clone git://github.com/stricaud/gtcaca.git gtcaca
$SUDO_CMD git clone https://github.com/MISP/mail_to_misp.git
$SUDO_CMD git clone git://github.com/stricaud/faup.git faup
$SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
sudo chown -R ${MISP_USER}:${MISP_USER} faup mail_to_misp gtcaca
cd gtcaca
$SUDO_USER mkdir -p build
$SUDO_CMD mkdir -p build
cd build
$SUDO_USER cmake .. && $SUDO_USER make
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
$SUDO_USER mkdir -p build
$SUDO_CMD mkdir -p build
cd build
$SUDO_USER cmake .. && $SUDO_USER make
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
cd ../../mail_to_misp
$SUDO_USER virtualenv -p python3 venv
$SUDO_USER ./venv/bin/pip install https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
$SUDO_USER ./venv/bin/pip install -r requirements.txt
$SUDO_USER cp mail_to_misp_config.py-example mail_to_misp_config.py
$SUDO_CMD virtualenv -p python3 venv
$SUDO_CMD ./venv/bin/pip install https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
$SUDO_CMD ./venv/bin/pip install -r requirements.txt
$SUDO_CMD cp mail_to_misp_config.py-example mail_to_misp_config.py
##$SUDO cp mail_to_misp_config.py-example mail_to_misp_config.py
$SUDO_USER sed -i "s/^misp_url\ =\ 'YOUR_MISP_URL'/misp_url\ =\ 'https:\/\/localhost'/g" /usr/local/src/mail_to_misp/mail_to_misp_config.py
$SUDO_USER sed -i "s/^misp_key\ =\ 'YOUR_KEY_HERE'/misp_key\ =\ '${AUTH_KEY}'/g" /usr/local/src/mail_to_misp/mail_to_misp_config.py
$SUDO_CMD sed -i "s/^misp_url\ =\ 'YOUR_MISP_URL'/misp_url\ =\ 'https:\/\/localhost'/g" /usr/local/src/mail_to_misp/mail_to_misp_config.py
$SUDO_CMD sed -i "s/^misp_key\ =\ 'YOUR_KEY_HERE'/misp_key\ =\ '${AUTH_KEY}'/g" /usr/local/src/mail_to_misp/mail_to_misp_config.py
}
ssdeep () {
debug "Install ssdeep 2.14.1"
cd /usr/local/src
$SUDO_USER wget https://github.com/ssdeep-project/ssdeep/releases/download/release-2.14.1/ssdeep-2.14.1.tar.gz
$SUDO_USER tar zxvf ssdeep-2.14.1.tar.gz
$SUDO_CMD wget https://github.com/ssdeep-project/ssdeep/releases/download/release-2.14.1/ssdeep-2.14.1.tar.gz
$SUDO_CMD tar zxvf ssdeep-2.14.1.tar.gz
cd ssdeep-2.14.1
$SUDO_USER ./configure --datadir=/usr --prefix=/usr --localstatedir=/var --sysconfdir=/etc
$SUDO_USER make
$SUDO_CMD ./configure --datadir=/usr --prefix=/usr --localstatedir=/var --sysconfdir=/etc
$SUDO_CMD make
sudo make install
#installing ssdeep_php
@ -1807,8 +1812,10 @@ ssdeep () {
sudo service apache2 restart
}
# viper-web is broken ATM
# Main Viper install function
viper () {
export PATH=$PATH:/home/misp/.local/bin
debug "Installing Viper dependencies"
cd /usr/local/src/
sudo apt-get install \
@ -1820,30 +1827,26 @@ viper () {
fi
fi
echo "Cloning Viper"
$SUDO_USER git clone https://github.com/viper-framework/viper.git
$SUDO_CMD git clone https://github.com/viper-framework/viper.git
$SUDO_CMD git clone https://github.com/viper-framework/viper-web.git
sudo chown -R $MISP_USER:$MISP_USER viper
sudo chown -R $MISP_USER:$MISP_USER viper-web
cd viper
echo "Creating virtualenv"
$SUDO_USER virtualenv -p python3 venv
$SUDO_CMD virtualenv -p python3 venv
echo "Submodule update"
# TODO: Check for current user install permissions
$SUDO_USER git submodule update --init --recursive
##$SUDO git submodule update --init --recursive
echo "Pip install deps"
$SUDO_USER ./venv/bin/pip install SQLAlchemy PrettyTable python-magic
echo "pip install scrapy"
$SUDO_USER ./venv/bin/pip install scrapy
echo "install lief"
$SUDO_USER ./venv/bin/pip install https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
echo "pip install reqs"
$SUDO_USER ./venv/bin/pip install -r requirements.txt
$SUDO_USER sed -i '1 s/^.*$/\#!\/usr\/local\/src\/viper\/venv\/bin\/python/' viper-cli
$SUDO_USER sed -i '1 s/^.*$/\#!\/usr\/local\/src\/viper\/venv\/bin\/python/' viper-web
echo "Launching viper-cli"
$SUDO_USER /usr/local/src/viper/viper-cli -h > /dev/null
$SUDO_CMD git submodule update --init --recursive
echo "pip install deps"
$SUDO_CMD ./venv/bin/pip install pefile olefile jbxapi Crypto pypdns pypssl r2pipe pdftools virustotal-api SQLAlchemy PrettyTable python-magic scrapy https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
$SUDO_CMD ./venv/bin/pip install .
echo 'update-modules' |/usr/local/src/viper/venv/bin/viper
cd /usr/local/src/viper-web
$SUDO_CMD sed -i '1 s/^.*$/\#!\/usr\/local\/src\/viper\/venv\/bin\/python/' viper-web
$SUDO_CMD /usr/local/src/viper/venv/bin/pip install -r requirements.txt
echo "Launching viper-web"
$SUDO_USER /usr/local/src/viper/viper-web -p 8888 -H 0.0.0.0 &
echo 'PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/usr/local/src/viper:/var/www/MISP/app/Console"' |sudo tee /etc/environment
$SUDO_CMD /usr/local/src/viper-web/viper-web -p 8888 -H 0.0.0.0 &
echo 'PATH="/home/misp/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/usr/local/src/viper:/var/www/MISP/app/Console"' |sudo tee -a /etc/environment
echo ". /etc/environment" >> /home/${MISP_USER}/.profile
# TODO: Perms, MISP_USER_HOME, nasty hack cuz Kali on R00t
@ -1854,8 +1857,8 @@ viper () {
fi
echo "Setting misp_url/misp_key"
$SUDO_USER sed -i "s/^misp_url\ =/misp_url\ =\ http:\/\/localhost/g" ${VIPER_HOME}/viper.conf
$SUDO_USER sed -i "s/^misp_key\ =/misp_key\ =\ $AUTH_KEY/g" ${VIPER_HOME}/viper.conf
$SUDO_CMD sed -i "s/^misp_url\ =/misp_url\ =\ http:\/\/localhost/g" ${VIPER_HOME}/viper.conf
$SUDO_CMD sed -i "s/^misp_key\ =/misp_key\ =\ $AUTH_KEY/g" ${VIPER_HOME}/viper.conf
# Reset admin password to: admin/Password1234
echo "Fixing admin.db with default password"
VIPER_COUNT=0
@ -2006,7 +2009,7 @@ installSupported () {
# TODO: Double check how the user is added and subsequently used during the install.
# TODO: Work on possibility to install as user X and install MISP for user Y
# TODO: Check if logout needed. (run SUDO_USER in installer)
# TODO: Check if logout needed. (run SUDO_CMD in installer)
# <snippet-begin add-user.sh>
# TODO: Double check how to properly handle postfix
# <snippet-begin postfix.sh>
@ -2110,8 +2113,9 @@ installSupported () {
progress 4
# Install Viper - functionLocation('generic/viper-debian.md')
[[ -n $VIPER ]] || [[ -n $ALL ]] && viper
progress 4
## FIXME: The current stat of Viper is broken, disabling any use.
##[[ -n $VIPER ]] || [[ -n $ALL ]] && viper
##progress 4
# Install ssdeep - functionLocation('generic/ssdeep-debian.md')
[[ -n $SSDEEP ]] || [[ -n $ALL ]] && ssdeep
@ -2230,6 +2234,7 @@ installMISPonKali () {
# install STIX2.0 library to support STIX 2.0 export
debug "Installing cti-python-stix2"
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I antlr4-python3-runtime==4.7.2 2> /dev/null > /dev/null
cd ${PATH_TO_MISP}/cti-python-stix2
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I . 2> /dev/null > /dev/null
@ -2385,8 +2390,9 @@ installMISPonKali () {
debug "Installing misp-modules"
mispmodules
debug "Installing Viper"
viper
## FIXME: The current stat of Viper is broken, disabling any use.
##debug "Installing Viper"
##viper
debug "Installing ssdeep"
ssdeep
@ -2463,7 +2469,14 @@ x86_64-fedora-30
x86_64-debian-stretch
x86_64-debian-buster
x86_64-ubuntu-bionic
x86_64-kali-2019.1
x86_64-kali-2019.2
x86_64-kali-2019.3
x86_64-kali-2019.4
x86_64-kali-2020.1
x86_64-kali-2020.2
x86_64-kali-2020.3
x86_64-kali-2020.4
armv6l-raspbian-stretch
armv7l-raspbian-stretch
armv7l-debian-jessie

6
INSTALL/INSTALL.sh.sfv
View File

@ -1,5 +1,5 @@
; Generated by RHash v1.3.8 on 2019-09-19 at 11:45.46
; Generated by RHash v1.3.9 on 2020-02-03 at 13:15.37
; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
;
; 98996 11:45.46 2019-09-19 INSTALL.sh
INSTALL.sh 590EBA6FCA2E8F2B5044843DF22FEED524ED3C7E 2B3E6DC0191D5D3AFD685E6ACF3E02D15261770E8E17A21F882EC2F07E908D55 13EB5B9EF3AEE3C96558DE068A0210F938491322F0AAD10DB958D9DCFA93B5313044013084E56647586AFCF8DE07C7D1 B169AA7096FAAF21E8CB8B7B7C9296C4CDB25E4B540637C15C040722EE52A2A8D2C4E8122CCEC25AFAD99F9CEC065DE14899E7C1DD571DE2B6127F08C6B79229
; 99767 13:15.37 2020-02-03 INSTALL.sh
INSTALL.sh F27125A2CD54B088B95072FF1D87C2A9EDDE3FDC 4A2F544B20AF69618BD0F744A433CBC442DAE17CC35028906DB20FB2C56DA2D6 CE625D7EB46FCC9B0228BFDD1AF62B9FC78F1D53B14CABA881C19C115F3E8F8D031EEF6A07E6E08A3B1F1D28C4EE8335 84D8940C178BFCFDA1A218D7AD7DEFA27EA62486DC7E06E165F1BE2C7738DCD384A95E62A95F7203AEDAD8593F648302E963BC3C00CCA987A415952A86192BC8

2
INSTALL/INSTALL.sh.sha1
View File

@ -1 +1 @@
590eba6fca2e8f2b5044843df22feed524ed3c7e INSTALL.sh
f27125a2cd54b088b95072ff1d87c2a9edde3fdc INSTALL.sh

2
INSTALL/INSTALL.sh.sha256
View File

@ -1 +1 @@
2b3e6dc0191d5d3afd685e6acf3e02d15261770e8e17a21f882ec2f07e908d55 INSTALL.sh
4a2f544b20af69618bd0f744a433cbc442dae17cc35028906db20fb2c56da2d6 INSTALL.sh

2
INSTALL/INSTALL.sh.sha384
View File

@ -1 +1 @@
13eb5b9ef3aee3c96558de068a0210f938491322f0aad10db958d9dcfa93b5313044013084e56647586afcf8de07c7d1 INSTALL.sh
ce625d7eb46fcc9b0228bfdd1af62b9fc78f1d53b14caba881c19c115f3e8f8d031eef6a07e6e08a3b1f1d28c4ee8335 INSTALL.sh

2
INSTALL/INSTALL.sh.sha512
View File

@ -1 +1 @@
b169aa7096faaf21e8cb8b7b7c9296c4cdb25e4b540637c15c040722ee52a2a8d2c4e8122ccec25afad99f9cec065de14899e7c1dd571de2b6127f08c6b79229 INSTALL.sh
84d8940c178bfcfda1a218d7ad7defa27ea62486dc7e06e165f1be2c7738dcd384a95e62a95f7203aedad8593f648302e963bc3c00cca987a415952a86192bc8 INSTALL.sh

20
INSTALL/INSTALL.tpl.sh
View File

@ -220,7 +220,7 @@ installSupported () {
# TODO: Double check how the user is added and subsequently used during the install.
# TODO: Work on possibility to install as user X and install MISP for user Y
# TODO: Check if logout needed. (run SUDO_USER in installer)
# TODO: Check if logout needed. (run SUDO_CMD in installer)
# <snippet-begin add-user.sh>
# TODO: Double check how to properly handle postfix
# <snippet-begin postfix.sh>
@ -324,8 +324,9 @@ installSupported () {
progress 4
# Install Viper - functionLocation('generic/viper-debian.md')
[[ -n $VIPER ]] || [[ -n $ALL ]] && viper
progress 4
## FIXME: The current stat of Viper is broken, disabling any use.
##[[ -n $VIPER ]] || [[ -n $ALL ]] && viper
##progress 4
# Install ssdeep - functionLocation('generic/ssdeep-debian.md')
[[ -n $SSDEEP ]] || [[ -n $ALL ]] && ssdeep
@ -444,6 +445,7 @@ installMISPonKali () {
# install STIX2.0 library to support STIX 2.0 export
debug "Installing cti-python-stix2"
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I antlr4-python3-runtime==4.7.2 2> /dev/null > /dev/null
cd ${PATH_TO_MISP}/cti-python-stix2
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I . 2> /dev/null > /dev/null
@ -599,8 +601,9 @@ installMISPonKali () {
debug "Installing misp-modules"
mispmodules
debug "Installing Viper"
viper
## FIXME: The current stat of Viper is broken, disabling any use.
##debug "Installing Viper"
##viper
debug "Installing ssdeep"
ssdeep
@ -677,7 +680,14 @@ x86_64-fedora-30
x86_64-debian-stretch
x86_64-debian-buster
x86_64-ubuntu-bionic
x86_64-kali-2019.1
x86_64-kali-2019.2
x86_64-kali-2019.3
x86_64-kali-2019.4
x86_64-kali-2020.1
x86_64-kali-2020.2
x86_64-kali-2020.3
x86_64-kali-2020.4
armv6l-raspbian-stretch
armv7l-raspbian-stretch
armv7l-debian-jessie

158
INSTALL/MYSQL.sql
View File

@ -57,6 +57,7 @@ CREATE TABLE IF NOT EXISTS `attribute_tags` (
`attribute_id` int(11) NOT NULL,
`event_id` int(11) NOT NULL,
`tag_id` int(11) NOT NULL,
`local` tinyint(1) NOT NULL DEFAULT 0,
PRIMARY KEY (`id`),
INDEX `attribute_id` (`attribute_id`),
INDEX `event_id` (`event_id`),
@ -120,6 +121,60 @@ CREATE TABLE IF NOT EXISTS `correlations` (
INDEX `a_sharing_group_id` (`a_sharing_group_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
CREATE TABLE IF NOT EXISTS decaying_models (
`id` int(11) NOT NULL AUTO_INCREMENT,
`uuid` varchar(40) COLLATE utf8_bin DEFAULT NULL,
`name` varchar(255) COLLATE utf8_bin NOT NULL,
`parameters` text,
`attribute_types` text,
`description` text,
`org_id` int(11),
`enabled` tinyint(1) NOT NULL DEFAULT 0,
`all_orgs` tinyint(1) NOT NULL DEFAULT 1,
`ref` text COLLATE utf8_unicode_ci,
`formula` varchar(255) COLLATE utf8_bin NOT NULL,
`version` varchar(255) COLLATE utf8_bin NOT NULL DEFAULT '',
`default` tinyint(1) NOT NULL DEFAULT 0,
PRIMARY KEY (id),
INDEX `uuid` (`uuid`),
INDEX `name` (`name`),
INDEX `org_id` (`org_id`),
INDEX `enabled` (`enabled`),
INDEX `all_orgs` (`all_orgs`),
INDEX `version` (`version`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE IF NOT EXISTS decaying_model_mappings (
`id` int(11) NOT NULL AUTO_INCREMENT,
`attribute_type` varchar(255) COLLATE utf8_bin NOT NULL,
`model_id` int(11) NOT NULL,
PRIMARY KEY (id),
INDEX `model_id` (`model_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
-- Table structure for table `event_graph`
--
CREATE TABLE IF NOT EXISTS event_graph (
`id` int(11) NOT NULL AUTO_INCREMENT,
`event_id` int(11) NOT NULL,
`user_id` int(11) NOT NULL,
`org_id` int(11) NOT NULL,
`timestamp` int(11) NOT NULL DEFAULT 0,
`network_name` varchar(255) CHARACTER SET utf8 COLLATE utf8_unicode_ci,
`network_json` MEDIUMTEXT NOT NULL,
`preview_img` MEDIUMTEXT,
PRIMARY KEY (id),
INDEX `event_id` (`event_id`),
INDEX `user_id` (`user_id`),
INDEX `org_id` (`org_id`),
INDEX `timestamp` (`timestamp`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
@ -144,6 +199,7 @@ CREATE TABLE IF NOT EXISTS `events` (
`locked` tinyint(1) NOT NULL DEFAULT 0,
`threat_level_id` int(11) NOT NULL,
`publish_timestamp` int(11) NOT NULL DEFAULT 0,
`sighting_timestamp` int(11) NOT NULL DEFAULT 0,
`disable_correlation` tinyint(1) NOT NULL DEFAULT 0,
`extends_uuid` varchar(40) COLLATE utf8_bin DEFAULT '',
PRIMARY KEY (`id`),
@ -219,6 +275,7 @@ CREATE TABLE IF NOT EXISTS `event_tags` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`event_id` int(11) NOT NULL,
`tag_id` int(11) NOT NULL,
`local` tinyint(1) NOT NULL DEFAULT 0,
PRIMARY KEY (`id`),
INDEX `event_id` (`event_id`),
INDEX `tag_id` (`tag_id`)
@ -268,6 +325,7 @@ CREATE TABLE IF NOT EXISTS `feeds` (
`lookup_visible` tinyint(1) DEFAULT 0,
`headers` TEXT COLLATE utf8_bin,
`caching_enabled` tinyint(1) NOT NULL DEFAULT 0,
`force_to_ids` tinyint(1) NOT NULL DEFAULT 0,
PRIMARY KEY (`id`),
INDEX `input_source` (`input_source`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
@ -303,6 +361,7 @@ CREATE TABLE IF NOT EXISTS `galaxies` (
`version` varchar(255) COLLATE utf8_bin NOT NULL,
`icon` VARCHAR(255) COLLATE utf8_bin NOT NULL DEFAULT '',
`namespace` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT "misp",
`kill_chain_order` text,
PRIMARY KEY (id),
INDEX `name` (`name`),
INDEX `uuid` (`uuid`),
@ -319,7 +378,8 @@ CREATE TABLE IF NOT EXISTS `galaxies` (
CREATE TABLE IF NOT EXISTS `galaxy_clusters` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`uuid` varchar(255) COLLATE utf8_bin NOT NULL,
`uuid` varchar(255) COLLATE utf8_bin NOT NULL default '',
`collection_uuid` varchar(255) COLLATE utf8_bin NOT NULL,
`type` varchar(255) COLLATE utf8_bin NOT NULL,
`value` text COLLATE utf8_bin NOT NULL,
`tag_name` varchar(255) COLLATE utf8_bin NOT NULL DEFAULT '',
@ -331,6 +391,7 @@ CREATE TABLE IF NOT EXISTS `galaxy_clusters` (
PRIMARY KEY (id),
INDEX `value` (`value`(255)),
INDEX `uuid` (`uuid`),
INDEX `collection_uuid` (`collection_uuid`),
INDEX `galaxy_id` (`galaxy_id`),
INDEX `version` (`version`),
INDEX `tag_name` (`tag_name`),
@ -467,6 +528,16 @@ CREATE TABLE IF NOT EXISTS `noticelist_entries` (
INDEX `noticelist_id` (`noticelist_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE IF NOT EXISTS `notification_logs` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`org_id` int(11) NOT NULL,
`type` varchar(255) COLLATE utf8_bin NOT NULL,
`timestamp` int(11) NOT NULL DEFAULT 0,
PRIMARY KEY (`id`),
KEY `org_id` (`org_id`),
KEY `type` (`type`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
-- -------------------------------------------------------
--
@ -532,7 +603,8 @@ CREATE TABLE IF NOT EXISTS `object_references` (
`referenced_type` int(11) NOT NULL DEFAULT 0,
`relationship_type` varchar(255) CHARACTER SET utf8 COLLATE utf8_unicode_ci,
`comment` text COLLATE utf8_bin NOT NULL,
`deleted` TINYINT NOT NULL DEFAULT 0,
`deleted` tinyint(1) NOT NULL DEFAULT 0,
PRIMARY KEY (id),
INDEX `source_uuid` (`source_uuid`),
INDEX `referenced_uuid` (`referenced_uuid`),
@ -662,6 +734,27 @@ CREATE TABLE IF NOT EXISTS `posts` (
INDEX `thread_id` (`thread_id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
CREATE TABLE IF NOT EXISTS `rest_client_histories` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`org_id` int(11) NOT NULL,
`user_id` int(11) NOT NULL,
`headers` text,
`body` text,
`url` text,
`http_method` varchar(255),
`timestamp` int(11) NOT NULL DEFAULT 0,
`use_full_path` tinyint(1) DEFAULT 0,
`show_result` tinyint(1) DEFAULT 0,
`skip_ssl` tinyint(1) DEFAULT 0,
`outcome` int(11) NOT NULL,
`bookmark` tinyint(1) NOT NULL DEFAUlT 0,
`bookmark_name` varchar(255) NULL DEFAULT '',
PRIMARY KEY (`id`),
KEY `org_id` (`org_id`),
KEY `user_id` (`user_id`),
KEY `timestamp` (`timestamp`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
@ -729,6 +822,7 @@ CREATE TABLE IF NOT EXISTS `servers` (
`org_id` int(11) NOT NULL,
`push` tinyint(1) NOT NULL,
`pull` tinyint(1) NOT NULL,
`push_sightings` tinyint(1) NOT NULL DEFAULT 0,
`lastpulledid` int(11) DEFAULT NULL,
`lastpushedid` int(11) DEFAULT NULL,
`organization` varchar(10) COLLATE utf8_bin DEFAULT NULL,
@ -741,8 +835,12 @@ CREATE TABLE IF NOT EXISTS `servers` (
`cert_file` varchar(255) COLLATE utf8_bin DEFAULT NULL,
`client_cert_file` varchar(255) COLLATE utf8_bin DEFAULT NULL,
`internal` tinyint(1) NOT NULL DEFAULT 0,
`skip_proxy` tinyint(1) NOT NULL DEFAULT 0,
`caching_enabled` tinyint(1) NOT NULL DEFAULT 0,
`priority` int(11) NOT NULL DEFAULT 0,
PRIMARY KEY (`id`),
INDEX `org_id` (`org_id`),
INDEX `priority` (`priority`),
INDEX `remote_org_id` (`remote_org_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
@ -895,6 +993,29 @@ CREATE TABLE IF NOT EXISTS `sightings` (
INDEX `type` (`type`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
CREATE TABLE IF NOT EXISTS tag_collections (
`id` int(11) NOT NULL AUTO_INCREMENT,
`uuid` varchar(40) COLLATE utf8_bin DEFAULT NULL,
`user_id` int(11) NOT NULL,
`org_id` int(11) NOT NULL,
`name` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`description` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL,
`all_orgs` tinyint(1) NOT NULL DEFAULT 0,
PRIMARY KEY (id),
INDEX `uuid` (`uuid`),
INDEX `user_id` (`user_id`),
INDEX `org_id` (`org_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE IF NOT EXISTS tag_collection_tags (
`id` int(11) NOT NULL AUTO_INCREMENT,
`tag_collection_id` int(11) NOT NULL,
`tag_id` int(11) NOT NULL,
PRIMARY KEY (id),
INDEX `uuid` (`tag_collection_id`),
INDEX `user_id` (`tag_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
@ -909,10 +1030,12 @@ CREATE TABLE IF NOT EXISTS `tags` (
`org_id` int(11) NOT NULL DEFAULT 0,
`user_id` int(11) NOT NULL DEFAULT 0,
`hide_tag` tinyint(1) NOT NULL DEFAULT 0,
`numerical_value` int(11) NULL,
PRIMARY KEY (`id`),
INDEX `name` (`name`(255)),
INDEX `org_id` (`org_id`),
INDEX `user_id` (`user_id`)
INDEX `user_id` (`user_id`),
INDEX `numerical_value` (`numerical_value`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
@ -946,6 +1069,8 @@ CREATE TABLE IF NOT EXISTS `taxonomies` (
`description` text COLLATE utf8_bin NOT NULL,
`version` int(11) NOT NULL,
`enabled` tinyint(1) NOT NULL DEFAULT 0,
`exclusive` tinyint(1) DEFAULT 0,
`required` tinyint(1) NOT NULL DEFAULT 0,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
@ -961,8 +1086,11 @@ CREATE TABLE IF NOT EXISTS `taxonomy_entries` (
`value` text COLLATE utf8_bin NOT NULL,
`expanded` text COLLATE utf8_bin,
`colour` varchar(7) CHARACTER SET utf8 COLLATE utf8_bin,
`description` text CHARACTER SET UTF8 collate utf8_bin,
`numerical_value` int(11) NULL,
PRIMARY KEY (`id`),
INDEX `taxonomy_predicate_id` (`taxonomy_predicate_id`)
INDEX `taxonomy_predicate_id` (`taxonomy_predicate_id`),
INDEX `numerical_value` (`numerical_value`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
-- --------------------------------------------------------
@ -977,8 +1105,12 @@ CREATE TABLE IF NOT EXISTS `taxonomy_predicates` (
`value` text COLLATE utf8_bin NOT NULL,
`expanded` text COLLATE utf8_bin,
`colour` varchar(7) CHARACTER SET utf8 COLLATE utf8_bin,
`description` text CHARACTER SET UTF8 collate utf8_bin,
`exclusive` tinyint(1) DEFAULT 0,
`numerical_value` int(11) NULL,
PRIMARY KEY (`id`),
INDEX `taxonomy_id` (`taxonomy_id`)
INDEX `taxonomy_id` (`taxonomy_id`),
INDEX `numerical_value` (`numerical_value`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
-- --------------------------------------------------------
@ -1113,6 +1245,18 @@ CREATE TABLE IF NOT EXISTS `threat_levels` (
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
CREATE TABLE IF NOT EXISTS `user_settings` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`setting` varchar(255) COLLATE utf8_bin NOT NULL,
`value` text COLLATE utf8_bin NOT NULL,
`user_id` int(11) NOT NULL,
`timestamp` int(11) NOT NULL,
INDEX `setting` (`setting`),
INDEX `user_id` (`user_id`),
INDEX `timestamp` (`timestamp`),
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-- --------------------------------------------------------
--
@ -1212,11 +1356,11 @@ CREATE TABLE IF NOT EXISTS `whitelist` (
--
INSERT INTO `admin_settings` (`id`, `setting`, `value`) VALUES
(1, 'db_version', '11');
(1, 'db_version', '40');
INSERT INTO `feeds` (`id`, `provider`, `name`, `url`, `distribution`, `default`, `enabled`) VALUES
(1, 'CIRCL', 'CIRCL OSINT Feed', 'https://www.circl.lu/doc/misp/feed-osint', 3, 1, 0),
(2, 'Botvrij.eu', 'The Botvrij.eu Data', 'http://www.botvrij.eu/data/feed-osint', 3, 1, 0);
(2, 'Botvrij.eu', 'The Botvrij.eu Data', 'https://www.botvrij.eu/data/feed-osint', 3, 1, 0);
INSERT INTO `regexp` (`id`, `regexp`, `replacement`, `type`) VALUES
(1, '/.:.ProgramData./i', '%ALLUSERSPROFILE%\\\\', 'ALL'),

6
INSTALL/POSTGRESQL-data-initial.sql
View File

@ -100,7 +100,7 @@ COPY public.event_tags (id, event_id, tag_id) FROM stdin;
-- Data for Name: events; Type: TABLE DATA; Schema: public; Owner: -
--
COPY public.events (id, org_id, date, info, user_id, uuid, published, analysis, attribute_count, orgc_id, "timestamp", distribution, sharing_group_id, proposal_email_lock, locked, threat_level_id, publish_timestamp, disable_correlation, extends_uuid) FROM stdin;
COPY public.events (id, org_id, date, info, user_id, uuid, published, analysis, attribute_count, orgc_id, "timestamp", distribution, sharing_group_id, proposal_email_lock, locked, threat_level_id, publish_timestamp, sighting_timestamp, disable_correlation, extends_uuid) FROM stdin;
\.
@ -118,7 +118,7 @@ COPY public.favourite_tags (id, tag_id, user_id) FROM stdin;
COPY public.feeds (id, name, provider, url, rules, enabled, distribution, sharing_group_id, tag_id, "default", source_format, fixed_event, delta_merge, event_id, publish, override_ids, settings, input_source, delete_local_file, lookup_visible, headers, caching_enabled) FROM stdin;
1 CIRCL OSINT Feed CIRCL https://www.circl.lu/doc/misp/feed-osint \N f 3 0 0 t misp f f 0 f f \N network f f \N f
2 The Botvrij.eu Data Botvrij.eu http://www.botvrij.eu/data/feed-osint \N f 3 0 0 t misp f f 0 f f \N network f f \N f
2 The Botvrij.eu Data Botvrij.eu https://www.botvrij.eu/data/feed-osint \N f 3 0 0 t misp f f 0 f f \N network f f \N f
\.
@ -323,7 +323,7 @@ COPY public.roles (id, name, created, modified, perm_add, perm_modify, perm_modi
-- Data for Name: servers; Type: TABLE DATA; Schema: public; Owner: -
--
COPY public.servers (id, name, url, authkey, org_id, push, pull, lastpulledid, lastpushedid, organization, remote_org_id, publish_without_email, unpublish_event, self_signed, pull_rules, push_rules, cert_file, client_cert_file, internal) FROM stdin;
COPY public.servers (id, name, url, authkey, org_id, push, pull, push_sightings, lastpulledid, lastpushedid, organization, remote_org_id, publish_without_email, unpublish_event, self_signed, pull_rules, push_rules, cert_file, client_cert_file, internal) FROM stdin;
\.

2
INSTALL/POSTGRESQL-structure.sql
View File

@ -335,6 +335,7 @@ CREATE TABLE public.events (
locked boolean DEFAULT false NOT NULL,
threat_level_id bigint NOT NULL,
publish_timestamp bigint DEFAULT '0'::bigint NOT NULL,
sighting_timestamp bigint DEFAULT '0'::bigint NOT NULL,
disable_correlation boolean DEFAULT false NOT NULL,
extends_uuid character varying(40) DEFAULT ''::character varying
);
@ -1171,6 +1172,7 @@ CREATE TABLE public.servers (
org_id bigint NOT NULL,
push boolean NOT NULL,
pull boolean NOT NULL,
push_sightings boolean DEFAULT false NOT NULL,
lastpulledid bigint,
lastpushedid bigint,
organization character varying(10),

12
INSTALL/misplogrotate.te
View File

@ -1,8 +1,16 @@
module misplogrotate 1.0;
module misplogrotate 1.2;
require {
type httpd_t;
type logrotate_t;
type httpd_log_t;
type httpd_sys_script_t;
type httpd_sys_content_t;
class dir { ioctl read getattr lock search open };
type httpd_sys_rw_content_t;
class dir { ioctl read getattr lock search open remove_name };
class file { unlink write };
}
#============= logrotate_t ==============
allow logrotate_t httpd_sys_content_t:dir { ioctl read getattr lock search open };
allow logrotate_t httpd_sys_rw_content_t:dir { ioctl read getattr lock search open };
allow httpd_t httpd_log_t:dir remove_name;
allow { httpd_t httpd_sys_script_t } httpd_log_t:file { unlink write };

BIN
INSTALL/screenshots/sh-misp-attack.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 218 KiB

BIN
INSTALL/screenshots/sh-misp-decaying-of-indicators.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 160 KiB

BIN
INSTALL/screenshots/sh-misp-eventgraph.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 248 KiB

BIN
INSTALL/screenshots/sh-misp-threat-actor-attack.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 215 KiB

2
PyMISP

@ -1 +1 @@
Subproject commit b67cec60a3c611efb96f69fda3ef92c3ce740eea
Subproject commit cb718b97f1e36e11a06870adb5368309e1c14912

2
VERSION.json
View File

@ -1 +1 @@
{"major":2, "minor":4, "hotfix":116}
{"major":2, "minor":4, "hotfix":120}

8
app/Config/config.default.php
View File

@ -17,7 +17,7 @@ $config = array(
'org' => 'ORGNAME',
'showorg' => true,
'threatlevel_in_email_subject' => true,
'email_subject_TLP_string' => 'TLP Amber',
'email_subject_TLP_string' => 'tlp:amber',
'email_subject_tag' => 'tlp',
'email_subject_include_tag_name' => true,
'background_jobs' => true,
@ -110,7 +110,6 @@ $config = array(
'ApacheShibbAuth' => // Configuration for shibboleth authentication
array(
'apacheEnv' => 'REMOTE_USER', // If proxy variable = HTTP_REMOTE_USER
'ssoAuth' => 'AUTH_TYPE',
'MailTag' => 'EMAIL_TAG',
'OrgTag' => 'FEDERATION_TAG',
'GroupTag' => 'GROUP_TAG',
@ -120,7 +119,6 @@ $config = array(
'group_two' => 2,
'group_one' => 1,
),
'DefaultRoleId' => 3,
'DefaultOrg' => 'DEFAULT_ORG',
),
*/
@ -141,14 +139,14 @@ $config = array(
/*
'ApacheSecureAuth' => // Configuration for kerberos authentication
array(
'apacheEnv' => 'REMOTE_USER', // If proxy variable = HTTP_REMOTE_USER
'apacheEnv' => 'REMOTE_USER', // If proxy variable = HTTP_REMOTE_USER, If BasicAuth ldap = PHP_AUTH_USER
'ldapServer' => 'ldap://example.com', // FQDN or IP
'ldapProtocol' => 3,
'ldapNetworkTimeout' => -1, // use -1 for unlimited network timeout
'ldapReaderUser' => 'cn=userWithReadAccess,ou=users,dc=example,dc=com', // DN ou RDN LDAP with reader user right
'ldapReaderPassword' => 'UserPassword', // the LDAP reader user password
'ldapDN' => 'dc=example,dc=com',
'ldapSearchFilter' => '', // Search filter to limit results from ldapsearh fx to specfic group. FX
'ldapSearchFilter' => '', // Search filter to limit results from ldapsearh fx to specific group. FX
//'ldapSearchFilter' => '(objectclass=InetOrgPerson)(!(nsaccountlock=True))(memberOf=cn=misp,cn=groups,cn=accounts,dc=example,dc=com)',
'ldapSearchAttribut' => 'uid', // filter for search
'ldapFilter' => array(

85
app/Console/Command/AdminShell.php
View File

@ -159,13 +159,12 @@ class AdminShell extends AppShell
}
}
public function updateWarningLists() {
$result = $this->Galaxy->update();
if ($result) {
echo 'Warning lists updated' . PHP_EOL;
} else {
echo 'Could not update warning lists' . PHP_EOL;
}
public function updateWarningLists()
{
$result = $this->Warninglist->update();
$success = count($result['success']);
$fails = count($result['fails']);
echo "$success warninglists updated, $fails fails" . PHP_EOL;
}
public function updateNoticeLists() {
@ -183,13 +182,7 @@ class AdminShell extends AppShell
echo 'Usage: ' . APP . '/cake ' . 'Admin updateObjectTemplates [user_id]' . PHP_EOL;
} else {
$userId = $this->args[0];
$user = $this->User->find('first', array(
'recursive' => -1,
'conditions' => array(
'User.id' => $userId,
),
'fields' => array('User.id', 'User.org_id')
));
$user = $this->User->getAuthUser($userId);
# If the user_id passed does not exist, do a global update.
if (empty($user)) {
echo 'User with ID: ' . $userId . ' not found' . PHP_EOL;
@ -287,6 +280,7 @@ class AdminShell extends AppShell
$setting = $this->Server->getSettingData($setting_name);
if (empty($setting)) {
echo 'Invalid setting "' . $setting_name . '". Please make sure that the setting that you are attempting to change exists and if a module parameter, the modules are running.' . PHP_EOL;
exit(1);
}
$result = $this->Server->serverSettingsEditValue($cli_user, $setting, $value);
if ($result === true) {
@ -314,49 +308,18 @@ class AdminShell extends AppShell
}
}
public function updateDatabase() {
public function runUpdates() {
$whoami = exec('whoami');
if ($whoami === 'httpd' || $whoami === 'www-data' || $whoami === 'apache') {
if ($whoami === 'httpd' || $whoami === 'www-data' || $whoami === 'apache' || $whoami === 'wwwrun' || $whoami === 'travis') {
echo 'Executing all updates to bring the database up to date with the current version.' . PHP_EOL;
$this->Server->runUpdates(true);
$processId = $this->args[0];
$this->Server->runUpdates(true, false, $processId);
echo 'All updates completed.' . PHP_EOL;
} else {
die('This OS user is not allowed to run this command.'. PHP_EOL. 'Run it under `www-data` or `httpd`.' . PHP_EOL . 'You tried to run this command as: ' . $whoami . PHP_EOL);
die('This OS user is not allowed to run this command.'. PHP_EOL. 'Run it under `www-data` or `httpd` or `apache` or `wwwrun`.' . PHP_EOL . 'You tried to run this command as: ' . $whoami . PHP_EOL);
}
}
public function updateApp() {
$whoami = exec('whoami');
if ($whoami === 'httpd' || $whoami === 'www-data' || $whoami === 'apache') {
$command = $this->args[0];
if (!empty($this->args[1])) {
$processId = $this->args[1];
$job = $this->Job->read(null, $processId);
} else { // create worker
$this->Job->create();
$job_data = array(
'worker' => 'prio',
'job_type' => 'update_app',
'job_input' => 'command: ' . $command,
'status' => 0,
'retries' => 0,
'org_id' => '',
'org' => '',
'message' => 'Updating.',
);
$this->Job->save($job_data);
$job = $this->Job->read(null, $this->Job->id);
}
$result = $this->Server->updateDatabase($command, false);
$job['Job']['progress'] = 100;
$job['Job']['message'] = 'Update done';
$this->Job->save($job);
} else {
die('This OS user is not allowed to run this command.' . PHP_EOL . 'Run it under `www-data` or `httpd`.' . PHP_EOL . 'You tried to run this command as: ' . $whoami . PHP_EOL);
}
}
public function getAuthkey() {
if (empty($this->args[0])) {
echo 'Invalid parameters. Usage: ' . APP . 'Console/cake Admin getAuthkey [user_email]' . PHP_EOL;
@ -551,4 +514,26 @@ class AdminShell extends AppShell
}
}
}
public function dumpCurrentDatabaseSchema()
{
$dbActualSchema = $this->Server->getActualDBSchema();
$dbVersion = $this->AdminSetting->find('first', array(
'conditions' => array('setting' => 'db_version')
));
if (!empty($dbVersion) && !empty($dbActualSchema['schema'])) {
$dbVersion = $dbVersion['AdminSetting']['value'];
$data = array(
'schema' => $dbActualSchema['schema'],
'indexes' => $dbActualSchema['indexes'],
'db_version' => $dbVersion
);
$file = new File(ROOT . DS . 'db_schema.json', true);
$file->write(json_encode($data, JSON_PRETTY_PRINT));
$file->close();
echo __("> Database schema dumped on disk") . PHP_EOL;
} else {
echo __("Something went wrong. Could not find the existing db version or fetch the current database schema.") . PHP_EOL;
}
}
}

50
app/Console/Command/EventShell.php
View File

@ -180,29 +180,6 @@ class EventShell extends AppShell
}
}
private function __recursiveEcho($array) {
$text = "";
foreach ($array as $k => $v) {
if (is_array($v)) {
if (empty($v)) $text .= '<' . $k . '/>';
else {
foreach ($v as $element) {
$text .= '<' . $k . '>';
$text .= $this->__recursiveEcho($element);
$text .= '</' . $k . '>';
}
}
} else {
if ($v === false) $v = 0;
if ($v === "" || $v === null) $text .= '<' . $k . '/>';
else {
$text .= '<' . $k . '>' . $v . '</' . $k . '>';
}
}
}
return $text;
}
public function cachehids() {
$timeStart = time();
$userId = $this->args[0];
@ -382,7 +359,6 @@ class EventShell extends AppShell
public function cachebro()
{
$timeStart = time();
$broHeader = "#fields\tindicator\tindicator_type\tmeta.source\tmeta.desc\tmeta.url\tmeta.do_notice\tmeta.if_in\n";
$userId = $this->args[0];
$user = $this->User->getAuthUser($userId);
$id = $this->args[1];
@ -400,8 +376,10 @@ class EventShell extends AppShell
}
$file->write('');
$skipHeader = false;
foreach ($types as $k => $type) {
$final = $this->Attribute->bro($user, $type);
$final = $this->Attribute->bro($user, $type, false, false, false, false, false, false, $skipHeader);
$skipHeader = true;
foreach ($final as $attribute) {
$file->append($attribute . PHP_EOL);
}
@ -531,6 +509,28 @@ class EventShell extends AppShell
$log->createLogEntry($user, 'publish', 'Event', $id, 'Event (' . $id . '): published.', 'published () => (1)');
}
public function publish_sightings() {
$id = $this->args[0];
$passAlong = $this->args[1];
$jobId = $this->args[2];
$userId = $this->args[3];
$user = $this->User->getAuthUser($userId);
$job = $this->Job->read(null, $jobId);
$this->Event->Behaviors->unload('SysLogLogable.SysLogLogable');
$result = $this->Event->publish_sightings($id, $passAlong);
$job['Job']['progress'] = 100;
$job['Job']['date_modified'] = date("Y-m-d H:i:s");
if ($result) {
$job['Job']['message'] = 'Sightings published.';
} else {
$job['Job']['message'] = 'Sightings published, but the upload to other instances may have failed.';
}
$this->Job->save($job);
$log = ClassRegistry::init('Log');
$log->create();
$log->createLogEntry($user, 'publish_sightings', 'Event', $id, 'Sightings for event (' . $id . '): published.', 'publish_sightings updated');
}
public function enrichment() {
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
die('Usage: ' . $this->Server->command_line_functions['enrichment'] . PHP_EOL);

37
app/Console/Command/ServerShell.php
View File

@ -6,12 +6,41 @@ class ServerShell extends AppShell
{
public $uses = array('Server', 'Task', 'Job', 'User', 'Feed');
public function listServers() {
$res = ['servers'=>[]];
$servers = $this->Server->find('all', [
'fields' => ['Server.id', 'Server.name', 'Server.url'],
'recursive' => 0
]);
foreach ($servers as $server)
$res['servers'][] = $server['Server'];
echo json_encode($res) . PHP_EOL;
}
public function test() {
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Test'] . PHP_EOL);
}
$serverId = intval($this->args[0]);
$res = @$this->Server->runConnectionTest($serverId);
if (!empty($res['message']))
$res['message'] = json_decode($res['message']);
echo json_encode($res) . PHP_EOL;
}
public function pull() {
if (empty($this->args[0]) || empty($this->args[1])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['pull'] . PHP_EOL);
}
$userId = $this->args[0];
$user = $this->User->getAuthUser($userId);
if (empty($user)) {
die('User ID do not match an existing user.' . PHP_EOL);
}
if (empty($this->args[1])) die();
$serverId = $this->args[1];
if (!empty($this->args[2])) {
@ -46,7 +75,7 @@ class ServerShell extends AppShell
'status' => 4
));
if (is_array($result)) {
$message = sprintf(__('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled.', count($result[0]), count($result[1]), $result[2]));
$message = sprintf(__('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled.', count($result[0]), count($result[1]), $result[2], $result[3]));
} else {
$message = sprintf(__('ERROR: %s'), $result);
}
@ -104,7 +133,7 @@ class ServerShell extends AppShell
public function fetchFeed() {
if (empty($this->args[0]) || empty($this->args[1])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['fetchFeed'] . PHP_EOL);
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Fetch feeds as local data'] . PHP_EOL);
}
$userId = $this->args[0];
$user = $this->User->getAuthUser($userId);
@ -195,7 +224,7 @@ class ServerShell extends AppShell
$data = array(
'worker' => 'default',
'job_type' => 'cache_servers',
'job_input' => 'Server: ' . $id,
'job_input' => 'Server: ' . $scopeid,
'status' => 0,
'retries' => 0,
'org' => $user['Organisation']['name'],
@ -230,7 +259,7 @@ class ServerShell extends AppShell
public function cacheFeed() {
if (empty($this->args[0]) || empty($this->args[1])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['cacheFeed'] . PHP_EOL);
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Cache feeds for quick lookups'] . PHP_EOL);
}
$userId = $this->args[0];
$user = $this->User->getAuthUser($userId);

1
app/Console/dumpDBSchema.sh Normal file
View File

@ -0,0 +1 @@
./cake admin dumpCurrentDatabaseSchema

1
app/Console/worker/start.sh
View File

@ -13,6 +13,7 @@ cd "${0%/*}"
../cake CakeResque.CakeResque start --interval 5 --queue prio
../cake CakeResque.CakeResque start --interval 5 --queue cache
../cake CakeResque.CakeResque start --interval 5 --queue email
../cake CakeResque.CakeResque start --interval 5 --queue update
../cake CakeResque.CakeResque startscheduler --interval 5
exit 0

3
app/Console/worker/start_dynamic.sh
View File

@ -28,7 +28,7 @@ else
fi
if [[ "$ADVANCED" == "1" ]]; then
for worker in `echo cache default email prio scheduler`; do
for worker in `echo cache default email prio scheduler update`; do
workerStatus=$(../cake Admin getWorkers |tail -n +7 |jq -r ".$worker" |jq -r '.ok')
PIDcount=$(../cake admin getWorkers |tail -n +7 |jq -r ".$worker.workers" |grep pid | wc -l)
echo -n "$worker has $PIDcount PID(s)"
@ -51,6 +51,7 @@ else
../cake CakeResque.CakeResque start --interval 5 --queue prio
../cake CakeResque.CakeResque start --interval 5 --queue cache
../cake CakeResque.CakeResque start --interval 5 --queue email
../cake CakeResque.CakeResque start --interval 5 --queue update
../cake CakeResque.CakeResque startscheduler --interval 5
exit 0

211
app/Controller/AppController.php
View File

@ -46,10 +46,12 @@ class AppController extends Controller
public $helpers = array('Utility', 'OrgImg', 'FontAwesome', 'UserName');
private $__queryVersion = '88';
public $pyMispVersion = '2.4.114';
public $phpmin = '7.0';
public $phprec = '7.2';
private $__queryVersion = '97';
public $pyMispVersion = '2.4.121';
public $phpmin = '7.2';
public $phprec = '7.4';
public $pythonmin = '3.6';
public $pythonrec = '3.7';
public $isApiAuthed = false;
public $baseurl = '';
@ -62,6 +64,8 @@ class AppController extends Controller
'attributes' => array('text', 'downloadAttachment', 'returnAttributes', 'restSearch', 'rpz', 'bro'),
);
protected $_legacyParams = array();
public function __construct($id = false, $table = null, $ds = null)
{
parent::__construct($id, $table, $ds);
@ -89,7 +93,11 @@ class AppController extends Controller
'ACL',
'RestResponse',
'Flash',
'Toolbox'
'Toolbox',
'RateLimit',
'IndexFilter',
'Deprecation',
'RestSearch'
//,'DebugKit.Toolbar'
);
@ -104,7 +112,14 @@ class AppController extends Controller
public function beforeFilter()
{
$this->Auth->loginRedirect = Configure::read('MISP.baseurl') . '/users/routeafterlogin';
$this->Auth->logoutRedirect = Configure::read('MISP.baseurl') . '/users/login';
$customLogout = Configure::read('Plugin.CustomAuth_custom_logout');
if ($customLogout) {
$this->Auth->logoutRedirect = $customLogout;
} else {
$this->Auth->logoutRedirect = Configure::read('MISP.baseurl') . '/users/login';
}
$this->__sessionMassage();
if (Configure::read('Security.allow_cors')) {
// Add CORS headers
@ -122,7 +137,7 @@ class AppController extends Controller
}
if (!empty($this->params['named']['sql'])) {
$this->sql_dump = 1;
$this->sql_dump = intval($this->params['named']['sql']);
}
$this->_setupDatabaseConnection();
@ -163,10 +178,10 @@ class AppController extends Controller
} else {
$this->Auth->authenticate['Form']['userFields'] = $auth_user_fields;
}
$versionArray = $this->{$this->modelClass}->checkMISPVersion();
if (!empty($this->params['named']['disable_background_processing'])) {
Configure::write('MISP.background_jobs', 0);
}
$versionArray = $this->{$this->modelClass}->checkMISPVersion();
$this->mispVersion = implode('.', array_values($versionArray));
$this->Security->blackHoleCallback = 'blackHole';
$this->_setupBaseurl();
@ -283,7 +298,9 @@ class AppController extends Controller
if ($this->Auth->user()) {
// update script
$this->{$this->modelClass}->runUpdates();
if ($this->Auth->user('Role')['perm_site_admin'] || (Configure::read('MISP.live') && !$this->_isRest())) {
$this->{$this->modelClass}->runUpdates();
}
$user = $this->Auth->user();
if (!isset($user['force_logout']) || $user['force_logout']) {
$this->loadModel('User');
@ -349,7 +366,7 @@ class AppController extends Controller
$this->Auth->logout();
throw new MethodNotAllowedException($message);//todo this should pb be removed?
} else {
$this->Flash->error('Warning: MISP is currently disabled for all users. Enable it in Server Settings (Administration -> Server Settings -> MISP tab -> live). An update might also be in progress, you can see the progress in ' , array('params' => array('url' => $this->baseurl . '/servers/updateProgress/', 'urlName' => __('Update Progress')), 'clear' => 1));
$this->Flash->error(__('Warning: MISP is currently disabled for all users. Enable it in Server Settings (Administration -> Server Settings -> MISP tab -> live). An update might also be in progress, you can see the progress in ') , array('params' => array('url' => $this->baseurl . '/servers/updateProgress/', 'urlName' => __('Update Progress')), 'clear' => 1));
}
}
if ($this->Session->check(AuthComponent::$sessionKey)) {
@ -358,7 +375,7 @@ class AppController extends Controller
if (!$this->User->exists()) {
$message = __('Something went wrong. Your user account that you are authenticated with doesn\'t exist anymore.');
if ($this->_isRest) {
$this->RestResponse->throwException(
echo $this->RestResponse->throwException(
401,
$message
);
@ -393,10 +410,8 @@ class AppController extends Controller
// instead of using checkAction(), like we normally do from controllers when trying to find out about a permission flag, we can use getActions()
// getActions returns all the flags in a single SQL query
if ($this->Auth->user()) {
$versionArray = $this->{$this->modelClass}->checkMISPVersion();
$this->mispVersionFull = implode('.', array_values($versionArray));
$this->set('mispVersion', implode('.', array($versionArray['major'], $versionArray['minor'], 0)));
$this->set('mispVersionFull', $this->mispVersionFull);
$this->set('mispVersionFull', $this->mispVersion);
$role = $this->getActions();
$this->set('me', $this->Auth->user());
$this->set('isAdmin', $role['perm_admin']);
@ -426,7 +441,7 @@ class AppController extends Controller
$this->Log->create();
$change = 'HTTP method: ' . $_SERVER['REQUEST_METHOD'] . PHP_EOL . 'Target: ' . $this->here;
if (($this->request->is('post') || $this->request->is('put')) && !empty(Configure::read('MISP.log_paranoid_include_post_body'))) {
$payload = $this->request->data;
$payload = $this->request->input();
if (!empty($payload['_Token'])) {
unset($payload['_Token']);
}
@ -463,17 +478,61 @@ class AppController extends Controller
}
$this->set('loggedInUserName', $this->__convertEmailToName($this->Auth->user('email')));
$notifications = $this->{$this->modelClass}->populateNotifications($this->Auth->user());
if ($this->request->params['controller'] === 'users' && $this->request->params['action'] === 'dashboard') {
$notifications = $this->{$this->modelClass}->populateNotifications($this->Auth->user());
} else {
$notifications = $this->{$this->modelClass}->populateNotifications($this->Auth->user(), 'fast');
}
$this->set('notifications', $notifications);
$this->ACL->checkAccess($this->Auth->user(), Inflector::variable($this->request->params['controller']), $this->action);
if ($this->_isRest()) {
$this->__rateLimitCheck();
}
if ($this->modelClass !== 'CakeError') {
$deprecationWarnings = $this->Deprecation->checkDeprecation($this->request->params['controller'], $this->action, $this->{$this->modelClass}, $this->Auth->user('id'));
if ($deprecationWarnings) {
$deprecationWarnings = __('WARNING: This functionality is deprecated and will be removed in the near future. ') . $deprecationWarnings;
if ($this->_isRest()) {
$this->response->header('X-Deprecation-Warning', $deprecationWarnings);
$this->components['RestResponse']['deprecationWarnings'] = $deprecationWarnings;
} else {
$this->Flash->warning($deprecationWarnings);
}
}
}
$this->components['RestResponse']['sql_dump'] = $this->sql_dump;
}
private function __rateLimitCheck()
{
$info = array();
$rateLimitCheck = $this->RateLimit->check(
$this->Auth->user(),
$this->request->params['controller'],
$this->action,
$this->{$this->modelClass},
$info,
$this->response->type()
);
if (!empty($info)) {
$this->RestResponse->setHeader('X-Rate-Limit-Limit', $info['limit']);
$this->RestResponse->setHeader('X-Rate-Limit-Remaining', $info['remaining']);
$this->RestResponse->setHeader('X-Rate-Limit-Reset', $info['reset']);
}
if ($rateLimitCheck !== true) {
$this->response->header('X-Rate-Limit-Limit', $info['limit']);
$this->response->header('X-Rate-Limit-Remaining', $info['remaining']);
$this->response->header('X-Rate-Limit-Reset', $info['reset']);
$this->response->body($rateLimitCheck);
$this->response->statusCode(429);
$this->response->send();
$this->_stop();
}
return true;
}
public function afterFilter()
{
if (Configure::read('debug') > 1 && !empty($this->sql_dump) && $this->_isRest()) {
$this->Log = ClassRegistry::init('Log');
echo json_encode($this->Log->getDataSource()->getLog(false, false), JSON_PRETTY_PRINT);
}
if ($this->isApiAuthed && $this->_isRest()) {
$this->Session->destroy();
}
@ -659,9 +718,8 @@ class AppController extends Controller
}
// generic function to standardise on the collection of parameters. Accepts posted request objects, url params, named url params
protected function _harvestParameters($options, &$exception)
protected function _harvestParameters($options, &$exception, $data = array())
{
$data = array();
if (!empty($options['request']->is('post'))) {
if (empty($options['request']->data)) {
$exception = $this->RestResponse->throwException(
@ -672,12 +730,25 @@ class AppController extends Controller
return false;
} else {
if (isset($options['request']->data['request'])) {
$data = $options['request']->data['request'];
$data = array_merge($data, $options['request']->data['request']);
} else {
$data = $options['request']->data;
$data = array_merge($data, $options['request']->data);
}
}
}
/*
* If we simply capture ordered URL params with func_get_args(), reassociate them.
* We can easily detect this by having ordered_url_params passed as a list instead of a dict.
*/
if (isset($options['ordered_url_params'][0])) {
$temp = array();
foreach ($options['ordered_url_params'] as $k => $url_param) {
if (!empty($options['paramArray'][$k])) {
$temp[$options['paramArray'][$k]] = $url_param;
}
}
$options['ordered_url_params'] = $temp;
}
if (!empty($options['paramArray'])) {
foreach ($options['paramArray'] as $p) {
if (
@ -1055,4 +1126,96 @@ class AppController extends Controller
}
}
protected function _legacyAPIRemap($options = array())
{
$ordered_url_params = array();
foreach ($options['paramArray'] as $k => $param) {
if (isset($options['ordered_url_params'][$k])) {
$ordered_url_params[$param] = $options['ordered_url_params'][$k];
} else {
$ordered_url_params[$param] = false;
}
}
$filterData = array(
'request' => $options['request'],
'named_params' => $options['named_params'],
'paramArray' => $options['paramArray'],
'ordered_url_params' => $ordered_url_params
);
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
if (!empty($options['injectedParams'])) {
foreach ($options['injectedParams'] as $injectedParam => $injectedValue) {
$filters[$injectedParam] = $injectedValue;
}
}
if (!empty($options['alias'])) {
foreach ($options['alias'] as $from => $to) {
if (!empty($filters[$from])) {
$filters[$to] = $filters[$from];
}
}
}
$this->_legacyParams = $filters;
return true;
}
public function restSearch()
{
$ordered_url_params = func_get_args();
if (empty($this->RestSearch->paramArray[$this->modelClass])) {
throw new NotFoundException(__('RestSearch is not implemented (yet) for this scope.'));
}
$scope = empty($this->scopeOverride) ? $this->modelClass : $this->scopeOverride;
if (!isset($this->$scope)) {
$this->loadModel($scope);
}
$filterData = array(
'request' => $this->request,
'named_params' => $this->params['named'],
'paramArray' => $this->RestSearch->paramArray[$scope],
'ordered_url_params' => func_get_args()
);
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception, $this->_legacyParams);
if (empty($filters['returnFormat'])) {
$filters['returnFormat'] = 'json';
}
unset($filterData);
if ($filters === false) {
return $exception;
}
$list = array();
$key = empty($filters['key']) ? $filters['returnFormat'] : $filters['key'];
$user = $this->_getApiAuthUser($key, $exception);
if ($user === false) {
return $exception;
}
if (isset($filters['returnFormat'])) {
$returnFormat = $filters['returnFormat'];
} else {
$returnFormat = 'json';
}
if ($returnFormat === 'download') {
$returnFormat = 'json';
}
if ($returnFormat === 'stix' && $this->_isJson()) {
$returnFormat = 'stix-json';
}
$elementCounter = 0;
$renderView = false;
$final = $this->$scope->restSearch($user, $returnFormat, $filters, false, false, $elementCounter, $renderView);
if (!empty($renderView) && !empty($final)) {
$this->layout = false;
$final = json_decode($final, true);
foreach ($final as $key => $data) {
$this->set($key, $data);
}
$this->render('/Events/module_views/' . $renderView);
} else {
$responseType = $this->$scope->validFormats[$returnFormat][0];
$filename = $this->RestSearch->getFilename($filters, $scope, $responseType);
return $this->RestResponse->viewData($final, $responseType, false, true, $filename, array('X-Result-Count' => $elementCounter, 'X-Export-Module-Used' => $returnFormat, 'X-Response-Format' => $responseType));
}
}
}

484
app/Controller/AttributesController.php
View File

@ -80,18 +80,10 @@ class AttributesController extends AppController
return $this->RestResponse->viewData($attributes, $this->response->type());
}
$org_ids = array();
$tag_ids = array();
$orgs = $this->Attribute->Event->Orgc->find('list', array(
'conditions' => array('Orgc.id' => $org_ids),
'fields' => array('Orgc.id', 'Orgc.name')
));
if (!empty($tag_ids)) {
$tags = $this->Attribute->AttributeTag->Tag->find('all', array(
'conditions' => array('Tag.id' => $tag_ids),
'recursive' => -1,
'fields' => array('Tag.id', 'Tag.name', 'Tag.colour')
));
}
if (!$this->_isRest()) {
$temp = $this->__searchUI($attributes);
$this->loadModel('Galaxy');
@ -247,13 +239,8 @@ class AttributesController extends AppController
$failed = 1;
$message = sprintf('Attributes saved, however, %s attributes could not be saved. Click %s for more info', count($fails), '$flashErrorMessage');
} else {
if (!empty($fails["attribute_0"])) {
$failed = 1;
$message = '0: ' . $v[0];
} else {
$failed = 1;
$message = 'Attribute could not be saved.';
}
$failed = 1;
$message = 'Attribute could not be saved.';
}
}
if (!empty($failKeys)) {
@ -275,8 +262,14 @@ class AttributesController extends AppController
$flashErrorMessage = implode('<br />', $flashErrorMessage);
$this->Session->write('flashErrorMessage', $flashErrorMessage);
}
if (empty($failed)) {
$this->Flash->success($message);
} else {
$this->Flash->error($message);
}
if ($this->request->is('ajax')) {
$this->autoRender = false;
$this->layout = false;
$errors = ($attributeCount > 1) ? $message : $this->Attribute->validationErrors;
if (!empty($successes)) {
return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => $message)),'status' => 200, 'type' => 'json'));
@ -284,11 +277,6 @@ class AttributesController extends AppController
return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => $errors)),'status' => 200, 'type' => 'json'));
}
} else {
if (empty($failed)) {
$this->Flash->success($message);
} else {
$this->Flash->error($message);
}
if ($successes > 0) {
$this->redirect(array('controller' => 'events', 'action' => 'view', $eventId));
}
@ -319,25 +307,31 @@ class AttributesController extends AppController
$this->loadModel('SharingGroup');
$sgs = $this->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1);
$this->set('sharingGroups', $sgs);
$info = array();
$initialDistribution = 5;
$configuredDistribution = Configure::check('MISP.default_attribute_distribution');
if ($configuredDistribution != null && $configuredDistribution != 'event') {
$initialDistribution = $configuredDistribution;
}
$this->set('initialDistribution', $initialDistribution);
$fieldDesc = array();
$distributionLevels = $this->Attribute->distributionLevels;
if (empty($sgs)) {
unset($distributionLevels[4]);
}
$this->set('distributionLevels', $distributionLevels);
foreach ($distributionLevels as $key => $value) {
$fieldDesc['distribution'][$key] = $this->Attribute->distributionDescriptions[$key]['formdesc'];
}
foreach ($this->Attribute->categoryDefinitions as $key => $value) {
$info['category'][$key] = array('key' => $key, 'desc' => isset($value['formdesc'])? $value['formdesc'] : $value['desc']);
$fieldDesc['category'][$key] = isset($value['formdesc']) ? $value['formdesc'] : $value['desc'];
}
foreach ($this->Attribute->typeDefinitions as $key => $value) {
$info['type'][$key] = array('key' => $key, 'desc' => isset($value['formdesc'])? $value['formdesc'] : $value['desc']);
}
foreach ($distributionLevels as $key => $value) {
$info['distribution'][$key] = array('key' => $value, 'desc' => $this->Attribute->distributionDescriptions[$key]['formdesc']);
$fieldDesc['type'][$key] = isset($value['formdesc']) ? $value['formdesc'] : $value['desc'];
}
$this->loadModel('Noticelist');
$notice_list_triggers = $this->Noticelist->getTriggerData();
$this->set('notice_list_triggers', json_encode($notice_list_triggers, true));
$this->set('info', $info);
$this->set('fieldDesc', $fieldDesc);
$this->set('typeDefinitions', $this->Attribute->typeDefinitions);
$this->set('categoryDefinitions', $this->Attribute->categoryDefinitions);
$this->set('published', $events['Event']['published']);
@ -870,7 +864,7 @@ class AttributesController extends AppController
$skipTimeCheck = true;
}
if ($skipTimeCheck || $this->request->data['Attribute']['timestamp'] > $existingAttribute['Attribute']['timestamp']) {
$recoverFields = array('value', 'to_ids', 'distribution', 'category', 'type', 'comment');
$recoverFields = array('value', 'to_ids', 'distribution', 'category', 'type', 'comment', 'first_seen', 'last_seen');
foreach ($recoverFields as $rF) {
if (!isset($this->request->data['Attribute'][$rF])) {
$this->request->data['Attribute'][$rF] = $existingAttribute['Attribute'][$rF];
@ -1070,7 +1064,7 @@ class AttributesController extends AppController
if (!$this->_isRest()) {
$this->Attribute->Event->insertLock($this->Auth->user(), $this->Attribute->data['Attribute']['event_id']);
}
$validFields = array('value', 'category', 'type', 'comment', 'to_ids', 'distribution');
$validFields = array('value', 'category', 'type', 'comment', 'to_ids', 'distribution', 'first_seen', 'last_seen');
$changed = false;
if (empty($this->request->data['Attribute'])) {
$this->request->data = array('Attribute' => $this->request->data);
@ -1104,6 +1098,9 @@ class AttributesController extends AppController
$event['Event']['timestamp'] = $date->getTimestamp();
$event['Event']['published'] = 0;
$this->Attribute->Event->save($event, array('fieldList' => array('published', 'timestamp', 'info')));
if ($attribute['Attribute']['object_id'] != 0) {
$this->Attribute->Object->updateTimestamp($attribute['Attribute']['object_id'], $date->getTimestamp());
}
$this->autoRender = false;
return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => 'Field updated.', 'check_publish' => true)), 'status'=>200, 'type' => 'json'));
} else {
@ -1157,53 +1154,43 @@ class AttributesController extends AppController
public function viewPicture($id, $thumbnail=false)
{
if (Validation::uuid($id)) {
$temp = $this->Attribute->find('first', array(
'recursive' => -1,
'conditions' => array('Attribute.uuid' => $id),
'fields' => array('Attribute.id', 'Attribute.uuid')
));
if (empty($temp)) {
throw new NotFoundException(__('Invalid attribute'));
}
$id = $temp['Attribute']['id'];
} elseif (!is_numeric($id)) {
$conditions = array('Attribute.uuid' => $id);
} elseif (is_numeric($id)) {
$conditions = array('Attribute.id' => $id);
} else {
throw new NotFoundException(__('Invalid attribute id.'));
}
$this->Attribute->id = $id;
if (!$this->Attribute->exists()) {
throw new NotFoundException('Invalid attribute');
}
$conditions = array(
'conditions' => array(
'Attribute.id' => $id,
'Attribute.type' => 'attachment'
),
$conditions['Attribute.type'] = 'attachment';
$options = array(
'conditions' => $conditions,
'includeAllTags' => false,
'includeAttributeUuid' => true,
'flatten' => true
'flatten' => true,
);
if ($this->_isRest()) {
$conditions['withAttachments'] = true;
$options['withAttachments'] = true;
}
$attribute = $this->Attribute->fetchAttributes($this->Auth->user(), $conditions);
$attribute = $this->Attribute->fetchAttributes($this->Auth->user(), $options);
if (empty($attribute)) {
throw new MethodNotAllowedException('Invalid attribute');
}
$attribute = $attribute[0];
if (!$this->Attribute->isImage($attribute['Attribute'])) {
throw new NotFoundException("Attribute is not an image.");
}
if ($this->_isRest()) {
return $this->RestResponse->viewData($attribute['Attribute']['data'], $this->response->type());
} else {
$width = isset($this->request->params['named']['width']) ? $this->request->params['named']['width'] : 200;
$height = isset($this->request->params['named']['height']) ? $this->request->params['named']['height'] : 200;
$image_data = $this->Attribute->getPictureData($attribute, $thumbnail, $width, $height);
$extension = explode('.', $attribute['Attribute']['value']);
$extension = end($extension);
$this->response->type(strtolower(h($extension)));
$this->response->body($image_data);
$this->autoRender = false;
$imageData = $this->Attribute->getPictureData($attribute, $thumbnail, $width, $height);
$extension = pathinfo($attribute['Attribute']['value'], PATHINFO_EXTENSION);
return new CakeResponse(array('body' => $imageData, 'type' => strtolower($extension)));
}
}
@ -1645,25 +1632,6 @@ class AttributesController extends AppController
}
}
// Deletes this specific attribute from all remote servers
private function __deleteAttributeFromServers($uuid)
{
// get a list of the servers with push active
$this->loadModel('Server');
$servers = $this->Server->find('all', array('conditions' => array('push' => 1)));
// iterate over the servers and upload the attribute
if (empty($servers)) {
return;
}
App::uses('SyncTool', 'Tools');
foreach ($servers as $server) {
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
$this->Attribute->deleteAttributeFromServer($uuid, $server, $HttpSocket);
}
}
public function search($continue = false)
{
$this->set('attrDescriptions', $this->Attribute->fieldDescriptions);
@ -1674,7 +1642,7 @@ class AttributesController extends AppController
if (isset($this->request->data['Attribute'])) {
$this->request->data = $this->request->data['Attribute'];
}
$checkForEmpty = array('value', 'tags', 'uuid', 'org', 'type', 'category');
$checkForEmpty = array('value', 'tags', 'uuid', 'org', 'type', 'category', 'first_seen', 'last_seen');
foreach ($checkForEmpty as $field) {
if (empty($this->request->data[$field]) || $this->request->data[$field] === 'ALL') {
unset($this->request->data[$field]);
@ -1684,12 +1652,12 @@ class AttributesController extends AppController
unset($this->request->data['to_ids']);
$this->request->data['ignore'] = 1;
}
$paramArray = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags');
$paramArray = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags', 'first_seen', 'last_seen');
$filterData = array(
'request' => $this->request,
'named_params' => $this->params['named'],
'paramArray' => $paramArray,
'ordered_url_params' => compact($paramArray),
'ordered_url_params' => @compact($paramArray),
'additional_delimiters' => PHP_EOL
);
$exception = false;
@ -1783,34 +1751,35 @@ class AttributesController extends AppController
private function __searchUI($attributes)
{
$sightingsData = array();
$sgids = $this->Attribute->Event->cacheSgids($this->Auth->user(), true);
$this->Feed = ClassRegistry::init('Feed');
if (!empty($options['overrideLimit'])) {
$overrideLimit = true;
} else {
$overrideLimit = false;
}
$this->loadModel('GalaxyCluster');
$cluster_names = $this->GalaxyCluster->find('list', array('fields' => array('GalaxyCluster.tag_name'), 'group' => array('GalaxyCluster.tag_name', 'GalaxyCluster.id')));
$this->loadModel('Sighting');
$user = $this->Auth->user();
foreach ($attributes as $k => $attribute) {
$attributes[$k]['Attribute']['AttributeTag'] = $attributes[$k]['AttributeTag'];
$attributes[$k]['Attribute'] = $this->Attribute->Event->massageTags($attributes[$k]['Attribute'], 'Attribute');
unset($attributes[$k]['AttributeTag']);
foreach ($attributes[$k]['Attribute']['AttributeTag'] as $k2 => $attributeTag) {
if (in_array($attributeTag['Tag']['name'], $cluster_names)) {
unset($attributes[$k]['Attribute']['AttributeTag'][$k2]);
$attributeId = $attribute['Attribute']['id'];
if ($this->Attribute->isImage($attribute['Attribute'])) {
if (extension_loaded('gd')) {
// if extension is loaded, the data is not passed to the view because it is asynchronously fetched
$attribute['Attribute']['image'] = true; // tell the view that it is an image despite not having the actual data
} else {
$attribute['Attribute']['image'] = $this->Attribute->base64EncodeAttachment($attribute['Attribute']);
}
$attributes[$k] = $attribute;
}
$attributes[$k]['Attribute']['AttributeTag'] = $attributes[$k]['AttributeTag'];
$attributes[$k]['Attribute'] = $this->Attribute->Event->massageTags($attributes[$k]['Attribute'], 'Attribute', $excludeGalaxy = false, $cullGalaxyTags = true);
unset($attributes[$k]['AttributeTag']);
$sightingsData = array_merge(
$sightingsData,
$this->Sighting->attachToEvent($attribute, $this->Auth->user(), $attributes[$k]['Attribute']['id'], $extraConditions = false)
$this->Sighting->attachToEvent($attribute, $user, $attributeId, $extraConditions = false)
);
$correlations = $this->Attribute->Event->getRelatedAttributes($this->Auth->user(), $attributes[$k]['Attribute']['id'], false, false, 'attribute');
$correlations = $this->Attribute->Event->getRelatedAttributes($user, $attributeId, false, false, 'attribute');
if (!empty($correlations)) {
$attributes[$k]['Attribute']['RelatedAttribute'] = $correlations[$attributes[$k]['Attribute']['id']];
$attributes[$k]['Attribute']['RelatedAttribute'] = $correlations[$attributeId];
}
$temp = $this->Feed->attachFeedCorrelations(array($attributes[$k]['Attribute']), $this->Auth->user, $attributes[$k]['Event'], $overrideLimit);
$temp = $this->Feed->attachFeedCorrelations(array($attributes[$k]['Attribute']), $user, $attributes[$k]['Event']);
if (!empty($temp)) {
$attributes[$k]['Attribute'] = $temp[0];
}
@ -1886,62 +1855,6 @@ class AttributesController extends AppController
$this->set('fails', $this->Attribute->checkComposites());
}
public function restSearch(
$returnFormat = false, $value = false, $type = false, $category = false, $org = false, $tags = false, $from = false,
$to = false, $last = false, $eventid = false, $withAttachments = false, $uuid = false, $publish_timestamp = false, $published = false,
$timestamp = false, $enforceWarninglist = false, $to_ids = false, $deleted = false, $includeEventUuid = false, $event_timestamp = false,
$threat_level_id = false
)
{
$paramArray = array(
'value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp',
'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags',
'includeProposals', 'returnFormat', 'published', 'limit', 'page', 'requested_attributes', 'includeContext', 'headerless',
'includeWarninglistHits', 'attackGalaxy', 'object_relation', 'includeSightings', 'includeCorrelations', 'includeDecayScore',
'decayingModel', 'excludeDecayed', 'modelOverrides', 'includeFullModel', 'score'
);
$filterData = array(
'request' => $this->request,
'named_params' => $this->params['named'],
'paramArray' => $paramArray,
'ordered_url_params' => compact($paramArray)
);
$validFormats = $this->Attribute->validFormats;
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
unset($filterData);
if ($filters === false) {
return $exception;
}
$list = array();
$user = $this->_getApiAuthUser($returnFormat, $exception);
if ($user === false) {
return $exception;
}
if (isset($filters['returnFormat'])) {
$returnFormat = $filters['returnFormat'];
} else {
$returnFormat = 'json';
}
if ($returnFormat === 'download') {
$returnFormat = 'json';
}
$elementCounter = 0;
$renderView = '';
$final = $this->Attribute->restSearch($user, $returnFormat, $filters, false, false, $elementCounter, $renderView);
if (!empty($renderView) && !empty($final)) {
$this->layout = false;
$final = json_decode($final, true);
foreach ($final as $key => $data) {
$this->set($key, $data);
}
$this->render('/Events/module_views/' . $renderView);
} else {
$responseType = $this->Attribute->validFormats[$returnFormat][0];
return $this->RestResponse->viewData($final, $responseType, false, true, false, array('X-Result-Count' => $elementCounter, 'X-Export-Module-Used' => $returnFormat, 'X-Response-Format' => $responseType));
}
}
// returns an XML with attributes that belong to an event. The type of attributes to be returned can be restricted by type using the 3rd parameter.
// Similar to the restSearch, this parameter can be chained with '&&' and negations are accepted too. For example filename&&!filename|md5 would return all filenames that don't have an md5
// The usage of returnAttributes is the following: [MISP-url]/attributes/returnAttributes/<API-key>/<type>/<signature flag>
@ -2087,165 +2000,50 @@ class AttributesController extends AppController
$this->__downloadAttachment($this->Attribute->data['Attribute']);
}
public function text($key='download', $type = 'all', $tags = false, $eventId = false, $allowNonIDS = false, $from = false, $to = false, $last = false, $enforceWarninglist = false, $allowNotPublished = false)
public function text()
{
$simpleFalse = array('eventId', 'allowNonIDS', 'tags', 'from', 'to', 'last', 'enforceWarninglist', 'allowNotPublished');
foreach ($simpleFalse as $sF) {
if (!is_array(${$sF}) && (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false')) {
${$sF} = false;
}
$this->_legacyAPIRemap(array(
'paramArray' => array(
'key', 'type', 'tags', 'eventId', 'allowNonIDS', 'from', 'to', 'last', 'enforceWarninglist', 'allowNotPublished'
),
'request' => $this->request,
'named_params' => $this->params['named'],
'ordered_url_params' => func_get_args(),
'injectedParams' => array(
'returnFormat' => 'text'
),
'alias' => array(
'eventId' => 'eventid'
)
));
if (!empty($this->_legacyParams['allowNonIDS'])) {
$this->_legacyParams['to_ids'] = [0,1];
}
if ($type === 'null' || $type === '0' || $type === 'false') {
$type = 'all';
if (!empty($this->_legacyParams['allowNotPublished'])) {
$this->_legacyParams['published'] = [0,1];
}
if ($this->request->is('post')) {
$params = array('type', 'tags', 'eventId', 'allowNonIDS', 'from', 'to', 'last', 'enforceWarninglist', 'allowNotPublished');
foreach ($params as $param) {
if (isset($this->request->data[$param])) {
${$param} = $this->request->data[$param];
}
}
if (!empty($this->_legacyParams['type']) && $this->_legacyParams['type'] === 'all') {
unset($this->_legacyParams['type']);
}
if ($from) {
$from = $this->Attribute->Event->dateFieldCheck($from);
}
if ($to) {
$to = $this->Attribute->Event->dateFieldCheck($to);
}
if ($last) {
$last = $this->Attribute->Event->resolveTimeDelta($last);
}
if ($key != 'download') {
// check if the key is valid -> search for users based on key
$user = $this->checkAuthUser($key);
if (!$user) {
throw new UnauthorizedException(__('This authentication key is not authorized to be used for exports. Contact your administrator.'));
}
} else {
if (!$this->Auth->user('id')) {
throw new UnauthorizedException(__('You have to be logged in to do that.'));
}
}
$this->response->type('txt'); // set the content type
$this->header('Content-Disposition: download; filename="misp.' . (is_array($type) ? 'multi' : $type) . '.txt"');
$this->layout = 'text/default';
$attributes = $this->Attribute->text($this->Auth->user(), $type, $tags, $eventId, $allowNonIDS, $from, $to, $last, $enforceWarninglist, $allowNotPublished);
$this->loadModel('Whitelist');
$attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true);
$this->set('attributes', $attributes);
$this->render('/Attributes/text');
return $this->restSearch();
}
public function rpz($key='download', $tags=false, $eventId=false, $from=false, $to=false, $policy=false, $walled_garden = false, $ns = false, $email = false, $serial = false, $refresh = false, $retry = false, $expiry = false, $minimum_ttl = false, $ttl = false, $enforceWarninglist = false, $ns_alt = false)
public function rpz()
{
// request handler for POSTed queries. If the request is a post, the parameters (apart from the key) will be ignored and replaced by the terms defined in the posted json or xml object.
// The correct format for both is a "request" root element, as shown by the examples below:
// For Json: {"request":{"policy": "Local-Data","walled_garden":"my.stop.page.net"}}
// For XML: <request><policy>Local-Data</policy><walled_garden>my.stop.page.net</walled_garden></request>
// the response type is used to determine the parsing method (xml/json)
if ($this->request->is('post')) {
if ($this->request->input('json_decode', true)) {
$data = $this->request->input('json_decode', true);
} else {
$data = $this->request->data;
}
if (empty($data)) {
throw new BadRequestException(__('Either specify the search terms in the url, or POST a json array / xml (with the root element being "request" and specify the correct headers based on content type).'));
}
$paramArray = array('eventId', 'tags', 'from', 'to', 'policy', 'walled_garden', 'ns', 'email', 'serial', 'refresh', 'retry', 'expiry', 'minimum_ttl', 'ttl', 'enforceWarninglist', 'ns_alt');
foreach ($paramArray as $p) {
if (isset($data['request'][$p])) {
${$p} = $data['request'][$p];
} else {
${$p} = false;
}
}
}
$simpleFalse = array('eventId', 'tags', 'from', 'to', 'policy', 'walled_garden', 'ns', 'email', 'serial', 'refresh', 'retry', 'expiry', 'minimum_ttl', 'ttl', 'enforceWarninglist', 'ns_alt');
foreach ($simpleFalse as $sF) {
if (!is_array(${$sF}) && (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false')) {
${$sF} = false;
}
}
if (!in_array($policy, array('NXDOMAIN', 'NODATA', 'DROP', 'Local-Data', 'PASSTHRU', 'TCP-only'))) {
$policy = false;
}
App::uses('RPZExport', 'Export');
$rpzExport = new RPZExport();
if ($policy) {
$policy = $rpzExport->getIdByPolicy($policy);
}
$this->loadModel('Server');
$rpzSettings = array();
$lookupData = array('policy', 'walled_garden', 'ns', 'ns_alt', 'email', 'serial', 'refresh', 'retry', 'expiry', 'minimum_ttl', 'ttl');
foreach ($lookupData as $v) {
if (${$v} !== false) {
$rpzSettings[$v] = ${$v};
} else {
$tempSetting = Configure::read('Plugin.RPZ_' . $v);
if (isset($tempSetting)) {
$rpzSettings[$v] = Configure::read('Plugin.RPZ_' . $v);
} else {
$rpzSettings[$v] = $this->Server->serverSettings['Plugin']['RPZ_' . $v]['value'];
}
}
}
if ($from) {
$from = $this->Attribute->Event->dateFieldCheck($from);
}
if ($to) {
$to = $this->Attribute->Event->dateFieldCheck($to);
}
if ($key != 'download') {
// check if the key is valid -> search for users based on key
$user = $this->checkAuthUser($key);
if (!$user) {
throw new UnauthorizedException(__('This authentication key is not authorized to be used for exports. Contact your administrator.'));
}
} else {
if (!$this->Auth->user('id')) {
throw new UnauthorizedException(__('You have to be logged in to do that.'));
}
}
if (false === $eventId || $eventId === null) {
$eventIds = $this->Attribute->Event->fetchEventIds($this->Auth->user(), false, false, false, true);
} elseif (is_numeric($eventId)) {
$eventIds = array($eventId);
} else {
throw new MethodNotAllowedException(__('Invalid event ID format.'));
}
$values = array();
foreach ($eventIds as $k => $eventId) {
$values = array_merge_recursive($values, $this->Attribute->rpz($this->Auth->user(), $tags, $eventId, $from, $to, $enforceWarninglist));
}
$this->response->type('txt'); // set the content type
$file = '';
if ($tags) {
$file = 'filtered.';
}
if ($eventId) {
$file .= 'event-' . $eventId . '.';
}
if ($from) {
$file .= 'from-' . $from . '.';
}
if ($to) {
$file .= 'to-' . $to . '.';
}
if ($file == '') {
$file = 'all.';
}
$this->header('Content-Disposition: download; filename="misp.rpz.' . $file . 'txt"');
$this->layout = 'text/default';
$this->loadModel('Whitelist');
foreach ($values as $key => $value) {
$values[$key] = $this->Whitelist->removeWhitelistedValuesFromArray($value);
}
$this->set('values', $values);
$this->set('rpzSettings', $rpzSettings);
$this->render('/Attributes/rpz');
$this->_legacyAPIRemap(array(
'paramArray' => array(
'key', 'tags', 'eventid', 'from', 'to', 'policy', 'walled_garden', 'ns',
'email', 'serial', 'refresh', 'retry', 'expiry', 'minimum_ttl', 'ttl',
'enforceWarninglist', 'ns_alt'
),
'request' => $this->request,
'named_params' => $this->params['named'],
'ordered_url_params' => func_get_args(),
'injectedParams' => array(
'returnFormat' => 'rpz'
)
));
return $this->restSearch();
}
public function bro($key = 'download', $type = 'all', $tags = false, $eventId = false, $from = false, $to = false, $last = false, $enforceWarninglist = false)
@ -2353,7 +2151,7 @@ class AttributesController extends AppController
public function fetchViewValue($id, $field = null)
{
$validFields = array('value', 'comment', 'type', 'category', 'to_ids', 'distribution', 'timestamp');
$validFields = array('value', 'comment', 'type', 'category', 'to_ids', 'distribution', 'timestamp', 'first_seen', 'last_seen');
if (!isset($field) || !in_array($field, $validFields)) {
throw new MethodNotAllowedException(__('Invalid field requested.'));
}
@ -2400,7 +2198,7 @@ class AttributesController extends AppController
public function fetchEditForm($id, $field = null)
{
$validFields = array('value', 'comment', 'type', 'category', 'to_ids', 'distribution');
$validFields = array('value', 'comment', 'type', 'category', 'to_ids', 'distribution', 'first_seen', 'last_seen');
if (!isset($field) || !in_array($field, $validFields)) {
throw new MethodNotAllowedException(__('Invalid field requested.'));
}
@ -2908,8 +2706,8 @@ class AttributesController extends AppController
if (isset($result['results']['Attribute'])) {
if (!empty($result['results']['Attribute'])) {
$attributes = array();
foreach($result['results']['Attribute'] as $attribute) {
array_push($attributes, array('type' => $attribute['type'], 'value' => $attribute['value']));
foreach($result['results']['Attribute'] as $result_attribute) {
$attributes[] = array('type' => $result_attribute['type'], 'value' => $result_attribute['value']);
}
$current_result['Attribute'] = $attributes;
}
@ -2986,6 +2784,7 @@ class AttributesController extends AppController
public function addTag($id = false, $tag_id = false)
{
$this->Taxonomy = $log = ClassRegistry::init('Taxonomy');
$rearrangeRules = array(
'request' => false,
'Attribute' => false,
@ -2995,14 +2794,11 @@ class AttributesController extends AppController
);
$RearrangeTool = new RequestRearrangeTool();
$this->request->data = $RearrangeTool->rearrangeArray($this->request->data, $rearrangeRules);
if ($id === false) {
$id = $this->request->data['attribute'];
}
if ($id === 'selected') {
$idList = json_decode($this->request->data['attribute_ids'], true);
}
$local = empty($this->params['named']['local']) ? 0 : 1;
if (!$this->request->is('post')) {
if ($id === false) {
throw new NotFoundException(__('Invalid attribute'));
}
$this->set('local', $local);
$this->set('object_id', $id);
$this->set('scope', 'Attribute');
@ -3010,7 +2806,22 @@ class AttributesController extends AppController
$this->autoRender = false;
$this->render('/Events/add_tag');
} else {
if ($id === false) {
if (!isset($this->request->data['attribute'])) {
throw new NotFoundException(__('Invalid attribute'));
}
$id = $this->request->data['attribute'];
}
if ($id === 'selected') {
if (!isset($this->request->data['attribute_ids'])) {
throw new NotFoundException(__('Invalid attribute'));
}
$idList = json_decode($this->request->data['attribute_ids'], true);
}
if ($tag_id === false) {
if (!isset($this->request->data['tag'])) {
throw new NotFoundException(__('Invalid tag'));
}
$tag_id = $this->request->data['tag'];
}
if (!is_numeric($tag_id)) {
@ -3067,6 +2878,7 @@ class AttributesController extends AppController
$this->Auth->user(),
array(
'conditions' => array('Attribute.id' => $id, 'Attribute.deleted' => 0),
'flatten' => 1,
'contain' => array('Event.orgc_id')
)
);
@ -3116,6 +2928,20 @@ class AttributesController extends AppController
$fails++;
continue;
}
$tagsOnAttribute = $this->Attribute->AttributeTag->find('all', array(
'conditions' => array(
'AttributeTag.attribute_id' => $id,
'AttributeTag.local' => $local
),
'contain' => 'Tag',
'fields' => array('Tag.name'),
'recursive' => -1
));
$exclusiveTestPassed = $this->Taxonomy->checkIfNewTagIsAllowedByTaxonomy($tag['Tag']['name'], Hash::extract($tagsOnAttribute, '{n}.Tag.name'));
if (!$exclusiveTestPassed) {
$fails++;
continue;
}
$this->Attribute->AttributeTag->create();
if ($this->Attribute->AttributeTag->save(array('attribute_id' => $id, 'tag_id' => $tag_id, 'event_id' => $eventId, 'local' => $local))) {
if (!$local) {
@ -3124,6 +2950,9 @@ class AttributesController extends AppController
$event['Event']['timestamp'] = $date->getTimestamp();
$result = $this->Attribute->Event->save($event);
$attribute['Attribute']['timestamp'] = $date->getTimestamp();
if ($attribute['Attribute']['object_id'] != 0) {
$this->Attribute->Object->updateTimestamp($attribute['Attribute']['object_id'], $date->getTimestamp());
}
$this->Attribute->save($attribute);
}
$log = ClassRegistry::init('Log');
@ -3191,9 +3020,15 @@ class AttributesController extends AppController
$RearrangeTool = new RequestRearrangeTool();
$this->request->data = $RearrangeTool->rearrangeArray($this->request->data, $rearrangeRules);
if ($id === false) {
if (!isset($this->request->data['attribute'])) {
throw new NotFoundException(__('Invalid attribute'));
}
$id = $this->request->data['attribute'];
}
if ($tag_id === false) {
if (!isset($this->request->data['tag'])) {
throw new NotFoundException(__('Invalid tag'));
}
$tag_id = $this->request->data['tag'];
}
$this->Attribute->id = $id;
@ -3259,15 +3094,20 @@ class AttributesController extends AppController
'fields' => array('Tag.name')
));
if ($this->Attribute->AttributeTag->delete($attributeTag['AttributeTag']['id'])) {
$event['Event']['published'] = 0;
$date = new DateTime();
$event['Event']['timestamp'] = $date->getTimestamp();
$this->Attribute->Event->save($event);
$this->Attribute->data['Attribute']['timestamp'] = $date->getTimestamp();
$this->Attribute->save($this->Attribute->data);
if (empty($attributeTag['AttributeTag']['local'])) {
$event['Event']['published'] = 0;
$date = new DateTime();
$event['Event']['timestamp'] = $date->getTimestamp();
$this->Attribute->Event->save($event);
if ($this->Attribute->data['Attribute']['object_id'] != 0) {
$this->Attribute->Object->updateTimestamp($this->Attribute->data['Attribute']['object_id'], $date->getTimestamp());
}
$this->Attribute->data['Attribute']['timestamp'] = $date->getTimestamp();
$this->Attribute->save($this->Attribute->data);
}
$log = ClassRegistry::init('Log');
$log->createLogEntry($this->Auth->user(), 'tag', 'Attribute', $id, 'Removed tag (' . $tag_id . ') "' . $tag['Tag']['name'] . '" from attribute (' . $id . ')', 'Attribute (' . $id . ') untagged of Tag (' . $tag_id . ')');
return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => 'Tag removed.', 'check_publish' => true)), 'status' => 200));
return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => 'Tag removed.', 'check_publish' => empty($attributeTag['AttributeTag']['local']))), 'status' => 200));
} else {
return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Tag could not be removed.')), 'status' => 200, 'type' => 'json'));
}

27
app/Controller/CommunitiesController.php
View File

@ -17,32 +17,16 @@ class CommunitiesController extends AppController
public function index()
{
$paramsToHarvest = array('context', 'value');
foreach ($paramsToHarvest as $param) {
if (!empty($this->params['named'][$param])) {
${$param} = $this->params['named'][$param];
} else if ($this->request->is('post') && !empty($this->request->data[$param])) {
${$param} = trim($this->request->data[$param]);
} else if ($param === 'context') {
${$param} = 'vetted';
}
}
$filterData = array(
'request' => $this->request,
'paramArray' => array('context', 'value'),
'named_params' => $this->params['named']
);
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
$filters = $this->IndexFilter->harvestParameters(array('context', 'value'));
if (empty($filters['context'])) {
$filters['context'] = 'vetted';
}
if (!empty($value)) {
$value = strtolower($value);
if (!empty($filters['value'])) {
$filters['value'] = strtolower($filters['value']);
} else {
$value = false;
$filters['value'] = false;
}
$community_list = $this->Community->getCommunityList($context, $value);
$community_list = $this->Community->getCommunityList($filters['context'], $filters['value']);
//foreach ($community)
if ($this->_isRest()) {
@ -53,7 +37,6 @@ class CommunitiesController extends AppController
$customPagination->truncateAndPaginate($community_list, $this->params, $this->modelClass, true);
$this->set('community_list', $community_list);
$this->set('context', $filters['context']);
$this->set('passedArgs', json_encode($this->passedArgs, true));
}
public function view($id)

33
app/Controller/Component/ACLComponent.php
View File

@ -25,6 +25,7 @@ class ACLComponent extends Component
'pruneDuplicateUUIDs' => array(),
'queryACL' => array(),
'removeDuplicateEvents' => array(),
'restSearch' => array('*'),
'updateDatabase' => array(),
'upgrade2324' => array(),
),
@ -122,8 +123,6 @@ class ACLComponent extends Component
'checkPublishedStatus' => array('*'),
'checkuuid' => array('perm_sync'),
'contact' => array('*'),
'create_dummy_event' => array(),
'create_massive_dummy_events' => array(),
'csv' => array('*'),
'cullEmptyEvents' => array(),
'delegation_index' => array('*'),
@ -132,7 +131,6 @@ class ACLComponent extends Component
'dot' => array(),
'downloadExport' => array('*'),
'downloadOpenIOCEvent' => array('*'),
'downloadSearchResult' => array('*'),
'edit' => array('perm_add'),
'enrichEvent' => array('perm_add'),
'export' => array('*'),
@ -146,6 +144,7 @@ class ACLComponent extends Component
'getEventGraphReferences' => array('*'),
'getEventGraphTags' => array('*'),
'getEventGraphGeneric' => array('*'),
'getEventTimeline' => array('*'),
'genDistributionGraph' => array('*'),
'getDistributionGraph' => array('*'),
'getReferenceData' => array('*'),
@ -161,6 +160,7 @@ class ACLComponent extends Component
'nids' => array('*'),
'proposalEventIndex' => array('*'),
'publish' => array('perm_publish'),
'publishSightings' => array('perm_sighting'),
'pushEventToZMQ' => array('perm_publish_zmq'),
'pushEventToKafka' => array('perm_publish_kafka'),
'pushProposals' => array('perm_sync'),
@ -278,6 +278,11 @@ class ACLComponent extends Component
'edit' => array('perm_add'),
'get_row' => array('perm_add'),
'orphanedObjectDiagnostics' => array(),
'editField' => array('perm_add'),
'fetchEditForm' => array('perm_add'),
'fetchViewValue' => array('*'),
'quickAddAttributeForm' => array('perm_add'),
'quickFetchTemplateWithValidObjectAttributes' => array('perm_add'),
'proposeObjectsFromAttributes' => array('*'),
'groupAttributesIntoObject' => array('perm_add'),
'revise_object' => array('perm_add'),
@ -356,9 +361,11 @@ class ACLComponent extends Component
),
'servers' => array(
'add' => array(),
'dbSchemaDiagnostic' => array(),
'cache' => array(),
'changePriority' => array(),
'checkout' => array(),
'clearWorkerQueue' => array(),
'createSync' => array('perm_sync'),
'delete' => array(),
'deleteFile' => array(),
@ -369,6 +376,7 @@ class ACLComponent extends Component
'getGit' => array(),
'getInstanceUUID' => array('perm_sync'),
'getPyMISPVersion' => array('*'),
'getRemoteUser' => array(),
'getSetting' => array(),
'getSubmodulesStatus' => array(),
'getSubmoduleQuickUpdateForm' => array(),
@ -383,6 +391,7 @@ class ACLComponent extends Component
'pull' => array(),
'purgeSessions' => array(),
'push' => array(),
'releaseUpdateLock' => array(),
'resetRemoteAuthKey' => array(),
'rest' => array('perm_auth'),
'restartWorkers' => array(),
@ -400,7 +409,7 @@ class ACLComponent extends Component
'updateProgress' => array(),
'updateSubmodule' => array(),
'uploadFile' => array(),
'clearWorkerQueue' => array()
'viewDeprecatedFunctionUse' => array()
),
'shadowAttributes' => array(
'accept' => array('perm_add'),
@ -440,8 +449,17 @@ class ACLComponent extends Component
'listSightings' => array('*'),
'quickDelete' => array('perm_sighting'),
'viewSightings' => array('*'),
'bulkSaveSightings' => array('OR' => array('perm_sync', 'perm_sighting')),
'quickAdd' => array('perm_sighting')
),
'sightingdb' => array(
'add' => array(),
'edit' => array(),
'delete' => array(),
'index' => array(),
'requestStatus' => array(),
'search' => array()
),
'tagCollections' => array(
'add' => array('perm_tag_editor'),
'addTag' => array('perm_tag_editor'),
@ -552,6 +570,13 @@ class ACLComponent extends Component
'verifyGPG' => array(),
'view' => array('*'),
),
'userSettings' => array(
'index' => array('*'),
'view' => array('*'),
'setSetting' => array('*'),
'getSetting' => array('*'),
'delete' => array('*')
),
'warninglists' => array(
'checkValue' => array('perm_auth'),
'delete' => array(),

95
app/Controller/Component/DeprecationComponent.php Normal file
View File

@ -0,0 +1,95 @@
<?php
class DeprecationComponent extends Component
{
public $redis = false;
/*
* Deprecated endpoints
* - simple controller->action structure
* - each endpoint can be set to to a deprecation warning message or false
*/
public $deprecatedEndpoints = false;
public function initialize(Controller $controller) {
$this->deprecatedEndpoints = array(
'attributes' => array(
'rpz' => __('Use /attributes/restSearch to export RPZ rules.'),
'text' => __('Use /attributes/restSearch to export flat indicator lists.')
),
'events' => array(
'addIOC' => __('Use MISP modules to import in OpenIOC format.'),
'csv' => __('Use /events/restSearch to export in CSV format.'),
'export' => __('Use the REST client to refine your search conditions and export in any of the given formats with much more control.'),
'hids' => __('Use /events/restSearch to export hashes.'),
'nids' => __('Use /events/restSearch to export in the various NIDS formats.'),
'stix' => __('Use /events/restSearch to export in STIX format.'),
'stix2' => __('Use /events/restSearch to export in STIX2 format.'),
'xml' => __('Use /events/restSearch to export in XML format. It is highly recommended to use JSON whenever possible.')
),
'posts' => array(
'add' => false,
'index' => false
),
'templates' => array(
'add' => false,
'populateEventFromTemplate' => false
),
'whitelists' => array(
'admin_add' => false
)
);
}
public function checkDeprecation($controller, $action, $model, $user_id)
{
if (isset($this->deprecatedEndpoints[$controller][$action])) {
$this->__logDeprecatedAccess($controller, $action, $model, $user_id);
if ($this->deprecatedEndpoints[$controller][$action]) {
return $this->deprecatedEndpoints[$controller][$action];
}
}
return false;
}
private function __logDeprecatedAccess($controller, $action, $model, $user_id)
{
$this->redis = $model->setupRedis();
if ($this->redis) {
@$this->redis->hincrby(
'misp:deprecation',
sprintf(
'%s:%s:%s',
$controller,
$action,
$user_id
),
1
);
$result = $this->redis->hGetAll('misp:deprecation');
}
return false;
}
public function getDeprecatedAccessList($model)
{
$this->redis = $model->setupRedis();
if ($this->redis) {
$rearranged = array();
@$result = $this->redis->hGetAll('misp:deprecation');
if (!empty($result)) {
foreach ($result as $key => $value) {
$key_components = explode(':', $key);
$rearranged[$key_components[0]][$key_components[1]][$key_components[2]] = (int)$value;
if (empty($rearranged[$key_components[0]][$key_components[1]]['total'])) {
$rearranged[$key_components[0]][$key_components[1]]['total'] = (int)$value;
} else {
$rearranged[$key_components[0]][$key_components[1]]['total'] += (int)$value;
}
}
}
}
return $rearranged;
}
}

80
app/Controller/Component/IndexFilterComponent.php Normal file
View File

@ -0,0 +1,80 @@
<?php
/**
* Get filter parameters from index searches
*/
class IndexFilterComponent extends Component
{
private $__Controller = false;
public function startup(Controller $controller) {
$this->__Controller = $controller;
}
// generic function to standardise on the collection of parameters. Accepts posted request objects, url params, named url params
public function harvestParameters($paramArray, &$exception = array())
{
$data = array();
if (!empty($this->__Controller->request->is('post'))) {
if (empty($this->__Controller->request->data)) {
$exception = $this->__Controller->RestResponse->throwException(
400,
__('Either specify the search terms in the url, or POST a json with the filter parameters.'),
'/' . $this->__Controller->request->params['controller'] . '/' . $this->__Controller->action
);
return false;
} else {
if (isset($this->__Controller->request->data['request'])) {
$data = $this->__Controller->request->data['request'];
} else {
$data = $this->__Controller->request->data;
}
}
}
if (!empty($paramArray)) {
foreach ($paramArray as $p) {
if (
isset($options['ordered_url_params'][$p]) &&
(!in_array(strtolower((string)$options['ordered_url_params'][$p]), array('null', '0', false, 'false', null)))
) {
$data[$p] = $options['ordered_url_params'][$p];
$data[$p] = str_replace(';', ':', $data[$p]);
}
if (isset($this->__Controller->params['named'][$p])) {
$data[$p] = str_replace(';', ':', $this->__Controller->params['named'][$p]);
}
}
}
foreach ($data as $k => $v) {
if (!is_array($data[$k])) {
$data[$k] = trim($data[$k]);
if (strpos($data[$k], '||')) {
$data[$k] = explode('||', $data[$k]);
}
}
}
if (!empty($options['additional_delimiters'])) {
if (!is_array($options['additional_delimiters'])) {
$options['additional_delimiters'] = array($options['additional_delimiters']);
}
foreach ($data as $k => $v) {
$found = false;
foreach ($options['additional_delimiters'] as $delim) {
if (strpos($v, $delim) !== false) {
$found = true;
}
}
if ($found) {
$data[$k] = explode($options['additional_delimiters'][0], str_replace($options['additional_delimiters'], $options['additional_delimiters'][0], $v));
foreach ($data[$k] as $k2 => $value) {
$data[$k][$k2] = trim($data[$k][$k2]);
}
}
}
}
$this->__Controller->set('passedArgs', json_encode($this->__Controller->passedArgs, true));
return $data;
}
}

62
app/Controller/Component/RateLimitComponent.php Normal file
View File

@ -0,0 +1,62 @@
<?php
App::uses('RandomTool', 'Tools');
App::uses('Component', 'Controller');
class RateLimitComponent extends Component
{
private $__limitedFunctions = array(
'attributes' => array(
'restSearch' => 1
),
'events' => array(
'restSearch' => 1
)
);
public $components = array('RestResponse');
public function check($user, $controller, $action, $Model, &$info = array(), $responseType)
{
if (!empty($user['Role']['enforce_rate_limit'])) {
$uuid = Configure::read('MISP.uuid');
if (empty($uuid)) {
$uuid = 'no-uuid';
}
$keyName = 'misp:' . $uuid . ':rate_limit:' . $user['id'];
if (!empty($this->__limitedFunctions[$controller][$action])) {
if ($user['Role']['rate_limit_count'] == 0) {
throw new MethodNotAllowedException(__('API searches are not allowed for this user role.'));
}
$redis = $Model->setupRedis();
$count = $redis->get($keyName);
if ($count !== false && $count >= $user['Role']['rate_limit_count']) {
$info = array(
'limit' => $user['Role']['rate_limit_count'],
'reset' => $redis->ttl($keyName),
'remaining'=> $user['Role']['rate_limit_count'] - $count
);
return $this->RestResponse->throwException(
429,
__('Rate limit exceeded.'),
'/' . $controller . '/' . $action,
$responseType
);
} else {
if ($count === false) {
$redis->setEx($keyName, 900, 1);
} else {
$redis->setEx($keyName, $redis->ttl($keyName), intval($count) + 1);
}
}
$count += 1;
$info = array(
'limit' => $user['Role']['rate_limit_count'],
'reset' => $redis->ttl($keyName),
'remaining'=> $user['Role']['rate_limit_count'] - $count
);
}
}
return true;
}
}

157
app/Controller/Component/RestResponseComponent.php
View File

@ -4,6 +4,8 @@ class RestResponseComponent extends Component
{
public $components = array('ACL');
public $headers = array();
private $__convertActionToMessage = array(
'SharingGroup' => array(
'addOrg' => 'add Organisation to',
@ -20,13 +22,13 @@ class RestResponseComponent extends Component
'add' => array(
'description' => "POST a MISP Attribute JSON to this API to create an Attribute.",
'mandatory' => array('value', 'type'),
'optional' => array('category', 'to_ids', 'uuid', 'distribution', 'sharing_group_id', 'timestamp', 'comment'),
'optional' => array('category', 'to_ids', 'uuid', 'distribution', 'sharing_group_id', 'timestamp', 'comment', 'data', 'encrypt', 'first_seen', 'last_seen'),
'params' => array('event_id')
),
'edit' => array(
'description' => "POST a MISP Attribute JSON to this API to update an Attribute. If the timestamp is set, it has to be newer than the existing Attribute.",
'mandatory' => array(),
'optional' => array('value', 'type', 'category', 'to_ids', 'uuid', 'distribution', 'sharing_group_id', 'timestamp', 'comment'),
'optional' => array('value', 'type', 'category', 'to_ids', 'uuid', 'distribution', 'sharing_group_id', 'timestamp', 'comment', 'date', 'encrypt', 'first_seen', 'last_seen'),
'params' => array('attribute_id')
),
'deleteSelected' => array(
@ -38,7 +40,7 @@ class RestResponseComponent extends Component
'restSearch' => array(
'description' => "Search MISP using a list of filter parameters and return the data in the selected format. The search is available on an event and an attribute level, just select the scope via the URL (/events/restSearch vs /attributes/restSearch). Besides the parameters listed, other, format specific ones can be passed along (for example: requested_attributes and includeContext for the CSV export). This API allows pagination via the page and limit parameters.",
'mandatory' => array('returnFormat'),
'optional' => array('page', 'limit', 'value' , 'type', 'category', 'org', 'tags', 'date', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'includeEventTags', 'event_timestamp', 'threat_level_id', 'eventinfo', 'includeProposals', 'includeDecayScore', 'includeFullModel', 'decayingModel', 'excludeDecayed', 'score'),
'optional' => array('page', 'limit', 'value' , 'type', 'category', 'org', 'tags', 'date', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'attribute_timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'includeEventTags', 'event_timestamp', 'threat_level_id', 'eventinfo', 'includeProposals', 'includeDecayScore', 'includeFullModel', 'decayingModel', 'excludeDecayed', 'score', 'first_seen', 'last_seen'),
'params' => array()
)
),
@ -169,11 +171,11 @@ class RestResponseComponent extends Component
'add' => array(
'description' => "POST an Server object in JSON format to this API to add a server.",
'mandatory' => array('url', 'name', 'remote_org_id', 'authkey'),
'optional' => array('push', 'pull', 'push_rules', 'pull_rules', 'submitted_cert', 'submitted_client_cert', 'json')
'optional' => array('push', 'pull', 'push_sightings', 'push_rules', 'pull_rules', 'submitted_cert', 'submitted_client_cert', 'json')
),
'edit' => array(
'description' => "POST an Server object in JSON format to this API to edit a server.",
'optional' => array('url', 'name', 'authkey', 'json', 'push', 'pull', 'push_rules', 'pull_rules', 'submitted_cert', 'submitted_client_cert', 'remote_org_id')
'optional' => array('url', 'name', 'authkey', 'json', 'push', 'pull', 'push_sightings', 'push_rules', 'pull_rules', 'submitted_cert', 'submitted_client_cert', 'remote_org_id')
),
'serverSettings' => array(
'description' => "Send a GET request to this endpoint to get a full diagnostic along with all currently set settings of the current instance. This will also include the worker status"
@ -249,6 +251,17 @@ class RestResponseComponent extends Component
'http_method' => 'GET'
)
),
'UserSetting' => array(
'setSetting' => array(
'description' => "POST a User setting object in JSON format to this API to create a new setting or update the equivalent existing setting. Admins/site admins can specify a user ID besides their own.",
'mandatory' => array('setting', 'value'),
'optional' => array('user_id')
),
'delete' => array(
'description' => "POST or DELETE to this API to delete an existing setting.",
'params' => array('id')
)
),
'Warninglist' => array(
'checkValue' => array(
'description' => "POST a JSON list with value(s) to check against the warninglists to get a JSON dictionary as a response with any hits, if there are any (with the key being the passed value triggering a warning).",
@ -265,6 +278,7 @@ class RestResponseComponent extends Component
public function initialize(Controller $controller) {
$this->__configureFieldConstraints();
$this->Controller = $controller;
}
public function getAllApisFieldsConstraint($user)
@ -378,6 +392,7 @@ class RestResponseComponent extends Component
if (isset($this->__convertActionToMessage[$controller][$action['action']])) {
$stringifiedAction = $this->__convertActionToMessage[$controller][$action['action']];
}
$response['saved'] = false;
$response['name'] = 'Could not ' . $stringifiedAction . ' ' . Inflector::singularize($controller);
$response['message'] = $response['name'];
$response['url'] = $this->__generateURL($action, $controller, $id);
@ -391,6 +406,8 @@ class RestResponseComponent extends Component
if (!$message) {
$message = Inflector::singularize($controller) . ' ' . $action['action'] . ((substr($action['action'], -1) == 'e') ? 'd' : 'ed');
}
$response['saved'] = true;
$response['success'] = true;
$response['name'] = $message;
$response['message'] = $response['name'];
$response['url'] = $this->__generateURL($action, $controller, $id);
@ -430,7 +447,28 @@ class RestResponseComponent extends Component
if (is_string($response)) {
$response = array('message' => $response);
}
if (Configure::read('debug') > 1 && !empty($this->Controller->sql_dump)) {
$this->Log = ClassRegistry::init('Log');
if ($this->Content->sql_dump === 2) {
$response = array('sql_dump' => $this->Log->getDataSource()->getLog(false, false));
} else {
$response['sql_dump'] = $this->Log->getDataSource()->getLog(false, false);
}
}
$response = json_encode($response, JSON_PRETTY_PRINT);
} else {
if (Configure::read('debug') > 1 && !empty($this->Controller->sql_dump)) {
$this->Log = ClassRegistry::init('Log');
if ($this->Controller->sql_dump === 2) {
$response = json_encode(array('sql_dump' => $this->Log->getDataSource()->getLog(false, false)));
} else {
$response = substr_replace(
$response,
sprintf(', "sql_dump": %s}', json_encode($this->Log->getDataSource()->getLog(false, false))),
-2
);
}
}
}
}
$cakeResponse = new CakeResponse(array('body'=> $response, 'status' => $code, 'type' => $type));
@ -441,13 +479,19 @@ class RestResponseComponent extends Component
$headers["Access-Control-Allow-Origin"] = explode(',', Configure::read('Security.cors_origins'));
$headers["Access-Control-Expose-Headers"] = ["X-Result-Count"];
}
if (!empty($this->headers)) {
foreach ($this->headers as $key => $value) {
$cakeResponse->header($key, $value);
}
}
if (!empty($headers)) {
foreach ($headers as $key => $value) {
$cakeResponse->header($key, $value);
}
}
if (!empty($deprecationWarnings)) {
$cakeResponse->header('X-Deprecation-Warning', $deprecationWarnings);
}
if ($download) {
$cakeResponse->download($download);
}
@ -489,14 +533,19 @@ class RestResponseComponent extends Component
return $cakeResponse;
}
public function throwException($code, $message, $url = '', $format = false, $raw = false)
public function throwException($code, $message, $url = '', $format = false, $raw = false, $headers = array())
{
$message = array(
'name' => $message,
'message' => $message,
'url' => $url
);
return $this->__sendResponse($message, $code, $format, $raw);
return $this->__sendResponse($message, $code, $format, $raw, false, $headers);
}
public function setHeader($header, $value)
{
$this->headers[$header] = $value;
}
public function describe($controller, $action, $id = false, $format = false)
@ -678,6 +727,12 @@ class RestResponseComponent extends Component
'autoclose' => true
),
),
'data' => array(
'input' => 'text',
'type' => 'string',
'operators' => array('equal'),
'help' => __('Base64 encoded file contents')
),
'date' => array(
'type' => 'date',
'validation' => array( 'format' => 'YYYY-MM-DD' ),
@ -777,6 +832,12 @@ class RestResponseComponent extends Component
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' )
),
'encrypt' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' ),
'help' => __('When uploading malicious samples, set this flag to tell MISP to encrpyt the sample and extract the file hashes. This will create a MISP object with the appropriate attributes.')
),
//'enforceWarningList' => array(
// 'input' => 'radio',
// 'type' => 'integer',
@ -799,7 +860,14 @@ class RestResponseComponent extends Component
'type' => 'integer',
'operators' => array('equal', 'not_equal'),
'validation' => array('min' => 0, 'step' => 1),
'help' => __('The timestamp at which the event was published')
'help' => __('The timestamp at which the event was last modified')
),
'attribute_timestamp' => array(
'input' => 'number',
'type' => 'integer',
'operators' => array('equal', 'not_equal'),
'validation' => array('min' => 0, 'step' => 1),
'help' => __('The timestamp at which the attribute was last modified')
),
'eventid' => array(
'input' => 'number',
@ -849,6 +917,12 @@ class RestResponseComponent extends Component
'operators' => array('equal'),
'help' => __('A valid external auth key')
),
'first_seen' => array(
'input' => 'text',
'type' => 'string',
'operators' => array('equal'),
'help' => 'A valid ISO 8601 datetime format, up to milli-seconds. i.e.: 2019-06-13T15:56:56.856074+02:00'
),
'fixed_event' => array(
'input' => 'select',
'type' => 'integer',
@ -971,6 +1045,12 @@ class RestResponseComponent extends Component
'operators' => array('equal', 'not_equal'),
'help' => __('Events published within the last x amount of time, where x can be defined in days, hours, minutes (for example 5d or 12h or 30m)')
),
'last_seen' => array(
'input' => 'text',
'type' => 'string',
'operators' => array('equal'),
'help' => 'A valid ISO 8601 datetime format, up to milli-seconds. i.e.: 2019-06-13T15:56:56.856074+02:00'
),
'limit' => array(
'input' => 'number',
'type' => 'integer',
@ -1226,6 +1306,12 @@ class RestResponseComponent extends Component
'values' => array(1 => 'True', 0 => 'False' ),
'help' => __('Allow the upload of events and their attribute to the server')
),
'push_sightings' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' ),
'help' => __('Allow the upload of sightings to the server')
),
'releasability' => array(
'input' => 'text',
'type' => 'string',
@ -1548,42 +1634,48 @@ class RestResponseComponent extends Component
$fieldsConstraint[$sf]['label'] = $label;
}
} else {
$fieldsConstraint[$field] = $this->__fieldsConstraint[$field];
$label = $scope . '.' . $field;
$fieldsConstraint[$field]['id'] = $label;
$fieldsConstraint[$field]['label'] = $label;
if (!empty($this->__fieldsConstraint[$field])) {
$fieldsConstraint[$field] = $this->__fieldsConstraint[$field];
$label = $scope . '.' . $field;
$fieldsConstraint[$field]['id'] = $label;
$fieldsConstraint[$field]['label'] = $label;
}
}
// add dynamic data and overwrite name collisions
switch($field) {
case "returnFormat":
$this->__overwriteReturnFormat($scope, $fieldsConstraint[$field]);
$this->__overwriteReturnFormat($scope, $action, $fieldsConstraint[$field]);
break;
case "type":
$this->__overwriteType($scope, $fieldsConstraint[$field]);
$this->__overwriteType($scope, $action, $fieldsConstraint[$field]);
break;
case "category":
$this->__overwriteCategory($scope, $fieldsConstraint[$field]);
$this->__overwriteCategory($scope, $action, $fieldsConstraint[$field]);
break;
case "decayingModel":
$this->__overwriteDecayingModel($scope, $fieldsConstraint[$field]);
break;
case "distribution":
$this->__overwriteDistribution($scope, $fieldsConstraint[$field]);
$this->__overwriteDistribution($scope, $action, $fieldsConstraint[$field]);
break;
case "tag":
case "tags":
case "EventTag":
$this->__overwriteTags($scope, $fieldsConstraint[$field]);
$this->__overwriteTags($scope, $action, $fieldsConstraint[$field]);
break;
case "nationality":
$this->__overwriteNationality($scope, $fieldsConstraint[$field]);
$this->__overwriteNationality($scope, $action, $fieldsConstraint[$field]);
break;
case "action":
$this->__overwriteAction($scope, $fieldsConstraint[$field]);
$this->__overwriteAction($scope, $action, $fieldsConstraint[$field]);
break;
case "role_id":
$this->__overwriteRoleId($scope, $fieldsConstraint[$field]);
$this->__overwriteRoleId($scope, $action, $fieldsConstraint[$field]);
break;
case "first_seen":
case "last_seen":
$this->__overwriteSeen($scope, $action, $fieldsConstraint[$field]);
break;
default:
break;
@ -1599,7 +1691,7 @@ class RestResponseComponent extends Component
}
// Fetch the correct values based on the scope, then overwrite default value
private function __overwriteReturnFormat($scope, &$field) {
private function __overwriteReturnFormat($scope, $action, &$field) {
switch($scope) {
case "Attribute":
$field['values'] = array_keys(ClassRegistry::init($scope)->validFormats);
@ -1609,7 +1701,7 @@ class RestResponseComponent extends Component
break;
}
}
private function __overwriteType($scope, &$field) {
private function __overwriteType($scope, $action, &$field) {
$field['input'] = 'select';
switch($scope) {
case "Attribute":
@ -1627,10 +1719,10 @@ class RestResponseComponent extends Component
}
}
private function __overwriteCategory($scope, &$field) {
private function __overwriteCategory($scope, $action, &$field) {
$field['values'] = array_keys(ClassRegistry::init("Attribute")->categoryDefinitions);
}
private function __overwriteDistribution($scope, &$field) {
private function __overwriteDistribution($scope, $action, &$field) {
$field['values'] = array();
foreach(ClassRegistry::init("Attribute")->distributionLevels as $d => $text) {
$field['values'][] = array('label' => $text, 'value' => $d);
@ -1647,7 +1739,7 @@ class RestResponseComponent extends Component
$field['values'][] = array('label' => h($model_name), 'value' => $i);
}
}
private function __overwriteTags($scope, &$field) {
private function __overwriteTags($scope, $action, &$field) {
$this->{$scope} = ClassRegistry::init("Tag");
$tags = $this->{$scope}->find('list', array(
'recursive' => -1,
@ -1660,13 +1752,13 @@ class RestResponseComponent extends Component
}
$field['values'] = $tags;
}
private function __overwriteNationality($scope, &$field) {
private function __overwriteNationality($scope, $action, &$field) {
$field['values'] = ClassRegistry::init("Organisation")->countries;
}
private function __overwriteAction($scope, &$field) {
private function __overwriteAction($scope, $action, &$field) {
$field['values'] = array_keys(ClassRegistry::init("Log")->actionDefinitions);
}
private function __overwriteRoleId($scope, &$field) {
private function __overwriteRoleId($scope, $action, &$field) {
$this->{$scope} = ClassRegistry::init("Role");
$roles = $this->{$scope}->find('list', array(
'recursive' => -1,
@ -1674,5 +1766,10 @@ class RestResponseComponent extends Component
));
$field['values'] = $roles;
}
private function __overwriteSeen($scope, $action, &$field) {
if ($action == 'restSearch') {
$field['help'] = __('Seen within the last x amount of time, where x can be defined in days, hours, minutes (for example 5d or 12h or 30m)');
}
}
}

46
app/Controller/Component/RestSearchComponent.php Normal file
View File

@ -0,0 +1,46 @@
<?php
App::uses('Component', 'Controller');
class RestSearchComponent extends Component
{
public $paramArray = array(
'Attribute' => array(
'returnFormat', 'value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp',
'published', 'timestamp','enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags',
'includeProposals', 'returnFormat', 'limit', 'page', 'requested_attributes', 'includeContext', 'headerless',
'includeWarninglistHits', 'attackGalaxy', 'object_relation', 'includeSightings', 'includeCorrelations', 'includeDecayScore',
'decayingModel', 'excludeDecayed', 'modelOverrides', 'includeFullModel', 'score', 'attribute_timestamp', 'first_seen', 'last_seen'
),
'Event' => array(
'returnFormat', 'value', 'type', 'category', 'org', 'tags', 'searchall', 'from', 'to', 'last', 'eventid', 'withAttachments',
'metadata', 'uuid', 'publish_timestamp', 'timestamp', 'published', 'enforceWarninglist', 'sgReferenceOnly',
'limit', 'page', 'requested_attributes', 'includeContext', 'headerless', 'includeWarninglistHits', 'attackGalaxy', 'deleted',
'excludeLocalTags', 'date', 'includeSightingdb', 'tag', 'object_relation'
),
'Sighting' => array(
'context', 'returnFormat', 'id', 'type', 'from', 'to', 'last', 'org_id', 'source', 'includeAttribute', 'includeEvent'
)
);
public function getFilename($filters, $scope, $responseType)
{
$filename = false;
if ($scope === 'Event') {
$filename = 'misp.event.';
if (!empty($filters['eventid']) && !is_array($filters['eventid'])) {
if (Validation::uuid(trim($filters['eventid']))) {
$filename .= trim($filters['eventid']);
} else if (!empty(intval(trim($filters['eventid'])))) {
$filename .= intval(trim($filters['eventid']));
}
} else {
$filename .= 'list';
}
}
if ($filename !== false) {
$filename .= '.' . $responseType;
}
return $filename;
}
}

3
app/Controller/DecayingModelController.php
View File

@ -591,7 +591,8 @@ class DecayingModelController extends AppController
$filters['uuid'] = $filters['id'];
} else {
$attributes = $this->User->Event->Attribute->fetchAttributes($this->Auth->user(), array(
'conditions' => array('Attribute.id' => $filters['id'])
'conditions' => array('Attribute.id' => $filters['id']),
'flatten' => 1
));
if (!empty($attributes)) {
$filters['uuid'] = $attributes[0]['Attribute']['uuid'];

2
app/Controller/EventGraphController.php
View File

@ -19,7 +19,7 @@ class EventGraphController extends AppController
throw new MethodNotAllowedException(__('No event ID set.'));
}
// retreive current org_id
// retrieve current org_id
$org_id = $this->_checkOrg();
// validate event

1063
app/Controller/EventsController.php
View File

File diff suppressed because it is too large Load Diff

16
app/Controller/FeedsController.php
View File

@ -132,7 +132,10 @@ class FeedsController extends AppController
public function importFeeds()
{
if ($this->request->is('post')) {
$results = $this->Feed->importFeeds($this->request->data['Feed']['json'], $this->Auth->user());
if (isset($this->request->data['Feed']['json'])) {
$this->request->data = $this->request->data['Feed']['json'];
}
$results = $this->Feed->importFeeds($this->request->data, $this->Auth->user());
if ($results['successes'] > 0) {
$flashType = 'success';
$message = $results['successes'] . ' new feeds added.';
@ -505,8 +508,15 @@ class FeedsController extends AppController
}
}
}
$this->Flash->success($message);
$this->redirect(array('action' => 'index'));
if (!isset($message)) {
$message = __('No feed enabled.');
}
if ($this->_isRest()) {
return $this->RestResponse->viewData(array('result' => $message), $this->response->type());
} else {
$this->Flash->success($message);
$this->redirect(array('action' => 'index'));
}
}
public function getEvent($feedId, $eventUuid, $all = false)

14
app/Controller/GalaxiesController.php
View File

@ -111,17 +111,19 @@ class GalaxiesController extends AppController
{
$mitreAttackGalaxyId = $this->Galaxy->getMitreAttackGalaxyId();
$local = !empty($this->params['named']['local']) ? $this->params['named']['local'] : '0';
$conditions = $namespace == '0' ? array() : array('namespace' => $namespace);
$conditions = $namespace === '0' ? array() : array('namespace' => $namespace);
$galaxies = $this->Galaxy->find('all', array(
'recursive' => -1,
'fields' => array('MAX(Galaxy.version) as latest_version', 'id', 'kill_chain_order', 'name', 'icon', 'description'),
'conditions' => $conditions,
'group' => array('name', 'id', 'kill_chain_order', 'icon', 'description'),
'order' => array('name asc')
));
$items = array();
$items[] = array(
'name' => __('All clusters'),
'value' => "/galaxies/selectCluster/" . h($target_id) . '/' . h($target_type) . '/0'. '/local:' . $local
$items = array(
array(
'name' => __('All clusters'),
'value' => "/galaxies/selectCluster/" . h($target_id) . '/' . h($target_type) . '/0'. '/local:' . $local
)
);
foreach ($galaxies as $galaxy) {
if (!isset($galaxy['Galaxy']['kill_chain_order'])) {

2
app/Controller/JobsController.php
View File

@ -25,7 +25,7 @@ class JobsController extends AppController
$issueCount = 0;
$workers = $this->Server->workerDiagnostics($issueCount);
$this->recursive = 0;
$queues = array('email', 'default', 'cache', 'prio');
$queues = array('email', 'default', 'cache', 'prio', 'update');
if ($queue && in_array($queue, $queues)) {
$this->paginate['conditions'] = array('Job.worker' => $queue);
}

2
app/Controller/LogsController.php
View File

@ -416,7 +416,7 @@ class LogsController extends AppController
$this->set('actions', $actions);
// combobox for models
$models = array('Attribute', 'Event', 'EventBlacklist', 'EventTag', 'DecayingModel', 'MispObject', 'Organisation', 'Post', 'Regexp', 'Role', 'Server', 'ShadowAttribute', 'SharingGroup', 'Tag', 'Task', 'Taxonomy', 'Template', 'Thread', 'User', 'Whitelist');
$models = array('Attribute', 'Event', 'EventBlacklist', 'EventTag', 'Feed', 'DecayingModel', 'MispObject', 'Organisation', 'Post', 'Regexp', 'Role', 'Server', 'ShadowAttribute', 'SharingGroup', 'Tag', 'Task', 'Taxonomy', 'Template', 'Thread', 'User', 'Whitelist');
$models = array('' => 'ALL') + $this->_arrayToValuesIndexArray($models);
$this->set('models', $models);
$this->set('actionDefinitions', $this->{$this->defaultModel}->actionDefinitions);

552
app/Controller/ObjectsController.php
View File

@ -198,6 +198,7 @@ class ObjectsController extends AppController
$this->MispObject->Event->insertLock($this->Auth->user(), $eventId);
}
$error = false;
$template = false;
if (!empty($templateId) || !$this->_isRest()) {
$templates = $this->MispObject->ObjectTemplate->find('all', array(
'conditions' => array('ObjectTemplate.id' => $templateId),
@ -247,10 +248,14 @@ class ObjectsController extends AppController
foreach ($object['Attribute'] as $k => $attribute) {
unset($object['Attribute'][$k]['id']);
$object['Attribute'][$k]['event_id'] = $eventId;
$this->MispObject->Event->Attribute->set($attribute);
$this->MispObject->Event->Attribute->set($object['Attribute'][$k]);
if (!$this->MispObject->Event->Attribute->validates()) {
if ($this->MispObject->Event->Attribute->validationErrors['value'][0] !== 'Composite type found but the value not in the composite (value1|value2) format.') {
$error = 'Could not save object as at least one attribute has failed validation (' . $attribute['object_relation'] . '). ' . json_encode($this->MispObject->Event->Attribute->validationErrors);
$error = sprintf(
'Could not save object as at least one attribute has failed validation (%s). %s',
isset($attribute['object_relation']) ? $attribute['object_relation'] : 'No object_relation',
json_encode($this->MispObject->Event->Attribute->validationErrors)
);
}
}
}
@ -281,6 +286,12 @@ class ObjectsController extends AppController
$result = $this->MispObject->saveObject($object, $eventId, $template, $this->Auth->user(), $errorBehaviour = 'halt');
if (is_numeric($result)) {
$this->MispObject->Event->unpublishEvent($eventId);
} else {
$object_validation_errors = array();
foreach($result as $field => $field_errors) {
$object_validation_errors[] = sprintf('%s: %s', $field, implode(', ', $field_errors));
}
$error = __('Object could not be saved.') . PHP_EOL . implode(PHP_EOL, $object_validation_errors);
}
} else {
$result = false;
@ -353,43 +364,20 @@ class ObjectsController extends AppController
$this->set('element', $element);
}
public function edit($id, $update_template_available=false)
public function edit($id, $update_template_available=false, $onlyAddNewAttribute=false)
{
if (Validation::uuid($id)) {
$conditions = array('Object.uuid' => $id);
} else {
$conditions = array('Object.id' => $id);
}
if (!$this->userRole['perm_modify']) {
throw new MethodNotAllowedException(__('You don\'t have permissions to edit objects.'));
}
$object = $this->MispObject->find('first', array(
'conditions' => $conditions,
'recursive' => -1,
'contain' => array(
'Attribute' => array(
'conditions' => array(
'Attribute.deleted' => 0
)
)
)
));
$id = $this->Toolbox->findIdByUuid($this->MispObject, $id);
$object = $this->MispObject->fetchObjects($this->Auth->user(), array('conditions' => array('Object.id' => $id)));
if (empty($object)) {
throw new NotFoundException(__('Invalid object.'));
}
$id = $object['Object']['id'];
$eventFindParams = array(
'recursive' => -1,
'fields' => array('Event.id', 'Event.uuid', 'Event.orgc_id'),
'conditions' => array('Event.id' => $object['Object']['event_id'])
);
$event = $this->MispObject->Event->find('first', $eventFindParams);
if (empty($event) || (!$this->_isSiteAdmin() && $event['Event']['orgc_id'] != $this->Auth->user('org_id'))) {
throw new NotFoundException(__('Invalid object.'));
$object = $object[0];
if ((!$this->_isSiteAdmin() && $object['Event']['orgc_id'] != $this->Auth->user('org_id'))) {
throw new MethodNotAllowedException(__('Insufficient permissions to edit this object.'));
}
$event = $this->MispObject->Event->fetchEvent($this->Auth->user(), array('eventid' => $object['Event']['id'], 'metadata' => 1))[0];
if (!$this->_isRest()) {
$this->MispObject->Event->insertLock($this->Auth->user(), $event['Event']['id']);
$this->MispObject->Event->insertLock($this->Auth->user(), $object['Event']['id']);
}
$template = $this->MispObject->ObjectTemplate->find('first', array(
'conditions' => array(
@ -401,130 +389,28 @@ class ObjectsController extends AppController
'ObjectTemplateElement'
)
));
if (empty($template)) {
if (empty($template) && !$this->_isRest()) {
$this->Flash->error('Object cannot be edited, no valid template found.');
$this->redirect(array('controller' => 'events', 'action' => 'view', $object['Object']['event_id']));
}
$newer_template = $this->MispObject->ObjectTemplate->find('first', array(
'conditions' => array(
'ObjectTemplate.uuid' => $object['Object']['template_uuid'],
'ObjectTemplate.version >' => $object['Object']['template_version'],
),
'recursive' => -1,
'contain' => array(
'ObjectTemplateElement'
),
'order' => array('ObjectTemplate.version DESC')
));
if (!empty($newer_template)) {
$newer_template_version = $newer_template['ObjectTemplate']['version'];
// ignore IDs for comparison
$cur_template_temp = Hash::remove(Hash::remove($template['ObjectTemplateElement'], '{n}.id'), '{n}.object_template_id');
$newer_template_temp = Hash::remove(Hash::remove($newer_template['ObjectTemplateElement'], '{n}.id'), '{n}.object_template_id');
$template_difference = array();
// check how current template is included in the newer
foreach ($cur_template_temp as $cur_obj_rel) {
$flag_sim = false;
foreach ($newer_template_temp as $newer_obj_rel) {
$tmp = Hash::diff($cur_obj_rel, $newer_obj_rel);
if (count($tmp) == 0) {
$flag_sim = true;
break;
}
}
if (!$flag_sim) {
$template_difference[] = $cur_obj_rel;
}
}
$updateable_attribute = $object['Attribute'];
$not_updateable_attribute = array();
if (!empty($template_difference)) { // older template not completely embeded in newer
foreach ($template_difference as $temp_diff_element) {
foreach ($object['Attribute'] as $i => $attribute) {
if (
$attribute['object_relation'] == $temp_diff_element['object_relation']
&& $attribute['type'] == $temp_diff_element['type']
) { // This attribute cannot be merged automatically
$attribute['merge-possible'] = false;
$not_updateable_attribute[] = $attribute;
unset($updateable_attribute[$i]);
}
}
}
}
$this->set('updateable_attribute', $updateable_attribute);
$this->set('not_updateable_attribute', $not_updateable_attribute);
if ($update_template_available) { // template version bump requested
$template = $newer_template; // bump the template version
}
} else {
$newer_template_version = false;
$templateData = $this->MispObject->resolveUpdatedTemplate($template, $object, $update_template_available);
$this->set('updateable_attribute', $templateData['updateable_attribute']);
$this->set('not_updateable_attribute', $templateData['not_updateable_attribute']);
if (isset($this->params['named']['revised_object'])) {
$revisedData = $this->MispObject->reviseObject($this->params['named']['revised_object'], $object);
$this->set('revised_object', $revisedData['revised_object_both']);
$object = $revisedData['object'];
}
if (isset($this->params['named']['revised_object'])) { // revised object data to be injected
$revised_object = json_decode(base64_decode($this->params['named']['revised_object']), true);
$revised_object_both = array('mergeable' => array(), 'notMergeable' => array());
// Loop through attributes to inject and perform the correct action
// (inject, duplicate, add warnings, ...) when applicable
foreach ($revised_object['Attribute'] as $attribute_to_inject) {
$flag_no_collision = true;
foreach ($object['Attribute'] as $attribute) {
if (
$attribute['object_relation'] == $attribute_to_inject['object_relation']
&& $attribute['type'] == $attribute_to_inject['type']
&& $attribute['value'] !== $attribute_to_inject['value']
) { // Collision on value
$multiple = !empty(Hash::extract($template['ObjectTemplateElement'], sprintf('{n}[object_relation=%s][type=%s][multiple=true]', $attribute['object_relation'], $attribute['type'])));
if ($multiple) { // if multiple is set, check if an entry exists already
$flag_entry_exists = false;
foreach ($object['Attribute'] as $attr) {
if (
$attr['object_relation'] == $attribute_to_inject['object_relation']
&& $attr['type'] == $attribute_to_inject['type']
&& $attr['value'] === $attribute_to_inject['value']
) {
$flag_entry_exists = true;
break;
}
}
if (!$flag_entry_exists) { // entry does no exists, can be duplicated
$attribute_to_inject['is_multiple'] = true;
$revised_object_both['mergeable'][] = $attribute_to_inject;
$object['Attribute'][] = $attribute_to_inject;
}
} else { // Collision on value, multiple not set => propose overwrite
$attribute_to_inject['current_value'] = $attribute['value'];
$attribute_to_inject['merge-possible'] = true; // the user can still swap value
$revised_object_both['notMergeable'][] = $attribute_to_inject;
}
$flag_no_collision = false;
} else if (
$attribute['object_relation'] == $attribute_to_inject['object_relation']
&& $attribute['type'] == $attribute_to_inject['type']
&& $attribute['value'] === $attribute_to_inject['value']
) { // all good, they are basically the same, do nothing
$revised_object_both['mergeable'][] = $attribute_to_inject;
$flag_no_collision = false;
}
}
if ($flag_no_collision) { // no collision, nor equalities => inject it straight away
$revised_object_both['mergeable'][] = $attribute_to_inject;
$object['Attribute'][] = $attribute_to_inject;
}
}
$this->set('revised_object', $revised_object_both);
if (!empty($templateData['template'])) {
$template = $this->MispObject->prepareTemplate($templateData['template'], $object);
}
$template = $this->MispObject->prepareTemplate($template, $object);
$enabledRows = false;
if ($this->request->is('post') || $this->request->is('put')) {
if (isset($this->request->data['request'])) {
$this->request->data = $this->request->data['request'];
}
if (empty($this->request->data['Object'])) {
$this->request->data['Object'] = $this->request->data;
}
if (isset($this->request->data['Object']['data'])) {
$this->request->data = json_decode($this->request->data['Object']['data'], true);
}
@ -533,34 +419,48 @@ class ObjectsController extends AppController
unset($this->request->data['Object']);
}
$objectToSave = $this->MispObject->attributeCleanup($this->request->data);
$objectToSave = $this->MispObject->deltaMerge($object, $objectToSave);
$objectToSave = $this->MispObject->deltaMerge($object, $objectToSave, $onlyAddNewAttribute);
$error_message = __('Object could not be saved.');
if (!is_numeric($objectToSave)){
$object_validation_errors = array();
foreach($objectToSave as $field => $field_errors) {
$object_validation_errors[] = sprintf('%s: %s', $field, implode(', ', $field_errors));
}
$error_message = __('Object could not be saved.') . PHP_EOL . implode(PHP_EOL, $object_validation_errors);
}
// we pre-validate the attributes before we create an object at this point
// This allows us to stop the process and return an error (API) or return
// to the add form
if (empty($error)) {
if ($this->_isRest()) {
if ($this->_isRest()) {
if (is_numeric($objectToSave)) {
$objectToSave = $this->MispObject->fetchObjects($this->Auth->user(), array('conditions' => array('Object.id' => $id)));
if (!empty($objectToSave)) {
$objectToSave = $objectToSave[0];
$objectToSave['Object']['Attribute'] = $objectToSave['Attribute'];
unset($objectToSave['Attribute']);
}
$this->MispObject->Event->unpublishEvent($objectToSave['Object']['event_id']);
return $this->RestResponse->viewData($objectToSave, $this->response->type());
} else {
return $this->RestResponse->saveFailResponse('Objects', 'edit', false, $id, $this->response->type());
}
} else {
$message = __('Object attributes saved.');
if ($this->request->is('ajax')) {
$this->autoRender = false;
if (is_numeric($objectToSave)) {
$objectToSave = $this->MispObject->find('first', array(
'recursive' => -1,
'conditions' => array('Object.id' => $id),
'contain' => array(
'Attribute' => array(
'fields' => $this->MispObject->Attribute->defaultFields
)
)
));
if (!empty($objectToSave)) {
$objectToSave['Object']['Attribute'] = $objectToSave['Attribute'];
unset($objectToSave['Attribute']);
}
$this->MispObject->Event->unpublishEvent($object['Object']['event_id']);
return $this->RestResponse->viewData($objectToSave, $this->response->type());
return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => $message)), 'status'=>200, 'type' => 'json'));
} else {
return $this->RestResponse->saveFailResponse('Objects', 'add', false, $id, $this->response->type());
return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'errors' => $error_message)), 'status'=>200, 'type' => 'json'));
}
} else {
$this->MispObject->Event->unpublishEvent($object['Object']['event_id']);
$this->Flash->success('Object saved.');
if (is_numeric($objectToSave)) {
$this->MispObject->Event->unpublishEvent($object['Object']['event_id']);
$this->Flash->success('Object saved.');
} else {
$this->Flash->error($error_message);
}
$this->redirect(array('controller' => 'events', 'action' => 'view', $object['Object']['event_id']));
}
}
@ -596,10 +496,324 @@ class ObjectsController extends AppController
$this->set('action', 'edit');
$this->set('object', $object);
$this->set('update_template_available', $update_template_available);
$this->set('newer_template_version', $newer_template_version);
$this->set('newer_template_version', empty($templateData['newer_template_version']) ? false : $templateData['newer_template_version']);
$this->render('add');
}
// ajax edit - post a single edited field and this method will attempt to save it and return a json with the validation errors if they occur.
public function editField($id)
{
if (Validation::uuid($id)) {
$this->MispObject->recursive = -1;
$temp = $this->MispObject->findByUuid($id);
if ($temp == null) {
throw new NotFoundException(__('Invalid object'));
}
$id = $temp['Object']['id'];
} elseif (!is_numeric($id)) {
throw new NotFoundException(__('Invalid event id.'));
}
if ((!$this->request->is('post') && !$this->request->is('put'))) {
throw new MethodNotAllowedException(__('This function can only be accessed via POST or PUT'));
}
$object = $this->MispObject->find('first', array(
'conditions' => array('Object.id' => $id),
'contain' => 'Event',
'recursive' => -1
));
if (empty($object)) {
return $this->RestResponse->saveFailResponse('Objects', 'edit', false, 'Invalid object');
}
if (!$this->_isSiteAdmin()) {
if ($this->MispObject->data['Event']['orgc_id'] == $this->Auth->user('org_id')
&& (($this->userRole['perm_modify'] && $this->MispObject->data['Event']['user_id'] != $this->Auth->user('id'))
|| $this->userRole['perm_modify_org'])) {
// Allow the edit
} else {
return $this->RestResponse->saveFailResponse('Objects', 'edit', false, 'Invalid attribute');
}
}
$validFields = array('comment', 'distribution', 'first_seen', 'last_seen');
$changed = false;
if (empty($this->request->data['Object'])) {
$this->request->data = array('Object' => $this->request->data);
if (empty($this->request->data['Object'])) {
throw new MethodNotAllowedException('Invalid input.');
}
}
$seen_changed = false;
foreach ($this->request->data['Object'] as $changedKey => $changedField) {
if (!in_array($changedKey, $validFields)) {
throw new MethodNotAllowedException('Invalid field.');
}
if ($object['Object'][$changedKey] == $changedField) {
$this->autoRender = false;
return $this->RestResponse->saveSuccessResponse('Objects', 'edit', $id, false, 'nochange');
}
$seen_changed = $changedKey == 'first_seen' || $changedKey == 'last_seen';
$object['Object'][$changedKey] = $changedField;
$changed = true;
}
$forcedSeenOnElements = array();
if (!$changed) {
return $this->RestResponse->saveSuccessResponse('Objects', 'edit', $id, false, 'nochange');
} elseif ($seen_changed) {
$forcedSeenOnElements[$changedKey] = $changedField;
}
$date = new DateTime();
$object['Object']['timestamp'] = $date->getTimestamp();
$object = $this->MispObject->syncObjectAndAttributeSeen($object, $forcedSeenOnElements);
if ($this->MispObject->save($object)) {
$event = $this->MispObject->Event->find('first', array(
'recursive' => -1,
'fields' => array('id', 'published', 'timestamp', 'info', 'uuid'),
'conditions' => array(
'id' => $object['Object']['event_id'],
)));
$event['Event']['timestamp'] = $date->getTimestamp();
$event['Event']['published'] = 0;
if ($seen_changed) {
$this->MispObject->Attribute->saveAttributes($object['Attribute']);
}
$this->MispObject->Event->save($event, array('fieldList' => array('published', 'timestamp', 'info')));
return $this->RestResponse->saveSuccessResponse('Objects', 'edit', $id, false, 'Field updated');
} else {
return $this->RestResponse->saveFailResponse('Objects', 'edit', false, $this->MispObject->validationErrors);
}
}
public function fetchViewValue($id, $field = null)
{
$validFields = array('timestamp', 'comment', 'distribution', 'first_seen', 'last_seen');
if (!isset($field) || !in_array($field, $validFields)) {
throw new MethodNotAllowedException('Invalid field requested.');
}
if (!$this->request->is('ajax')) {
throw new MethodNotAllowedException('This function can only be accessed via AJAX.');
}
$params = array(
'conditions' => array('Object.id' => $id),
'fields' => array('id', 'distribution', 'event_id', $field),
'contain' => array(
'Event' => array(
'fields' => array('distribution', 'id', 'org_id'),
)
),
'flatten' => 1
);
$object = $this->MispObject->fetchObjectSimple($this->Auth->user(), $params);
if (empty($object)) {
throw new NotFoundException(__('Invalid object'));
}
$object = $object[0];
$result = $object['Object'][$field];
if ($field == 'distribution') {
$result=$this->MispObject->shortDist[$result];
}
$this->set('value', $result);
$this->layout = 'ajax';
$this->render('ajax/objectViewFieldForm');
}
public function fetchEditForm($id, $field = null)
{
$validFields = array('distribution', 'comment', 'first_seen', 'last_seen');
if (!isset($field) || !in_array($field, $validFields)) {
throw new MethodNotAllowedException('Invalid field requested.');
}
if (!$this->request->is('ajax')) {
throw new MethodNotAllowedException('This function can only be accessed via AJAX.');
}
$fields = array('id', 'distribution', 'event_id');
$fields[] = $field;
$params = array(
'conditions' => array('Object.id' => $id),
'fields' => $fields,
'contain' => array(
'Event' => array(
'fields' => array('distribution', 'id', 'user_id', 'orgc_id', 'org_id'),
)
)
);
$object = $this->MispObject->fetchObjectSimple($this->Auth->user(), $params);
if (empty($object)) {
throw new NotFoundException(__('Invalid attribute'));
}
$object = $object[0];
if (!$this->_isSiteAdmin()) {
if ($object['Event']['orgc_id'] == $this->Auth->user('org_id')
&& (($this->userRole['perm_modify'] && $object['Event']['user_id'] != $this->Auth->user('id'))
|| $this->userRole['perm_modify_org'])) {
// Allow the edit
} else {
throw new NotFoundException(__('Invalid object'));
}
}
$this->layout = 'ajax';
if ($field == 'distribution') {
$distributionLevels = $this->MispObject->shortDist;
unset($distributionLevels[4]);
$this->set('distributionLevels', $distributionLevels);
}
$this->set('object', $object['Object']);
$fieldURL = ucfirst($field);
$this->render('ajax/objectEdit' . $fieldURL . 'Form');
}
// Construct a template with valid object attributes to add to an object
public function quickFetchTemplateWithValidObjectAttributes($id) {
$this->MispObject->id = $id;
if (!$this->MispObject->exists()) {
if ($this->request->is('ajax')) {
return $this->RestResponse->saveFailResponse('Objects', 'add', false, 'Invalid object', $this->response->type());
} else {
throw new NotFoundException(__('Invalid object'));
}
}
$fields = array('template_uuid', 'template_version', 'id');
$params = array(
'conditions' => array('Object.id' => $id),
'fields' => $fields,
'flatten' => 1,
);
// fetchObjects restrict access based on user
$object = $this->MispObject->fetchObjects($this->Auth->user(), $params);
if (empty($object)) {
if ($this->request->is('ajax')) {
return $this->RestResponse->saveFailResponse('Objects', 'add', false, 'Invalid object', $this->response->type());
} else {
throw new NotFoundException(__('Invalid object'));
}
} else {
$object = $object[0];
}
// get object attributes already set
$objectRelation = array();
foreach($object['Attribute'] as $attr) {
$objectRelation[$attr['object_relation']] = 1;
}
$objectRelation = array_keys($objectRelation);
// get object attribute defined in the object's template
$template = $this->MispObject->ObjectTemplate->find('first', array(
'conditions' => array(
'ObjectTemplate.uuid' => $object['Object']['template_uuid'],
'ObjectTemplate.version' => $object['Object']['template_version'],
),
'recursive' => -1,
'flatten' => 1,
'contain' => 'ObjectTemplateElement'
));
if (empty($template)) {
if ($this->request->is('ajax')) {
return $this->RestResponse->saveFailResponse('Objects', 'add', false, 'Invalid template', $this->response->type());
} else {
throw new NotFoundException(__('Invalid template'));
}
}
// unset object invalid object attribute
foreach($template['ObjectTemplateElement'] as $i => $objAttr) {
if (in_array($objAttr['object_relation'], $objectRelation) && !$objAttr['multiple']) {
unset($template['ObjectTemplateElement'][$i]);
}
}
if ($this->request->is('get') || $this->request->is('post')) {
$this->set('template', $template);
$this->set('objectId', $object['Object']['id']);
$items = array();
foreach ($template['ObjectTemplateElement'] as $objectAttribute) {
$name = sprintf('%s :: %s', $objectAttribute['object_relation'], $objectAttribute['type']);
$items[] = array(
'name' => $name,
'value' => '/objects/quickAddAttributeForm/' . $object['Object']['id'] . '/' . $objectAttribute['object_relation'],
'template' => array(
'name' => $name,
'infoExtra' => $objectAttribute['description'],
)
);
}
$this->set('options', array(
'flag_redraw_chosen' => true
));
$this->set('items', $items);
$this->render('/Elements/generic_picker');
} else {
return $template;
}
}
/**
* GET: Returns a form allowing to add a valid object attribute to an object
* POST/PUT: Add the attribute to the object
*/
public function quickAddAttributeForm($id, $fieldName = null) {
if ($this->request->is('GET')) {
if (!isset($fieldName)) {
throw new MethodNotAllowedException('No field requested.');
}
$this->MispObject->id = $id;
if (!$this->MispObject->exists()) {
throw new NotFoundException(__('Invalid object'));
}
$fields = array('template_uuid', 'template_version', 'id', 'event_id');
$params = array(
'conditions' => array('Object.id' => $id),
'fields' => $fields,
'flatten' => 1,
);
// fetchObjects restrict access based on user
$object = $this->MispObject->fetchObjects($this->Auth->user(), $params);
if (empty($object)) {
throw new NotFoundException(__('Invalid object'));
} else {
$object = $object[0];
}
$template = $this->MispObject->ObjectTemplate->find('first', array(
'conditions' => array(
'ObjectTemplate.uuid' => $object['Object']['template_uuid'],
'ObjectTemplate.version' => $object['Object']['template_version'],
),
'recursive' => -1,
'flatten' => 1,
'contain' => array(
'ObjectTemplateElement' => array('conditions' => array(
'object_relation' => $fieldName
))
)
));
if (empty($template)) {
throw new NotFoundException(__('Invalid object'));
}
if (empty($template['ObjectTemplateElement'])) {
throw new NotFoundException(__('Invalid fields') . ' `' . h($fieldName) . '`');
}
// check if fields can be added
foreach($object['Attribute'] as $i => $objAttr) {
$objectAttrFromTemplate = $template['ObjectTemplateElement'][0];
if ($objAttr['object_relation'] == $fieldName && !$objectAttrFromTemplate['multiple']) {
throw new NotFoundException(__('Invalid field'));
}
}
$template = $this->MispObject->prepareTemplate($template, $object);
$this->layout = 'ajax';
$this->set('object', $object['Object']);
$template_element = $template['ObjectTemplateElement'][0];
unset($template_element['value']); // avoid filling if multiple
$this->set('template_element', $template_element);
$distributionData = $this->MispObject->Event->Attribute->fetchDistributionData($this->Auth->user());
$this->set('distributionData', $distributionData);
$info = array();
foreach ($distributionData['levels'] as $key => $value) {
$info['distribution'][$key] = array('key' => $value, 'desc' => $this->MispObject->Event->Attribute->distributionDescriptions[$key]['formdesc']);
}
$this->set('info', $info);
$this->render('ajax/quickAddAttributeForm');
} else if ($this->request->is('post') || $this->request->is('put')) {
return $this->edit($this->request->data['Object']['id'], false, true);
}
}
public function delete($id, $hard = false)
{
$id = $this->Toolbox->findIdByUuid($this->MispObject, $id);

197
app/Controller/ServersController.php
View File

@ -110,8 +110,13 @@ class ServersController extends AppController
if (empty($combinedArgs['limit'])) {
$combinedArgs['limit'] = 60;
}
$total_count = 0;
$events = $this->Server->previewIndex($id, $this->Auth->user(), $combinedArgs, $total_count);
try {
list($events, $total_count) = $this->Server->previewIndex($id, $this->Auth->user(), $combinedArgs);
} catch (Exception $e) {
$this->Flash->error(__('Download failed.') . ' ' . $e->getMessage());
$this->redirect(array('action' => 'index'));
}
$this->loadModel('Event');
$threat_levels = $this->Event->ThreatLevel->find('all');
$this->set('threatLevels', Set::combine($threat_levels, '{n}.ThreatLevel.id', '{n}.ThreatLevel.name'));
@ -122,11 +127,9 @@ class ServersController extends AppController
$params['pageCount'] = ceil($total_count / $params['limit']);
}
$this->params->params['paging'] = array($this->modelClass => $params);
if (is_array($events)) {
if (count($events) > 60) {
$customPagination->truncateByPagination($events, $params);
}
} else ($events = array());
if (count($events) > 60) {
$customPagination->truncateByPagination($events, $params);
}
$this->set('events', $events);
$this->set('eventDescriptions', $this->Event->fieldDescriptions);
$this->set('analysisLevels', $this->Event->analysisLevels);
@ -150,12 +153,15 @@ class ServersController extends AppController
if (empty($server)) {
throw new NotFoundException('Invalid server ID.');
}
$event = $this->Server->previewEvent($serverId, $eventId);
// work on this in the future to improve the feedback
// 2 = wrong error code
if (is_numeric($event)) {
throw new NotFoundException('Invalid event.');
try {
$event = $this->Server->previewEvent($serverId, $eventId);
} catch (NotFoundException $e) {
throw new NotFoundException(__("Event '$eventId' not found."));
} catch (Exception $e) {
$this->Flash->error(__('Download failed.') . ' ' . $e->getMessage());
$this->redirect(array('action' => 'previewIndex', $serverId));
}
$this->loadModel('Event');
$params = $this->Event->rearrangeEventForView($event, $this->passedArgs, $all);
$this->params->params['paging'] = array('Server' => $params);
@ -250,6 +256,7 @@ class ServersController extends AppController
$defaults = array(
'push' => 0,
'pull' => 0,
'push_sightings' => 0,
'caching_enabled' => 0,
'json' => '[]',
'push_rules' => '[]',
@ -393,6 +400,7 @@ class ServersController extends AppController
}
$this->set('allTags', $allTags);
$this->set('host_org_id', Configure::read('MISP.host_org_id'));
$this->render('edit');
}
}
@ -444,7 +452,7 @@ class ServersController extends AppController
}
if (!$fail) {
// say what fields are to be updated
$fieldList = array('id', 'url', 'push', 'pull', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
$fieldList = array('id', 'url', 'push', 'pull', 'push_sightings', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
$this->request->data['Server']['id'] = $id;
if (isset($this->request->data['Server']['authkey']) && "" != $this->request->data['Server']['authkey']) {
$fieldList[] = 'authkey';
@ -663,13 +671,14 @@ class ServersController extends AppController
if (!Configure::read('MISP.background_jobs')) {
$result = $this->Server->pull($this->Auth->user(), $id, $technique, $s);
if (is_array($result)) {
$success = sprintf(__('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled.', count($result[0]), count($result[1]), $result[2]));
$success = sprintf(__('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled.', count($result[0]), count($result[1]), $result[2], $result[3]));
} else {
$error = $result;
}
$this->set('successes', $result[0]);
$this->set('fails', $result[1]);
$this->set('pulledProposals', $result[2]);
$this->set('pulledSightings', $result[3]);
} else {
$this->loadModel('Job');
$this->Job->create();
@ -907,7 +916,7 @@ class ServersController extends AppController
);
$writeableErrors = array(0 => __('OK'), 1 => __('not found'), 2 => __('is not writeable'));
$readableErrors = array(0 => __('OK'), 1 => __('not readable'));
$gpgErrors = array(0 => __('OK'), 1 => __('FAIL: settings not set'), 2 => __('FAIL: Failed to load GnuPG'), 3 => __('FAIL: Issues with the key/passphrase'), 4 => __('FAIL: encrypt failed'));
$gpgErrors = array(0 => __('OK'), 1 => __('FAIL: settings not set'), 2 => __('FAIL: Failed to load GnuPG'), 3 => __('FAIL: Issues with the key/passphrase'), 4 => __('FAIL: sign failed'));
$proxyErrors = array(0 => __('OK'), 1 => __('not configured (so not tested)'), 2 => __('Getting URL via proxy failed'));
$zmqErrors = array(0 => __('OK'), 1 => __('not enabled (so not tested)'), 2 => __('Python ZeroMQ library not installed correctly.'), 3 => __('ZeroMQ script not running.'));
$stixOperational = array(0 => __('Some of the libraries related to STIX are not installed. Make sure that all libraries listed below are correctly installed.'), 1 => __('OK'));
@ -1052,6 +1061,9 @@ class ServersController extends AppController
// get the DB diagnostics
$dbDiagnostics = $this->Server->dbSpaceUsage();
$dbSchemaDiagnostics = $this->Server->dbSchemaDiagnostic();
$redisInfo = $this->Server->redisInfo();
$moduleTypes = array('Enrichment', 'Import', 'Export', 'Cortex');
foreach ($moduleTypes as $type) {
@ -1063,7 +1075,7 @@ class ServersController extends AppController
$sessionStatus = $this->Server->sessionDiagnostics($diagnostic_errors, $sessionCount);
$this->set('sessionCount', $sessionCount);
$additionalViewVars = array('gpgStatus', 'sessionErrors', 'proxyStatus', 'sessionStatus', 'zmqStatus', 'stixVersion', 'cyboxVersion', 'mixboxVersion', 'maecVersion', 'stix2Version', 'pymispVersion', 'moduleStatus', 'yaraStatus', 'gpgErrors', 'proxyErrors', 'zmqErrors', 'stixOperational', 'stix', 'moduleErrors', 'moduleTypes', 'dbDiagnostics');
$additionalViewVars = array('gpgStatus', 'sessionErrors', 'proxyStatus', 'sessionStatus', 'zmqStatus', 'stixVersion', 'cyboxVersion', 'mixboxVersion', 'maecVersion', 'stix2Version', 'pymispVersion', 'moduleStatus', 'yaraStatus', 'gpgErrors', 'proxyErrors', 'zmqErrors', 'stixOperational', 'stix', 'moduleErrors', 'moduleTypes', 'dbDiagnostics', 'dbSchemaDiagnostics', 'redisInfo');
}
// check whether the files are writeable
$writeableDirs = $this->Server->writeableDirsDiagnostics($diagnostic_errors);
@ -1104,6 +1116,9 @@ class ServersController extends AppController
'writeableDirs' => $writeableDirs,
'writeableFiles' => $writeableFiles,
'readableFiles' => $readableFiles,
'dbDiagnostics' => $dbDiagnostics,
'dbSchemaDiagnostics' => $dbSchemaDiagnostics,
'redisInfo' => $redisInfo,
'finalSettings' => $dumpResults,
'extensions' => $extensions,
'workers' => $worker_array
@ -1127,6 +1142,9 @@ class ServersController extends AppController
$this->set('phpversion', phpversion());
$this->set('phpmin', $this->phpmin);
$this->set('phprec', $this->phprec);
$this->set('pythonmin', $this->pythonmin);
$this->set('pythonrec', $this->pythonrec);
$this->set('pymisp', $this->pymisp);
}
}
@ -1135,12 +1153,23 @@ class ServersController extends AppController
if (!$this->_isSiteAdmin() || !$this->request->is('post')) {
throw new MethodNotAllowedException();
}
$validTypes = array('default', 'email', 'scheduler', 'cache', 'prio');
$validTypes = array('default', 'email', 'scheduler', 'cache', 'prio', 'update');
if (!in_array($type, $validTypes)) {
throw new MethodNotAllowedException('Invalid worker type.');
}
$prepend = '';
if ($type != 'scheduler') {
$workerIssueCount = 0;
$workerDiagnostic = $this->Server->workerDiagnostics($workerIssueCount);
if ($type == 'update' && isset($workerDiagnostic['update']['ok']) && $workerDiagnostic['update']['ok']) {
$message = __('Only one `update` worker can run at a time');
if ($this->_isRest()) {
return $this->RestResponse->saveFailResponse('Servers', 'startWorker', false, $message, $this->response->type());
} else {
$this->Flash->error($message);
$this->redirect('/servers/serverSettings/workers');
}
}
shell_exec($prepend . APP . 'Console' . DS . 'cake CakeResque.CakeResque start --interval 5 --queue ' . $type .' > /dev/null 2>&1 &');
} else {
shell_exec($prepend . APP . 'Console' . DS . 'cake CakeResque.CakeResque startscheduler -i 5 > /dev/null 2>&1 &');
@ -1453,6 +1482,20 @@ class ServersController extends AppController
}
}
public function getRemoteUser($id)
{
$this->Server->id = $id;
if (!$this->Server->exists()) {
throw new NotFoundException(__('Invalid server'));
}
$user = $this->Server->getRemoteUser($id);
if (empty($user)) {
throw new NotFoundException(__('Invalid user or user not found.'));
} else {
return $this->RestResponse->viewData($user);
}
}
public function testConnection($id = false)
{
if (!$this->Auth->user('Role')['perm_sync'] && !$this->Auth->user('Role')['perm_site_admin']) {
@ -1470,6 +1513,10 @@ class ServersController extends AppController
if (isset($version['perm_sync'])) {
$perm_sync = $version['perm_sync'];
}
$perm_sighting = false;
if (isset($version['perm_sighting'])) {
$perm_sighting = $version['perm_sighting'];
}
App::uses('Folder', 'Utility');
$file = new File(ROOT . DS . 'VERSION.json', true);
$local_version = json_decode($file->read(), true);
@ -1496,10 +1543,14 @@ class ServersController extends AppController
if (!$mismatch && $version[2] < 111) {
$mismatch = 'proposal';
}
if (!$perm_sync) {
if (!$perm_sync && !$perm_sighting) {
$result['status'] = 7;
return new CakeResponse(array('body'=> json_encode($result), 'type' => 'json'));
}
if (!$perm_sync && $perm_sighting) {
$result['status'] = 8;
return new CakeResponse(array('body'=> json_encode($result), 'type' => 'json'));
}
return new CakeResponse(
array(
'body'=> json_encode(
@ -1597,7 +1648,7 @@ class ServersController extends AppController
throw new MethodNotAllowedException('This action requires API access.');
}
$versionArray = $this->Server->checkMISPVersion();
$this->set('response', array('version' => $versionArray['major'] . '.' . $versionArray['minor'] . '.' . $versionArray['hotfix'], 'perm_sync' => $this->userRole['perm_sync']));
$this->set('response', array('version' => $versionArray['major'] . '.' . $versionArray['minor'] . '.' . $versionArray['hotfix'], 'perm_sync' => $this->userRole['perm_sync'], 'perm_sighting' => $this->userRole['perm_sighting']));
$this->set('_serialize', 'response');
}
@ -1664,36 +1715,49 @@ class ServersController extends AppController
$this->set('updateLocked', $this->Server->isUpdateLocked());
}
public function updateProgress()
public function updateProgress($ajaxHtml=false)
{
if (!$this->_isSiteAdmin()) {
throw new MethodNotAllowedException('You are not authorised to do that.');
}
$update_progress = $this->Server->getUpdateProgress();
$current_index = $update_progress['current'];
$current_command = !isset($update_progress['commands'][$current_index]) ? '' : $update_progress['commands'][$current_index];
$lookup_string = preg_replace('/\s{2,}/', '', substr($current_command, 0, -1));
$sql_info = $this->Server->query("SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;");
if (empty($sql_info)) {
$update_progress['process_list'] = array();
$this->AdminSetting = ClassRegistry::init('AdminSetting');
$dbVersion = $this->AdminSetting->getSetting('db_version');
$updateProgress = $this->Server->getUpdateProgress();
$updateProgress['db_version'] = $dbVersion;
$maxUpdateNumber = max(array_keys($this->Server->db_changes));
$updateProgress['complete_update_remaining'] = max($maxUpdateNumber - $dbVersion, 0);
$updateProgress['update_locked'] = $this->Server->isUpdateLocked();
$updateProgress['lock_remaining_time'] = $this->Server->getLockRemainingTime();
$updateProgress['update_fail_number_reached'] = $this->Server->UpdateFailNumberReached();
$currentIndex = $updateProgress['current'];
$currentCommand = !isset($updateProgress['commands'][$currentIndex]) ? '' : $updateProgress['commands'][$currentIndex];
$lookupString = preg_replace('/\s{2,}/', '', substr($currentCommand, 0, -1));
$sqlInfo = $this->Server->query("SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST;");
if (empty($sqlInfo)) {
$updateProgress['process_list'] = array();
} else {
// retreive current update process
foreach($sql_info as $row) {
if (preg_replace('/\s{2,}/', '', $row['PROCESSLIST']['INFO']) == $lookup_string) {
$sql_info = $row['PROCESSLIST'];
// retrieve current update process
foreach($sqlInfo as $row) {
if (preg_replace('/\s{2,}/', '', $row['PROCESSLIST']['INFO']) == $lookupString) {
$sqlInfo = $row['PROCESSLIST'];
break;
}
}
$update_progress['process_list'] = array();
$update_progress['process_list']['STATE'] = isset($sql_info['STATE']) ? $sql_info['STATE'] : '';
$update_progress['process_list']['PROGRESS'] = isset($sql_info['PROGRESS']) ? $sql_info['PROGRESS'] : 0;
$update_progress['process_list']['STAGE'] = isset($sql_info['STAGE']) ? $sql_info['STAGE'] : 0;
$update_progress['process_list']['MAX_STAGE'] = isset($sql_info['MAX_STAGE']) ? $sql_info['MAX_STAGE'] : 0;
$updateProgress['process_list'] = array();
$updateProgress['process_list']['STATE'] = isset($sqlInfo['STATE']) ? $sqlInfo['STATE'] : '';
$updateProgress['process_list']['PROGRESS'] = isset($sqlInfo['PROGRESS']) ? $sqlInfo['PROGRESS'] : 0;
$updateProgress['process_list']['STAGE'] = isset($sqlInfo['STAGE']) ? $sqlInfo['STAGE'] : 0;
$updateProgress['process_list']['MAX_STAGE'] = isset($sqlInfo['MAX_STAGE']) ? $sqlInfo['MAX_STAGE'] : 0;
}
if ($this->request->is('ajax')) {
return $this->RestResponse->viewData(h($update_progress), $this->response->type());
$this->set('ajaxHtml', $ajaxHtml);
if ($this->request->is('ajax') && $ajaxHtml) {
$this->set('updateProgress', $updateProgress);
$this->layout = false;
} elseif ($this->request->is('ajax') || $this->_isRest()) {
return $this->RestResponse->viewData(h($updateProgress), $this->response->type());
} else {
$this->set('updateProgress', $update_progress);
$this->set('updateProgress', $updateProgress);
}
}
@ -1897,16 +1961,16 @@ misp_verifycert = %s
relative_path = \'%s\'
body = %s
from pymisp import PyMISP
from pymisp import ExpandedPyMISP
misp = PyMISP(misp_url, misp_key, misp_verifycert)
misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
misp.direct_call(relative_path, body)
',
$baseurl,
$request['header']['Authorization'],
$verifyCert,
$relative,
(empty($request['body']) ? 'Null' : $request['body'])
(empty($request['body']) ? 'None' : $request['body'])
);
return $python_script;
}
@ -2142,4 +2206,53 @@ misp.direct_call(relative_path, body)
return $this->RestResponse->saveFailResponse('Servers', 'changePriority', $id, $message, $this->response->type());
}
}
public function releaseUpdateLock()
{
if (!$this->request->is('post')) {
throw new MethodNotAllowedException(__('This endpoint expects POST requests.'));
}
if (!$this->_isSiteAdmin()) {
throw new MethodNotAllowedException(__('Only site admin accounts can release the update lock.'));
}
$this->Server->changeLockState(false);
$this->Server->resetUpdateFailNumber();
$this->redirect(array('action' => 'updateProgress'));
}
public function dbSchemaDiagnostic()
{
if (!$this->_isSiteAdmin()) {
throw new MethodNotAllowedException(__('Only site admin accounts get the DB schema diagnostic.'));
}
$dbSchemaDiagnostics = $this->Server->dbSchemaDiagnostic();
if ($this->_isRest()) {
return $this->RestResponse->viewData($dbSchemaDiagnostics, $this->response->type());
} else {
$this->set('checkedTableColumn', $dbSchemaDiagnostics['checked_table_column']);
$this->set('dbSchemaDiagnostics', $dbSchemaDiagnostics['diagnostic']);
$this->set('dbIndexDiagnostics', $dbSchemaDiagnostics['diagnostic_index']);
$this->set('expectedDbVersion', $dbSchemaDiagnostics['expected_db_version']);
$this->set('actualDbVersion', $dbSchemaDiagnostics['actual_db_version']);
$this->set('error', $dbSchemaDiagnostics['error']);
$this->set('remainingLockTime', $dbSchemaDiagnostics['remaining_lock_time']);
$this->set('updateFailNumberReached', $dbSchemaDiagnostics['update_fail_number_reached']);
$this->set('updateLocked', $dbSchemaDiagnostics['update_locked']);
$this->set('dataSource', $dbSchemaDiagnostics['dataSource']);
$this->set('columnPerTable', $dbSchemaDiagnostics['columnPerTable']);
$this->set('indexes', $dbSchemaDiagnostics['indexes']);
$this->render('/Elements/healthElements/db_schema_diagnostic');
}
}
public function viewDeprecatedFunctionUse()
{
$data = $this->Deprecation->getDeprecatedAccessList($this->Server);
if ($this->_isRest()) {
return $this->RestResponse->viewData($data, $this->response->type());
} else {
$this->layout = false;
$this->set('data', $data);
}
}
}

36
app/Controller/ShadowAttributesController.php
View File

@ -96,7 +96,7 @@ class ShadowAttributesController extends AppController
$this->Attribute->delete($activeAttribute['Attribute']['id']);
} else {
// Update the live attribute with the shadow data
$fieldsToUpdate = array('value1', 'value2', 'value', 'type', 'category', 'comment', 'to_ids');
$fieldsToUpdate = array('value1', 'value2', 'value', 'type', 'category', 'comment', 'to_ids', 'first_seen', 'last_seen');
foreach ($fieldsToUpdate as $f) {
$activeAttribute['Attribute'][$f] = $shadow[$f];
}
@ -197,24 +197,6 @@ class ShadowAttributesController extends AppController
}
}
// If we accept a proposed attachment, then the attachment itself needs to be moved from files/eventId/shadow/shadowId to files/eventId/attributeId
private function _moveFile($shadowId, $newId, $eventId)
{
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$attachments_dir = $this->ShadowAttribute->getDefaultAttachments_dir();
}
$pathOld = $attachments_dir . DS . 'shadow' . DS . $shadowId;
$pathNew = $attachments_dir . DS . $newId;
if (rename($pathOld, $pathNew)) {
return true;
} else {
$this->Flash->error(__('Moving of the file that this attachment references failed.', true), 'default', array());
$this->redirect(array('controller' => 'events', 'action' => 'view', $eventId));
}
}
private function __discard($id)
{
$sa = $this->ShadowAttribute->find(
@ -433,7 +415,7 @@ class ShadowAttributesController extends AppController
array(
'conditions' => array('ShadowAttribute.id' => $this->ShadowAttribute->id),
'recursive' => -1,
'fields' => array('id', 'old_id', 'event_id', 'type', 'category', 'value', 'comment','to_ids', 'uuid', 'event_org_id', 'email', 'deleted', 'timestamp')
'fields' => array('id', 'old_id', 'event_id', 'type', 'category', 'value', 'comment','to_ids', 'uuid', 'event_org_id', 'email', 'deleted', 'timestamp', 'first_seen', 'last_seen')
)
);
$this->set('ShadowAttribute', $sa['ShadowAttribute']);
@ -707,12 +689,12 @@ class ShadowAttributesController extends AppController
if ($attachment) {
$fields = array(
'static' => array('old_id' => 'Attribute.id', 'uuid' => 'Attribute.uuid', 'event_id' => 'Attribute.event_id', 'event_uuid' => 'Event.uuid', 'event_org_id' => 'Event.orgc_id', 'category' => 'Attribute.category', 'type' => 'Attribute.type'),
'optional' => array('value', 'to_ids', 'comment')
'optional' => array('value', 'to_ids', 'comment', 'first_seen', 'last_seen')
);
} else {
$fields = array(
'static' => array('old_id' => 'Attribute.id', 'uuid' => 'Attribute.uuid', 'event_id' => 'Attribute.event_id', 'event_uuid' => 'Event.uuid', 'event_org_id' => 'Event.orgc_id'),
'optional' => array('category', 'type', 'value', 'to_ids', 'comment')
'optional' => array('category', 'type', 'value', 'to_ids', 'comment', 'first_seen', 'last_seen')
);
if ($existingAttribute['Attribute']['object_id']) {
unset($fields['optional']['type']);
@ -749,7 +731,7 @@ class ShadowAttributesController extends AppController
array(
'conditions' => array('ShadowAttribute.id' => $this->ShadowAttribute->id),
'recursive' => -1,
'fields' => array('id', 'old_id', 'event_id', 'type', 'category', 'value', 'comment','to_ids', 'uuid', 'event_org_id', 'email', 'deleted', 'timestamp')
'fields' => array('id', 'old_id', 'event_id', 'type', 'category', 'value', 'comment','to_ids', 'uuid', 'event_org_id', 'email', 'deleted', 'timestamp', 'first_seen', 'last_seen')
)
);
$this->set('ShadowAttribute', $sa['ShadowAttribute']);
@ -851,6 +833,8 @@ class ShadowAttributesController extends AppController
'type' => $existingAttribute['Attribute']['type'],
'to_ids' => $existingAttribute['Attribute']['to_ids'],
'value' => $existingAttribute['Attribute']['value'],
'first_seen' => $existingAttribute['Attribute']['first_seen'],
'last_seen' => $existingAttribute['Attribute']['last_seen'],
'email' => $this->Auth->user('email'),
'org_id' => $this->Auth->user('org_id'),
'proposal_to_delete' => true,
@ -891,7 +875,7 @@ class ShadowAttributesController extends AppController
'recursive' => -1,
'contain' => 'Event',
'fields' => array(
'ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id',
'ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.first_seen', 'ShadowAttribute.last_seen',
'Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.distribution', 'Event.uuid'
),
'conditions' => array('AND' => array('ShadowAttribute.id' => $id, $distConditions, 'ShadowAttribute.deleted' => 0))
@ -962,7 +946,7 @@ class ShadowAttributesController extends AppController
}
$params = array(
'conditions' => $conditions,
'fields' => array('ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.timestamp'),
'fields' => array('ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.timestamp', 'ShadowAttribute.first_seen', 'ShadowAttribute.last_seen'),
'contain' => array(
'Event' => array(
'fields' => array('id', 'org_id', 'info', 'orgc_id', 'uuid'),
@ -1142,7 +1126,7 @@ class ShadowAttributesController extends AppController
}
}
$keys = array_flip(array('uuid', 'event_id', 'value', 'type', 'category', 'to_ids'));
$keys = array_flip(array('uuid', 'event_id', 'value', 'type', 'category', 'to_ids', 'first_seen', 'last_seen'));
$proposal = array_intersect_key($attribute['Attribute'], $keys);
$proposal['email'] = $this->Auth->user('email');

224
app/Controller/SightingdbController.php Normal file
View File

@ -0,0 +1,224 @@
<?php
App::uses('AppController', 'Controller');
class SightingdbController extends AppController
{
public $components = array('Session', 'RequestHandler');
public $paginate = array(
'limit' => 60,
'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page.
'order' => array(
'Sightingdb.id' => 'DESC'
),
'recursive' => -1,
'contain' => array('SightingdbOrg' => 'Organisation')
);
public function beforeFilter()
{
parent::beforeFilter();
$this->Security->unlockedActions = array('search');
}
public function add()
{
if ($this->request->is('post')) {
if (empty($this->request->data['Sightingdb'])) {
$this->request->data = array('Sightingdb' => $this->request->data);
}
$this->Sightingdb->create();
$result = $this->Sightingdb->save($this->request->data);
$message = $result ? __('SightingDB connection added.') : __('SightingDB connection could not be added.');
if ($result) {
if (isset($this->request->data['Sightingdb']['org_id'])) {
$this->Sightingdb->SightingdbOrg->resetOrgs($this->Sightingdb->id, $this->request->data['Sightingdb']['org_id']);
}
}
if ($this->_isRest()) {
if ($result) {
return $this->RestResponse->saveSuccessResponse('Sightingdb', 'add', $this->response->type(), $message);
} else {
return $this->RestResponse->saveFailResponse('Sightingdb', 'add', $message, $this->response->type());
}
} else {
if ($result) {
$this->Flash->success($message);
$this->redirect(array('action' => 'index'));
} else {
$message .= __(' Reason: %s', json_encode($this->Sightingdb->validationErrors, true));
$this->Flash->error($message);
}
}
}
$orgs = $this->Sightingdb->SightingdbOrg->Organisation->find('list', array(
'conditions' => array('Organisation.local' => 1),
'order' => array('LOWER(Organisation.name)'),
'fields' => array('Organisation.id', 'Organisation.name')
));
$this->set('orgs', $orgs);
}
public function edit($id)
{
$existingEntry = $this->Sightingdb->find('first', array(
'recursive' => -1,
'conditions' => array('Sightingdb.id' => $id),
'contain' => array('SightingdbOrg.org_id')
));
$existingEntry = $this->Sightingdb->extractOrgIds($existingEntry);
if (empty($id) || empty($existingEntry)) {
throw new NotFoundException(__('Invalid SightingDB entry.'));
}
if ($this->request->is('post') || $this->request->is('put')) {
if (empty($this->request->data['Sightingdb'])) {
$this->request->data = array('Sightingdb' => $this->request->data);
}
$keys = array('host', 'port', 'description', 'name', 'owner', 'enabled', 'skip_proxy', 'ssl_skip_verification', 'namespace');
foreach ($keys as $key) {
if (!empty($this->request->data['Sightingdb'][$key])) {
$existingEntry['Sightingdb'][$key] = $this->request->data['Sightingdb'][$key];
}
}
$result = $this->Sightingdb->save($existingEntry);
if (isset($this->request->data['Sightingdb']['org_id'])) {
$this->Sightingdb->SightingdbOrg->resetOrgs($this->Sightingdb->id, $this->request->data['Sightingdb']['org_id']);
}
$message = $result ? __('SightingDB connection updated.') : __('SightingDB connection could not be updated.');
if ($this->_isRest()) {
if ($result) {
return $this->RestResponse->saveSuccessResponse('Sightingdb', 'edit', $id, $this->response->type(), $message);
} else {
return $this->RestResponse->saveFailResponse('Sightingdb', 'edit', $id, $message, $this->response->type());
}
} else {
if ($result) {
$this->Flash->success($message);
$this->redirect(array('action' => 'index'));
} else {
$message .= __(' Reason: %s', json_encode($this->Sightingdb->validationErrors, true));
$this->Flash->error($message);
}
}
} else {
$this->request->data = $existingEntry;
}
$orgs = $this->Sightingdb->SightingdbOrg->Organisation->find('list', array(
'conditions' => array('Organisation.local' => 1),
'order' => array('LOWER(Organisation.name)'),
'fields' => array('Organisation.id', 'Organisation.name')
));
$this->set('id', $id);
$this->set('orgs', $orgs);
$this->render('/Sightingdb/add');
}
public function delete($id)
{
$existingEntry = $this->Sightingdb->find('first', array(
'recursive' => -1,
'conditions' => array('Sightingdb.id' => $id)
));
if (empty($id) || empty($existingEntry)) {
throw new NotFoundException(__('Invalid SightingDB entry.'));
}
if ($this->request->is('post') || $this->request->is('delete')) {
$result = $this->Sightingdb->delete($existingEntry['Sightingdb']['id']);
if ($result) {
$message = __('SightingDB connection removed.');
} else {
$message = __('SightingDB connection could not be removed.');
}
if ($this->_isRest()) {
if ($result) {
return $this->RestResponse->saveSuccessResponse('Sightingdb', 'edit', $id, $this->response->type(), $message);
} else {
return $this->RestResponse->saveFailResponse('Sightingdb', 'edit', $id, $message, $this->response->type());
}
} else {
if ($result) {
$this->Flash->success($message);
$this->redirect(array('action' => 'index'));
} else {
$message .= __(' Reason: %s', json_encode($this->Sightingdb->validationErrors, true));
$this->Flash->error($message);
}
$this->redirect(array('action' => 'index'));
}
}
}
public function index()
{
$filters = $this->IndexFilter->harvestParameters(array('value'));
if (!empty($filters['value'])) {
if (is_array($filters['value'])) {
foreach ($filters['value'] as &$value) {
$value = '%' . strtolower($value) . '%';
}
} else {
$filters['value'] = '%' . strtolower($filters['value']) . '%';
}
$this->paginate['conditions']['AND'][] = array(
'OR' => array(
'Sightingdb.name LIKE' => $filters['value'],
'Sightingdb.owner LIKE' => $filters['value'],
'Sightingdb.host LIKE' => $filters['value']
)
);
}
if ($this->_isRest()) {
$params = array(
'contain' => $this->paginate['contain'],
'conditions' => empty($this->paginate['conditions']) ? array() : $this->paginate['conditions'],
);
$data = $this->Sightingdb->find('all', $params);
$data = $this->Sightingdb->extractOrgIdsFromList($data);
return $this->RestResponse->viewData($data, $this->response->type());
} else {
$this->set('data', $this->paginate());
}
}
public function requestStatus($id)
{
$result = $this->Sightingdb->requestStatus($id);
if (is_array($result)) {
return $this->RestResponse->viewData($result, $this->response->type());
} else {
return $this->RestResponse->saveFailResponse('Sightingdb', 'requestStatus', $id, $result, $this->response->type());
}
}
public function search($id)
{
if (empty($id)) {
throw new InvalidArgumentException(__('Pass a valid SightingDB ID'));
}
$sightingdb = $this->Sightingdb->find('first', array(
'recursive' => -1,
'conditions' => array('Sightingdb.id' => $id),
'contain' => array('SightingdbOrg')
));
if (empty($sightingdb)) {
throw new NotFoundException('Invalid sightingDB');
}
if (!empty($this->request->data['value'])) {
$requestValue = trim($this->request->data['value']);
$result = $this->Sightingdb->queryValues(array($requestValue => array()), $sightingdb);
if (!empty($result[$requestValue][$sightingdb['Sightingdb']['id']])) {
$result = $result[$requestValue][$sightingdb['Sightingdb']['id']];
$result = array(
'first_seen' => date('Y-m-d H:i:s', $result['first_seen']),
'last_seen' => date('Y-m-d H:i:s', $result['last_seen']),
'count' => $result['count']
);
} else {
$result = array('count' => 0);
}
} else {
$result = array('count' => 0);
}
return $this->RestResponse->viewData($result, $this->response->type());
}
}

62
app/Controller/SightingsController.php
View File

@ -66,7 +66,7 @@ class SightingsController extends AppController
$source = isset($this->request->data['source']) ? trim($this->request->data['source']) : '';
}
if (!$error) {
$result = $this->Sighting->saveSightings($id, $values, $timestamp, $this->Auth->user(), $type, $source);
$result = $this->Sighting->saveSightings($id, $values, $timestamp, $this->Auth->user(), $type, $source, false, true);
}
if (!is_numeric($result)) {
$error = $result;
@ -276,42 +276,6 @@ class SightingsController extends AppController
return $this->RestResponse->viewData($sightings);
}
public function restSearch($context = false)
{
$allowedContext = array(false, 'event', 'attribute');
$paramArray = array('returnFormat', 'id', 'type', 'from', 'to', 'last', 'org_id', 'source', 'includeAttribute', 'includeEvent');
$filterData = array(
'request' => $this->request,
'named_params' => $this->params['named'],
'paramArray' => $paramArray,
'ordered_url_params' => compact($paramArray)
);
$filters = $this->_harvestParameters($filterData, $exception);
// validate context
if (!in_array($context, $allowedContext, true)) {
throw new MethodNotAllowedException(_('Invalid context.'));
}
// ensure that an id is provided if context is set
if ($context !== false && !isset($filters['id'])) {
throw new MethodNotAllowedException(_('An id must be provided if the context is set.'));
}
$filters['context'] = $context;
if (isset($filters['returnFormat'])) {
$returnFormat = $filters['returnFormat'];
}
if ($returnFormat === 'download') {
$returnFormat = 'json';
}
$sightings = $this->Sighting->restSearch($this->Auth->user(), $returnFormat, $filters);
$validFormats = $this->Sighting->validFormats;
$responseType = $validFormats[$returnFormat][0];
return $this->RestResponse->viewData($sightings, $responseType, false, true);
}
public function listSightings($id = false, $context = 'attribute', $org_id = false)
{
$rawId = $id;
@ -419,4 +383,28 @@ class SightingsController extends AppController
$this->layout = 'ajax';
$this->render('ajax/view_sightings');
}
// Save sightings synced over, restricted to sync users
public function bulkSaveSightings($eventId = false)
{
if ($this->request->is('post')) {
if (empty($this->request->data['Sighting'])) {
$sightings = $this->request->data;
} else {
$sightings = $this->request->data['Sighting'];
}
$saved = $this->Sighting->bulkSaveSightings($eventId, $sightings, $this->Auth->user());
if (is_numeric($saved)) {
if ($saved > 0) {
return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => $saved . ' sightings added.')), 'status' => 200, 'type' => 'json'));
} else {
return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'success' => 'No sightings added.')), 'status' => 200, 'type' => 'json'));
}
} else {
throw new MethodNotAllowedException($saved);
}
} else {
throw new MethodNotAllowedException('This method is only accessible via POST requests.');
}
}
}

6
app/Controller/TagCollectionsController.php
View File

@ -237,6 +237,9 @@ class TagCollectionsController extends AppController
$RearrangeTool = new RequestRearrangeTool();
$this->request->data = $RearrangeTool->rearrangeArray($this->request->data, $rearrangeRules);
if ($id === false) {
if (!isset($this->request->data['tag_collection'])) {
throw new NotFoundException(__('Invalid tag collection'));
}
$id = $this->request->data['tag_collection'];
}
if (!$this->request->is('post')) {
@ -247,6 +250,9 @@ class TagCollectionsController extends AppController
$this->render('/Events/add_tag');
} else {
if ($tag_id === false) {
if (!isset($this->request->data['tag'])) {
throw new NotFoundException(__('Invalid tag'));
}
$tag_id = $this->request->data['tag'];
}
$conditions = array();

121
app/Controller/TagsController.php
View File

@ -459,6 +459,7 @@ class TagsController extends AppController
public function showEventTag($id)
{
$this->loadModel('EventTag');
$this->loadModel('Taxonomy');
if (!$this->EventTag->Event->checkIfAuthorised($this->Auth->user(), $id)) {
throw new MethodNotAllowedException('Invalid event.');
}
@ -487,6 +488,8 @@ class TagsController extends AppController
'conditions' => array('Event.id' => $id)
));
$this->set('required_taxonomies', $this->EventTag->Event->getRequiredTaxonomies());
$tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($tags);
$this->set('tagConflicts', $tagConflicts);
$this->set('event', $event);
$this->layout = 'ajax';
$this->render('/Events/ajax/ajaxTags');
@ -496,6 +499,7 @@ class TagsController extends AppController
{
$this->helpers[] = 'TextColour';
$this->loadModel('AttributeTag');
$this->loadModel('Taxonomy');
$this->Tag->AttributeTag->Attribute->id = $id;
if (!$this->Tag->AttributeTag->Attribute->exists()) {
@ -528,6 +532,8 @@ class TagsController extends AppController
$this->set('event', $event);
$this->set('attributeTags', $attributeTags);
$this->set('attributeId', $id);
$tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attributeTags);
$this->set('tagConflicts', $tagConflicts);
$this->layout = 'ajax';
$this->render('/Attributes/ajax/ajaxAttributeTags');
}
@ -854,47 +860,62 @@ class TagsController extends AppController
$this->render('/Servers/json/simple');
}
private function __findObjectByUuid($object_uuid, &$type)
private function __findObjectByUuid($object_uuid, &$type, $scope = 'modify')
{
$this->loadModel('Event');
$object = $this->Event->find('first', array(
'conditions' => array(
'Event.uuid' => $object_uuid,
),
'fields' => array('Event.orgc_id', 'Event.id'),
'recursive' => -1
if (!$this->userRole['perm_tagger']) {
throw new MethodNotAllowedException(__('This functionality requires tagging permission.'));
}
$object = $this->Event->fetchEvent($this->Auth->user(), array(
'event_uuid' => $object_uuid,
'metadata' => 1
));
$type = 'Event';
if (!empty($object)) {
$object = $object[0];
if (
$scope !== 'view' &&
!$this->_isSiteAdmin() &&
!$this->userRole['perm_tagger'] &&
$object['Event']['orgc_id'] != $this->Auth->user('org_id')
) {
throw new MethodNotAllowedException('Invalid Target.');
$message = __('Cannot alter the tags of this data, only the organisation that has created the data (orgc) can modify global tags.');
if ($this->Auth->user('org_id') === Configure::read('MISP.host_org_id')) {
$message .= ' ' . __('Please consider using local tags if you are in the host organisation of the instance.');
}
throw new MethodNotAllowedException($message);
}
} else {
$type = 'Attribute';
$object = $this->Event->Attribute->find('first', array(
'conditions' => array(
'Attribute.uuid' => $object_uuid,
),
'fields' => array('Attribute.id'),
'recursive' => -1,
'contain' => array('Event.orgc_id')
));
$object = $this->Event->Attribute->fetchAttributes(
$this->Auth->user(),
array(
'conditions' => array(
'Attribute.uuid' => $object_uuid
),
'flatten' => 1
)
);
if (!empty($object)) {
if (!$this->_isSiteAdmin() && !$this->userRole['perm_tagger'] && $object['Event']['orgc_id'] != $this->Auth->user('org_id')) {
throw new MethodNotAllowedException('Invalid Target.');
$object = $object[0];
if (
$scope !== 'view' &&
!$this->_isSiteAdmin() &&
$object['Event']['orgc_id'] != $this->Auth->user('org_id')
) {
$message = __('Cannot alter the tags of this data, only the organisation that has created the data (orgc) can modify global tags.');
if ($this->Auth->user('org_id') === Configure::read('MISP.host_org_id')) {
$message .= ' ' . __('Please consider using local tags if you are in the host organisation of the instance.');
}
throw new MethodNotAllowedException($message);
}
} else {
throw new MethodNotAllowedException('Invalid Target.');
throw new MethodNotAllowedException(__('Invalid Target.'));
}
}
return $object;
}
public function attachTagToObject($uuid = false, $tag = false)
public function attachTagToObject($uuid = false, $tag = false, $local = false)
{
if (!$this->request->is('post')) {
throw new MethodNotAllowedException('This method is only accessible via POST requests.');
@ -921,8 +942,16 @@ class TagsController extends AppController
} else {
$conditions = array('LOWER(Tag.name) LIKE' => strtolower(trim($tag)));
}
if (empty($local)) {
if (!empty($this->request->data['local'])) {
$local = $this->request->data['local'];
}
}
if (!empty($local) && $this->Auth->user('org_id') != Configure::read('MISP.host_org_id')) {
throw new MethodNotAllowedException(__('Local tags can only be added by users of the host organisation.'));
}
$objectType = '';
$object = $this->__findObjectByUuid($uuid, $objectType);
$object = $this->__findObjectByUuid($uuid, $objectType, $local ? 'view' : 'modify');
$existingTag = $this->Tag->find('first', array('conditions' => $conditions, 'recursive' => -1));
if (empty($existingTag)) {
if (!is_numeric($tag)) {
@ -930,7 +959,10 @@ class TagsController extends AppController
throw new MethodNotAllowedException('Tag not found and insufficient privileges to create it.');
}
$this->Tag->create();
$this->Tag->save(array('Tag' => array('name' => $tag, 'colour' => $this->Tag->random_color())));
$result = $this->Tag->save(array('Tag' => array('name' => $tag, 'colour' => $this->Tag->random_color())));
if (!$result) {
return $this->RestResponse->saveFailResponse('Tags', 'attachTagToObject', false, __('Unable to create tag. Reason: ' . json_encode($this->Tag->validationErrors)), $this->response->type());
}
$existingTag = $this->Tag->find('first', array('recursive' => -1, 'conditions' => array('Tag.id' => $this->Tag->id)));
} else {
throw new NotFoundException('Invalid Tag.');
@ -938,33 +970,28 @@ class TagsController extends AppController
}
if (!$this->_isSiteAdmin()) {
if (!in_array($existingTag['Tag']['org_id'], array(0, $this->Auth->user('org_id')))) {
throw new MethodNotAllowedException('Invalid Tag.');
throw new MethodNotAllowedException('Invalid Tag. This tag can only be set by a fixed organisation.');
}
if (!in_array($existingTag['Tag']['user_id'], array(0, $this->Auth->user('id')))) {
throw new MethodNotAllowedException('Invalid Tag.');
throw new MethodNotAllowedException('Invalid Tag. This tag can only be set by a fixed user.');
}
}
$this->loadModel($objectType);
$connectorObject = $objectType . 'Tag';
$conditions = array(
strtolower($objectType) . '_id' => $object[$objectType]['id'],
'tag_id' => $existingTag['Tag']['id']
'tag_id' => $existingTag['Tag']['id'],
'local' => ($local ? 1 : 0)
);
$existingAssociation = $this->$objectType->$connectorObject->find('first', array(
'conditions' => array(
strtolower($objectType) . '_id' => $object[$objectType]['id'],
'tag_id' => $existingTag['Tag']['id']
)
'conditions' => $conditions
));
if (!empty($existingAssociation)) {
return $this->RestResponse->saveSuccessResponse('Tags', 'attachTagToObject', false, $this->response->type(), $objectType . ' already has the requested tag attached, no changes had to be made.');
}
$this->$objectType->$connectorObject->create();
$data = array(
$connectorObject => array(
strtolower($objectType) . '_id' => $object[$objectType]['id'],
'tag_id' => $existingTag['Tag']['id']
)
$connectorObject => $conditions
);
if ($objectType == 'Attribute') {
$data[$connectorObject]['event_id'] = $object['Event']['id'];
@ -978,12 +1005,16 @@ class TagsController extends AppController
$date = new DateTime();
$tempObject[$objectType]['timestamp'] = $date->getTimestamp();
$this->$objectType->save($tempObject);
if ($objectType === 'Attribute') {
$this->$objectType->Event->unpublishEvent($object['Event']['id']);
} else if ($objectType === 'Event') {
$this->Event->unpublishEvent($object['Event']['id']);
if($local) {
$message = 'Local tag ' . $existingTag['Tag']['name'] . '(' . $existingTag['Tag']['id'] . ') successfully attached to ' . $objectType . '(' . $object[$objectType]['id'] . ').';
} else {
if ($objectType === 'Attribute') {
$this->$objectType->Event->unpublishEvent($object['Event']['id']);
} else if ($objectType === 'Event') {
$this->Event->unpublishEvent($object['Event']['id']);
}
$message = 'Global tag ' . $existingTag['Tag']['name'] . '(' . $existingTag['Tag']['id'] . ') successfully attached to ' . $objectType . '(' . $object[$objectType]['id'] . ').';
}
$message = 'Tag ' . $existingTag['Tag']['name'] . '(' . $existingTag['Tag']['id'] . ') successfully attached to ' . $objectType . '(' . $object[$objectType]['id'] . ').';
return $this->RestResponse->saveSuccessResponse('Tags', 'attachTagToObject', false, $this->response->type(), $message);
} else {
return $this->RestResponse->saveFailResponse('Tags', 'attachTagToObject', false, 'Failed to attach tag to object.', $this->response->type());
@ -1022,9 +1053,9 @@ class TagsController extends AppController
throw new MethodNotAllowedException('Invalid Tag.');
}
$objectType = '';
$object = $this->__findObjectByUuid($uuid, $objectType);
$object = $this->__findObjectByUuid($uuid, $objectType, 'view');
if (empty($object)) {
throw new MethodNotAllowedException('Invalid Target.');
throw new MethodNotAllowedException(__('Invalid Target.'));
}
$connectorObject = $objectType . 'Tag';
$this->loadModel($objectType);
@ -1036,6 +1067,14 @@ class TagsController extends AppController
));
if (empty($existingAssociation)) {
throw new MethodNotAllowedException('Could not remove tag as it is not attached to the target ' . $objectType);
} else {
if (empty($existingAssociation[$objectType . 'Tag']['local'])) {
$object = $this->__findObjectByUuid($uuid, $objectType);
} else {
if ($object['Event']['orgc_id'] !== $this->Auth->user('org_id') && $this->Auth->user('org_id') != Configure::read('MISP.host_org_id')) {
throw new MethodNotAllowedException(__('Insufficient privileges to remove local tags from events you do not own.'));
}
}
}
$result = $this->$objectType->$connectorObject->delete($existingAssociation[$connectorObject]['id']);
if ($result) {

13
app/Controller/TaxonomiesController.php
View File

@ -22,7 +22,18 @@ class TaxonomiesController extends AppController
public function index()
{
$this->paginate['recursive'] = -1;
$taxonomies = $this->paginate();
if ($this->_isRest()) {
$keepFields = array('conditions', 'contain', 'recursive', 'sort');
$searchParams = array();
foreach ($keepFields as $field) {
if (!empty($this->paginate[$field])) {
$searchParams[$field] = $this->paginate[$field];
}
}
$taxonomies = $this->Taxonomy->find('all', $searchParams);
} else {
$taxonomies = $this->paginate();
}
$this->loadModel('Tag');
foreach ($taxonomies as $key => $taxonomy) {
$total = 0;

361
app/Controller/UserSettingsController.php Normal file
View File

@ -0,0 +1,361 @@
<?php
/*
*
* Feature developed as part of a training given by CIRCL in Luxembourg on 26/09/2019
* Verbose comments for educational purposes only
*
*/
App::uses('AppController', 'Controller');
class UserSettingsController extends AppController
{
public $components = array('Session', 'RequestHandler');
public $paginate = array(
'limit' => 60,
'maxLimit' => 9999,
'order' => array(
'UserSetting.id' => 'DESC'
),
'contain' => array(
'User.id',
'User.email'
)
);
public function index()
{
$filterData = array(
'request' => $this->request,
'paramArray' => array('setting', 'user_id', 'sort', 'direction', 'page', 'limit'),
'named_params' => $this->params['named']
);
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
$conditions = array();
if (!empty($filters['setting'])) {
$conditions['AND'][] = array(
'setting' => $filters['setting']
);
}
if (!empty($filters['user_id'])) {
if ($filters['user_id'] === 'all') {
$context = 'all';
} else if ($filters['user_id'] === 'me') {
$conditions['AND'][] = array(
'user_id' => $this->Auth->user('id')
);
$context = 'me';
} else if ($filters['user_id'] === 'org') {
$conditions['AND'][] = array(
'user_id' => $this->UserSetting->User->find(
'list', array(
'conditions' => array(
'User.org_id' => $this->Auth->user('org_id')
),
'fields' => array(
'User.id', 'User.id'
)
)
)
);
$context = 'org';
} else {
$conditions['AND'][] = array(
'user_id' => $filters['user_id']
);
}
}
if (!$this->_isSiteAdmin()) {
if ($this->_isAdmin()) {
$conditions['AND'][] = array(
'UserSetting.user_id' => $this->UserSetting->User->find(
'list', array(
'conditions' => array(
'User.org_id' => $this->Auth->user('org_id')
),
'fields' => array(
'User.id', 'User.id'
)
)
)
);
} else {
$conditions['AND'][] = array(
'UserSetting.user_id' => $this->Auth->user('id')
);
}
}
if ($this->_isRest()) {
$params = array(
'conditions' => $conditions
);
if (!empty($filters['page'])) {
$params['page'] = $filters['page'];
$params['limit'] = $this->paginate['limit'];
}
if (!empty($filters['limit'])) {
$params['limit'] = $filters['limit'];
}
$userSettings = $this->UserSetting->find('all', $params);
return $this->RestResponse->viewData($userSettings, $this->response->type());
} else {
$this->paginate['conditions'] = $conditions;
$data = $this->paginate();
foreach ($data as $k => $v) {
if (!empty($this->UserSetting->validSettings[$v['UserSetting']['setting']])) {
$data[$k]['UserSetting']['restricted'] = empty($this->UserSetting->validSettings[$v['UserSetting']['setting']]['restricted']) ? '' : $this->UserSetting->validSettings[$v['UserSetting']['setting']]['restricted'];
} else {
$data[$k]['UserSetting']['restricted'] = array();
}
}
$this->set('data', $data);
$this->set('context', empty($context) ? 'null' : $context);
}
}
public function view($id)
{
// check if the ID is valid and whether a user setting with the given ID exists
if (empty($id) || !is_numeric($id)) {
throw new InvalidArgumentException(__('Invalid ID passed.'));
}
$userSetting = $this->UserSetting->find('first', array(
'recursive' => -1,
'conditions' => array(
'UserSetting.id' => $id
),
'contain' => array('User.id', 'User.org_id')
));
if (empty($userSetting)) {
throw new NotFoundException(__('Invalid user setting.'));
}
$checkAccess = $this->UserSetting->checkAccess($this->Auth->user(), $userSetting);
if (!$checkAccess) {
throw new NotFoundException(__('Invalid user setting.'));
}
if ($this->_isRest()) {
unset($userSetting['User']);
return $this->RestResponse->viewData($userSetting, $this->response->type());
} else {
$this->set($data, $userSetting);
}
}
public function setSetting($user_id = false, $setting = false)
{
if (!empty($setting)) {
if (!$this->UserSetting->checkSettingValidity($setting)) {
throw new MethodNotAllowedException(__('Invalid setting.'));
}
$settingPermCheck = $this->UserSetting->checkSettingAccess($this->Auth->user(), $setting);
if ($settingPermCheck !== true) {
throw new MethodNotAllowedException(__('This setting is restricted and requires the following permission(s): %s', $settingPermCheck));
}
}
// handle POST requests
if ($this->request->is('post')) {
// massage the request to allow for unencapsulated POST requests via the API
// {"key": "value"} instead of {"UserSetting": {"key": "value"}}
if (empty($this->request->data['UserSetting'])) {
$this->request->data = array('UserSetting' => $this->request->data);
}
if (!empty($user_id)) {
$this->request->data['UserSetting']['user_id'] = $user_id;
}
if (!empty($setting)) {
$this->request->data['UserSetting']['setting'] = $setting;
}
// force our user's ID as the user ID in all cases
$userSetting = array(
'user_id' => $this->Auth->user('id')
);
if (!empty($this->request->data['UserSetting']['user_id']) && is_numeric($this->request->data['UserSetting']['user_id'])) {
$user = $this->UserSetting->User->find('first', array(
'recursive' => -1,
'conditions' => array('User.id' => $this->request->data['UserSetting']['user_id']),
'fields' => array('User.org_id')
));
if (
$this->_isSiteAdmin() ||
($this->_isAdmin() && ($user['User']['org_id'] == $this->Auth->user('org_id')))
) {
$userSetting['user_id'] = $this->request->data['UserSetting']['user_id'];
}
}
if (empty($this->request->data['UserSetting']['setting']) || !isset($this->request->data['UserSetting']['setting'])) {
throw new MethodNotAllowedException(__('This endpoint expects both a setting and a value to be set.'));
}
if (!$this->UserSetting->checkSettingValidity($this->request->data['UserSetting']['setting'])) {
throw new MethodNotAllowedException(__('Invalid setting.'));
}
$settingPermCheck = $this->UserSetting->checkSettingAccess($this->Auth->user(), $this->request->data['UserSetting']['setting']);
if ($settingPermCheck !== true) {
throw new MethodNotAllowedException(__('This setting is restricted and requires the following permission(s): %s', $settingPermCheck));
}
$userSetting['setting'] = $this->request->data['UserSetting']['setting'];
if ($this->request->data['UserSetting']['value'] !== '') {
$userSetting['value'] = json_encode(json_decode($this->request->data['UserSetting']['value'], true));
} else {
$userSetting['value'] = '';
}
$existingSetting = $this->UserSetting->find('first', array(
'recursive' => -1,
'conditions' => array(
'UserSetting.user_id' => $userSetting['user_id'],
'UserSetting.setting' => $userSetting['setting']
)
));
if (empty($existingSetting)) {
$this->UserSetting->create();
} else {
$userSetting['id'] = $existingSetting['UserSetting']['id'];
}
// save the setting
$result = $this->UserSetting->save(array('UserSetting' => $userSetting));
if ($result) {
// if we've managed to save our setting
if ($this->_isRest()) {
// if we are dealing with an API request
$userSetting = $this->UserSetting->find('first', array(
'recursive' => -1,
'conditions' => array('UserSetting.id' => $this->UserSetting->id)
));
return $this->RestResponse->viewData($userSetting, $this->response->type());
} else {
// if we are dealing with a UI request, redirect the user to the user view with the proper flash message
$this->Flash->success(__('Setting saved.'));
$this->redirect(array('controller' => 'user_settings', 'action' => 'index', $this->Auth->User('id')));
}
} else {
// if we've failed saving our setting
if ($this->_isRest()) {
// if we are dealing with an API request
return $this->RestResponse->saveFailResponse('UserSettings', 'add', false, $this->UserSetting->validationErrors, $this->response->type());
} else {
/*
* if we are dealing with a UI request, simply set an error in a flash message
* and render the view of this endpoint, pre-populated with the submitted values.
*/
$this->Flash->error(__('Setting could not be saved.'));
}
}
}
if ($this->_isRest()) {
// GET request via the API should describe the endpoint
return $this->RestResponse->describe('UserSettings', 'setSetting', false, $this->response->type());
} else {
// load the valid settings from the model
$validSettings = $this->UserSetting->validSettings;
if ($this->_isSiteAdmin()) {
$users = $this->UserSetting->User->find('list', array(
'recursive' => -1,
'fields' => array('User.id', 'User.email')
));
} else if ($this->_isAdmin()) {
$users = $this->UserSetting->User->find('list', array(
'recursive' => -1,
'conditions' => array('User.org_id' => $this->Auth->user('org_id')),
'fields' => array('User.id', 'User.email')
));
} else {
$users = array($this->Auth->user('id') => $this->Auth->user('email'));
}
if (!empty($user_id) && $this->request->is('get')) {
$this->request->data['UserSetting']['user_id'] = $user_id;
}
$this->set('setting', $setting);
$this->set('users', $users);
$this->set('validSettings', $validSettings);
}
}
public function getSetting($user_id, $setting)
{
if (!$this->UserSetting->checkSettingValidity($setting)) {
throw new MethodNotAllowedException(__('Invalid setting.'));
}
$userSetting = $this->UserSetting->find('first', array(
'recursive' => -1,
'conditions' => array(
'UserSetting.user_id' => $user_id,
'UserSetting.setting' => $setting
),
'contain' => array('User.id', 'User.org_id')
));
$checkAccess = $this->UserSetting->checkAccess($this->Auth->user(), $userSetting, $user_id);
if (empty($checkAccess)) {
throw new MethodNotAllowedException(__('Invalid setting.'));
}
if (!empty($userSetting)) {
$userSetting = json_encode($userSetting['UserSetting']['value']);
} else {
$userSetting = '[]';
}
return $this->RestResponse->viewData($userSetting, $this->response->type(), false, true);
}
public function delete($id = false)
{
if ($this->request->is('get') && $this->_isRest()) {
/*
* GET request via the API should describe the endpoint
* Unlike with the add() endpoint, we want to run this check before doing anything else,
* in order to allow us to reach this endpoint without passing a valid ID
*/
return $this->RestResponse->describe('UserSettings', 'delete', false, $this->response->type());
}
// check if the ID is valid and whether a user setting with the given ID exists
if (empty($id) || !is_numeric($id)) {
throw new InvalidArgumentException(__('Invalid ID passed.'));
}
$userSetting = $this->UserSetting->find('first', array(
'recursive' => -1,
'conditions' => array(
'UserSetting.id' => $id
),
'contain' => array('User.id', 'User.org_id')
));
if (empty($userSetting)) {
throw new NotFoundException(__('Invalid user setting.'));
}
$checkAccess = $this->UserSetting->checkAccess($this->Auth->user(), $userSetting);
if (!$checkAccess) {
throw new NotFoundException(__('Invalid user setting.'));
}
$settingPermCheck = $this->UserSetting->checkSettingAccess($this->Auth->user(), $userSetting['UserSetting']['setting']);
if ($settingPermCheck !== true) {
throw new MethodNotAllowedException(__('This setting is restricted and requires the following permission(s): %s', $settingPermCheck));
}
if ($this->request->is('post') || $this->request->is('delete')) {
// Delete the setting that we were after.
$result = $this->UserSetting->delete($userSetting['UserSetting']['id']);
if ($result) {
// set the response for both the UI and API
$message = __('Setting deleted.');
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('UserSettings', 'delete', $id, $this->response->type(), $message);
} else {
$this->Flash->success($message);
}
} else {
// set the response for both the UI and API
$message = __('Setting could not be deleted.');
if ($this->_isRest()) {
return $this->RestResponse->saveFailResponse('UserSettings', 'delete', $id, $message, $this->response->type());
} else {
$this->Flash->error($message);
}
}
/*
* The API responses stopped executing this function and returned a serialised response to the user.
* For UI users, redirect to where they issued the request from.
*/
$this->redirect($this->referer());
} else {
throw new MethodNotAllowedException(__('Expecting POST or DELETE request.'));
}
}
}

149
app/Controller/UsersController.php
View File

@ -48,12 +48,19 @@ class UsersController extends AppController
));
$id = $userid['User']['id'];
}
$this->User->id = $id;
$this->User->recursive = 0;
if (!$this->User->exists()) {
$user = $this->User->read(null, $id);
$user = $this->User->find('first', array(
'recursive' => -1,
'conditions' => array('User.id' => $id),
'contain' => array(
'UserSetting',
'Role',
'Organisation'
)
));
if (empty($user)) {
throw new NotFoundException(__('Invalid user'));
}
$user = $this->User->read(null, $id);
if (!empty($user['User']['gpgkey'])) {
$pgpDetails = $this->User->verifySingleGPG($user);
$user['User']['pgp_status'] = isset($pgpDetails[2]) ? $pgpDetails[2] : 'OK';
@ -62,12 +69,30 @@ class UsersController extends AppController
if ($this->_isRest()) {
unset($user['User']['server_id']);
$user['User']['password'] = '*****';
return $this->RestResponse->viewData(array('User' => $user['User']), $this->response->type());
$temp = array();
foreach ($user['UserSetting'] as $k => $v) {
$temp[$v['setting']] = $v['value'];
}
$user['UserSetting'] = $temp;
return $this->RestResponse->viewData($this->__massageUserObject($user), $this->response->type());
} else {
$this->set('user', $user);
}
}
private function __massageUserObject($user)
{
unset($user['User']['server_id']);
$user['User']['password'] = '*****';
$objectsToInclude = array('User', 'Role', 'UserSetting', 'Organisation');
foreach ($objectsToInclude as $objectToInclude) {
if (isset($user[$objectToInclude])) {
$temp[$objectToInclude] = $user[$objectToInclude];
}
}
return $temp;
}
public function request_API()
{
if (Configure::read('MISP.disable_emailing')) {
@ -92,14 +117,20 @@ class UsersController extends AppController
if (!$this->_isAdmin() && Configure::read('MISP.disableUserSelfManagement')) {
throw new MethodNotAllowedException('User self-management has been disabled on this instance.');
}
$id = $this->Auth->user('id');
$this->User->read(null, $id);
if (!$this->User->exists()) {
$currentUser = $this->User->find('first', array(
'conditions' => array('User.id' => $this->Auth->user('id')),
'recursive' => -1
));
if (empty($currentUser)) {
throw new NotFoundException('Something went wrong. Your user account could not be accessed.');
}
$id = $currentUser['User']['id'];
if ($this->request->is('post') || $this->request->is('put')) {
if (empty($this->request->data['User'])) {
$this->request->data = array('User' => $this->request->data);
}
$abortPost = false;
if (!$this->_isSiteAdmin() && !empty($this->request->data['User']['email'])) {
if (!empty($this->request->data['User']['email']) && !$this->_isSiteAdmin()) {
$organisation = $this->User->Organisation->find('first', array(
'conditions' => array('Organisation.id' => $this->Auth->user('org_id')),
'recursive' => -1
@ -116,7 +147,7 @@ class UsersController extends AppController
}
}
if ($abortPost) {
$this->Flash->error(__('Invalid e-mail domain. Your user is restricted to creating users for the following domain(s): ') . implode(', ', $organisation['Organisation']['restricted_to_domain']));
$message = __('Invalid e-mail domain. Your user is restricted to creating users for the following domain(s): ') . implode(', ', $organisation['Organisation']['restricted_to_domain']);
}
}
}
@ -138,17 +169,49 @@ class UsersController extends AppController
if (!$abortPost) {
// What fields should be saved (allowed to be saved)
$fieldList = array('email', 'autoalert', 'gpgkey', 'certif_public', 'nids_sid', 'contactalert', 'disabled');
if ("" != $this->request->data['User']['password']) {
if (!empty($this->request->data['User']['password'])) {
$fieldList[] = 'password';
$fieldList[] = 'confirm_password';
}
foreach ($this->request->data['User'] as $k => $v) {
$currentUser['User'][$k] = $v;
}
// Save the data
if ($this->User->save($this->request->data, true, $fieldList)) {
$this->Flash->success(__('The profile has been updated'));
$this->_refreshAuth();
$this->redirect(array('action' => 'view', $id));
} else {
$this->Flash->error(__('The profile could not be updated. Please, try again.'));
if ($this->_isRest()) {
if (!empty($this->request->data['User']['password'])) {
if ($this->request->data['User']['password'] === '*****') {
unset($this->request->data['User']['password']);
} else {
$currentUser['User']['confirm_password'] = $this->request->data['User']['password'];
}
}
}
if ($this->User->save($currentUser, true, $fieldList)) {
if ($this->_isRest()) {
$user = $this->User->find('first', array(
'conditions' => array('User.id' => $id),
'recursive' => -1,
'contain' => array(
'Organisation',
'Role',
'UserSetting'
)
));
return $this->RestResponse->viewData($this->__massageUserObject($user), $this->response->type());
} else {
$this->Flash->success(__('The profile has been updated'));
$this->_refreshAuth();
$this->redirect(array('action' => 'view', $id));
}
} else {
$message = __('The profile could not be updated. Please, try again.');
$abortPost = true;
}
}
if ($abortPost) {
return $this->RestResponse->saveFailResponse('Users', 'edit', $id, $message, $this->response->type());
} else {
$this->Flash->error($message);
}
} else {
$this->User->set('password', '');
@ -462,11 +525,18 @@ class UsersController extends AppController
public function admin_view($id = null)
{
$this->User->id = $id;
if (!$this->User->exists()) {
$user = $this->User->find('first', array(
'recursive' => -1,
'conditions' => array('User.id' => $id),
'contain' => array(
'UserSetting',
'Role',
'Organisation'
)
));
if (empty($user)) {
throw new NotFoundException(__('Invalid user'));
}
$user = $this->User->read(null, $id);
if (!empty($user['User']['gpgkey'])) {
$pgpDetails = $this->User->verifySingleGPG($user);
$user['User']['pgp_status'] = isset($pgpDetails[2]) ? $pgpDetails[2] : 'OK';
@ -482,11 +552,21 @@ class UsersController extends AppController
}
if ($this->_isRest()) {
$user['User']['password'] = '*****';
$temp = array();
foreach ($user['UserSetting'] as $k => $v) {
$temp[$v['setting']] = $v['value'];
}
$user['UserSetting'] = $temp;
return $this->RestResponse->viewData(array(
'User' => $user['User'],
'Role' => $user['Role'],
'UserSetting' => $user['UserSetting']
), $this->response->type());
return $this->RestResponse->viewData(array('User' => $user['User']), $this->response->type());
} else {
$temp = $this->User->data['User']['invited_by'];
$user2 = $this->User->find('first', array('conditions' => array('User.id' => $user['User']['invited_by']), 'recursive' => -1));
$this->set('id', $id);
$this->set('user2', $this->User->read(null, $temp));
$this->set('user2', $user2);
}
}
@ -806,9 +886,9 @@ class UsersController extends AppController
continue;
}
if ($field != 'confirm_password') {
array_push($fieldsOldValues, $this->User->field($field));
$fieldsOldValues[$field] = $this->User->field($field);
} else {
array_push($fieldsOldValues, $this->User->field('password'));
$fieldsOldValues[$field] = $this->User->field('password');
}
}
if (
@ -849,31 +929,28 @@ class UsersController extends AppController
}
$cP++;
}
array_push($fieldsNewValues, $newValueStr);
$fieldsNewValues[$field] = $newValueStr;
} else {
array_push($fieldsNewValues, $newValue);
$fieldsNewValues[$field] = $newValue;
}
} else {
array_push($fieldsNewValues, $this->data['User']['password']);
$fieldsNewValues[$field] = $this->data['User']['password'];
}
}
// compare
$fieldsResultStr = '';
$c = 0;
$fieldsResult = array();
foreach ($fields as $field) {
if (isset($fieldsOldValues[$c]) && $fieldsOldValues[$c] != $fieldsNewValues[$c]) {
if (isset($fieldsOldValues[$field]) && $fieldsOldValues[$field] != $fieldsNewValues[$field]) {
if ($field != 'confirm_password' && $field != 'enable_password') {
$fieldsResultStr = $fieldsResultStr . ', ' . $field . ' (' . $fieldsOldValues[$c] . ') => (' . $fieldsNewValues[$c] . ')';
$fieldsResult[$field] = array($fieldsOldValues[$field], $fieldsNewValues[$field]);
}
}
$c++;
}
$fieldsResultStr = substr($fieldsResultStr, 2);
$user = $this->User->find('first', array(
'recursive' => -1,
'conditions' => array('User.id' => $this->User->id)
));
$this->User->extralog($this->Auth->user(), "edit", "user", $fieldsResultStr, $user);
$this->User->extralog($this->Auth->user(), "edit", "user", $fieldsResult, $user);
if ($this->_isRest()) {
$user['User']['password'] = '******';
return $this->RestResponse->viewData($user, $this->response->type());
@ -1724,10 +1801,12 @@ class UsersController extends AppController
'group' => 'Event.orgc_id',
'conditions' => array('Event.orgc_id' => array_keys($orgs)),
'recursive' => -1,
'fields' => array('Event.orgc_id', 'count(*)')
'fields' => array('Event.orgc_id', 'count(*)', 'sum(Event.attribute_count) as attributeCount')
));
foreach ($events as $event) {
$orgs[$event['Event']['orgc_id']]['eventCount'] = $event[0]['count(*)'];
$orgs[$event['Event']['orgc_id']]['attributeCount'] = $event[0]['attributeCount'];
$orgs[$event['Event']['orgc_id']]['orgActivity'] = $this->User->getOrgActivity($event['Event']['orgc_id'], array('event_timestamp' => '365d'));
}
unset($events);
$orgs = Set::combine($orgs, '{n}.name', '{n}');

60
app/Lib/Export/HashesExport.php Normal file
View File

@ -0,0 +1,60 @@
<?php
class HashesExport
{
public $additional_params = array(
'flatten' => 1
);
public $validTypes = array(
'simple' => array(
'md5', 'sha1', 'sha256', 'sha224', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'tlsh',
'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'pehash', 'authentihash',
'impfuzzy'
),
'composite' => array(
'malware-sample', 'filename|md5', 'filename|sha1', 'filename|sha256', 'filename|sha224', 'filename|sha512',
'filename|sha512/224', 'filename|sha512/256', 'filename|ssdeep', 'filename|imphash', 'filename|tlsh',
'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'filename|pehash',
'filename|authentihash', 'filename|impfuzzy'
)
);
public function handler($data, $options = array())
{
if ($options['scope'] === 'Attribute') {
if (in_array($data['Attribute']['type'], $this->validTypes['composite'])) {
return explode('|', $data['Attribute']['value'])[1];
} else if (in_array($data['Attribute']['type'], $this->validTypes['simple'])) {
return $data['Attribute']['value'];
}
}
if ($options['scope'] === 'Event') {
$result = array();
foreach ($data['Attribute'] as $attribute) {
if (in_array($attribute['type'], $this->validTypes['composite'])) {
$result[] = explode('|', $attribute['value'])[1];
} else if (in_array($attribute['type'], $this->validTypes['simple'])) {
$result[] = $attribute['value'];
}
}
return implode($this->separator(), $result);
}
return '';
}
public function header($options = array())
{
return '';
}
public function footer()
{
return "\n";
}
public function separator()
{
return "\n";
}
}

78
app/Lib/Export/NetfilterExport.php Normal file
View File

@ -0,0 +1,78 @@
<?php
class NetfilterExport
{
public $additional_params = array(
'flatten' => 1,
'conditions' => array(
'AND' => array(
'Attribute.type' => array(
'ip-dst', 'ip-src', 'domain|ip', 'ip-dst|port', 'ip-src|port'
)
)
)
);
public $non_restrictive_export = true;
private $__attributeTypeMappings = array(
'ip-dst' => 'full',
'ip-src' => 'full',
'domain|ip' => 1,
'ip-dst|port' => 0,
'ip-src|port' => 0
);
public function handler($data, $options = array())
{
$action = empty($options['filters']['netfilter_action']) ? 'DROP' : $options['filters']['netfilter_action'];
if ($options['scope'] === 'Attribute') {
if (in_array($data['Attribute']['type'], array_keys($this->__attributeTypeMappings))) {
return $this->__convertToRule($data['Attribute'], $action) . "\n";
} else {
return '';
}
}
if ($options['scope'] === 'Event') {
$result = array();
foreach ($data['Attribute'] as $attribute) {
if (in_array($data['Attribute']['type'], array_keys($this->__attributeTypeMappings))) {
$result[] = $this->__convertToRule($data['Attribute'], $action);
}
}
return implode($this->separator(), $result) . "\n";
}
return '';
}
private function __convertToRule($attribute, $action)
{
$ip = false;
if ($this->__attributeTypeMappings[$attribute['type']] === 'full') {
$ip = $attribute['value'];
} else {
$ip = explode('|', $attribute['value']);
$ip = $ip[$this->__attributeTypeMappings[$attribute['type']]];
}
return sprintf(
'iptables -A INPUT -s %s -j %s',
$ip,
$action
);
}
public function header($options = array())
{
return '';
}
public function footer()
{
return "";
}
public function separator()
{
return "";
}
}

4
app/Lib/Export/NidsExport.php
View File

@ -581,7 +581,7 @@ class NidsExport
$explodedNames = explode('.', $name);
// for each part
foreach ($explodedNames as $explodedName) {
// count the lenght of the part, and add |length| before
// count the length of the part, and add |length| before
$length = strlen($explodedName);
if ($length > 255) {
log('WARNING: DNS name is too long for RFC: '.$name);
@ -613,7 +613,7 @@ class NidsExport
$explodedNames = explode('.', $name);
// for each part
foreach ($explodedNames as $explodedName) {
// count the lenght of the part, and add |length| before
// count the length of the part, and add |length| before
$length = strlen($explodedName);
if ($length > 255) {
log('WARNING: DNS name is too long for RFC: '.$name);

4
app/Lib/Export/Stix1Export.php
View File

@ -15,13 +15,13 @@ class Stix1Export extends StixExport
$this->__org = escapeshellarg(Configure::read('MISP.org'));
$framing_file = $this->__scripts_dir . 'misp_framing.py ';
$my_server = ClassRegistry::init('Server');
return $my_server->getPythonVersion() . ' ' . $framing_file . $this->__return_type . ' ' . $this->__baseurl . ' ' . $this->__org . ' xml' . $this->__end_of_cmd;
return $my_server->getPythonVersion() . ' ' . $framing_file . $this->__return_type . ' ' . $this->__baseurl . ' ' . $this->__org . ' ' . $this->__return_format . ' ' . $this->__end_of_cmd;
}
protected function __parse_misp_events($filename)
{
$scriptFile = $this->__scripts_dir . $this->__script_name;
$my_server = ClassRegistry::init('Server');
return shell_exec($my_server->getPythonVersion() . ' ' . $scriptFile . ' ' . $filename . ' xml ' . $this->__baseurl . ' ' . $this->__org . $this->__end_of_cmd);
return shell_exec($my_server->getPythonVersion() . ' ' . $scriptFile . ' ' . $filename . ' ' . $this->__return_format . ' ' . $this->__baseurl . ' ' . $this->__org . $this->__end_of_cmd);
}
}

35
app/Lib/Export/StixExport.php
View File

@ -2,6 +2,11 @@
class StixExport
{
public $additional_params = array(
'includeEventTags' => 1,
'includeGalaxy' => 1
);
protected $__return_format = 'json';
protected $__scripts_dir = APP . 'files/scripts/';
protected $__end_of_cmd = ' 2>' . APP . 'tmp/logs/exec-errors.log';
protected $__return_type = null;
@ -34,7 +39,7 @@ class StixExport
$this->__empty_file = false;
} else {
if ($attributes_count > $this->__attributes_limit) {
$randomFileName = $this->generateRandomFileName();
$randomFileName = $this->__generateRandomFileName();
$tmpFile = new File($this->__tmp_dir . $randomFileName, true, 0644);
$tmpFile->write($event);
$tmpFile->close();
@ -54,8 +59,13 @@ class StixExport
public function header($options = array())
{
$this->__return_type = $options['returnFormat'];
if ($this->__return_type == 'stix-json') {
$this->__return_type = 'stix';
} else if ($this->__return_type == 'stix') {
$this->__return_format = 'xml';
}
$framing_cmd = $this->initiate_framing_params();
$randomFileName = $this->generateRandomFileName();
$randomFileName = $this->__generateRandomFileName();
$this->__tmp_dir = $this->__scripts_dir . 'tmp/';
$this->__framing = json_decode(shell_exec($framing_cmd), true);
$this->__stix_file = new File($this->__tmp_dir . $randomFileName . '.' . $this->__return_type);
@ -75,11 +85,13 @@ class StixExport
$this->__tmp_file->close();
array_push($this->__filenames, $this->__current_filename);
}
foreach ($this->__filenames as $filename) {
foreach ($this->__filenames as $f => $filename) {
$result = $this->__parse_misp_events($filename);
$decoded = json_decode($result, true);
if (!isset($decoded['success']) || !$decoded['success']) {
return '';
$this->__delete_temporary_files($f);
$error = $decoded && !empty($decoded['error']) ? $decoded['error'] : $result;
return 'Error while processing your query: ' . $error;
}
$file = new File($this->__tmp_dir . $filename . '.out');
$stix_event = ($this->__return_type == 'stix') ? $file->read() : substr($file->read(), 1, -1);
@ -104,14 +116,25 @@ class StixExport
private function __initialize_misp_file()
{
$this->__current_filename = $this->generateRandomFileName();
$this->__current_filename = $this->__generateRandomFileName();
$this->__tmp_file = new File($this->__tmp_dir . $this->__current_filename, true, 0644);
$this->__tmp_file->write('{"response": [');
$this->__empty_file = true;
}
public function generateRandomFileName()
private function __generateRandomFileName()
{
return (new RandomTool())->random_str(false, 12);
}
private function __delete_temporary_files($index)
{
foreach ($this->__filenames as $f => $filename) {
if ($index >= $f) {
unlink($this->__tmp_dir . $filename);
}
}
$this->__stix_file->close();
$this->__stix_file->delete();
}
}

2
app/Lib/Tools/ComplexTypeTool.php
View File

@ -41,7 +41,7 @@ class ComplexTypeTool
array(
'from' => '/\.+/',
'to' => '.',
'types' => array('link', 'url', 'ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname')
'types' => array('ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname')
),
array(
'from' => '/\[hxxp:\/\/\]/',

26
app/Lib/Tools/EventGraphTool.php
View File

@ -40,7 +40,7 @@
private function __get_event($id)
{
$this->__json['available_rotation_key'] = $this->__authorized_JSON_key;
$this->__json['available_pivot_key'] = $this->__authorized_JSON_key;
$fullevent = $this->__eventModel->fetchEvent($this->__user, array('eventid' => $id, 'flatten' => 0, 'includeTagRelations' => 1, 'extended' => $this->__extended_view));
$event = array();
@ -108,7 +108,7 @@
}
}
// value rule - search in the object's atribute value
// value rule - search in the object's attribute value
$valueMatch = true;
if (isset($obj['Attribute'])) {
foreach ($obj['Attribute'] as $attr) {
@ -232,12 +232,12 @@
$event = $this->__get_filtered_event($id);
$this->__json['items'] = array();
$this->__json['relations'] = array();
$this->__json['existing_object_relation'] = array();
if (empty($event)) {
return $this->__json;
}
if (!empty($event['Object'])) {
$object = $event['Object'];
} else {
@ -265,7 +265,7 @@
foreach ($object as $obj) {
$toPush = array(
'id' => $obj['id'],
'id' => sprintf('o-%s', $obj['id']),
'uuid' => $obj['uuid'],
'type' => $obj['name'],
'label' => '',
@ -290,8 +290,8 @@
$toPush = array(
'id' => $rel['id'],
'uuid' => $rel['uuid'],
'from' => $obj['id'],
'to' => $rel['referenced_id'],
'from' => sprintf('o-%s', $obj['id']),
'to' => $rel['referenced_type'] == 1 ? sprintf('o-%s', $rel['referenced_id']) : $rel['referenced_id'],
'type' => $rel['relationship_type'],
'comment' => $rel['comment'],
'event_id' => $rel['event_id'],
@ -312,7 +312,7 @@
if (empty($event)) {
return $this->__json;
}
if (!empty($event['Object'])) {
$object = $event['Object'];
} else {
@ -356,7 +356,7 @@
foreach ($object as $obj) {
$toPush = array(
'id' => $obj['id'],
'id' => sprintf('o-%s', $obj['id']),
'uuid' => $obj['uuid'],
'type' => $obj['name'],
'Attribute' => $obj['Attribute'],
@ -382,7 +382,7 @@
if (!in_array($tag['name'], $added_value)) {
$toPush = array(
'id' => "tag_edge_id_" . $i,
'from' => $obj['id'],
'from' => sprintf('o-%s', $obj['id']),
'to' => $tag['name'],
);
$tagSet[$tag['name']] = $tag;
@ -419,7 +419,7 @@
if (empty($event)) {
return $this->__json;
}
if (!empty($event['Object'])) {
$object = $event['Object'];
} else {
@ -466,7 +466,7 @@
foreach ($object as $obj) {
$toPush = array(
'id' => $obj['id'],
'id' => sprintf('o-%s', $obj['id']),
'uuid' => $obj['uuid'],
'type' => $obj['name'],
'Attribute' => $obj['Attribute'],
@ -491,7 +491,7 @@
if (!in_array($keyVal, $added_value)) {
$toPush = array(
'id' => "keyType_edge_id_" . $i,
'from' => $obj['id'],
'from' => sprintf('o-%s', $obj['id']),
'to' => "keyType_" . $keyVal,
);
array_push($added_value, $keyVal);

138
app/Lib/Tools/EventTimelineTool.php Normal file
View File

@ -0,0 +1,138 @@
<?php
class EventTimelineTool
{
private $__lookupTables = array();
private $__user = false;
private $__json = array();
private $__eventModel = false;
private $__refModel = false;
# Will be use latter on
private $__related_events = array();
private $__related_attributes = array();
public function construct($eventModel, $user, $filterRules, $extended_view=0)
{
$this->__eventModel = $eventModel;
$this->__objectTemplateModel = $eventModel->Object->ObjectTemplate;
$this->__user = $user;
$this->__filterRules = $filterRules;
$this->__json = array();
$this->__extended_view = $extended_view;
$this->__lookupTables = array(
'analysisLevels' => $this->__eventModel->analysisLevels,
'distributionLevels' => $this->__eventModel->Attribute->distributionLevels
);
return true;
}
public function construct_for_ref($refModel, $user)
{
$this->__refModel = $refModel;
$this->__user = $user;
$this->__json = array();
return true;
}
private function __get_event($id)
{
$fullevent = $this->__eventModel->fetchEvent($this->__user, array('eventid' => $id, 'flatten' => 0, 'includeTagRelations' => 1, 'extended' => $this->__extended_view));
$event = array();
if (empty($fullevent)) {
return $event;
}
if (!empty($fullevent[0]['Object'])) {
$event['Object'] = $fullevent[0]['Object'];
} else {
$event['Object'] = array();
}
if (!empty($fullevent[0]['Attribute'])) {
$event['Attribute'] = $fullevent[0]['Attribute'];
} else {
$event['Attribute'] = array();
}
return $event;
}
public function get_timeline($id)
{
$event = $this->__get_event($id);
$this->__json['items'] = array();
if (empty($event)) {
return $this->__json;
}
if (!empty($event['Object'])) {
$object = $event['Object'];
} else {
$object = array();
}
if (!empty($event['Attribute'])) {
$attribute = $event['Attribute'];
} else {
$attribute = array();
}
// extract links and node type
foreach ($attribute as $attr) {
$toPush = array(
'id' => $attr['id'],
'uuid' => $attr['uuid'],
'content' => $attr['value'],
'event_id' => $attr['event_id'],
'group' => 'attribute',
'timestamp' => $attr['timestamp'],
'first_seen' => $attr['first_seen'],
'last_seen' => $attr['last_seen'],
);
$this->__json['items'][] = $toPush;
}
foreach ($object as $obj) {
$toPush_obj = array(
'id' => $obj['id'],
'uuid' => $obj['uuid'],
'content' => $obj['name'],
'group' => 'object',
'meta-category' => $obj['meta-category'],
'template_uuid' => $obj['template_uuid'],
'event_id' => $obj['event_id'],
'timestamp' => $obj['timestamp'],
'Attribute' => array(),
);
$toPush_obj['first_seen'] = $obj['first_seen'];
$toPush_obj['last_seen'] = $obj['last_seen'];
$toPush_obj['first_seen_overwrite'] = false;
$toPush_obj['last_seen_overwrite'] = false;
foreach ($obj['Attribute'] as $obj_attr) {
// replaced *_seen based on object attribute
if ($obj_attr['object_relation'] == 'first-seen' && is_null($toPush_obj['first_seen'])) {
$toPush_obj['first_seen'] = $obj_attr['value']; // replace first_seen of the object to seen of the element
$toPush_obj['first_seen_overwrite'] = true;
} elseif ($obj_attr['object_relation'] == 'last-seen' && is_null($toPush_obj['last_seen'])) {
$toPush_obj['last_seen'] = $obj_attr['value']; // replace last_seen of the object to seen of the element
$toPush_obj['last_seen_overwrite'] = true;
}
$toPush_attr = array(
'id' => $obj_attr['id'],
'uuid' => $obj_attr['uuid'],
'content' => $obj_attr['value'],
'contentType' => $obj_attr['object_relation'],
'event_id' => $obj_attr['event_id'],
'group' => 'object_attribute',
'timestamp' => $obj_attr['timestamp'],
);
$toPush_obj['Attribute'][] = $toPush_attr;
}
$this->__json['items'][] = $toPush_obj;
}
return $this->__json;
}
}

2
app/Lib/Tools/FileAccessTool.php
View File

@ -2,7 +2,7 @@
class FileAccessTool
{
private $__fileErrorMsgPrefix = 'An error has occured while attempting to ';
private $__fileErrorMsgPrefix = 'An error has occurred while attempting to ';
public function createTempFile($dir, $prefix = 'MISP')
{

4
app/Lib/Tools/FinancialTool.php
View File

@ -111,12 +111,12 @@ class FinancialTool
*
* my_bcmod - get modulus (substitute for bcmod)
* string my_bcmod ( string left_operand, int modulus )
* left_operand can be really big, but be carefull with modulus :(
* left_operand can be really big, but be careful with modulus :(
* by Andrius Baranauskas and Laurynas Butkus :) Vilnius, Lithuania
**/
private function my_bcmod($x, $y)
{
// how many numbers to take at once? carefull not to exceed (int)
// how many numbers to take at once? careful not to exceed (int)
$take = 5;
$mod = '';

50
app/Lib/Tools/JSONConverterTool.php
View File

@ -43,34 +43,19 @@ class JSONConverterTool
//
// cleanup the array from things we do not want to expose
//
$tempSightings = array();
if (!empty($event['Sighting'])) {
foreach ($event['Sighting'] as $sighting) {
$tempSightings[$sighting['attribute_id']][] = $sighting;
}
unset($event['Sighting']);
}
unset($event['Event']['user_id']);
if (isset($event['Event']['Attribute'])) {
$event['Event']['Attribute'] = $this->__cleanAttributes($event['Event']['Attribute']);
if (!empty($event['Sighting'])) {
foreach ($event['Event']['Attribute'] as $ak => $attribute) {
foreach ($event['Sighting'] as $as => $sighting) {
if ($attribute['id'] == $sighting['attribute_id']) {
$event['Event']['Attribute'][$ak]['Sighting'][] = $sighting;
}
}
}
}
$event['Event']['Attribute'] = $this->__cleanAttributes($event['Event']['Attribute'], $tempSightings);
}
if (isset($event['Event']['Object'])) {
$event['Event']['Object'] = $this->__cleanObjects($event['Event']['Object']);
if (!empty($event['Sighting'])) {
foreach ($event['Event']['Object'] as $k => $object) {
if (!empty($object['Attribute'])) {
foreach ($object['Attribute'] as $ak => $attribute) {
foreach ($event['Sighting'] as $as => $sighting) {
if ($attribute['id'] == $sighting['attribute_id']) {
$event['Event']['Object'][$k]['Attribute'][$ak]['Sighting'][] = $sighting;
}
}
}
}
}
}
$event['Event']['Object'] = $this->__cleanObjects($event['Event']['Object'], $tempSightings);
}
if (!empty($event['Sighting'])) {
unset($event['Sighting']);
@ -92,18 +77,18 @@ class JSONConverterTool
return json_encode($result, JSON_PRETTY_PRINT);
}
private function __cleanAttributes($attributes)
private function __cleanAttributes($attributes, $tempSightings = array())
{
// remove value1 and value2 from the output and remove invalid utf8 characters for the xml parser
foreach ($attributes as $key => $value) {
if (isset($value['SharingGroup']) && empty($value['SharingGroup'])) {
foreach ($attributes as $key => $attribute) {
if (isset($attribute['SharingGroup']) && empty($attribute['SharingGroup'])) {
unset($attributes[$key]['SharingGroup']);
}
unset($attributes[$key]['value1']);
unset($attributes[$key]['value2']);
unset($attributes[$key]['category_order']);
if (isset($event['RelatedAttribute'][$value['id']])) {
$attributes[$key]['RelatedAttribute'] = $event['Event']['RelatedAttribute'][$value['id']];
if (isset($event['RelatedAttribute'][$attribute['id']])) {
$attributes[$key]['RelatedAttribute'] = $event['Event']['RelatedAttribute'][$attribute['id']];
foreach ($attributes[$key]['RelatedAttribute'] as &$ra) {
$ra = array('Attribute' => $ra);
}
@ -115,15 +100,18 @@ class JSONConverterTool
}
unset($attributes[$key]['AttributeTag']);
}
if (!empty($tempSightings[$attribute['id']])) {
$attributes[$key]['Sighting'] = $tempSightings[$attribute['id']];
}
}
return $attributes;
}
private function __cleanObjects($objects)
private function __cleanObjects($objects, $tempSightings = array())
{
foreach ($objects as $k => $object) {
if (!empty($object['Attribute'])) {
$objects[$k]['Attribute'] = $this->__cleanAttributes($object['Attribute']);
$objects[$k]['Attribute'] = $this->__cleanAttributes($object['Attribute'], $tempSightings);
} else {
unset($objects[$k]);
}

35
app/Lib/Tools/SyncTool.php
View File

@ -6,7 +6,6 @@ class SyncTool
public function setupHttpSocket($server = null)
{
$params = array();
App::uses('HttpSocket', 'Network/Http');
if (!empty($server)) {
if ($server['Server']['cert_file']) {
$params['ssl_cafile'] = APP . "files" . DS . "certs" . DS . $server['Server']['id'] . '.pem';
@ -21,23 +20,39 @@ class SyncTool
$params['ssl_verify_peer'] = false;
}
}
}
$HttpSocket = new HttpSocket($params);
if (empty($server['Server']['skip_proxy'])) {
$proxy = Configure::read('Proxy');
if (isset($proxy['host']) && !empty($proxy['host'])) {
$HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']);
if (!empty($server['Server']['skip_proxy'])) {
$params['skip_proxy'] = 1;
}
}
return $HttpSocket;
return $this->createHttpSocket($params);
}
public function setupHttpSocketFeed($feed = null)
{
return $this->setupHttpSocket();
}
/**
* @param array $params
* @return HttpSocket
* @throws Exception
*/
public function createHttpSocket($params = array())
{
// Use own CA PEM file
$caPath = Configure::read('MISP.ca_path');
if (!isset($params['ssl_cafile']) && $caPath) {
if (!file_exists($caPath)) {
throw new Exception("CA file '$caPath' doesn't exists.");
}
$params['ssl_cafile'] = $caPath;
}
App::uses('HttpSocket', 'Network/Http');
$HttpSocket = new HttpSocket();
$HttpSocket = new HttpSocket($params);
$proxy = Configure::read('Proxy');
if (isset($proxy['host']) && !empty($proxy['host'])) {
if (empty($params['skip_proxy']) && isset($proxy['host']) && !empty($proxy['host'])) {
$HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']);
}
return $HttpSocket;

2
app/Lib/cakephp

@ -1 +1 @@
Subproject commit c4a51509c554a0762e8b4d9b3985fe042b445fe7
Subproject commit d2e1681eb8ec75e6c2819fa113834843fed6995a

2602
app/Locale/cze/LC_MESSAGES/default.po
View File

File diff suppressed because it is too large Load Diff

3154
app/Locale/dan/LC_MESSAGES/default.po
View File

File diff suppressed because it is too large Load Diff

2774
app/Locale/fra/LC_MESSAGES/default.po
View File

File diff suppressed because it is too large Load Diff

6268
app/Locale/ita/LC_MESSAGES/default.po
View File

File diff suppressed because it is too large Load Diff

2694
app/Locale/jpn/LC_MESSAGES/default.po
View File

File diff suppressed because it is too large Load Diff

2602
app/Locale/kor/LC_MESSAGES/default.po
View File

File diff suppressed because it is too large Load Diff

10
app/Locale/no/LC_MESSAGES/default.po
View File

@ -1,7 +1,7 @@
msgid ""
msgstr ""
"Project-Id-Version: misp\n"
"PO-Revision-Date: 2019-07-04 09:30\n"
"PO-Revision-Date: 2019-11-30 07:05\n"
"Last-Translator: Steve Clement (SteveClement)\n"
"Language-Team: Norwegian\n"
"MIME-Version: 1.0\n"
@ -3891,8 +3891,8 @@ msgid "The debug level of the instance for site admins. This feature allows site
msgstr "Feilsøkingsnivået for forekomsten for nettstedadministratorer. Denne funksjonen tillater at webansvarlige kan kjøre feilsøkingsmodus på en levende forekomst uten å utsette den for andre brukere. Det mest fordelaktige alternativet for feilsøking og site_admin_debug brukes til administratorer på nettstedet."
#: Model/Server.php:2254
msgid "Failed (partially?) because of errors: "
msgstr "Mislyktes (delvis?) På grunn av feil: "
msgid "Failed (partially?) because of validation errors: "
msgstr "Mislyktes (delvis?) På grunn av valideringsfeil: "
#: Model/Server.php:2258
msgid "Blocked an edit to an event that was created locally. This can happen if a synchronised event that was created on this instance was modified by an administrator on the remote side."
@ -15945,8 +15945,8 @@ msgstr "Synkroniser brukeren for"
#: View/Users/admin_add.ctp:70
#: View/Users/admin_edit.ctp:64
#: View/Users/edit.ctp:23
msgid "Paste the user's GnuPG key here or try to retrieve it from the CIRCL key server by clicking on \"Fetch GnuPG key\" below."
msgstr "Lim inn brukerens GnuPG-nøkkel her, eller prøv å hente den fra CIRCL-nøkkelserveren ved å klikke på \"Hent GnuPG-nøkkel\" nedenfor."
msgid "Paste the user's GnuPG key here or try to retrieve it from the MIT key server by clicking on \"Fetch GnuPG key\" below."
msgstr "Lim inn brukerens GnuPG-nøkkel her, eller prøv å hente den fra MIT-nøkkelserveren ved å klikke på \"Hent GnuPG-nøkkel\" nedenfor."
#: View/Users/admin_add.ctp:72
#: View/Users/admin_edit.ctp:66

6608
app/Locale/pol/LC_MESSAGES/default.po
View File

File diff suppressed because it is too large Load Diff

2672
app/Locale/pt_BR/LC_MESSAGES/default.po
View File

File diff suppressed because it is too large Load Diff

2644
app/Locale/spa/LC_MESSAGES/default.po
View File

File diff suppressed because it is too large Load Diff

6290
app/Locale/swe/LC_MESSAGES/default.po
View File

File diff suppressed because it is too large Load Diff

6452
app/Locale/zh-s/LC_MESSAGES/default.po
View File

File diff suppressed because it is too large Load Diff

10
app/Model/AdminSetting.php
View File

@ -21,12 +21,9 @@ class AdminSetting extends AppModel
$setting_object = $this->find('first', array(
'conditions' => array('setting' => $setting)
));
if (!empty($setting_object)) {
$setting_object['AdminSetting']['value'] = $value;
} else {
$this->create();
$setting_object['AdminSetting'] = array('setting' => $setting, 'value' => $value);
}
$this->deleteAll(array('setting' => $setting));
$this->create();
$setting_object['AdminSetting'] = array('setting' => $setting, 'value' => $value);
if ($this->save($setting_object)) {
return true;
} else {
@ -34,6 +31,7 @@ class AdminSetting extends AppModel
}
}
public function getSetting($setting)
{
$setting_object = $this->find('first', array(

693
app/Model/AppModel.php
View File

File diff suppressed because it is too large Load Diff

442
app/Model/Attribute.php
View File

@ -35,7 +35,7 @@ class Attribute extends AppModel
);
public $defaultFields = array(
'id', 'event_id', 'object_id', 'object_relation', 'category', 'type', 'value', 'to_ids', 'uuid', 'timestamp', 'distribution', 'sharing_group_id', 'comment', 'deleted', 'disable_correlation'
'id', 'event_id', 'object_id', 'object_relation', 'category', 'type', 'value', 'to_ids', 'uuid', 'timestamp', 'distribution', 'sharing_group_id', 'comment', 'deleted', 'disable_correlation', 'first_seen', 'last_seen'
);
public $distributionDescriptions = array(
@ -70,7 +70,7 @@ class Attribute extends AppModel
//
// NOTE WHEN MODIFYING: please ensure to run the script 'tools/gen_misp_types_categories.py' to update the new definitions everywhere. (docu, website, RFC, ...)
//
//
$this->categoryDefinitions = array(
'Internal reference' => array(
'desc' => __('Reference used by the publishing party (e.g. ticket number)'),
@ -89,16 +89,16 @@ class Attribute extends AppModel
'Payload delivery' => array(
'desc' => __('Information about how the malware is delivered'),
'formdesc' => __('Information about the way the malware payload is initially delivered, for example information about the email or web-site, vulnerability used, originating IP etc. Malware sample itself should be attached here.'),
'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|authentihash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash','filename|impfuzzy', 'filename|pehash', 'mac-address', 'mac-eui-64', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'hostname', 'domain', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'email-body', 'url', 'user-agent', 'AS', 'pattern-in-file', 'pattern-in-traffic', 'stix2-pattern', 'yara', 'sigma', 'mime-type', 'attachment', 'malware-sample', 'link', 'malware-type', 'comment', 'text', 'hex', 'vulnerability', 'weakness', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'other', 'hostname|port', 'email-dst-display-name', 'email-src-display-name', 'email-header', 'email-reply-to', 'email-x-mailer', 'email-mime-boundary', 'email-thread-index', 'email-message-id', 'mobile-application-id', 'whois-registrant-email', 'anonymised')
'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|authentihash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash','filename|impfuzzy', 'filename|pehash', 'mac-address', 'mac-eui-64', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'hostname', 'domain', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'email-body', 'url', 'user-agent', 'AS', 'pattern-in-file', 'pattern-in-traffic', 'stix2-pattern', 'yara', 'sigma', 'mime-type', 'attachment', 'malware-sample', 'link', 'malware-type', 'comment', 'text', 'hex', 'vulnerability', 'weakness', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'other', 'hostname|port', 'email-dst-display-name', 'email-src-display-name', 'email-header', 'email-reply-to', 'email-x-mailer', 'email-mime-boundary', 'email-thread-index', 'email-message-id', 'mobile-application-id', 'chrome-extension-id', 'whois-registrant-email', 'anonymised')
),
'Artifacts dropped' => array(
'desc' => __('Any artifact (files, registry keys etc.) dropped by the malware or other modifications to the system'),
'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|authentihash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy','filename|pehash', 'regkey', 'regkey|value', 'pattern-in-file', 'pattern-in-memory','pdb', 'stix2-pattern', 'yara', 'sigma', 'attachment', 'malware-sample', 'named pipe', 'mutex', 'windows-scheduled-task', 'windows-service-name', 'windows-service-displayname', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'cookie', 'gene', 'mime-type', 'anonymised')
'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|authentihash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy','filename|pehash', 'regkey', 'regkey|value', 'pattern-in-file', 'pattern-in-memory','pdb', 'stix2-pattern', 'yara', 'sigma', 'attachment', 'malware-sample', 'named pipe', 'mutex', 'windows-scheduled-task', 'windows-service-name', 'windows-service-displayname', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'cookie', 'gene', 'kusto-query', 'mime-type', 'anonymised')
),
'Payload installation' => array(
'desc' => __('Info on where the malware gets installed in the system'),
'formdesc' => __('Location where the payload was placed in the system and the way it was installed. For example, a filename|md5 type attribute can be added here like this: c:\\windows\\system32\\malicious.exe|41d8cd98f00b204e9800998ecf8427e.'),
'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|authentihash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy', 'filename|pehash', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'stix2-pattern', 'yara', 'sigma', 'vulnerability', 'weakness', 'attachment', 'malware-sample', 'malware-type', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'mobile-application-id', 'other', 'mime-type', 'anonymised')
'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|authentihash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy', 'filename|pehash', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'stix2-pattern', 'yara', 'sigma', 'vulnerability', 'weakness', 'attachment', 'malware-sample', 'malware-type', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'mobile-application-id', 'chrome-extension-id', 'other', 'mime-type', 'anonymised')
),
'Persistence mechanism' => array(
'desc' => __('Mechanisms used by the malware to start at boot'),
@ -107,7 +107,7 @@ class Attribute extends AppModel
),
'Network activity' => array(
'desc' => __('Information about network traffic generated by the malware'),
'types' => array('ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'port', 'hostname', 'domain', 'domain|ip', 'mac-address', 'mac-eui-64', 'email-dst', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'pattern-in-file', 'stix2-pattern', 'pattern-in-traffic', 'attachment', 'comment', 'text', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'other', 'hex', 'cookie', 'hostname|port', 'bro', 'zeek', 'anonymised', 'community-id', 'email-subject')
'types' => array('ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'port', 'hostname', 'domain', 'domain|ip', 'mac-address', 'mac-eui-64', 'email-dst', 'email-src', 'eppn', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'pattern-in-file', 'stix2-pattern', 'pattern-in-traffic', 'attachment', 'comment', 'text', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'other', 'hex', 'cookie', 'hostname|port', 'bro', 'zeek', 'anonymised', 'community-id', 'email-subject')
),
'Payload type' => array(
'desc' => __('Information about the final payload(s)'),
@ -126,7 +126,7 @@ class Attribute extends AppModel
'Financial fraud' => array(
'desc' => __('Financial Fraud indicators'),
'formdesc' => __('Financial Fraud indicators, for example: IBAN Numbers, BIC codes, Credit card numbers, etc.'),
'types' => array('btc', 'xmr', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'phone-number', 'comment', 'text', 'other', 'hex', 'anonymised'),
'types' => array('btc', 'dash', 'xmr', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'phone-number', 'comment', 'text', 'other', 'hex', 'anonymised'),
),
'Support Tool' => array(
'desc' => __('Tools supporting analysis or detection of the event'),
@ -135,7 +135,7 @@ class Attribute extends AppModel
'Social network' => array(
'desc' => __('Social networks and platforms'),
// email-src and email-dst or should we go with a new email type that is neither / both?
'types' => array('github-username', 'github-repository', 'github-organisation', 'jabber-id', 'twitter-id', 'email-src', 'email-dst', 'comment', 'text', 'other', 'whois-registrant-email', 'anonymised')
'types' => array('github-username', 'github-repository', 'github-organisation', 'jabber-id', 'twitter-id', 'email-src', 'email-dst', 'eppn','comment', 'text', 'other', 'whois-registrant-email', 'anonymised')
),
'Person' => array(
'desc' => __('A human being - natural person'),
@ -149,7 +149,7 @@ class Attribute extends AppModel
//
// NOTE WHEN MODIFYING: please ensure to run the script 'tools/gen_misp_types_categories.py' to update the new definitions everywhere. (docu, website, RFC, ...)
//
//
$this->typeDefinitions = array(
'md5' => array('desc' => __('A checksum in md5 format'), 'formdesc' => __("You are encouraged to use filename|md5 instead. A checksum in md5 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'sha1' => array('desc' => __('A checksum in sha1 format'), 'formdesc' => __("You are encouraged to use filename|sha1 instead. A checksum in sha1 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
@ -165,6 +165,7 @@ class Attribute extends AppModel
'domain' => array('desc' => __('A domain name used in the malware'), 'formdesc' => __("A domain name used in the malware. Use this instead of hostname when the upper domain is important or can be used to create links between events."), 'default_category' => 'Network activity', 'to_ids' => 1),
'domain|ip' => array('desc' => __('A domain name and its IP address (as found in DNS lookup) separated by a |'),'formdesc' => __("A domain name and its IP address (as found in DNS lookup) separated by a | (no spaces)"), 'default_category' => 'Network activity', 'to_ids' => 1),
'email-src' => array('desc' => __("The email address used to send the malware."), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'eppn' => array('desc' => __("eduPersonPrincipalName - eppn - the NetId of the person for the purposes of inter-institutional authentication. Should be stored in the form of user@univ.edu, where univ.edu is the name of the local security domain."), 'default_category' => 'Network activity', 'to_ids' => 1),
'email-dst' => array('desc' => __("A recipient email address"), 'formdesc' => __("A recipient email address that is not related to your constituency."), 'default_category' => 'Network activity', 'to_ids' => 1),
'email-subject' => array('desc' => __("The subject of the email"), 'default_category' => 'Payload delivery', 'to_ids' => 0),
'email-attachment' => array('desc' => __("File name of the email attachment."), 'default_category' => 'Payload delivery', 'to_ids' => 1),
@ -190,6 +191,7 @@ class Attribute extends AppModel
'stix2-pattern' => array('desc' => __('STIX 2 pattern'), 'default_category' => 'Payload installation', 'to_ids' => 1),
'sigma' => array('desc' => __('Sigma - Generic Signature Format for SIEM Systems'), 'default_category' => 'Payload installation', 'to_ids' => 1),
'gene' => array('desc' => __('GENE - Go Evtx sigNature Engine'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
'kusto-query' => array('desc' => __('Kusto query - Kusto from Microsoft Azure is a service for storing and running interactive analytics over Big Data.'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
'mime-type' => array('desc' => __('A media type (also MIME type and content type) is a two-part identifier for file formats and format contents transmitted on the Internet'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
'identity-card-number' => array('desc' => __('Identity card number'), 'default_category' => 'Person', 'to_ids' => 0),
'cookie' => array('desc' => __('HTTP cookie as often stored on the user web client. This can include authentication cookie or session cookie.'), 'default_category' => 'Network activity', 'to_ids' => 0),
@ -211,6 +213,7 @@ class Attribute extends AppModel
'target-location' => array('desc' => __('Attack Targets Physical Location(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0),
'target-external' => array('desc' => __('External Target Organizations Affected by this Attack'), 'default_category' => 'Targeting data', 'to_ids' => 0),
'btc' => array('desc' => __('Bitcoin Address'), 'default_category' => 'Financial fraud', 'to_ids' => 1),
'dash' => array('desc' => __('Dash Address'), 'default_category' => 'Financial fraud', 'to_ids' => 1),
'xmr' => array('desc' => __('Monero Address'), 'default_category' => 'Financial fraud', 'to_ids' => 1),
'iban' => array('desc' => __('International Bank Account Number'), 'default_category' => 'Financial fraud', 'to_ids' => 1),
'bic' => array('desc' => __('Bank Identifier Code Number also known as SWIFT-BIC, SWIFT code or ISO 9362 code'), 'default_category' => 'Financial fraud', 'to_ids' => 1),
@ -270,9 +273,9 @@ class Attribute extends AppModel
'datetime' => array('desc' => __('Datetime in the ISO 8601 format'), 'default_category' => 'Other', 'to_ids' => 0),
'cpe' => array('desc' => __('Common platform enumeration'), 'default_category' => 'Other', 'to_ids' => 0),
'port' => array('desc' => __('Port number'), 'default_category' => 'Network activity', 'to_ids' => 0),
'ip-dst|port' => array('desc' => __('IP destination and port number seperated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1),
'ip-src|port' => array('desc' => __('IP source and port number seperated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1),
'hostname|port' => array('desc' => __('Hostname and port number seperated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1),
'ip-dst|port' => array('desc' => __('IP destination and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1),
'ip-src|port' => array('desc' => __('IP source and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1),
'hostname|port' => array('desc' => __('Hostname and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1),
'mac-address' => array('desc' => __('Mac address'), 'default_category' => 'Network activity', 'to_ids' => 0),
'mac-eui-64' => array('desc' => __('Mac EUI-64 address'), 'default_category' => 'Network activity', 'to_ids' => 0),
// verify IDS flag defaults for these
@ -315,6 +318,7 @@ class Attribute extends AppModel
'place-port-of-onward-foreign-destination' => array('desc' => __('A Port where the passenger is transiting to'), 'default_category' => 'Person', 'to_ids' => 0),
'passenger-name-record-locator-number' => array('desc' => __('The Passenger Name Record Locator is a key under which the reservation for a trip is stored in the system. The PNR contains, among other data, the name, flight segments and address of the passenger. It is defined by a combination of five or six letters and numbers.'), 'default_category' => 'Person', 'to_ids' => 0),
'mobile-application-id' => array('desc' => __('The application id of a mobile application'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'chrome-extension-id' => array('desc' => __('Chrome extension id'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'cortex' => array('desc' => __('Cortex analysis result'), 'default_category' => 'External analysis', 'to_ids' => 0),
'boolean' => array('desc' => __('Boolean value - to be used in objects'), 'default_category' => 'Other', 'to_ids' => 0),
'anonymised' => array('desc' => __('Anonymised value - described with the anonymisation object via a relationship'), 'formdesc' => __('Anonymised value - described with the anonymisation object via a relationship.'), 'default_category' => 'Other', 'to_ids' => 0)
@ -372,7 +376,9 @@ class Attribute extends AppModel
'deleted',
'disable_correlation',
'object_id',
'object_relation'
'object_relation',
'first_seen',
'last_seen'
);
public $searchResponseTypes = array(
@ -400,12 +406,14 @@ class Attribute extends AppModel
'suricata' => array('txt', 'NidsSuricataExport', 'rules'),
'snort' => array('txt', 'NidsSnortExport', 'rules'),
'text' => array('txt', 'TextExport', 'txt'),
'hashes' => array('txt', 'HashesExport', 'txt'),
'yara' => array('txt', 'YaraExport', 'yara'),
'yara-json' => array('json', 'YaraExport', 'json'),
'rpz' => array('txt', 'RPZExport', 'rpz'),
'csv' => array('csv', 'CsvExport', 'csv'),
'cache' => array('txt', 'CacheExport', 'cache'),
'attack-sightings' => array('json', 'AttackSightingsExport', 'json')
'attack-sightings' => array('json', 'AttackSightingsExport', 'json'),
'netfilter' => array('txt', 'NetfilterExport', 'sh')
);
// FIXME we need a better way to list the defaultCategories knowing that new attribute types will continue to appear in the future. We should generate this dynamically or use a function using the default_category of the $typeDefinitions
@ -435,6 +443,7 @@ class Attribute extends AppModel
'mac-eui-64' => 'Network activity',
'hostname' => 'Network activity',
'domain' => 'Network activity',
'eppn' => 'Network activity',
'url' => 'Network activity',
'ja3-fingerprint-md5' => 'Network activity',
'hassh-md5' => 'Network activity',
@ -521,6 +530,16 @@ class Attribute extends AppModel
'rule' => array('inList', array('0', '1', '2', '3', '4', '5')),
'message' => 'Options: Your organisation only, This community only, Connected communities, All communities, Sharing group, Inherit event',
'required' => true
),
'first_seen' => array(
'rule' => array('datetimeOrNull'),
'required' => false,
'message' => array('Invalid ISO 8601 format')
),
'last_seen' => array(
'rule' => array('datetimeOrNull'),
'required' => false,
'message' => array('Invalid ISO 8601 format')
)
);
@ -599,12 +618,21 @@ class Attribute extends AppModel
if (isset($v['Attribute']['object_relation']) && $v['Attribute']['object_relation'] === null) {
$results[$k]['Attribute']['object_relation'] = '';
}
$results[$k] = $this->UTCToISODatetime($results[$k], $this->alias);
}
return $results;
}
public function beforeSave($options = array())
{
if (!empty($this->data['Attribute']['id'])) {
$this->old = $this->find('first', array(
'recursive' => -1,
'conditions' => array('Attribute.id' => $this->data['Attribute']['id'])
));
} else {
$this->old = false;
}
// explode value of composite type in value1 and value2
// or copy value to value1 if not composite type
if (!empty($this->data['Attribute']['type'])) {
@ -623,6 +651,8 @@ class Attribute extends AppModel
}
}
$this->data = $this->ISODatetimeToUTC($this->data, $this->alias);
// update correlation... (only needed here if there's an update)
if ($this->id || !empty($this->data['Attribute']['id'])) {
$this->__beforeSaveCorrelation($this->data['Attribute']);
@ -669,7 +699,26 @@ class Attribute extends AppModel
$this->__alterAttributeCount($this->data['Attribute']['event_id'], false, $passedEvent);
}
} else {
$this->__afterSaveCorrelation($this->data['Attribute'], false, $passedEvent);
/*
* Only recorrelate if:
* - We are dealing with a new attribute OR
* - The existing attribute's previous state is known AND
* value, type or disable correlation have changed
* This will avoid recorrelations when it's not really needed, such as adding a tag
*/
if (!$created) {
if (
empty($this->old) ||
$this->data['Attribute']['value'] != $this->old['Attribute']['value'] ||
$this->data['Attribute']['disable_correlation'] != $this->old['Attribute']['disable_correlation'] ||
$this->data['Attribute']['type'] != $this->old['Attribute']['type']
) {
$this->__beforeSaveCorrelation($this->data['Attribute']);
$this->__afterSaveCorrelation($this->data['Attribute'], false, $passedEvent);
}
} else {
$this->__afterSaveCorrelation($this->data['Attribute'], false, $passedEvent);
}
}
$result = true;
// if the 'data' field is set on the $this->data then save the data to the correct file
@ -789,17 +838,19 @@ class Attribute extends AppModel
{
parent::beforeValidate();
if (!isset($this->data['Attribute']['type'])) {
$this->validationErrors['type'] = ['No type set.'];
return false;
}
if (is_array($this->data['Attribute']['value'])) {
$this->validationErrors['type'] = ['Value is an array.'];
return false;
}
App::uses('ComplexTypeTool', 'Tools');
$this->complexTypeTool = new ComplexTypeTool();
$this->data['Attribute']['value'] = $this->complexTypeTool->refangValue($this->data['Attribute']['value'], $this->data['Attribute']['type']);
if (!empty($this->data['Attribute']['object_id']) && empty($this->data['Attribute']['object_relation'])) {
$this->validationErrors['type'] = ['Object attribute sent, but no object_relation set.'];
return false;
}
// remove leading and trailing blanks
@ -824,6 +875,16 @@ class Attribute extends AppModel
$date = new DateTime();
$this->data['Attribute']['timestamp'] = $date->getTimestamp();
}
// parse first_seen different formats
if (isset($this->data['Attribute']['first_seen'])) {
$this->data['Attribute']['first_seen'] = $this->data['Attribute']['first_seen'] === '' ? null : $this->data['Attribute']['first_seen'];
}
// parse last_seen different formats
if (isset($this->data['Attribute']['last_seen'])) {
$this->data['Attribute']['last_seen'] = $this->data['Attribute']['last_seen'] === '' ? null : $this->data['Attribute']['last_seen'];
}
// TODO: add explanatory comment
// TODO: i18n?
$result = $this->runRegexp($this->data['Attribute']['type'], $this->data['Attribute']['value']);
@ -956,6 +1017,20 @@ class Attribute extends AppModel
return $this->runValidation($value, $this->data['Attribute']['type']);
}
// check whether the variable is null or datetime
public function datetimeOrNull($fields)
{
$k = array_keys($fields)[0];
$seen = $fields[$k];
try {
new DateTime($seen);
$returnValue = true;
} catch (Exception $e) {
$returnValue = false;
}
return $returnValue || is_null($seen);
}
private $__hexHashLengths = array(
'authentihash' => 64,
'md5' => 32,
@ -1177,6 +1252,7 @@ class Attribute extends AppModel
}
break;
case 'email-src':
case 'eppn':
case 'email-dst':
case 'target-email':
case 'whois-registrant-email':
@ -1232,6 +1308,7 @@ class Attribute extends AppModel
case 'stix2-pattern':
case 'sigma':
case 'gene':
case 'kusto-query':
case 'mime-type':
case 'identity-card-number':
case 'cookie':
@ -1315,6 +1392,7 @@ class Attribute extends AppModel
case 'github-organisation':
case 'cpe':
case 'twitter-id':
case 'chrome-extension-id':
case 'mobile-application-id':
// no newline
if (!preg_match("#\n#", $value)) {
@ -1350,6 +1428,7 @@ class Attribute extends AppModel
case 'iban':
case 'bic':
case 'btc':
case 'dash':
case 'xmr':
if (preg_match('/^[a-zA-Z0-9]+$/', $value)) {
$returnValue = true;
@ -1703,60 +1782,100 @@ class Attribute extends AppModel
return $this->saveAttachment($attribute);
}
public function getPictureData($attribute, $thumbnail=false, $width=200, $height=200)
/**
* Currently, as image are considered files with JPG (JPEG), PNG or GIF extension.
* @param array $attribute
* @return bool
*/
public function isImage(array $attribute)
{
$extension = explode('.', $attribute['Attribute']['value']);
$extension = end($extension);
if (extension_loaded('gd')) {
if (!$thumbnail) {
$data = $this->getAttachment($attribute['Attribute']);
$image = ImageCreateFromString($data);
ob_start ();
switch ($extension) {
case 'gif':
// php-gd doesn't support animated gif. Skipping...
break;
case 'jpg':
case 'jpeg':
imagejpeg($image);
break;
case 'png':
imagepng($image);
break;
default:
break;
}
$image_data = $extension != 'gif' ? ob_get_contents() : $data;
ob_end_clean ();
} else { // thumbnail requested, resample picture with desired dimension and save result
$thumbnail_exists = $this->getAttachment($attribute['Attribute'], $path_suffix='_thumbnail');
if ($width == 200 && $height == 200 && $thumbnail_exists !== '') { // check if thumbnail already exists
$image_data = $thumbnail_exists;
} else {
$data = $this->getAttachment($attribute['Attribute']);
if ($extension == 'gif') {
$image_data = $data;
} else {
$image = ImageCreateFromString($data);
$extension = 'jpg';
$imageTC = ImageCreateTrueColor($width, $height);
ImageCopyResampled($imageTC, $image, 0, 0, 0, 0, $width, $height, ImageSX($image), ImageSY($image));
ob_start ();
imagejpeg ($imageTC);
$image_data = ob_get_contents();
ob_end_clean ();
imagedestroy($image);
imagedestroy($imageTC);
}
// save thumbnail for later reuse
$attribute['Attribute']['data'] = $image_data;
$this->saveAttachment($attribute['Attribute'], '_thumbnail');
return $attribute['type'] === 'attachment' &&
Validation::extension($attribute['value'], array('jpg', 'jpeg', 'png', 'gif'));
}
/**
* @param array $attribute
* @param bool $thumbnail
* @param int $maxWidth - When $thumbnail is true
* @param int $maxHeight - When $thumbnail is true
* @return string
* @throws Exception
*/
public function getPictureData(array $attribute, $thumbnail=false, $maxWidth=200, $maxHeight=200)
{
if ($thumbnail && extension_loaded('gd')) {
if ($maxWidth == 200 && $maxHeight == 200) {
// Return thumbnail directly if already exists
$imageData = $this->getAttachment($attribute['Attribute'], $path_suffix='_thumbnail');
if ($imageData !== '') {
return $imageData;
}
}
// Thumbnail doesn't exists, we need to generate it
$imageData = $this->getAttachment($attribute['Attribute']);
$imageData = $this->resizeImage($imageData, $maxWidth, $maxHeight);
// Save just when requested default thumbnail size
if ($maxWidth == 200 && $maxHeight == 200) {
$attribute['Attribute']['data'] = $imageData;
$this->saveAttachment($attribute['Attribute'], $path_suffix='_thumbnail');
}
} else {
$image_data = $this->getAttachment($attribute['Attribute']);
$imageData = $this->getAttachment($attribute['Attribute']);
}
return $image_data;
return $imageData;
}
/**
* @param string $data
* @param int $maxWidth
* @param int $maxHeight
* @return string
* @throws Exception
*/
private function resizeImage($data, $maxWidth, $maxHeight)
{
$image = imagecreatefromstring($data);
if ($image === false) {
throw new Exception("Image is not valid.");
}
$currentWidth = imagesx($image);
$currentHeight = imagesy($image);
// Compute thumbnail size with keeping ratio
if ($currentWidth > $currentHeight) {
$newWidth = min($currentWidth, $maxWidth);
$divisor = $currentWidth / $newWidth;
$newHeight = floor($currentHeight / $divisor);
} else {
$newHeight = min($currentHeight, $maxHeight);
$divisor = $currentHeight / $newHeight;
$newWidth = floor($currentWidth / $divisor);
}
$imageThumbnail = imagecreatetruecolor($newWidth, $newHeight);
// Allow transparent background
imagealphablending($imageThumbnail, false);
imagesavealpha($imageThumbnail, true);
$transparent = imagecolorallocatealpha($imageThumbnail, 255, 255, 255, 127);
imagefilledrectangle($imageThumbnail, 0, 0, $newWidth, $newHeight, $transparent);
// Resize image
imagecopyresampled($imageThumbnail, $image, 0, 0, 0, 0, $newWidth, $newHeight, $currentWidth, $currentHeight);
imagedestroy($image);
// Output image to string
ob_start();
imagepng($imageThumbnail, null, 9);
$imageData = ob_get_contents();
ob_end_clean();
imagedestroy($imageThumbnail);
return $imageData;
}
public function __beforeSaveCorrelation($a)
@ -2106,6 +2225,50 @@ class Attribute extends AppModel
return $fails;
}
public function ISODatetimeToUTC($data, $alias)
{
// convert into utc and micro sec
if (!empty($data[$alias]['first_seen'])) {
$d = new DateTime($data[$alias]['first_seen']);
$d->setTimezone(new DateTimeZone('GMT'));
$fs_sec = $d->format('U');
$fs_micro = $d->format('u');
$fs_micro = str_pad($fs_micro, 6, "0", STR_PAD_LEFT);
$fs = $fs_sec . $fs_micro;
$data[$alias]['first_seen'] = $fs;
}
if (!empty($data[$alias]['last_seen'])) {
$d = new DateTime($data[$alias]['last_seen']);
$d->setTimezone(new DateTimeZone('GMT'));
$ls_sec = $d->format('U');
$ls_micro = $d->format('u');
$ls_micro = str_pad($ls_micro, 6, "0", STR_PAD_LEFT);
$ls = $ls_sec . $ls_micro;
$data[$alias]['last_seen'] = $ls;
}
return $data;
}
public function UTCToISODatetime($data, $alias)
{
if (!empty($data[$alias]['first_seen'])) {
$fs = $data[$alias]['first_seen'];
$fs_sec = intval($fs / 1000000); // $fs is in micro (10^6)
$fs_micro = $fs % 1000000;
$fs_micro = str_pad($fs_micro, 6, "0", STR_PAD_LEFT);
$fs = $fs_sec . '.' . $fs_micro;
$data[$alias]['first_seen'] = DateTime::createFromFormat('U.u', $fs)->format('Y-m-d\TH:i:s.uP');
}
if (!empty($data[$alias]['last_seen'])) {
$ls = $data[$alias]['last_seen'];
$ls_sec = intval($ls / 1000000); // $ls is in micro (10^6)
$ls_micro = $ls % 1000000;
$ls_micro = str_pad($ls_micro, 6, "0", STR_PAD_LEFT);
$ls = $ls_sec . '.' . $ls_micro;
$data[$alias]['last_seen'] = DateTime::createFromFormat('U.u', $ls)->format('Y-m-d\TH:i:s.uP');
}
return $data;
}
public function hids($user, $type, $tags = '', $from = false, $to = false, $last = false, $jobId = false, $enforceWarninglist = false)
{
@ -2220,7 +2383,7 @@ class Attribute extends AppModel
$rules = array();
foreach ($eventIds as $event) {
$conditions['AND'] = array('Attribute.to_ids' => 1, "Event.published" => 1, 'Attribute.event_id' => $event['Event']['id']);
$valid_types = array('ip-dst', 'ip-src', 'ip-dst|port', 'ip-src|port', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'domain', 'domain|ip', 'hostname', 'url', 'user-agent', 'snort');
$valid_types = array('ip-dst', 'ip-src', 'ip-dst|port', 'ip-src|port', 'eppn', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'domain', 'domain|ip', 'hostname', 'url', 'user-agent', 'snort');
$conditions['AND']['Attribute.type'] = $valid_types;
if (!empty($type)) {
$conditions['AND'][] = array('Attribute.type' => $type);
@ -2496,7 +2659,7 @@ class Attribute extends AppModel
return $values;
}
public function bro($user, $type, $tags = false, $eventId = false, $from = false, $to = false, $last = false, $enforceWarninglist = false)
public function bro($user, $type, $tags = false, $eventId = false, $from = false, $to = false, $last = false, $enforceWarninglist = false, $skipHeader = false)
{
App::uses('BroExport', 'Export');
$export = new BroExport();
@ -2564,7 +2727,9 @@ class Attribute extends AppModel
}
natsort($intel);
$intel = array_unique($intel);
array_unshift($intel, $export->header);
if (empty($skipHeader)) {
array_unshift($intel, $export->header);
}
return $intel;
}
@ -2579,7 +2744,8 @@ class Attribute extends AppModel
'fields' => array('Attribute.id', 'Attribute.event_id', 'Attribute.type', 'Attribute.category', 'Attribute.comment', 'Attribute.to_ids', 'Attribute.value', 'Attribute.value' . $valueField),
'contain' => array('Event' => array('fields' => array('Event.id', 'Event.threat_level_id', 'Event.orgc_id', 'Event.uuid'))),
'group' => array('Attribute.type', 'Attribute.value' . $valueField), // fields to GROUP BY
'enforceWarninglist' => $enforceWarninglist
'enforceWarninglist' => $enforceWarninglist,
'flatten' => 1
)
);
$orgs = $this->Event->Orgc->find('list', array(
@ -3102,7 +3268,10 @@ class Attribute extends AppModel
if (!empty($options['includeGalaxy'])) {
$this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
}
if (Configure::read('MISP.proposals_block_attributes') && isset($options['conditions']['AND']['Attribute.to_ids']) && $options['conditions']['AND']['Attribute.to_ids'] == 1) {
if (
Configure::read('MISP.proposals_block_attributes') &&
!empty($options['allow_proposal_blocking'])
) {
$this->bindModel(array('hasMany' => array('ShadowAttribute' => array('foreignKey' => 'old_id'))));
$proposalRestriction = array(
'ShadowAttribute' => array(
@ -3224,6 +3393,8 @@ class Attribute extends AppModel
if (empty($at['Tag'])) {
unset($results[$k]['AttributeTag'][$k2]);
$tagCulled = true;
} else {
$results[$k]['AttributeTag'][$k2]['Tag']['local'] = $results[$k]['AttributeTag'][$k2]['local'];
}
}
if ($tagCulled) {
@ -3233,6 +3404,7 @@ class Attribute extends AppModel
if (isset($result['Event']['EventTag'])) {
$results[$k]['Event']['Tag'] = array();
foreach ($result['Event']['EventTag'] as $et) {
$et['Tag']['local'] = $et['local'];
$results[$k]['Event']['Tag'][] = $et['Tag'];
}
unset($results[$k]['Event']['EventTag']);
@ -3489,6 +3661,7 @@ class Attribute extends AppModel
$defaultDistribution = Configure::read('MISP.default_attribute_distribution');
}
}
$saveResult = true;
foreach ($attributes as $k => $attribute) {
if (!empty($attribute['encrypt']) && $attribute['encrypt']) {
$attribute = $this->onDemandEncrypt($attribute);
@ -3498,9 +3671,9 @@ class Attribute extends AppModel
}
unset($attribute['Attachment']);
$this->create();
$this->save($attribute);
$saveResult = $saveResult && $this->save($attribute);
}
return true;
return $saveResult;
}
public function onDemandEncrypt($attribute)
@ -3551,11 +3724,6 @@ class Attribute extends AppModel
return true;
}
public function convertToOpenIOC($user, $attributes)
{
return $this->IOCExport->buildAll($user, $event);
}
private function __createTagSubQuery($tag_id, $blocked = false, $scope = 'Event', $limitAttributeHitsTo = 'event')
{
$conditionKey = $blocked ? array('NOT' => array('EventTag.tag_id' => $tag_id)) : array('EventTag.tag_id' => $tag_id);
@ -3623,6 +3791,32 @@ class Attribute extends AppModel
return $conditions;
}
public function setTimestampSeenConditions($timestamp, $conditions, $scope = 'Attribute.first_seen', $returnRaw = false)
{
if (is_array($timestamp)) {
$timestamp[0] = intval($this->Event->resolveTimeDelta($timestamp[0])) * 1000000; // seen in stored in micro-seconds in the DB
$timestamp[1] = intval($this->Event->resolveTimeDelta($timestamp[1])) * 1000000; // seen in stored in micro-seconds in the DB
if ($timestamp[0] > $timestamp[1]) {
$temp = $timestamp[0];
$timestamp[0] = $timestamp[1];
$timestamp[1] = $temp;
}
$conditions['AND'][] = array($scope . ' >=' => $timestamp[0]);
$conditions['AND'][] = array($scope . ' <=' => $timestamp[1]);
} else {
$timestamp = intval($this->Event->resolveTimeDelta($timestamp)) * 1000000; // seen in stored in micro-seconds in the DB
if ($scope == 'Attribute.first_seen') {
$conditions['AND'][] = array($scope . ' >=' => $timestamp);
} else {
$conditions['AND'][] = array($scope . ' <=' => $timestamp);
}
}
if ($returnRaw) {
return $timestamp;
}
return $conditions;
}
public function setToIDSConditions($to_ids, $conditions)
{
if ($to_ids === 'exclude') {
@ -3931,7 +4125,8 @@ class Attribute extends AppModel
if (isset($attribute['uuid'])) {
$existingAttribute = $this->find('first', array(
'conditions' => array('Attribute.uuid' => $attribute['uuid']),
'recursive' => -1
'contain' => array('AttributeTag' => 'Tag'),
'recursive' => -1,
));
$this->Log = ClassRegistry::init('Log');
if (count($existingAttribute)) {
@ -3950,7 +4145,7 @@ class Attribute extends AppModel
return true;
}
// If a field is not set in the request, just reuse the old value
$recoverFields = array('value', 'to_ids', 'distribution', 'category', 'type', 'comment', 'sharing_group_id', 'object_id', 'object_relation');
$recoverFields = array('value', 'to_ids', 'distribution', 'category', 'type', 'comment', 'sharing_group_id', 'object_id', 'object_relation', 'first_seen', 'last_seen');
foreach ($recoverFields as $rF) {
if (!isset($attribute[$rF])) {
$attribute[$rF] = $existingAttribute['Attribute'][$rF];
@ -4020,7 +4215,9 @@ class Attribute extends AppModel
'comment',
'sharing_group_id',
'deleted',
'disable_correlation'
'disable_correlation',
'first_seen',
'last_seen'
);
if ($objectId) {
$fieldList[] = 'object_id';
@ -4042,28 +4239,36 @@ class Attribute extends AppModel
));
return $this->validationErrors;
} else {
if (isset($attribute['Tag']) && $user['Role']['perm_tagger']) {
foreach ($attribute['Tag'] as $tag) {
$tag_id = $this->AttributeTag->Tag->captureTag($tag, $user);
if ($tag_id) {
// fix the IDs here
$this->AttributeTag->attachTagToAttribute($this->id, $attribute['event_id'], $tag_id);
} else {
// If we couldn't attach the tag it is most likely because we couldn't create it - which could have many reasons
// However, if a tag couldn't be added, it could also be that the user is a tagger but not a tag editor
// In which case if no matching tag is found, no tag ID is returned. Logging these is pointless as it is the correct behaviour.
if ($user['Role']['perm_tag_editor']) {
$this->Log->create();
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Attrubute',
'model_id' => $this->id,
'email' => $user['email'],
'action' => 'edit',
'user_id' => $user['id'],
'title' => 'Failed create or attach Tag ' . $tag['name'] . ' to the attribute.',
'change' => ''
));
if ($user['Role']['perm_tagger']) {
/*
We should uncomment the line below in the future once we have tag soft-delete
A solution to still keep the behavior for previous instance could be to not soft-delete the Tag if the remote instance
has a version below x
*/
// $this->AttributeTag->pruneOutdatedAttributeTagsFromSync(isset($attribute['Tag']) ? $attribute['Tag'] : array(), $existingAttribute['AttributeTag']);
if (isset($attribute['Tag'])) {
foreach ($attribute['Tag'] as $tag) {
$tag_id = $this->AttributeTag->Tag->captureTag($tag, $user);
if ($tag_id) {
// fix the IDs here
$this->AttributeTag->attachTagToAttribute($this->id, $attribute['event_id'], $tag_id);
} else {
// If we couldn't attach the tag it is most likely because we couldn't create it - which could have many reasons
// However, if a tag couldn't be added, it could also be that the user is a tagger but not a tag editor
// In which case if no matching tag is found, no tag ID is returned. Logging these is pointless as it is the correct behaviour.
if ($user['Role']['perm_tag_editor']) {
$this->Log->create();
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Attrubute',
'model_id' => $this->id,
'email' => $user['email'],
'action' => 'edit',
'user_id' => $user['id'],
'title' => 'Failed create or attach Tag ' . $tag['name'] . ' to the attribute.',
'change' => ''
));
}
}
}
}
@ -4081,6 +4286,7 @@ class Attribute extends AppModel
$result = $this->fetchAttributes($user, array(
'conditions' => array('Attribute.id' => $id),
'flatten' => 1,
'deleted' => [0,1],
'recursive' => -1,
'contain' => array('Event')
));
@ -4180,6 +4386,9 @@ class Attribute extends AppModel
'uuid' => array('function' => 'set_filter_uuid'),
'deleted' => array('function' => 'set_filter_deleted'),
'timestamp' => array('function' => 'set_filter_timestamp'),
'attribute_timestamp' => array('function' => 'set_filter_timestamp'),
'first_seen' => array('function' => 'set_filter_seen'),
'last_seen' => array('function' => 'set_filter_seen'),
'to_ids' => array('function' => 'set_filter_to_ids'),
'comment' => array('function' => 'set_filter_comment')
),
@ -4235,6 +4444,7 @@ class Attribute extends AppModel
if (!isset($filters['published'])) {
$filters['published'] = 1;
}
$filters['allow_proposal_blocking'] = 1;
}
if (!empty($filters['quickFilter'])) {
$filters['searchall'] = $filters['quickFilter'];
@ -4253,6 +4463,10 @@ class Attribute extends AppModel
$filters['wildcard'] = $filters['searchall'];
}
}
$subqueryElements = $this->Event->harvestSubqueryElements($filters);
$filters = $this->Event->addFiltersFromSubqueryElements($filters, $subqueryElements);
$conditions = $this->buildFilterConditions($user, $filters);
$params = array(
'conditions' => $conditions,
@ -4267,9 +4481,11 @@ class Attribute extends AppModel
'includeWarninglistHits' => !empty($filters['includeWarninglistHits']) ? $filters['includeWarninglistHits'] : 0,
'includeContext' => !empty($filters['includeContext']) ? $filters['includeContext'] : 0,
'includeSightings' => !empty($filters['includeSightings']) ? $filters['includeSightings'] : 0,
'includeSightingdb' => !empty($filters['includeSightingdb']) ? $filters['includeSightingdb'] : 0,
'includeCorrelations' => !empty($filters['includeCorrelations']) ? $filters['includeCorrelations'] : 0,
'includeDecayScore' => !empty($filters['includeDecayScore']) ? $filters['includeDecayScore'] : 0,
'includeFullModel' => !empty($filters['includeFullModel']) ? $filters['includeFullModel'] : 0
'includeFullModel' => !empty($filters['includeFullModel']) ? $filters['includeFullModel'] : 0,
'allow_proposal_blocking' => !empty($filters['allow_proposal_blocking']) ? $filters['allow_proposal_blocking'] : 0
);
if (!empty($filters['attackGalaxy'])) {
$params['attackGalaxy'] = $filters['attackGalaxy'];
@ -4279,6 +4495,9 @@ class Attribute extends AppModel
}
if (isset($filters['limit'])) {
$params['limit'] = $filters['limit'];
if (!isset($filters['page'])) {
$filters['page'] = 1;
}
}
if (isset($filters['page'])) {
$params['page'] = $filters['page'];
@ -4313,14 +4532,18 @@ class Attribute extends AppModel
'filters' => $filters
);
if (!empty($exportTool->additional_params)) {
$params = array_merge($params, $exportTool->additional_params);
$params = array_merge_recursive(
$params,
$exportTool->additional_params
);
}
$tmpfile = tmpfile();
fwrite($tmpfile, $exportTool->header($exportToolParams));
$loop = false;
if (empty($params['limit'])) {
$memory_in_mb = $this->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
$memory_scaling_factor = isset($exportTool->memory_scaling_factor) ? $exportTool->memory_scaling_factor : 100;
$default_attribute_memory_coefficient = Configure::check('MISP.default_attribute_memory_coefficient') ? Configure::read('MISP.default_attribute_memory_coefficient') : 80;
$memory_scaling_factor = isset($exportTool->memory_scaling_factor) ? $exportTool->memory_scaling_factor : $default_attribute_memory_coefficient;
$params['limit'] = $memory_in_mb * $memory_scaling_factor;
$loop = true;
$params['page'] = 1;
@ -4343,6 +4566,10 @@ class Attribute extends AppModel
while ($continue) {
$this->Whitelist = ClassRegistry::init('Whitelist');
$results = $this->fetchAttributes($user, $params, $continue);
if ($params['includeSightingdb']) {
$this->Sightingdb = ClassRegistry::init('Sightingdb');
$results = $this->Sightingdb->attachToAttributes($results, $user);
}
$params['page'] += 1;
$results = $this->Whitelist->removeWhitelistedFromArray($results, true);
$results = array_values($results);
@ -4350,8 +4577,9 @@ class Attribute extends AppModel
$temp = '';
foreach ($results as $attribute) {
$elementCounter++;
$temp .= $exportTool->handler($attribute, $exportToolParams);
if ($temp !== '') {
$handlerResult = $exportTool->handler($attribute, $exportToolParams);
$temp .= $handlerResult;
if ($handlerResult !== '') {
if ($i != count($results) -1) {
$temp .= $exportTool->separator($exportToolParams);
}

17
app/Model/AttributeTag.php
View File

@ -103,6 +103,22 @@ class AttributeTag extends AppModel
return true;
}
// This function help mirroring the tags at attribute level. It will delete tags that are not present on the remote attribute
public function pruneOutdatedAttributeTagsFromSync($newerTags, $originalAttributeTags)
{
$newerTagsName = array();
foreach ($newerTags as $tag) {
$newerTagsName[] = strtolower($tag['name']);
}
foreach ($originalAttributeTags as $k => $attributeTag) {
if (!$attributeTag['local']) { //
if (!in_array(strtolower($attributeTag['Tag']['name']), $newerTagsName)) {
$this->softDelete($attributeTag['id']);
}
}
}
}
public function countForTag($tag_id, $user)
{
return $this->find('count', array(
@ -172,6 +188,7 @@ class AttributeTag extends AppModel
$attributes = $this->Attribute->fetchAttributes($user, array(
'conditions' => $conditions,
'flatten' => 1,
'includeAllTags' => 1
));
if (empty($attributes)) {

31
app/Model/Behavior/RegexpBehavior.php
View File

@ -4,37 +4,40 @@ App::uses('Regexp', 'Model');
/**
* Behavior to regexp all string fields in a model
*
*/
class RegexpBehavior extends ModelBehavior
{
private $__allRegexp = array();
private $__allRegexp = null;
public $excluded_types = array('sigma', 'float');
public function setup(Model $model, $config = null)
{
$regexp = new Regexp();
$this->__allRegexp = $regexp->find('all', array('order' => 'id ASC'));
}
/**
* replace the current value according to the regexp rules, or block blacklisted regular expressions
*
* @param Model $Model
* @param unknown_type $array
* @param string $type
* @param string $value
* @return string
*/
public function runRegexp(Model $Model, $type, $value)
{
if (in_array($type, $this->excluded_types)) {
return $value;
}
if ($this->__allRegexp === null) {
$regexp = new Regexp();
$this->__allRegexp = $regexp->find('all', array('order' => 'id ASC'));
}
foreach ($this->__allRegexp as $regexp) {
if (!empty($regexp['Regexp']['replacement']) && !empty($regexp['Regexp']['regexp']) && ($regexp['Regexp']['type'] === 'ALL' || $regexp['Regexp']['type'] === $type)) {
$value = preg_replace($regexp['Regexp']['regexp'], $regexp['Regexp']['replacement'], $value);
}
if (empty($regexp['Regexp']['replacement']) && preg_match($regexp['Regexp']['regexp'], $value) && ($regexp['Regexp']['type'] === 'ALL' || $regexp['Regexp']['type'] === $type)) {
return false;
if ($regexp['Regexp']['type'] === 'ALL' || $regexp['Regexp']['type'] === $type) {
if (!empty($regexp['Regexp']['replacement']) && !empty($regexp['Regexp']['regexp'])) {
$value = preg_replace($regexp['Regexp']['regexp'], $regexp['Regexp']['replacement'], $value);
}
if (empty($regexp['Regexp']['replacement']) && preg_match($regexp['Regexp']['regexp'], $value)) {
return false;
}
}
}
return $value;

921
app/Model/Datasource/Database/MysqlExtendedLogging.php Normal file
View File

@ -0,0 +1,921 @@
<?php
/**
* MySQL layer for DBO
*
* CakePHP(tm) : Rapid Development Framework (https://cakephp.org)
* Copyright (c) Cake Software Foundation, Inc. (https://cakefoundation.org)
*
* Licensed under The MIT License
* For full copyright and license information, please see the LICENSE.txt
* Redistributions of files must retain the above copyright notice.
*
* @copyright Copyright (c) Cake Software Foundation, Inc. (https://cakefoundation.org)
* @link https://cakephp.org CakePHP(tm) Project
* @package Cake.Model.Datasource.Database
* @since CakePHP(tm) v 0.10.5.1790
* @license https://opensource.org/licenses/mit-license.php MIT License
*/
App::uses('DboSource', 'Model/Datasource');
/**
* MySQL DBO driver object
*
* Provides connection and SQL generation for MySQL RDMS
*
* @package Cake.Model.Datasource.Database
*/
class MysqlExtendedLogging extends DboSource {
/**
* Datasource description
*
* @var string
*/
public $description = "MySQL DBO Driver";
/**
* Base configuration settings for MySQL driver
*
* @var array
*/
protected $_baseConfig = array(
'persistent' => true,
'host' => 'localhost',
'login' => 'root',
'password' => '',
'database' => 'cake',
'port' => '3306',
'flags' => array()
);
/**
* Reference to the PDO object connection
*
* @var PDO
*/
protected $_connection = null;
/**
* Start quote
*
* @var string
*/
public $startQuote = "`";
/**
* End quote
*
* @var string
*/
public $endQuote = "`";
/**
* use alias for update and delete. Set to true if version >= 4.1
*
* @var bool
*/
protected $_useAlias = true;
/**
* List of engine specific additional field parameters used on table creating
*
* @var array
*/
public $fieldParameters = array(
'charset' => array('value' => 'CHARACTER SET', 'quote' => false, 'join' => ' ', 'column' => false, 'position' => 'beforeDefault'),
'collate' => array('value' => 'COLLATE', 'quote' => false, 'join' => ' ', 'column' => 'Collation', 'position' => 'beforeDefault'),
'comment' => array('value' => 'COMMENT', 'quote' => true, 'join' => ' ', 'column' => 'Comment', 'position' => 'afterDefault'),
'unsigned' => array(
'value' => 'UNSIGNED',
'quote' => false,
'join' => ' ',
'column' => false,
'position' => 'beforeDefault',
'noVal' => true,
'options' => array(true),
'types' => array('integer', 'smallinteger', 'tinyinteger', 'float', 'decimal', 'biginteger')
)
);
/**
* List of table engine specific parameters used on table creating
*
* @var array
*/
public $tableParameters = array(
'charset' => array('value' => 'DEFAULT CHARSET', 'quote' => false, 'join' => '=', 'column' => 'charset'),
'collate' => array('value' => 'COLLATE', 'quote' => false, 'join' => '=', 'column' => 'Collation'),
'engine' => array('value' => 'ENGINE', 'quote' => false, 'join' => '=', 'column' => 'Engine'),
'comment' => array('value' => 'COMMENT', 'quote' => true, 'join' => '=', 'column' => 'Comment'),
);
/**
* MySQL column definition
*
* @var array
* @link https://dev.mysql.com/doc/refman/5.7/en/data-types.html MySQL Data Types
*/
public $columns = array(
'primary_key' => array('name' => 'NOT NULL AUTO_INCREMENT'),
'string' => array('name' => 'varchar', 'limit' => '255'),
'text' => array('name' => 'text'),
'enum' => array('name' => 'enum'),
'biginteger' => array('name' => 'bigint', 'limit' => '20'),
'integer' => array('name' => 'int', 'limit' => '11', 'formatter' => 'intval'),
'smallinteger' => array('name' => 'smallint', 'limit' => '6', 'formatter' => 'intval'),
'tinyinteger' => array('name' => 'tinyint', 'limit' => '4', 'formatter' => 'intval'),
'float' => array('name' => 'float', 'formatter' => 'floatval'),
'decimal' => array('name' => 'decimal', 'formatter' => 'floatval'),
'datetime' => array('name' => 'datetime', 'format' => 'Y-m-d H:i:s', 'formatter' => 'date'),
'timestamp' => array('name' => 'timestamp', 'format' => 'Y-m-d H:i:s', 'formatter' => 'date'),
'time' => array('name' => 'time', 'format' => 'H:i:s', 'formatter' => 'date'),
'date' => array('name' => 'date', 'format' => 'Y-m-d', 'formatter' => 'date'),
'binary' => array('name' => 'blob'),
'boolean' => array('name' => 'tinyint', 'limit' => '1')
);
/**
* Mapping of collation names to character set names
*
* @var array
*/
protected $_charsets = array();
/**
* Connects to the database using options in the given configuration array.
*
* MySQL supports a few additional options that other drivers do not:
*
* - `unix_socket` Set to the path of the MySQL sock file. Can be used in place
* of host + port.
* - `ssl_key` SSL key file for connecting via SSL. Must be combined with `ssl_cert`.
* - `ssl_cert` The SSL certificate to use when connecting via SSL. Must be
* combined with `ssl_key`.
* - `ssl_ca` The certificate authority for SSL connections.
*
* @return bool True if the database could be connected, else false
* @throws MissingConnectionException
*/
public function connect() {
$config = $this->config;
$this->connected = false;
$flags = $config['flags'] + array(
PDO::ATTR_PERSISTENT => $config['persistent'],
PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => true,
PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION
);
if (!empty($config['encoding'])) {
$flags[PDO::MYSQL_ATTR_INIT_COMMAND] = 'SET NAMES ' . $config['encoding'];
}
if (!empty($config['ssl_key']) && !empty($config['ssl_cert'])) {
$flags[PDO::MYSQL_ATTR_SSL_KEY] = $config['ssl_key'];
$flags[PDO::MYSQL_ATTR_SSL_CERT] = $config['ssl_cert'];
}
if (!empty($config['ssl_ca'])) {
$flags[PDO::MYSQL_ATTR_SSL_CA] = $config['ssl_ca'];
}
if (empty($config['unix_socket'])) {
$dsn = "mysql:host={$config['host']};port={$config['port']};dbname={$config['database']}";
} else {
$dsn = "mysql:unix_socket={$config['unix_socket']};dbname={$config['database']}";
}
try {
$this->_connection = new PDO(
$dsn,
$config['login'],
$config['password'],
$flags
);
$this->connected = true;
if (!empty($config['settings'])) {
foreach ($config['settings'] as $key => $value) {
$this->_execute("SET $key=$value");
}
}
} catch (PDOException $e) {
throw new MissingConnectionException(array(
'class' => get_class($this),
'message' => $e->getMessage()
));
}
$this->_charsets = array();
$this->_useAlias = (bool)version_compare($this->getVersion(), "4.1", ">=");
return $this->connected;
}
/**
* Check whether the MySQL extension is installed/loaded
*
* @return bool
*/
public function enabled() {
return in_array('mysql', PDO::getAvailableDrivers());
}
/**
* Returns an array of sources (tables) in the database.
*
* @param mixed $data List of tables.
* @return array Array of table names in the database
*/
public function listSources($data = null) {
$cache = parent::listSources();
if ($cache) {
return $cache;
}
$result = $this->_execute('SHOW TABLES FROM ' . $this->name($this->config['database']));
if (!$result) {
$result->closeCursor();
return array();
}
$tables = array();
while ($line = $result->fetch(PDO::FETCH_NUM)) {
$tables[] = $line[0];
}
$result->closeCursor();
parent::listSources($tables);
return $tables;
}
/**
* Builds a map of the columns contained in a result
*
* @param PDOStatement $results The results to format.
* @return void
*/
public function resultSet($results) {
$this->map = array();
$numFields = $results->columnCount();
$index = 0;
while ($numFields-- > 0) {
$column = $results->getColumnMeta($index);
if ($column['len'] === 1 && (empty($column['native_type']) || $column['native_type'] === 'TINY')) {
$type = 'boolean';
} else {
$type = empty($column['native_type']) ? 'string' : $column['native_type'];
}
if (!empty($column['table']) && strpos($column['name'], $this->virtualFieldSeparator) === false) {
$this->map[$index++] = array($column['table'], $column['name'], $type);
} else {
$this->map[$index++] = array(0, $column['name'], $type);
}
}
}
/**
* Fetches the next row from the current result set
*
* @return mixed array with results fetched and mapped to column names or false if there is no results left to fetch
*/
public function fetchResult() {
if ($row = $this->_result->fetch(PDO::FETCH_NUM)) {
$resultRow = array();
foreach ($this->map as $col => $meta) {
list($table, $column, $type) = $meta;
$resultRow[$table][$column] = $row[$col];
if ($type === 'boolean' && $row[$col] !== null) {
$resultRow[$table][$column] = $this->boolean($resultRow[$table][$column]);
}
}
return $resultRow;
}
$this->_result->closeCursor();
return false;
}
/**
* Gets the database encoding
*
* @return string The database encoding
*/
public function getEncoding() {
return $this->_execute('SHOW VARIABLES LIKE ?', array('character_set_client'))->fetchObject()->Value;
}
/**
* Query charset by collation
*
* @param string $name Collation name
* @return string|false Character set name
*/
public function getCharsetName($name) {
if ((bool)version_compare($this->getVersion(), "5", "<")) {
return false;
}
if (isset($this->_charsets[$name])) {
return $this->_charsets[$name];
}
$r = $this->_execute(
'SELECT CHARACTER_SET_NAME FROM INFORMATION_SCHEMA.COLLATIONS WHERE COLLATION_NAME = ?',
array($name)
);
$cols = $r->fetch(PDO::FETCH_ASSOC);
if (isset($cols['CHARACTER_SET_NAME'])) {
$this->_charsets[$name] = $cols['CHARACTER_SET_NAME'];
} else {
$this->_charsets[$name] = false;
}
return $this->_charsets[$name];
}
/**
* Returns an array of the fields in given table name.
*
* @param Model|string $model Name of database table to inspect or model instance
* @return array Fields in table. Keys are name and type
* @throws CakeException
*/
public function describe($model) {
$key = $this->fullTableName($model, false);
$cache = parent::describe($key);
if ($cache) {
return $cache;
}
$table = $this->fullTableName($model);
$fields = false;
$cols = $this->_execute('SHOW FULL COLUMNS FROM ' . $table);
if (!$cols) {
throw new CakeException(__d('cake_dev', 'Could not describe table for %s', $table));
}
while ($column = $cols->fetch(PDO::FETCH_OBJ)) {
$fields[$column->Field] = array(
'type' => $this->column($column->Type),
'null' => ($column->Null === 'YES' ? true : false),
'default' => $column->Default,
'length' => $this->length($column->Type)
);
if (in_array($fields[$column->Field]['type'], $this->fieldParameters['unsigned']['types'], true)) {
$fields[$column->Field]['unsigned'] = $this->_unsigned($column->Type);
}
if (in_array($fields[$column->Field]['type'], array('timestamp', 'datetime')) &&
in_array(strtoupper($column->Default), array('CURRENT_TIMESTAMP', 'CURRENT_TIMESTAMP()'))
) {
$fields[$column->Field]['default'] = null;
}
if (!empty($column->Key) && isset($this->index[$column->Key])) {
$fields[$column->Field]['key'] = $this->index[$column->Key];
}
foreach ($this->fieldParameters as $name => $value) {
if (!empty($column->{$value['column']})) {
$fields[$column->Field][$name] = $column->{$value['column']};
}
}
if (isset($fields[$column->Field]['collate'])) {
$charset = $this->getCharsetName($fields[$column->Field]['collate']);
if ($charset) {
$fields[$column->Field]['charset'] = $charset;
}
}
}
$this->_cacheDescription($key, $fields);
$cols->closeCursor();
return $fields;
}
/**
* Generates and executes an SQL UPDATE statement for given model, fields, and values.
*
* @param Model $model The model to update.
* @param array $fields The fields to update.
* @param array $values The values to set.
* @param mixed $conditions The conditions to use.
* @return bool
*/
public function update(Model $model, $fields = array(), $values = null, $conditions = null) {
if (!$this->_useAlias) {
return parent::update($model, $fields, $values, $conditions);
}
if (!$values) {
$combined = $fields;
} else {
$combined = array_combine($fields, $values);
}
$alias = $joins = false;
$fields = $this->_prepareUpdateFields($model, $combined, empty($conditions), !empty($conditions));
$fields = implode(', ', $fields);
$table = $this->fullTableName($model);
if (!empty($conditions)) {
$alias = $this->name($model->alias);
if ($model->name === $model->alias) {
$joins = implode(' ', $this->_getJoins($model));
}
}
$conditions = $this->conditions($this->defaultConditions($model, $conditions, $alias), true, true, $model);
if ($conditions === false) {
return false;
}
if (!$this->execute($this->renderStatement('update', compact('table', 'alias', 'joins', 'fields', 'conditions')))) {
$model->onError();
return false;
}
return true;
}
/**
* Generates and executes an SQL DELETE statement for given id/conditions on given model.
*
* @param Model $model The model to delete from.
* @param mixed $conditions The conditions to use.
* @return bool Success
*/
public function delete(Model $model, $conditions = null) {
if (!$this->_useAlias) {
return parent::delete($model, $conditions);
}
$alias = $this->name($model->alias);
$table = $this->fullTableName($model);
$joins = implode(' ', $this->_getJoins($model));
if (empty($conditions)) {
$alias = $joins = false;
}
$complexConditions = $this->_deleteNeedsComplexConditions($model, $conditions);
if (!$complexConditions) {
$joins = false;
}
$conditions = $this->conditions($this->defaultConditions($model, $conditions, $alias), true, true, $model);
if ($conditions === false) {
return false;
}
if ($this->execute($this->renderStatement('delete', compact('alias', 'table', 'joins', 'conditions'))) === false) {
$model->onError();
return false;
}
return true;
}
/**
* Checks whether complex conditions are needed for a delete with the given conditions.
*
* @param Model $model The model to delete from.
* @param mixed $conditions The conditions to use.
* @return bool Whether or not complex conditions are needed
*/
protected function _deleteNeedsComplexConditions(Model $model, $conditions) {
$fields = array_keys($this->describe($model));
foreach ((array)$conditions as $key => $value) {
if (in_array(strtolower(trim($key)), $this->_sqlBoolOps, true)) {
if ($this->_deleteNeedsComplexConditions($model, $value)) {
return true;
}
} elseif (strpos($key, $model->alias) === false && !in_array($key, $fields, true)) {
return true;
}
}
return false;
}
/**
* Sets the database encoding
*
* @param string $enc Database encoding
* @return bool
*/
public function setEncoding($enc) {
return $this->_execute('SET NAMES ' . $enc) !== false;
}
/**
* Returns an array of the indexes in given datasource name.
*
* @param string $model Name of model to inspect
* @return array Fields in table. Keys are column and unique
*/
public function index($model) {
$index = array();
$table = $this->fullTableName($model);
$old = version_compare($this->getVersion(), '4.1', '<=');
if ($table) {
$indexes = $this->_execute('SHOW INDEX FROM ' . $table);
// @codingStandardsIgnoreStart
// MySQL columns don't match the cakephp conventions.
while ($idx = $indexes->fetch(PDO::FETCH_OBJ)) {
if ($old) {
$idx = (object)current((array)$idx);
}
if (!isset($index[$idx->Key_name]['column'])) {
$col = array();
$index[$idx->Key_name]['column'] = $idx->Column_name;
if ($idx->Index_type === 'FULLTEXT') {
$index[$idx->Key_name]['type'] = strtolower($idx->Index_type);
} else {
$index[$idx->Key_name]['unique'] = (int)($idx->Non_unique == 0);
}
} else {
if (!empty($index[$idx->Key_name]['column']) && !is_array($index[$idx->Key_name]['column'])) {
$col[] = $index[$idx->Key_name]['column'];
}
$col[] = $idx->Column_name;
$index[$idx->Key_name]['column'] = $col;
}
if (!empty($idx->Sub_part)) {
if (!isset($index[$idx->Key_name]['length'])) {
$index[$idx->Key_name]['length'] = array();
}
$index[$idx->Key_name]['length'][$idx->Column_name] = $idx->Sub_part;
}
}
// @codingStandardsIgnoreEnd
$indexes->closeCursor();
}
return $index;
}
/**
* Generate a MySQL Alter Table syntax for the given Schema comparison
*
* @param array $compare Result of a CakeSchema::compare()
* @param string $table The table name.
* @return string|false String of alter statements to make.
*/
public function alterSchema($compare, $table = null) {
if (!is_array($compare)) {
return false;
}
$out = '';
$colList = array();
foreach ($compare as $curTable => $types) {
$indexes = $tableParameters = $colList = array();
if (!$table || $table === $curTable) {
$out .= 'ALTER TABLE ' . $this->fullTableName($curTable) . " \n";
foreach ($types as $type => $column) {
if (isset($column['indexes'])) {
$indexes[$type] = $column['indexes'];
unset($column['indexes']);
}
if (isset($column['tableParameters'])) {
$tableParameters[$type] = $column['tableParameters'];
unset($column['tableParameters']);
}
switch ($type) {
case 'add':
foreach ($column as $field => $col) {
$col['name'] = $field;
$alter = 'ADD ' . $this->buildColumn($col);
if (isset($col['after'])) {
$alter .= ' AFTER ' . $this->name($col['after']);
}
$colList[] = $alter;
}
break;
case 'drop':
foreach ($column as $field => $col) {
$col['name'] = $field;
$colList[] = 'DROP ' . $this->name($field);
}
break;
case 'change':
foreach ($column as $field => $col) {
if (!isset($col['name'])) {
$col['name'] = $field;
}
$alter = 'CHANGE ' . $this->name($field) . ' ' . $this->buildColumn($col);
if (isset($col['after'])) {
$alter .= ' AFTER ' . $this->name($col['after']);
}
$colList[] = $alter;
}
break;
}
}
$colList = array_merge($colList, $this->_alterIndexes($curTable, $indexes));
$colList = array_merge($colList, $this->_alterTableParameters($curTable, $tableParameters));
$out .= "\t" . implode(",\n\t", $colList) . ";\n\n";
}
}
return $out;
}
/**
* Generate a "drop table" statement for the given table
*
* @param type $table Name of the table to drop
* @return string Drop table SQL statement
*/
protected function _dropTable($table) {
return 'DROP TABLE IF EXISTS ' . $this->fullTableName($table) . ";";
}
/**
* Generate MySQL table parameter alteration statements for a table.
*
* @param string $table Table to alter parameters for.
* @param array $parameters Parameters to add & drop.
* @return array Array of table property alteration statements.
*/
protected function _alterTableParameters($table, $parameters) {
if (isset($parameters['change'])) {
return $this->buildTableParameters($parameters['change']);
}
return array();
}
/**
* Format indexes for create table
*
* @param array $indexes An array of indexes to generate SQL from
* @param string $table Optional table name, not used
* @return array An array of SQL statements for indexes
* @see DboSource::buildIndex()
*/
public function buildIndex($indexes, $table = null) {
$join = array();
foreach ($indexes as $name => $value) {
$out = '';
if ($name === 'PRIMARY') {
$out .= 'PRIMARY ';
$name = null;
} else {
if (!empty($value['unique'])) {
$out .= 'UNIQUE ';
}
$name = $this->startQuote . $name . $this->endQuote;
}
if (isset($value['type']) && strtolower($value['type']) === 'fulltext') {
$out .= 'FULLTEXT ';
}
$out .= 'KEY ' . $name . ' (';
if (is_array($value['column'])) {
if (isset($value['length'])) {
$vals = array();
foreach ($value['column'] as $column) {
$name = $this->name($column);
if (isset($value['length'])) {
$name .= $this->_buildIndexSubPart($value['length'], $column);
}
$vals[] = $name;
}
$out .= implode(', ', $vals);
} else {
$out .= implode(', ', array_map(array(&$this, 'name'), $value['column']));
}
} else {
$out .= $this->name($value['column']);
if (isset($value['length'])) {
$out .= $this->_buildIndexSubPart($value['length'], $value['column']);
}
}
$out .= ')';
$join[] = $out;
}
return $join;
}
/**
* Generate MySQL index alteration statements for a table.
*
* @param string $table Table to alter indexes for
* @param array $indexes Indexes to add and drop
* @return array Index alteration statements
*/
protected function _alterIndexes($table, $indexes) {
$alter = array();
if (isset($indexes['drop'])) {
foreach ($indexes['drop'] as $name => $value) {
$out = 'DROP ';
if ($name === 'PRIMARY') {
$out .= 'PRIMARY KEY';
} else {
$out .= 'KEY ' . $this->startQuote . $name . $this->endQuote;
}
$alter[] = $out;
}
}
if (isset($indexes['add'])) {
$add = $this->buildIndex($indexes['add']);
foreach ($add as $index) {
$alter[] = 'ADD ' . $index;
}
}
return $alter;
}
/**
* Format length for text indexes
*
* @param array $lengths An array of lengths for a single index
* @param string $column The column for which to generate the index length
* @return string Formatted length part of an index field
*/
protected function _buildIndexSubPart($lengths, $column) {
if ($lengths === null) {
return '';
}
if (!isset($lengths[$column])) {
return '';
}
return '(' . $lengths[$column] . ')';
}
/**
* Returns a detailed array of sources (tables) in the database.
*
* @param string $name Table name to get parameters
* @return array Array of table names in the database
*/
public function listDetailedSources($name = null) {
$condition = '';
if (is_string($name)) {
$condition = ' WHERE name = ' . $this->value($name);
}
$result = $this->_connection->query('SHOW TABLE STATUS ' . $condition, PDO::FETCH_ASSOC);
if (!$result) {
$result->closeCursor();
return array();
}
$tables = array();
foreach ($result as $row) {
$tables[$row['Name']] = (array)$row;
unset($tables[$row['Name']]['queryString']);
if (!empty($row['Collation'])) {
$charset = $this->getCharsetName($row['Collation']);
if ($charset) {
$tables[$row['Name']]['charset'] = $charset;
}
}
}
$result->closeCursor();
if (is_string($name) && isset($tables[$name])) {
return $tables[$name];
}
return $tables;
}
/**
* Converts database-layer column types to basic types
*
* @param string $real Real database-layer column type (i.e. "varchar(255)")
* @return string Abstract column type (i.e. "string")
*/
public function column($real) {
if (is_array($real)) {
$col = $real['name'];
if (isset($real['limit'])) {
$col .= '(' . $real['limit'] . ')';
}
return $col;
}
$col = str_replace(')', '', $real);
$limit = $this->length($real);
if (strpos($col, '(') !== false) {
list($col, $vals) = explode('(', $col);
}
if (in_array($col, array('date', 'time', 'datetime', 'timestamp'))) {
return $col;
}
if (($col === 'tinyint' && $limit === 1) || $col === 'boolean') {
return 'boolean';
}
if (strpos($col, 'bigint') !== false || $col === 'bigint') {
return 'biginteger';
}
if (strpos($col, 'tinyint') !== false) {
return 'tinyinteger';
}
if (strpos($col, 'smallint') !== false) {
return 'smallinteger';
}
if (strpos($col, 'int') !== false) {
return 'integer';
}
if (strpos($col, 'char') !== false || $col === 'tinytext') {
return 'string';
}
if (strpos($col, 'text') !== false) {
return 'text';
}
if (strpos($col, 'blob') !== false || $col === 'binary') {
return 'binary';
}
if (strpos($col, 'float') !== false || strpos($col, 'double') !== false) {
return 'float';
}
if (strpos($col, 'decimal') !== false || strpos($col, 'numeric') !== false) {
return 'decimal';
}
if (strpos($col, 'enum') !== false) {
return "enum($vals)";
}
if (strpos($col, 'set') !== false) {
return "set($vals)";
}
return 'text';
}
/**
* {@inheritDoc}
*/
public function value($data, $column = null, $null = true) {
$value = parent::value($data, $column, $null);
if (is_numeric($value) && substr($column, 0, 3) === 'set') {
return $this->_connection->quote($value);
}
return $value;
}
/**
* Gets the schema name
*
* @return string The schema name
*/
public function getSchemaName() {
return $this->config['database'];
}
/**
* Check if the server support nested transactions
*
* @return bool
*/
public function nestedTransactionSupported() {
return $this->useNestedTransactions && version_compare($this->getVersion(), '4.1', '>=');
}
/**
* Check if column type is unsigned
*
* @param string $real Real database-layer column type (i.e. "varchar(255)")
* @return bool True if column is unsigned, false otherwise
*/
protected function _unsigned($real) {
return strpos(strtolower($real), 'unsigned') !== false;
}
/**
* Inserts multiple values into a table. Uses a single query in order to insert
* multiple rows.
*
* @param string $table The table being inserted into.
* @param array $fields The array of field/column names being inserted.
* @param array $values The array of values to insert. The values should
* be an array of rows. Each row should have values keyed by the column name.
* Each row must have the values in the same order as $fields.
* @return bool
*/
public function insertMulti($table, $fields, $values) {
$table = $this->fullTableName($table);
$holder = implode(', ', array_fill(0, count($fields), '?'));
$fields = implode(', ', array_map(array($this, 'name'), $fields));
$pdoMap = array(
'integer' => PDO::PARAM_INT,
'float' => PDO::PARAM_STR,
'boolean' => PDO::PARAM_BOOL,
'string' => PDO::PARAM_STR,
'text' => PDO::PARAM_STR
);
$columnMap = array();
$rowHolder = "({$holder})";
$sql = "INSERT INTO {$table} ({$fields}) VALUES ";
$countRows = count($values);
for ($i = 0; $i < $countRows; $i++) {
if ($i !== 0) {
$sql .= ',';
}
$sql .= " $rowHolder";
}
$statement = $this->_connection->prepare($sql);
foreach ($values[key($values)] as $key => $val) {
$type = $this->introspectType($val);
$columnMap[$key] = $pdoMap[$type];
}
$valuesList = array();
$i = 1;
foreach ($values as $value) {
foreach ($value as $col => $val) {
$valuesList[] = $val;
$statement->bindValue($i, $val, $columnMap[$col]);
$i++;
}
}
$result = $statement->execute();
$statement->closeCursor();
if ($this->fullDebug) {
$this->logQuery($sql, $valuesList);
}
return $result;
}
}

50
app/Model/DecayingModelsFormulas/Base.php
View File

@ -6,8 +6,30 @@ abstract class DecayingModelBase
return 'BONFIRE LIT';
}
protected function __extractTagBasename($tagName) {
$pieces = array();
if (preg_match('/^[^:="]+:[^:="]+="[^:="]+"$/i', $tagName)) {
$temp = explode(':', $tagName);
$pieces = array_merge(array($temp[0]), explode('=', $temp[1]));
$pieces['complete'] = $tagName;
$pieces['namespace'] = $pieces[0];
$pieces['predicate'] = $pieces[1];
$pieces['2tag'] = sprintf('%s:%s', $pieces[0], $pieces[1]);
$pieces['base'] = sprintf('%s:%s', $pieces[0], $pieces[1]);
} elseif (preg_match('/^[^:="]+:[^:="]+$/i', $tagName)) {
$pieces = explode(':', $tagName);
$pieces['complete'] = $tagName;
$pieces['namespace'] = $pieces[0];
$pieces['predicate'] = $pieces[1];
$pieces['2tag'] = sprintf('%s:%s', $pieces[0], $pieces[1]);
$pieces['base'] = $pieces[0];
}
return $pieces;
}
// Get effective taxonomy ratio based on taxonomies attached to the attribute
// Basically, it adapts the ratio defined in the model to fit the actual attached tags
// Basically, it adapts the ratio defined in the model to fit the actual attached tags
protected function __getRatioScore($model, $tags)
{
$ratioScore = array();
@ -17,21 +39,21 @@ abstract class DecayingModelBase
}
$total_score = 0.0;
foreach ($tags as $tag) {
$namespace_predicate = explode('=', $tag['Tag']['name'])[0];
if (isset($taxonomy_base_ratio[$namespace_predicate]) && is_numeric($tag['Tag']['numerical_value'])) {
$total_score += floatval($taxonomy_base_ratio[$namespace_predicate]);
$tagBaseName = $this->__extractTagBasename($tag['Tag']['name'])['base'];
if (isset($taxonomy_base_ratio[$tagBaseName]) && is_numeric($tag['Tag']['numerical_value'])) {
$total_score += floatval($taxonomy_base_ratio[$tagBaseName]);
}
}
foreach ($tags as $i => $tag) {
$namespace_predicate = explode('=', $tag['Tag']['name'])[0];
if (isset($taxonomy_base_ratio[$namespace_predicate]) && is_numeric($tag['Tag']['numerical_value'])) {
$ratioScore[$namespace_predicate] = floatval($taxonomy_base_ratio[$namespace_predicate]) / $total_score;
$tagBaseName = $this->__extractTagBasename($tag['Tag']['name'])['base'];
if (isset($taxonomy_base_ratio[$tagBaseName]) && is_numeric($tag['Tag']['numerical_value'])) {
$ratioScore[$tagBaseName] = floatval($taxonomy_base_ratio[$tagBaseName]) / $total_score;
}
}
return $ratioScore;
}
// return attribute tag with event tag matching the namespace+predicate overridden
// return attribute tag with event tag matching the tag basename overridden
protected function __getPrioritisedTag($attribute)
{
$tags = array();
@ -40,15 +62,15 @@ abstract class DecayingModelBase
if (isset($attribute['EventTag'])) {
foreach ($attribute['EventTag'] as $i => $tag) {
$tags[] = $tag;
$namespace_predicate = explode('=', $tag['Tag']['name'])[0];
$temp_mapping[$namespace_predicate][] = $i;
$tagBaseName = $this->__extractTagBasename($tag['Tag']['name'])['base'];
$temp_mapping[$tagBaseName][] = $i;
}
}
if (isset($attribute['AttributeTag'])) {
foreach ($attribute['AttributeTag'] as $tag) {
$namespace_predicate = explode('=', $tag['Tag']['name'])[0];
if (!empty($temp_mapping[$namespace_predicate])) { // need to override event tag
foreach ($temp_mapping[$namespace_predicate] as $i => $eventtag_index) {
$tagBaseName = $this->__extractTagBasename($tag['Tag']['name'])['base'];
if (!empty($temp_mapping[$tagBaseName])) { // need to override event tag
foreach ($temp_mapping[$tagBaseName] as $i => $eventtag_index) {
$overridden_tags[] = array(
'EventTag' => $tags[$eventtag_index],
'AttributeTag' => $tag
@ -124,5 +146,3 @@ abstract class DecayingModelBase
// Return a True if the attribute should be marked as decayed
abstract public function isDecayed($model, $attribute, $score);
}
?>

1
app/Model/DecayingModelsFormulas/Polynomial.php
View File

@ -24,4 +24,3 @@ class Polynomial extends DecayingModelBase
return $threshold > $score;
}
}
?>

1
app/Model/DecayingModelsFormulas/PolynomialExtended.php
View File

@ -51,4 +51,3 @@ class PolynomialExtended extends Polynomial
return parent::isDecayed($model, $attribute, $score);
}
}
?>

697
app/Model/Event.php
View File

File diff suppressed because it is too large Load Diff

2
app/Model/EventTag.php
View File

@ -212,7 +212,7 @@ class EventTag extends AppModel
return array('scores' => $scores, 'maxScore' => $maxScore);
}
// Fetch all tags contained in an event (both event and attributes) ignoring the occurence. No ACL
// Fetch all tags contained in an event (both event and attributes) ignoring the occurrence. No ACL
public function getTagScoresUniform($eventId=0, $allowedTags=array())
{
$conditions = array('Tag.id !=' => null);

58
app/Model/Feed.php
View File

@ -365,19 +365,12 @@ class Feed extends AppModel
if ($scope === 'Server' || $source[$scope]['source_format'] == 'misp') {
$pipe = $redis->multi(Redis::PIPELINE);
$eventUuidHitPosition = array();
$i = 0;
foreach ($objects as $k => $object) {
if (isset($object[$scope])) {
foreach ($object[$scope] as $currentFeed) {
if ($source[$scope]['id'] == $currentFeed['id']) {
$eventUuidHitPosition[$i] = $k;
$i++;
if (in_array($object['type'], $compositeTypes)) {
$value = explode('|', $object['value']);
$redis->smembers($cachePrefix . 'event_uuid_lookup:' . md5($value[0]));
} else {
$redis->smembers($cachePrefix . 'event_uuid_lookup:' . md5($object['value']));
}
$eventUuidHitPosition[] = $k;
$redis->smembers($cachePrefix . 'event_uuid_lookup:' . $hashTable[$k]);
}
}
}
@ -422,7 +415,7 @@ class Feed extends AppModel
}
} catch (Exception $e) {
CakeLog::error($this->exceptionAsMessage("Could not add event '$uuid' from feed {$feed['Feed']['id']}.", $e));
$this->logException("Could not add event '$uuid' from feed {$feed['Feed']['id']}.", $e);
$results['add']['fail'] = array('uuid' => $uuid, 'reason' => $e->getMessage());
}
@ -439,7 +432,7 @@ class Feed extends AppModel
$results['edit']['success'] = $uuid;
}
} catch (Exception $e) {
CakeLog::error($this->exceptionAsMessage("Could not edit event '$uuid' from feed {$feed['Feed']['id']}.", $e));
$this->logException("Could not edit event '$uuid' from feed {$feed['Feed']['id']}.", $e);
$results['edit']['fail'] = array('uuid' => $uuid, 'reason' => $e->getMessage());
}
@ -780,7 +773,7 @@ class Feed extends AppModel
try {
$actions = $this->getNewEventUuids($this->data, $HttpSocket);
} catch (Exception $e) {
CakeLog::error($this->exceptionAsMessage("Could not get new event uuids for feed $feedId.", $e));
$this->logException("Could not get new event uuids for feed $feedId.", $e);
$this->jobProgress($jobId, 'Could not fetch event manifest. See log for more details.');
return false;
}
@ -798,7 +791,7 @@ class Feed extends AppModel
try {
$temp = $this->getFreetextFeed($this->data, $HttpSocket, $this->data['Feed']['source_format'], 'all');
} catch (Exception $e) {
CakeLog::error($this->exceptionAsMessage("Could not get freetext feed $feedId", $e));
$this->logException("Could not get freetext feed $feedId", $e);
$this->jobProgress($jobId, 'Could not fetch freetext feed. See log for more details.');
return false;
}
@ -821,7 +814,7 @@ class Feed extends AppModel
try {
$result = $this->saveFreetextFeedData($this->data, $data, $user);
} catch (Exception $e) {
CakeLog::error($this->exceptionAsMessage("Could not save freetext feed data for feed $feedId.", $e));
$this->logException("Could not save freetext feed data for feed $feedId.", $e);
return false;
}
@ -964,7 +957,7 @@ class Feed extends AppModel
* @param $user Not used
* @param int|bool $jobId
* @param string $scope
* @return bool Returns true if at least one feed was cached sucessfully.
* @return bool Returns true if at least one feed was cached successfully.
* @throws Exception
*/
public function cacheFeedInitiator($user, $jobId = false, $scope = 'freetext')
@ -1037,14 +1030,15 @@ class Feed extends AppModel
try {
$values = $this->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format'], 'all');
} catch (Exception $e) {
CakeLog::error($this->exceptionAsMessage("Could not get freetext feed $feedId", $e));
$this->logException("Could not get freetext feed $feedId", $e);
$this->jobProgress($jobId, 'Could not fetch freetext feed. See log for more details.');
return false;
}
foreach ($values as $k => $value) {
$redis->sAdd('misp:feed_cache:' . $feedId, md5($value['value']));
$redis->sAdd('misp:feed_cache:combined', md5($value['value']));
$md5Value = md5($value['value']);
$redis->sAdd('misp:feed_cache:' . $feedId, $md5Value);
$redis->sAdd('misp:feed_cache:combined', $md5Value);
if ($k % 1000 == 0) {
$this->jobProgress($jobId, "Feed $feedId: $k/" . count($values) . " values cached.");
}
@ -1060,7 +1054,7 @@ class Feed extends AppModel
try {
$manifest = $this->getManifest($feed, $HttpSocket);
} catch (Exception $e) {
CakeLog::error($this->exceptionAsMessage("Could not get manifest for feed $feedId.", $e));
$this->logException("Could not get manifest for feed $feedId.", $e);
return false;
}
@ -1071,7 +1065,7 @@ class Feed extends AppModel
try {
$event = $this->downloadAndParseEventFromFeed($feed, $uuid, $HttpSocket);
} catch (Exception $e) {
CakeLog::error($this->exceptionAsMessage("Could not get and parse event '$uuid' for feed $feedId.", $e));
$this->logException("Could not get and parse event '$uuid' for feed $feedId.", $e);
return false;
}
@ -1112,7 +1106,7 @@ class Feed extends AppModel
try {
$cache = $this->getCache($feed, $HttpSocket);
} catch (Exception $e) {
CakeLog::notice($this->exceptionAsMessage("Could not get cache file for $feedId.", $e));
$this->logException("Could not get cache file for $feedId.", $e, LOG_NOTICE);
return false;
}
@ -1230,8 +1224,10 @@ class Feed extends AppModel
public function importFeeds($feeds, $user, $default = false)
{
$feeds = json_decode($feeds, true);
if (!isset($feeds[0])) {
if (is_string($feeds)) {
$feeds = json_decode($feeds, true);
}
if ($feeds && !isset($feeds[0])) {
$feeds = array($feeds);
}
$results = array('successes' => 0, 'fails' => 0);
@ -1625,22 +1621,6 @@ class Feed extends AppModel
}
}
/**
* @param string $message
* @param Exception $exception
* @return string
*/
private function exceptionAsMessage($message, $exception)
{
$message = sprintf("%s\n[%s] %s",
$message,
get_class($exception),
$exception->getMessage()
);
$message .= "\nStack Trace:\n" . $exception->getTraceAsString();
return $message;
}
/**
* remove all events tied to a feed. Returns int on success, error message
* as string on failure

10
app/Model/Galaxy.php
View File

@ -378,8 +378,12 @@ class Galaxy extends AppModel
{
$galaxy = $this->find('first', array(
'recursive' => -1,
'fields' => 'id',
'conditions' => array('Galaxy.type' => $type, 'Galaxy.namespace' => $namespace),
'fields' => array('MAX(Galaxy.version) as latest_version', 'id'),
'conditions' => array(
'Galaxy.type' => $type,
'Galaxy.namespace' => $namespace
),
'group' => array('name', 'id')
));
return empty($galaxy) ? 0 : $galaxy['Galaxy']['id'];
}
@ -456,7 +460,7 @@ class Galaxy extends AppModel
$matrixData['tabs'] = $cols;
$this->sortMatrixByScore($matrixData['tabs'], $scores);
// #FIXME temporary fix: retreive tag name of deprecated mitre galaxies (for the stats)
// #FIXME temporary fix: retrieve tag name of deprecated mitre galaxies (for the stats)
if ($galaxy['Galaxy']['id'] == $this->getMitreAttackGalaxyId()) {
$names = array('Enterprise Attack - Attack Pattern', 'Pre Attack - Attack Pattern', 'Mobile Attack - Attack Pattern');
$tag_names = array();

145
app/Model/GalaxyCluster.php
View File

@ -142,32 +142,98 @@ class GalaxyCluster extends AppModel
*/
public function getCluster($name)
{
$conditions = array('LOWER(GalaxyCluster.tag_name)' => strtolower($name));
if (is_numeric($name)) {
$conditions = array('GalaxyCluster.id' => $name);
}
if (isset($this->__clusterCache[$name])) {
return $this->__clusterCache[$name];
}
$objects = array('Galaxy', 'GalaxyElement');
if (is_numeric($name)) {
$conditions = array('GalaxyCluster.id' => $name);
} else {
$conditions = array('LOWER(GalaxyCluster.tag_name)' => strtolower($name));
}
$cluster = $this->find('first', array(
'conditions' => $conditions,
'contain' => array('Galaxy', 'GalaxyElement')
));
if (!empty($cluster)) {
if (isset($cluster['Galaxy'])) {
$cluster['GalaxyCluster']['Galaxy'] = $cluster['Galaxy'];
unset($cluster['Galaxy']);
}
$elements = array();
foreach ($cluster['GalaxyElement'] as $element) {
if (!isset($elements[$element['key']])) {
$elements[$element['key']] = array($element['value']);
} else {
$elements[$element['key']][] = $element['value'];
$cluster = $this->postprocess($cluster);
}
$this->__clusterCache[$name] = $cluster;
return $cluster;
}
/**
* @param array $events
* @param bool $replace
* @return array
*/
public function attachClustersToEventIndex(array $events, $replace = false)
{
$clusterTagNames = array();
foreach ($events as $event) {
foreach ($event['EventTag'] as $k2 => $eventTag) {
if (substr($eventTag['Tag']['name'], 0, strlen('misp-galaxy:')) === 'misp-galaxy:') {
$clusterTagNames[] = strtolower($eventTag['Tag']['name']);
}
}
unset($cluster['GalaxyElement']);
}
$clusters = $this->find('all', array(
'conditions' => array('LOWER(GalaxyCluster.tag_name)' => $clusterTagNames),
'contain' => array('Galaxy', 'GalaxyElement'),
));
$clustersByTagName = array();
foreach ($clusters as $cluster) {
$clustersByTagName[strtolower($cluster['GalaxyCluster']['tag_name'])] = $cluster;
}
foreach ($events as $k => $event) {
foreach ($event['EventTag'] as $k2 => $eventTag) {
$tagName = strtolower($eventTag['Tag']['name']);
if (isset($clustersByTagName[$tagName])) {
$cluster = $this->postprocess($clustersByTagName[$tagName], $eventTag['Tag']['id']);
$cluster['GalaxyCluster']['tag_id'] = $eventTag['Tag']['id'];
$cluster['GalaxyCluster']['local'] = $eventTag['local'];
$events[$k]['GalaxyCluster'][] = $cluster['GalaxyCluster'];
if ($replace) {
unset($events[$k]['EventTag'][$k2]);
}
}
}
}
return $events;
}
/**
* @param array $cluster
* @param int|null $tagId
* @return array
*/
private function postprocess(array $cluster, $tagId = null)
{
if (isset($cluster['Galaxy'])) {
$cluster['GalaxyCluster']['Galaxy'] = $cluster['Galaxy'];
unset($cluster['Galaxy']);
}
$elements = array();
foreach ($cluster['GalaxyElement'] as $element) {
if (!isset($elements[$element['key']])) {
$elements[$element['key']] = array($element['value']);
} else {
$elements[$element['key']][] = $element['value'];
}
}
unset($cluster['GalaxyElement']);
$cluster['GalaxyCluster']['meta'] = $elements;
if ($tagId) {
$cluster['GalaxyCluster']['tag_id'] = $tagId;
} else {
$this->Tag = ClassRegistry::init('Tag');
$tag_id = $this->Tag->find(
'first',
@ -182,29 +248,40 @@ class GalaxyCluster extends AppModel
if (!empty($tag_id)) {
$cluster['GalaxyCluster']['tag_id'] = $tag_id['Tag']['id'];
}
$cluster['GalaxyCluster']['meta'] = $elements;
}
$this->__clusterCache[$name] = $cluster;
return $cluster;
}
public function attachClustersToEventIndex($events, $replace = false)
public function getClusterTagsFromMeta($galaxyElements)
{
foreach ($events as $k => $event) {
foreach ($event['EventTag'] as $k2 => $eventTag) {
if (substr($eventTag['Tag']['name'], 0, strlen('misp-galaxy:')) === 'misp-galaxy:') {
$cluster = $this->getCluster($eventTag['Tag']['name']);
if ($cluster) {
$cluster['GalaxyCluster']['tag_id'] = $eventTag['Tag']['id'];
$cluster['GalaxyCluster']['local'] = $eventTag['local'];
$events[$k]['GalaxyCluster'][] = $cluster['GalaxyCluster'];
if ($replace) {
unset($events[$k]['EventTag'][$k2]);
}
}
}
}
// AND operator between cluster metas
$tmpResults = array();
foreach ($galaxyElements as $galaxyElementKey => $galaxyElementValue) {
$tmpResults[] = array_values($this->GalaxyElement->find('list', array(
'conditions' => array(
'key' => $galaxyElementKey,
'value' => $galaxyElementValue,
),
'fields' => array('galaxy_cluster_id'),
'recursive' => -1
)));
}
return $events;
$clusterTags = array();
if (!empty($tmpResults)) {
// Get all Clusters matching all conditions
$matchingClusters = $tmpResults[0];
array_shift($tmpResults);
foreach ($tmpResults as $tmpResult) {
$matchingClusters = array_intersect($matchingClusters, $tmpResult);
}
$clusterTags = $this->find('list', array(
'conditions' => array('id' => $matchingClusters),
'fields' => array('GalaxyCluster.tag_name'),
'recursive' => -1
));
}
return array_values($clusterTags);
}
}

152
app/Model/Log.php
View File

@ -16,51 +16,56 @@ class Log extends AppModel
);
public $validate = array(
'action' => array(
'rule' => array('inList', array(
'accept',
'accept_delegation',
'add',
'admin_email',
'auth',
'auth_fail',
'blacklisted',
'change_pw',
'delete',
'disable',
'discard',
'edit',
'email',
'enable',
'error',
'export',
'file_upload',
'galaxy',
'include_formula',
'login',
'login_fail',
'logout',
'merge',
'pruneUpdateLogs',
'publish',
'publish alert',
'pull',
'purge_events',
'push',
'remove_dead_workers',
'request',
'request_delegation',
'reset_auth_key',
'security',
'serverSettingsEdit',
'tag',
'undelete',
'update',
'update_database',
'upgrade_24',
'upload_sample',
'version_warning',
'warning'
)),
'rule' => array(
'inList',
array( // ensure that the length of the rules is < 20 in length
'accept',
'accept_delegation',
'add',
'admin_email',
'auth',
'auth_fail',
'blacklisted',
'change_pw',
'delete',
'disable',
'discard',
'edit',
'email',
'enable',
'error',
'export',
'file_upload',
'galaxy',
'include_formula',
'login',
'login_fail',
'logout',
'merge',
'pruneUpdateLogs',
'publish',
'publish_sightings',
'publish alert',
'pull',
'purge_events',
'push',
'remove_dead_workers',
'request',
'request_delegation',
'reset_auth_key',
'security',
'serverSettingsEdit',
'tag',
'undelete',
'update',
'update_database',
'update_db_worker',
'upgrade_24',
'upload_sample',
'version_warning',
'warning'
)
),
'message' => 'Options : ...'
)
);
@ -83,7 +88,7 @@ class Log extends AppModel
public $logMetaAdmin = array(
'update' => array('values' => array('update_database'), 'name' => 'MISP Update results'),
'settings' => array('values' => array('serverSettingsEdit', 'remove_dead_workers'), 'name' => 'Setting changes'),
'errors' => array('values' => array('warning', 'errors', 'version_warning'), 'name' => 'Warnings and errors'),
'errors' => array('values' => array('warning', 'error', 'version_warning'), 'name' => 'Warnings and errors'),
'email' => array('values' => array('admin_email'))
);
@ -177,19 +182,54 @@ class Log extends AppModel
return $data;
}
public function createLogEntry($user = array('Organisation' => array('name' => 'SYSTEM'), 'email' => 'SYSTEM', 'id' => 0), $action, $model, $model_id = 0, $title = '', $change = '')
/**
* @param string|array $user
* @param string $action
* @param string $model
* @param int $modelId
* @param string $title
* @param string|array $change
* @return array
* @throws Exception
*/
public function createLogEntry($user, $action, $model, $modelId = 0, $title = '', $change = '')
{
if ($user === 'SYSTEM') {
$user = array('Organisation' => array('name' => 'SYSTEM'), 'email' => 'SYSTEM', 'id' => 0);
} else if (!is_array($user)) {
throw new InvalidArgumentException("User must be array or 'SYSTEM' string.");
}
if (is_array($change)) {
$output = array();
foreach ($change as $field => $values) {
if (strpos($field, 'password') !== false) { // if field name contains password, replace value with asterisk
$oldValue = $newValue = "*****";
} else {
list($oldValue, $newValue) = $values;
}
$output[] = "$field ($oldValue) => ($newValue)";
}
$change = implode(", ", $output);
}
$this->create();
$this->save(array(
'org' => $user['Organisation']['name'],
'email' =>$user['email'],
'user_id' => $user['id'],
'action' => $action,
'title' => $title,
'change' => $change,
'model' => $model,
'model_id' => $model_id,
$result = $this->save(array(
'org' => $user['Organisation']['name'],
'email' => $user['email'],
'user_id' => $user['id'],
'action' => $action,
'title' => $title,
'change' => $change,
'model' => $model,
'model_id' => $modelId,
));
if (!$result) {
throw new Exception("Cannot save log because of validation errors: " . json_encode($this->validationErrors));
}
return $result;
}
// to combat a certain bug that causes the upgrade scripts to loop without being able to set the correct version

444
app/Model/MispObject.php
View File

@ -46,6 +46,8 @@ class MispObject extends AppModel
),
);
public $shortDist = array(0 => 'Organisation', 1 => 'Community', 2 => 'Connected', 3 => 'All', 4 => ' Sharing Group', 5 => 'Inherit');
public $validate = array(
'uuid' => array(
'uuid' => array(
@ -56,10 +58,46 @@ class MispObject extends AppModel
'rule' => 'isUnique',
'message' => 'The UUID provided is not unique',
'required' => 'create'
)
),
),
'first_seen' => array(
'rule' => array('datetimeOrNull'),
'required' => false,
'message' => array('Invalid ISO 8601 format')
),
'last_seen' => array(
'rule' => array('datetimeOrNull'),
'required' => false,
'message' => array('Invalid ISO 8601 format')
)
);
// check whether the variable is null or datetime
public function datetimeOrNull($fields)
{
$k = array_keys($fields)[0];
$seen = $fields[$k];
try {
new DateTime($seen);
$returnValue = true;
} catch (Exception $e) {
$returnValue = false;
}
return $returnValue || is_null($seen);
}
public function afterFind($results, $primary = false)
{
foreach ($results as $k => $v) {
$results[$k] = $this->Attribute->UTCToISODatetime($results[$k], $this->alias);
}
return $results;
}
public function beforeSave($options = array()) {
$this->data = $this->Attribute->ISODatetimeToUTC($this->data, $this->alias);
}
public function beforeValidate($options = array())
{
parent::beforeValidate();
@ -75,6 +113,14 @@ class MispObject extends AppModel
$date = new DateTime();
$this->data[$this->alias]['timestamp'] = $date->getTimestamp();
}
// parse first_seen different formats
if (isset($this->data[$this->alias]['first_seen'])) {
$this->data[$this->alias]['first_seen'] = $this->data[$this->alias]['first_seen'] === '' ? null : $this->data[$this->alias]['first_seen'];
}
// parse last_seen different formats
if (isset($this->data[$this->alias]['last_seen'])) {
$this->data[$this->alias]['last_seen'] = $this->data[$this->alias]['last_seen'] === '' ? null : $this->data[$this->alias]['last_seen'];
}
if (empty($this->data[$this->alias]['template_version'])) {
$this->data[$this->alias]['template_version'] = 1;
}
@ -184,6 +230,18 @@ class MispObject extends AppModel
$result = $this->id;
foreach ($object['Attribute'] as $k => $attribute) {
$object['Attribute'][$k]['object_id'] = $this->id;
if (
(!array_key_exists('first_seen', $object['Attribute'][$k]) || is_null($object['Attribute'][$k]['first_seen'])) &&
(array_key_exists('first_seen', $object['Object']) && !is_null($object['Object']['first_seen']))
) {
$object['Attribute'][$k]['first_seen'] = $object['Object']['first_seen'];
}
if (
(!array_key_exists('last_seen', $object['Attribute'][$k]) || is_null($object['Attribute'][$k]['last_seen'])) &&
(array_key_exists('last_seen', $object['Object']) && !is_null($object['Object']['last_seen']))
) {
$object['Attribute'][$k]['last_seen'] = $object['Object']['last_seen'];
}
}
$this->Attribute->saveAttributes($object['Attribute']);
} else {
@ -254,6 +312,28 @@ class MispObject extends AppModel
return $conditions;
}
public function fetchObjectSimple($user, $options = array())
{
$params = array(
'conditions' => $this->buildConditions($user),
'fields' => array(),
'recursive' => -1
);
if (isset($options['conditions'])) {
$params['conditions']['AND'][] = $options['conditions'];
}
if (isset($options['fields'])) {
$params['fields'] = $options['fields'];
}
$results = $this->find('all', array(
'conditions' => $params['conditions'],
'recursive' => -1,
'fields' => $params['fields'],
'contain' => array('Event' => array('distribution', 'id', 'user_id', 'orgc_id', 'org_id')),
'sort' => false
));
return $results;
}
// Method that fetches all objects
// very flexible, it's basically a replacement for find, with the addition that it restricts access based on user
@ -489,12 +569,57 @@ class MispObject extends AppModel
$tmpfile->delete();
$tmpfile->close();
}
if (!isset($attributes['Attribute'][$k]['first_seen'])) {
$attributes['Attribute'][$k]['first_seen'] = null;
}
if (!isset($attributes['Attribute'][$k]['last_seen'])) {
$attributes['Attribute'][$k]['last_seen'] = null;
}
unset($attributes['Attribute'][$k]['save']);
}
return $attributes;
}
public function deltaMerge($object, $objectToSave)
// Set Object's *-seen (and ObjectAttribute's *-seen and ObjectAttribute's value if requested) to the provided *-seen value
// Therefore, synchronizing the 3 values
public function syncObjectAndAttributeSeen($object, $forcedSeenOnElements, $applyOnAttribute=True) {
if (empty($forcedSeenOnElements)) {
return $object;
}
if (isset($forcedSeenOnElements['first_seen'])) {
$object['Object']['first_seen'] = $forcedSeenOnElements['first_seen'];
}
if (isset($forcedSeenOnElements['last_seen'])) {
$object['Object']['last_seen'] = $forcedSeenOnElements['last_seen'];
}
if ($applyOnAttribute) {
if (isset($object['Attribute'])) {
$attributes = $object['Attribute'];
} else {
$attributes = $this->find('first', array(
'conditions' => array('id' => $object['Object']['id']),
'contain' => array('Attribute')
))['Attribute'];
}
foreach($attributes as $i => $attribute) {
if (isset($forcedSeenOnElements['first_seen'])) {
$attributes[$i]['first_seen'] = $forcedSeenOnElements['first_seen'];
if ($attribute['object_relation'] == 'first-seen') {
$attributes[$i]['value'] = $forcedSeenOnElements['first_seen'];
}
} elseif (isset($forcedSeenOnElements['last_seen'])) {
$attributes[$i]['last_seen'] = $forcedSeenOnElements['last_seen'];
if ($attribute['object_relation'] == 'last-seen') {
$attributes[$i]['value'] = $forcedSeenOnElements['last_seen'];
}
}
}
$object['Attribute'] = $attributes;
}
return $object;
}
public function deltaMerge($object, $objectToSave, $onlyAddNewAttribute=false)
{
if (!isset($objectToSave['Object'])) {
$dataToBackup = array('ObjectReferences', 'Attribute', 'ShadowAttribute');
@ -513,69 +638,144 @@ class MispObject extends AppModel
}
unset($dataToBackup);
}
$object['Object']['comment'] = $objectToSave['Object']['comment'];
$object['Object']['distribution'] = $objectToSave['Object']['distribution'];
if ($object['Object']['distribution'] == 4) {
$object['Object']['sharing_group_id'] = $objectToSave['Object']['sharing_group_id'];
if (isset($objectToSave['Object']['comment'])) {
$object['Object']['comment'] = $objectToSave['Object']['comment'];
}
if (isset($objectToSave['Object']['distribution'])) {
$object['Object']['distribution'] = $objectToSave['Object']['distribution'];
if ($object['Object']['distribution'] == 4) {
$object['Object']['sharing_group_id'] = $objectToSave['Object']['sharing_group_id'];
}
}
$date = new DateTime();
$object['Object']['timestamp'] = $date->getTimestamp();
$this->save($object);
$checkFields = array('category', 'value', 'to_ids', 'distribution', 'sharing_group_id', 'comment', 'disable_correlation');
if (!empty($objectToSave['Attribute'])) {
foreach ($objectToSave['Attribute'] as $newKey => $newAttribute) {
foreach ($object['Attribute'] as $origKey => $originalAttribute) {
if (!empty($newAttribute['uuid'])) {
if ($newAttribute['uuid'] == $originalAttribute['uuid']) {
$different = false;
foreach ($checkFields as $f) {
if ($f == 'sharing_group_id' && empty($newAttribute[$f])) {
$newAttribute[$f] = 0;
$forcedSeenOnElements = array();
if (isset($objectToSave['Object']['first_seen'])) {
$forcedSeenOnElements['first_seen'] = $objectToSave['Object']['first_seen'];
}
if (isset($objectToSave['Object']['last_seen'])) {
$forcedSeenOnElements['last_seen'] = $objectToSave['Object']['last_seen'];
}
$object = $this->syncObjectAndAttributeSeen($object, $forcedSeenOnElements, false);
$saveResult = $this->save($object);
if ($saveResult === false) {
return $this->validationErrors;
}
if (!$onlyAddNewAttribute) {
$checkFields = array('category', 'value', 'to_ids', 'distribution', 'sharing_group_id', 'comment', 'disable_correlation', 'first_seen', 'last_seen');
if (!empty($objectToSave['Attribute'])) {
foreach ($objectToSave['Attribute'] as $newKey => $newAttribute) {
foreach ($object['Attribute'] as $origKey => $originalAttribute) {
if (!empty($newAttribute['uuid'])) {
if ($newAttribute['uuid'] == $originalAttribute['uuid']) {
$different = false;
foreach ($checkFields as $f) {
if ($f == 'sharing_group_id' && empty($newAttribute[$f])) {
$newAttribute[$f] = 0;
}
if ($newAttribute[$f] != $originalAttribute[$f]) {
$different = true;
}
// Set seen of object at attribute level
if (isset($forcedSeenOnElements['first_seen'])) {
$newAttribute['first_seen'] = $forcedSeenOnElements['first_seen'];
if ($newAttribute['object_relation'] == 'first-seen') {
// $newAttribute['value'] = $forcedSeenOnElements['first_seen'];
}
$different = true;
}
if (isset($forcedSeenOnElements['last_seen'])) {
$newAttribute['last_seen'] = $forcedSeenOnElements['last_seen'];
if ($newAttribute['object_relation'] == 'last-seen') {
// $newAttribute['value'] = $forcedSeenOnElements['last_seen'];
}
$different = true;
}
}
if ($newAttribute[$f] != $originalAttribute[$f]) {
$different = true;
if ($different) {
$newAttribute['id'] = $originalAttribute['id'];
$newAttribute['event_id'] = $object['Object']['event_id'];
$newAttribute['object_id'] = $object['Object']['id'];
$newAttribute['timestamp'] = $date->getTimestamp();
$result = $this->Event->Attribute->save(array('Attribute' => $newAttribute), array(
'category',
'value',
'to_ids',
'distribution',
'sharing_group_id',
'comment',
'timestamp',
'object_id',
'event_id',
'disable_correlation',
'first_seen',
'last_seen'
));
}
unset($object['Attribute'][$origKey]);
continue 2;
}
if ($different) {
$newAttribute['id'] = $originalAttribute['id'];
$newAttribute['event_id'] = $object['Object']['event_id'];
$newAttribute['object_id'] = $object['Object']['id'];
$newAttribute['timestamp'] = $date->getTimestamp();
$result = $this->Event->Attribute->save(array('Attribute' => $newAttribute), array(
'category',
'value',
'to_ids',
'distribution',
'sharing_group_id',
'comment',
'timestamp',
'object_id',
'event_id',
'disable_correlation'
));
}
unset($object['Attribute'][$origKey]);
continue 2;
}
}
}
$this->Event->Attribute->create();
$newAttribute['event_id'] = $object['Object']['event_id'];
$newAttribute['object_id'] = $object['Object']['id'];
if (!isset($newAttribute['timestamp'])) {
$newAttribute['distribution'] = Configure::read('MISP.default_attribute_distribution');
if ($newAttribute['distribution'] == 'event') {
$newAttribute['distribution'] = 5;
$this->Event->Attribute->create();
$newAttribute['event_id'] = $object['Object']['event_id'];
$newAttribute['object_id'] = $object['Object']['id'];
// Set seen of object at attribute level
if (isset($forcedSeenOnElements['first_seen'])) {
$newAttribute['first_seen'] = $forcedSeenOnElements['first_seen'];
if ($newAttribute['object_relation'] == 'first-seen') {
$newAttribute['value'] = $forcedSeenOnElements['first_seen'];
}
}
if (isset($forcedSeenOnElements['last_seen'])) {
$newAttribute['last_seen'] = $forcedSeenOnElements['last_seen'];
if ($newAttribute['object_relation'] == 'last-seen') {
$newAttribute['value'] = $forcedSeenOnElements['last_seen'];
}
}
if (!isset($newAttribute['timestamp'])) {
$newAttribute['distribution'] = Configure::read('MISP.default_attribute_distribution');
if ($newAttribute['distribution'] == 'event') {
$newAttribute['distribution'] = 5;
}
}
$this->Event->Attribute->save($newAttribute);
$attributeArrays['add'][] = $newAttribute;
unset($objectToSave['Attribute'][$newKey]);
}
foreach ($object['Attribute'] as $origKey => $originalAttribute) {
$originalAttribute['deleted'] = 1;
$this->Event->Attribute->save($originalAttribute);
}
$this->Event->Attribute->save($newAttribute);
$attributeArrays['add'][] = $newAttribute;
unset($objectToSave['Attribute'][$newKey]);
}
foreach ($object['Attribute'] as $origKey => $originalAttribute) {
$originalAttribute['deleted'] = 1;
$this->Event->Attribute->save($originalAttribute);
} else { // we only add the new attribute
$newAttribute = $objectToSave['Attribute'][0];
$this->Event->Attribute->create();
$newAttribute['event_id'] = $object['Object']['event_id'];
$newAttribute['object_id'] = $object['Object']['id'];
// Set seen of object at attribute level
if (
(!array_key_exists('first_seen', $newAttribute) || is_null($newAttribute['first_seen'])) &&
(!array_key_exists('first_seen', $object['Object']) && !is_null($object['Object']['first_seen']))
) {
$newAttribute['first_seen'] = $object['Object']['first_seen'];
}
if (
(!array_key_exists('last_seen', $newAttribute) || is_null($newAttribute['last_seen'])) &&
(!array_key_exists('last_seen', $object['Object']) && !is_null($object['Object']['last_seen']))
) {
$newAttribute['last_seen'] = $object['Object']['last_seen'];
$different = true;
}
if (!isset($newAttribute['timestamp'])) {
$newAttribute['distribution'] = Configure::read('MISP.default_attribute_distribution');
if ($newAttribute['distribution'] == 'event') {
$newAttribute['distribution'] = 5;
}
}
$saveAttributeResult = $this->Attribute->saveAttributes(array($newAttribute));
return $saveAttributeResult ? $this->id : $this->validationErrors;
}
return $this->id;
}
@ -597,8 +797,10 @@ class MispObject extends AppModel
$this->Event->unpublishEvent($eventId);
$objectId = $this->id;
$partialFails = array();
foreach ($object['Object']['Attribute'] as $attribute) {
$this->Attribute->captureAttribute($attribute, $eventId, $user, $objectId, $log);
if (!empty($object['Object']['Attribute'])) {
foreach ($object['Object']['Attribute'] as $attribute) {
$this->Attribute->captureAttribute($attribute, $eventId, $user, $objectId, $log);
}
}
return true;
} else {
@ -690,8 +892,6 @@ class MispObject extends AppModel
'change' => 'Validation errors: ' . json_encode($this->validationErrors) . ' Full Object: ' . json_encode($attribute),
));
return $this->validationErrors;
} else {
$this->Event->unpublishEvent($eventId);
}
if (!empty($object['Attribute'])) {
foreach ($object['Attribute'] as $attribute) {
@ -701,14 +901,14 @@ class MispObject extends AppModel
return true;
}
public function updateTimestamp($id)
public function updateTimestamp($id, $timestamp = false)
{
$date = new DateTime();
$object = $this->find('first', array(
'recursive' => -1,
'conditions' => array('Object.id' => $id)
));
$object['Object']['timestamp'] = $date->getTimestamp();
$object['Object']['timestamp'] = $timestamp == false ? $date->getTimestamp() : $timestamp;
$object['Object']['skip_zmq'] = 1;
$object['Object']['skip_kafka'] = 1;
$result = $this->save($object);
@ -837,4 +1037,130 @@ class MispObject extends AppModel
return $saved_object['Object']['id'];
}
public function resolveUpdatedTemplate($template, $object, $update_template_available = false)
{
$toReturn = array(
'updateable_attribute' => false,
'not_updateable_attribute' => false,
'newer_template_version' => false,
'template' => $template
);
if (!empty($template)) {
$newer_template = $this->ObjectTemplate->find('first', array(
'conditions' => array(
'ObjectTemplate.uuid' => $object['Object']['template_uuid'],
'ObjectTemplate.version >' => $object['Object']['template_version'],
),
'recursive' => -1,
'contain' => array(
'ObjectTemplateElement'
),
'order' => array('ObjectTemplate.version DESC')
));
if (!empty($newer_template)) {
$toReturn['newer_template_version'] = $newer_template['ObjectTemplate']['version'];
// ignore IDs for comparison
$cur_template_temp = Hash::remove(Hash::remove($template['ObjectTemplateElement'], '{n}.id'), '{n}.object_template_id');
$newer_template_temp = Hash::remove(Hash::remove($newer_template['ObjectTemplateElement'], '{n}.id'), '{n}.object_template_id');
$template_difference = array();
// check how current template is included in the newer
foreach ($cur_template_temp as $cur_obj_rel) {
$flag_sim = false;
foreach ($newer_template_temp as $newer_obj_rel) {
$tmp = Hash::diff($cur_obj_rel, $newer_obj_rel);
if (count($tmp) == 0) {
$flag_sim = true;
break;
}
}
if (!$flag_sim) {
$template_difference[] = $cur_obj_rel;
}
}
$toReturn['updateable_attribute'] = $object['Attribute'];
$toReturn['not_updateable_attribute'] = array();
} else {
$toReturn['newer_template_version'] = false;
}
if (!empty($template_difference)) { // older template not completely embeded in newer
foreach ($template_difference as $temp_diff_element) {
foreach ($object['Attribute'] as $i => $attribute) {
if (
$attribute['object_relation'] == $temp_diff_element['object_relation']
&& $attribute['type'] == $temp_diff_element['type']
) { // This attribute cannot be merged automatically
$attribute['merge-possible'] = false;
$toReturn['not_updateable_attribute'][] = $attribute;
unset($toReturn['updateable_attribute'][$i]);
}
}
}
}
if ($update_template_available) { // template version bump requested
$toReturn['template'] = $newer_template; // bump the template version
}
}
return $toReturn;
}
public function reviseObject($revised_object, $object) {
$revised_object = json_decode(base64_decode($revised_object), true);
$revised_object_both = array('mergeable' => array(), 'notMergeable' => array());
// Loop through attributes to inject and perform the correct action
// (inject, duplicate, add warnings, ...) when applicable
foreach ($revised_object['Attribute'] as $attribute_to_inject) {
$flag_no_collision = true;
foreach ($object['Attribute'] as $attribute) {
if (
$attribute['object_relation'] == $attribute_to_inject['object_relation']
&& $attribute['type'] == $attribute_to_inject['type']
&& $attribute['value'] !== $attribute_to_inject['value']
) { // Collision on value
$multiple = !empty(Hash::extract($template['ObjectTemplateElement'], sprintf('{n}[object_relation=%s][type=%s][multiple=true]', $attribute['object_relation'], $attribute['type'])));
if ($multiple) { // if multiple is set, check if an entry exists already
$flag_entry_exists = false;
foreach ($object['Attribute'] as $attr) {
if (
$attr['object_relation'] == $attribute_to_inject['object_relation']
&& $attr['type'] == $attribute_to_inject['type']
&& $attr['value'] === $attribute_to_inject['value']
) {
$flag_entry_exists = true;
break;
}
}
if (!$flag_entry_exists) { // entry does no exists, can be duplicated
$attribute_to_inject['is_multiple'] = true;
$revised_object_both['mergeable'][] = $attribute_to_inject;
$object['Attribute'][] = $attribute_to_inject;
}
} else { // Collision on value, multiple not set => propose overwrite
$attribute_to_inject['current_value'] = $attribute['value'];
$attribute_to_inject['merge-possible'] = true; // the user can still swap value
$revised_object_both['notMergeable'][] = $attribute_to_inject;
}
$flag_no_collision = false;
} else if (
$attribute['object_relation'] == $attribute_to_inject['object_relation']
&& $attribute['type'] == $attribute_to_inject['type']
&& $attribute['value'] === $attribute_to_inject['value']
) { // all good, they are basically the same, do nothing
$revised_object_both['mergeable'][] = $attribute_to_inject;
$flag_no_collision = false;
}
}
if ($flag_no_collision) { // no collision, nor equalities => inject it straight away
$revised_object_both['mergeable'][] = $attribute_to_inject;
$object['Attribute'][] = $attribute_to_inject;
}
}
return array(
'object' => $object,
'revised_object_both' => $revised_object_both
);
}
}

13
app/Model/Organisation.php
View File

@ -415,4 +415,17 @@ class Organisation extends AppModel
$event['Org'] = $this->__orgCache[$event['Event']['org_id']];
return $event;
}
public function getOrgIdsFromMeta($metaConditions)
{
$orgIds = $this->find('list', array(
'conditions' => $metaConditions,
'fields' => array('id'),
'recursive' => -1
));
if (empty($orgIds)) {
return array(-1);
}
return array_values($orgIds);
}
}

2
app/Model/Post.php
View File

@ -118,7 +118,7 @@ class Post extends AppModel
$bodyDetail .= "The following message was added: \n";
$bodyDetail .= "\n";
$bodyDetail .= $message . "\n";
$tplColorString = !empty(Configure::read('MISP.email_subject_TLP_string')) ? Configure::read('MISP.email_subject_TLP_string') : "TLP Amber";
$tplColorString = !empty(Configure::read('MISP.email_subject_TLP_string')) ? Configure::read('MISP.email_subject_TLP_string') : "tlp:amber";
$subject = "[" . Configure::read('MISP.org') . " MISP] New post in discussion " . $post['Post']['thread_id'] . " - ".$tplColorString;
foreach ($orgMembers as $recipient) {
$this->User->sendEmail($recipient, $bodyDetail, $body, $subject);

3
app/Model/Role.php
View File

@ -220,6 +220,9 @@ class Role extends AppModel
$this->data['Role']['memory_limit'] = '';
}
}
if (empty($this->data['Role']['rate_limit_count'])) {
$this->data['Role']['rate_limit_count'] = 0;
}
return true;
}

1028
app/Model/Server.php
View File

File diff suppressed because it is too large Load Diff

44
app/Model/ShadowAttribute.php
View File

@ -10,6 +10,8 @@ class ShadowAttribute extends AppModel
public $name = 'ShadowAttribute'; // TODO general
public $recursive = -1;
public $actsAs = array(
'SysLogLogable.SysLogLogable' => array( // TODO Audit, logable
'userModel' => 'User',
@ -131,6 +133,16 @@ class ShadowAttribute extends AppModel
'rule' => array('boolean'),
),
),
'first_seen' => array(
'rule' => array('datetimeOrNull'),
'required' => false,
'message' => array('Invalid ISO 8601 format')
),
'last_seen' => array(
'rule' => array('datetimeOrNull'),
'required' => false,
'message' => array('Invalid ISO 8601 format')
)
);
public function __construct($id = false, $table = null, $ds = null)
@ -175,6 +187,9 @@ class ShadowAttribute extends AppModel
if ($this->data['ShadowAttribute']['deleted']) {
$this->__beforeDeleteCorrelation($this->data['ShadowAttribute']);
}
// convert into utc and micro sec
$this->data = $this->Attribute->ISODatetimeToUTC($this->data, $this->alias);
return true;
}
@ -356,6 +371,14 @@ class ShadowAttribute extends AppModel
return true;
}
public function afterFind($results, $primary = false)
{
foreach ($results as $k => $v) {
$results[$k] = $this->Attribute->UTCToISODatetime($results[$k], $this->alias);
}
return $results;
}
public function validateTypeValue($fields)
{
$category = $this->data['ShadowAttribute']['category'];
@ -472,6 +495,20 @@ class ShadowAttribute extends AppModel
return $fails;
}
// check whether the variable is null or datetime
public function datetimeOrNull($fields)
{
$k = array_keys($fields)[0];
$seen = $fields[$k];
try {
new DateTime($seen);
$returnValue = true;
} catch (Exception $e) {
$returnValue = false;
}
return $returnValue || is_null($seen);
}
public function setDeleted($id)
{
$this->Behaviors->detach('SysLogLogable.SysLogLogable');
@ -508,7 +545,12 @@ class ShadowAttribute extends AppModel
public function getEventContributors($id)
{
$orgs = $this->find('all', array('fields' => array('DISTINCT(org_id)'), 'conditions' => array('event_id' => $id), 'order' => false));
$orgs = $this->find('all', array('fields' => array(
'DISTINCT(ShadowAttribute.org_id)'),
'conditions' => array('event_id' => $id),
'recursive' => -1,
'order' => false
));
$org_ids = array();
$this->Organisation = ClassRegistry::init('Organisation');
foreach ($orgs as $org) {

Some files were not shown because too many files have changed in this diff Show More