MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'develop' of github.com:MISP/MISP into develop

pull/9538/head
Alexandre Dulaunoy 2024-01-31 13:34:26 +01:00
parent 7b49980b5c 9425c99894
commit 4be80d39a8
No known key found for this signature in database
GPG Key ID: 09E2CD4944E6CBCD
98 changed files with 2866 additions and 37744 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

381
.github/workflows/main.yml vendored
View File

@ -6,9 +6,9 @@ name: misp
# events but only for the 2.4 and develop branches # events but only for the 2.4 and develop branches
on: on:
push: push:
branches: [ 2.4, develop, misp-stix, taxii ] branches: [ '2.4', develop, misp-stix, taxii ]
pull_request: pull_request:
branches: [ 2.4, develop, misp-stix ] branches: [ '2.4', develop, misp-stix ]
# A workflow run is made up of one or more jobs that can run sequentially or in parallel # A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs: jobs:
@ -62,186 +62,178 @@ jobs:
php_version: ${{ matrix.php }} php_version: ${{ matrix.php }}
run: | run: |
sudo apt-get -y update sudo apt-get -y update
# Repo is missing for unknown reason
LC_ALL=C.UTF-8 sudo apt-add-repository ppa:ondrej/php -y
if [[ $php_version == "7.2" ]]; then
# hotfix due to: https://bugs.php.net/bug.php?id=81640 TODO: remove after libpcre2-8-0:10.36 gets to stable channel
sudo apt-get --fix-broken install
fi
sudo apt-get -y install curl python3 python3-pip python3-virtualenv apache2 libapache2-mod-php$php_version sudo apt-get -y install curl python3 python3-pip python3-virtualenv apache2 libapache2-mod-php$php_version
# Runs a set of commands using the runners shell # Runs a set of commands using the runners shell
- name: Install deps - name: Install deps
run: | run: |
sudo chown $USER:www-data $HOME/.composer sudo chown $USER:www-data $HOME/.composer
pushd app pushd app
sudo -H -u $USER composer config --no-plugins allow-plugins.composer/installers true composer config --no-plugins allow-plugins.composer/installers true
sudo -H -u $USER composer install --no-progress composer install --no-progress
popd popd
cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php
# Set perms
sudo chown -R $USER:www-data `pwd` # Set perms
sudo chmod -R 775 `pwd` sudo chown -R $USER:www-data `pwd`
sudo chmod -R g+ws `pwd`/app/tmp sudo chmod -R 775 `pwd`
sudo chmod -R g+ws `pwd`/app/tmp/cache sudo chmod -R g+ws `pwd`/app/tmp
sudo chmod -R g+ws `pwd`/app/tmp/cache/persistent sudo chmod -R g+ws `pwd`/app/tmp/cache
sudo chmod -R g+ws `pwd`/app/tmp/cache/models sudo chmod -R g+ws `pwd`/app/tmp/cache/persistent
sudo chmod -R g+ws `pwd`/app/tmp/logs sudo chmod -R g+ws `pwd`/app/tmp/cache/models
sudo chmod -R g+ws `pwd`/app/files sudo chmod -R g+ws `pwd`/app/tmp/logs
sudo chmod -R g+ws `pwd`/app/files/scripts/tmp sudo chmod -R g+ws `pwd`/app/files
sudo chown -R $USER:www-data `pwd` sudo chmod -R g+ws `pwd`/app/files/scripts/tmp
# Resque perms sudo chown -R $USER:www-data `pwd`
sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp
sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp # Resque perms
# install MySQL sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp
sudo chmod -R 777 `pwd`/INSTALL sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';"
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant usage on *.* to misp@'%' identified by 'blah';" # Fill database with basic MISP schema
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant all privileges on misp.* to misp@'%';" mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';"
mysql -h 127.0.0.1 --port 3306 -u misp -pblah misp < INSTALL/MYSQL.sql mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant usage on *.* to misp@'%' identified by 'blah';"
# configure apache virtual hosts mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant all privileges on misp.* to misp@'%';"
sudo chmod -R 777 `pwd`/build mysql -h 127.0.0.1 --port 3306 -u misp -pblah misp < INSTALL/MYSQL.sql
sudo mkdir -p /etc/apache2/sites-available
sudo cp -f build/github-action-ci-apache /etc/apache2/sites-available/misp.conf # configure apache virtual hosts
sudo sed -e "s?%GITHUB_WORKSPACE%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.conf sudo mkdir -p /etc/apache2/sites-available
sudo sed -e "s?%HOST%?${HOST}?g" --in-place /etc/apache2/sites-available/misp.conf sudo cp -f build/github-action-ci-apache /etc/apache2/sites-available/misp.conf
sudo a2dissite 000-default sudo sed -e "s?%GITHUB_WORKSPACE%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.conf
sudo a2ensite misp.conf sudo sed -e "s?%HOST%?${HOST}?g" --in-place /etc/apache2/sites-available/misp.conf
cat /etc/apache2/sites-enabled/misp.conf sudo a2dissite 000-default
sudo a2enmod rewrite sudo a2ensite misp.conf
sudo systemctl restart apache2 cat /etc/apache2/sites-enabled/misp.conf
# MISP configuration sudo a2enmod rewrite
sudo chmod -R 777 `pwd`/travis sudo systemctl start --no-block apache2
sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php
sudo cp travis/database.php app/Config/database.php # MISP configuration
sudo cp app/Config/core.default.php app/Config/core.php sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php
sudo cp app/Config/config.default.php app/Config/config.php sudo cp build/database.php app/Config/database.php
sudo cp travis/email.php app/Config/email.php sudo cp app/Config/core.default.php app/Config/core.php
# Ensure the perms sudo cp app/Config/config.default.php app/Config/config.php
sudo chown -R $USER:www-data `pwd`/app/Config sudo cp build/email.php app/Config/email.php
sudo chmod -R 777 `pwd`/app/Config
# GPG setup # GPG setup
sudo mkdir `pwd`/.gnupg sudo mkdir `pwd`/.gnupg
# /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS # /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS
sudo cp -a /dev/urandom /dev/random sudo cp -a /dev/urandom /dev/random
sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/travis/gpg sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/build/gpg
sudo gpg --list-secret-keys --homedir `pwd`/.gnupg sudo gpg --list-secret-keys --homedir `pwd`/.gnupg
# change perms
sudo chown -R $USER:www-data `pwd` # change perms
sudo chown -R www-data:www-data `pwd`/.gnupg sudo chown -R $USER:www-data `pwd`
sudo chmod -R 700 `pwd`/.gnupg sudo chown -R www-data:www-data `pwd`/.gnupg
sudo usermod -a -G www-data $USER sudo chmod -R 700 `pwd`/.gnupg
sudo chmod -R 777 `pwd`/app/Plugin/CakeResque/tmp/ sudo usermod -a -G www-data $USER
# Ensure the perms of config files sudo chmod -R 777 `pwd`/app/Plugin/CakeResque/tmp/
sudo chown -R $USER:www-data `pwd`/app/Config # Ensure the perms of config files
sudo chmod -R 777 `pwd`/app/Config sudo chown -R $USER:www-data `pwd`/app/Config
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.server_settings_skip_backup_rotate" 1' sudo chmod -R 777 `pwd`/app/Config
sudo chown -R $USER:www-data `pwd`/app/Config app/Console/cake Admin setSetting "MISP.server_settings_skip_backup_rotate" 1
sudo chmod -R 777 `pwd`/app/Config sudo chown -R $USER:www-data `pwd`/app/Config
sudo chmod -R 777 `pwd`/app/Config
# fix perms (?) # fix perms (?)
namei -m /home/runner/work namei -m /home/runner/work
sudo chmod +x /home/runner/work sudo chmod +x /home/runner/work
sudo chmod +x /home/runner sudo chmod +x /home/runner
sudo chmod +x /home sudo chmod +x /home
sudo chmod +x / sudo chmod +x /
- name: DB Update
run: |
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.osuser" $USER'
sudo -E su $USER -c 'app/Console/cake Admin runUpdates'
sudo -E su $USER -c 'app/Console/cake Admin schemaDiagnostics'
- name: Configure MISP
run: |
sudo -u $USER app/Console/cake userInit -q | sudo tee ./key.txt
echo "AUTH=`cat key.txt`" >> $GITHUB_ENV
sudo -u $USER app/Console/cake Admin setSetting "Session.autoRegenerate" 0
sudo -u $USER app/Console/cake Admin setSetting "Session.timeout" 600
sudo -u $USER app/Console/cake Admin setSetting "Session.cookieTimeout" 3600
sudo -u $USER app/Console/cake Admin setSetting "MISP.host_org_id" 1
sudo -u $USER app/Console/cake Admin setSetting "MISP.email" "info@admin.test"
sudo -u $USER app/Console/cake Admin setSetting "MISP.disable_emailing" false
sudo -u $USER app/Console/cake Admin setSetting --force "debug" true
sudo -u $USER app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false
sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1"
sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_port" 6379
sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_database" 13
sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_password" ""
sudo -u $USER app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test"
sudo -u $USER app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg"
sudo -u $USER app/Console/cake Admin setSetting "GnuPG.password" "travistest"
sudo -u $USER app/Console/cake Admin setSetting "MISP.download_gpg_from_homedir" 1
- name: Configure ZMQ
run: |
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_host" "127.0.0.1"
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_port" 6379
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_database" 1
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_password" ""
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" 1
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_audit_notifications_enable" 1
- name: Update Galaxies
run: sudo -E su $USER -c 'app/Console/cake Admin updateGalaxies'
- name: Update Taxonomies
run: sudo -E su $USER -c 'app/Console/cake Admin updateTaxonomies'
- name: Update Warninglists
run: sudo -E su $USER -c 'app/Console/cake Admin updateWarningLists --verbose'
- name: Update Noticelists
run: sudo -E su $USER -c 'app/Console/cake Admin updateNoticeLists'
- name: Update Object Templates
run: sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates 1'
- name: Turn MISP live
run: sudo -E su $USER -c 'app/Console/cake Live 1'
- name: Check if Redis is ready
run: sudo -E su $USER -c 'app/Console/cake Admin redisReady'
- name: Start workers
run: |
sudo chmod +x app/Console/worker/start.sh
sudo -u www-data 'app/Console/worker/start.sh'
- name: Python setup - name: Python setup
run: | run: |
sudo chmod 777 ./key.txt # Dirty install python stuff
sudo chmod -R 777 ./tests python3 -m virtualenv -p python3 ./venv
# Start workers app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python"
# Dirty install python stuff . ./venv/bin/activate
python3 -m virtualenv -p python3 ./venv export PYTHONPATH=$PYTHONPATH:./app/files/scripts
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python"' pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara pytest
. ./venv/bin/activate deactivate
export PYTHONPATH=$PYTHONPATH:./app/files/scripts
pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara pytest - name: DB Update
deactivate run: |
app/Console/cake Admin setSetting "MISP.osuser" $USER
app/Console/cake Admin runUpdates
app/Console/cake Admin schemaDiagnostics
- name: Configure MISP
run: |
app/Console/cake User init | sudo tee ./key.txt
echo "AUTH=`cat key.txt`" >> $GITHUB_ENV
app/Console/cake Admin setSetting "Session.autoRegenerate" 0
app/Console/cake Admin setSetting "Session.timeout" 600
app/Console/cake Admin setSetting "Session.cookieTimeout" 3600
app/Console/cake Admin setSetting "MISP.host_org_id" 1
app/Console/cake Admin setSetting "MISP.email" "info@admin.test"
app/Console/cake Admin setSetting "MISP.disable_emailing" false
app/Console/cake Admin setSetting --force "debug" true
app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false
app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1"
app/Console/cake Admin setSetting "MISP.redis_port" 6379
app/Console/cake Admin setSetting "MISP.redis_database" 13
app/Console/cake Admin setSetting "MISP.redis_password" ""
app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test"
app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg"
app/Console/cake Admin setSetting "GnuPG.password" "travistest"
app/Console/cake Admin setSetting "MISP.download_gpg_from_homedir" 1
app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_host" "127.0.0.1"
app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_port" 6379
app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_database" 1
app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_password" ""
app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" 1
app/Console/cake Admin setSetting "Plugin.ZeroMQ_audit_notifications_enable" 1
- name: Update Galaxies
run: app/Console/cake Admin updateGalaxies
- name: Update Taxonomies
run: app/Console/cake Admin updateTaxonomies
- name: Update Warninglists
run: app/Console/cake Admin updateWarningLists --verbose
- name: Update Noticelists
run: app/Console/cake Admin updateNoticeLists
- name: Update Object Templates
run: app/Console/cake Admin updateObjectTemplates 1
- name: Turn MISP live
run: app/Console/cake Admin live 1
- name: Check if Redis is ready
run: app/Console/cake Admin redisReady
- name: Start workers
run: |
sudo chmod +x app/Console/worker/start.sh
sudo -u www-data 'app/Console/worker/start.sh'
- name: Test if apache is working - name: Test if apache is working
run: | run: |
sudo systemctl status apache2 --no-pager -l sudo systemctl status apache2 --no-pager -l
sudo apache2ctl -S sudo apache2ctl -S
curl http://${HOST} curl -sS http://${HOST}
sudo chmod -R 777 PyMISP
pushd PyMISP - name: Check if dependencies working as expected
echo 'url = "http://'${HOST}'"' >> tests/keys.py run: |
echo 'key = "'${AUTH}'"' >> tests/keys.py sudo chmod -R 777 PyMISP
cat tests/keys.py pushd PyMISP
popd echo 'url = "http://'${HOST}'"' >> tests/keys.py
. ./venv/bin/activate echo 'key = "'${AUTH}'"' >> tests/keys.py
pushd tests cat tests/keys.py
bash ./build-test.sh popd
popd . ./venv/bin/activate
deactivate pushd tests
bash ./build-test.sh
popd
deactivate
- name: Run PHP tests - name: Run PHP tests
run: | run: |
./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/ ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ -e php,ctp app/
sudo -u www-data ./app/Vendor/bin/phpunit app/Test/ sudo -u www-data ./app/Vendor/bin/phpunit app/Test/
- name: Clone test files - name: Clone test files
uses: actions/checkout@v4 uses: actions/checkout@v4
@ -249,31 +241,30 @@ jobs:
repository: viper-framework/viper-test-files repository: viper-framework/viper-test-files
path: PyMISP/tests/viper-test-files path: PyMISP/tests/viper-test-files
- name: Run tests - name: Run tests
run: | run: |
pushd tests pushd tests
./curl_tests_GH.sh $AUTH $HOST ./curl_tests_GH.sh $AUTH $HOST
popd popd
sudo chmod -R g+ws `pwd`/app/tmp/logs sudo chmod -R g+ws `pwd`/app/tmp/logs
. ./venv/bin/activate . ./venv/bin/activate
pushd PyMISP pushd PyMISP
cp tests/keys.py . cp tests/keys.py .
python -m pytest -v --durations=0 tests/test_mispevent.py python -m pytest -v --durations=0 tests/test_mispevent.py
python -m pytest -v --durations=0 tests/testlive_comprehensive.py python -m pytest -v --durations=0 tests/testlive_comprehensive.py
popd popd
python tests/testlive_security.py -v python tests/testlive_security.py -v
python tests/testlive_sync.py python tests/testlive_sync.py
python tests/testlive_comprehensive_local.py -v python tests/testlive_comprehensive_local.py -v
cp PyMISP/tests/keys.py PyMISP/examples/events/ cp PyMISP/tests/keys.py PyMISP/examples/events/
pushd PyMISP/examples/events/ pushd PyMISP/examples/events/
python ./create_massive_dummy_events.py -l 5 -a 30 python ./create_massive_dummy_events.py -l 5 -a 30
popd popd
pip install jsonschema pip install jsonschema
python tools/misp-feed/validate.py python tools/misp-feed/validate.py
deactivate deactivate
- name: Check requirements.txt - name: Check requirements.txt
run: python tests/check_requirements.py run: python tests/check_requirements.py
@ -282,13 +273,13 @@ jobs:
if: ${{ always() }} if: ${{ always() }}
# update logs_test.sh when adding more logsources here # update logs_test.sh when adding more logsources here
run: | run: |
tail -n +1 `pwd`/app/tmp/logs/* tail -n +1 `pwd`/app/tmp/logs/*
tail -n +1 /var/log/apache2/*.log tail -n +1 /var/log/apache2/*.log
sudo -u $USER app/Console/cake Log export /tmp/logs.json.gz --without-changes app/Console/cake Log export /tmp/logs.json.gz --without-changes
zcat /tmp/logs.json.gz zcat /tmp/logs.json.gz
- name: Errors in Logs - name: Errors in Logs
if: ${{ always() }} if: ${{ always() }}
run: | run: |
./tests/logs_tests.sh ./tests/logs_tests.sh

195
.travis.yml
View File

@ -1,195 +0,0 @@
language: php
php:
- 7.2
- 7.3
- 7.4
- nightly
services:
- redis
sudo: required
dist: bionic
addons:
mariadb: '10.2'
hosts:
- misp.local
- localhost
before_install:
- git config --global user.name "TravisCI"
- export PATH="$HOME/.local/bin:$PATH"
install:
- date
- sudo apt-get -y update
# Install haveged, because Travis lacks entropy.
- sudo apt-get -y install haveged python3 python3-venv python3-pip python3-dev python3-nose python3-redis python3-lxml python3-dateutil python3-msgpack libxml2-dev libzmq3-dev zlib1g-dev apache2 curl php-mysql php-dev php-cli libapache2-mod-php libfuzzy-dev php-mbstring libonig4 php-json php-xml php-opcache php-readline php-redis php-gnupg php-gd
- sudo pip3 install --upgrade pip setuptools requests
- sudo pip3 install --upgrade -r requirements.txt
- sudo pip3 install --upgrade -r requirements-dev.txt
- pip3 install --user poetry
- phpenv rehash
- sudo mkdir $HOME/.composer ; sudo chown $USER:www-data $HOME/.composer
- pushd app
- sudo -H -u $USER php composer.phar install --no-progress
- sudo phpenmod redis
- sudo phpenmod gnupg
- popd
- cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php
# Set perms
- sudo chown -R $USER:www-data `pwd`
- sudo chmod -R 775 `pwd`
- sudo chmod -R g+ws `pwd`/app/tmp
- sudo chmod -R g+ws `pwd`/app/tmp/cache
- sudo chmod -R g+ws `pwd`/app/tmp/cache/persistent
- sudo chmod -R g+ws `pwd`/app/tmp/cache/models
- sudo chmod -R g+ws `pwd`/app/tmp/logs
- sudo chmod -R g+ws `pwd`/app/files
- sudo chmod -R g+ws `pwd`/app/files/scripts/tmp
- sudo chown -R $USER:www-data `pwd`
# Resque perms
- sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp
- sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp
# install MySQL
- sudo chmod -R 777 `pwd`/INSTALL
- mysql -u root -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';"
- mysql -u root -e 'create database misp;'
- mysql -u root -e "grant usage on *.* to misp@localhost identified by 'blah'";
- mysql -u root -e "grant all privileges on misp.* to misp@localhost;"
- mysql -u misp -pblah misp < INSTALL/MYSQL.sql
# configure apache virtual hosts
- sudo chmod -R 777 `pwd`/build
- sudo mkdir -p /etc/apache2/sites-available
- sudo cp -f build/travis-ci-apache /etc/apache2/sites-available/misp.local.conf
- sudo sed -e "s?%TRAVIS_BUILD_DIR%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.local.conf
- sudo a2dissite 000-default
- sudo a2ensite misp.local.conf
- sudo a2enmod rewrite
- sudo service apache2 restart
# MISP configuration
- sudo chmod -R 777 `pwd`/travis
- sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php
- sudo cp travis/database.php app/Config/database.php
- sudo cp app/Config/core.default.php app/Config/core.php
- sudo cp app/Config/config.default.php app/Config/config.php
- sudo cp travis/email.php app/Config/email.php
# Ensure the perms
- sudo chown -R $USER:www-data `pwd`/app/Config
- sudo chmod -R 770 `pwd`/app/Config
# GPG setup
- sudo mkdir `pwd`/.gnupg
# /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS
- sudo cp -a /dev/urandom /dev/random
- sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/travis/gpg
- sudo gpg --list-secret-keys --homedir `pwd`/.gnupg
# change perms
- sudo chown -R $USER:www-data `pwd`
- sudo chmod +x /home/travis/build
- sudo chmod +x /home/travis
- sudo chmod +x /home
- sudo chmod -R 770 `pwd`/.gnupg
# Get authkey
- sudo usermod -a -G www-data $USER
- sudo -E su $USER -c 'app/Console/cake Admin runUpdates'
- sudo -E su $USER -c 'app/Console/cake userInit -q | sudo tee ./key.txt'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.autoRegenerate" 0'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.timeout" 600'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.cookieTimeout" 3600'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.host_org_id" 1'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.email" "info@admin.test"'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.disable_emailing" false'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "debug" true'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1"'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_port" 6379'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_database" 13'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_password" ""'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test"'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg"'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.password" "travistest"'
- sudo -E su $USER -c 'app/Console/cake Admin updateGalaxies'
- sudo -E su $USER -c 'app/Console/cake Admin updateTaxonomies'
- sudo -E su $USER -c 'app/Console/cake Admin updateWarningLists'
- sudo -E su $USER -c 'app/Console/cake Admin updateNoticeLists'
- sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates 1'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" true'
- sudo -E su $USER -c 'app/Console/cake Live 1'
- sudo chmod 777 ./key.txt
- sudo chmod -R 777 ./tests
# Start workers
- sudo chmod +x app/Console/worker/start.sh
- sudo -E su $USER -c 'app/Console/worker/start.sh &'
- sleep 10
# Dirty install python stuff
- virtualenv -p python3.6 ./venv
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$TRAVIS_BUILD_DIR/venv/bin/python"'
- . ./venv/bin/activate
- pushd cti-python-stix2
- pip install .
- popd
- pushd PyMISP
- pip install .[fileobjects]
- popd
- pip install stix zmq redis plyara
- deactivate
before_script:
- curl http://misp.local
- AUTH=`cat key.txt`
- sudo chmod -R 777 PyMISP
- pushd PyMISP
- echo 'url = "http://misp.local"' >> tests/keys.py
- echo 'key = "'${AUTH}'"' >> tests/keys.py
- cat tests/keys.py
- popd
script:
- ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
- ./app/Vendor/bin/phpunit app/Test/ComplexTypeToolTest.php
- ./app/Vendor/bin/phpunit app/Test/JSONConverterToolTest.php
# Ensure the perms
- sudo chown -R $USER:www-data `pwd`/app/Config
- sudo chmod -R 770 `pwd`/app/Config
- pushd tests
- ./curl_tests.sh $AUTH
- popd
- pushd PyMISP
- git submodule init
- git submodule update
- travis_retry poetry install -E fileobjects -E openioc -E virustotal -E docs -E pdfexport
- poetry run python tests/testlive_comprehensive.py
- poetry run python tests/test_mispevent.py
- popd
- cp PyMISP/tests/keys.py PyMISP/examples/events/
- pushd PyMISP/examples/events/
- poetry run python ./create_massive_dummy_events.py -l 5 -a 30
- popd
- python3 tools/misp-feed/validate.py
after_failure:
- curl http://misp.local
- cat /etc/apache2/sites-available/misp.local.conf
- sudo tail -n +1 `pwd`/app/tmp/logs/*
- sudo ls -l /var/log/apache2
- sudo cat /var/log/apache2/error.log
- sudo cat /var/log/apache2/misp.local_error.log
- sudo cat /var/log/apache2/misp.local_access.log
- pwd
notifications:
webhooks:
urls:
- https://webhooks.gitter.im/e/05e30284086a8e948d31
on_success: change # options: [always|never|change] default: always
on_failure: always # options: [always|never|change] default: always
on_start: never # options: [always|never|change] default: always
after_success:
- sudo tail -n +1 `pwd`/app/tmp/logs/*
- coveralls
- coverage report
- coverage xml
- codecov

116
app/Console/Command/AdminShell.php
View File

@ -46,11 +46,11 @@ class AdminShell extends AppShell
'help' => __('Update the JSON definition of taxonomies.'), 'help' => __('Update the JSON definition of taxonomies.'),
)); ));
$parser->addSubcommand('setSetting', [ $parser->addSubcommand('setSetting', [
'help' => __('Set setting in PHP config file.'), 'help' => __('Set setting in MISP config'),
'parser' => [ 'parser' => [
'arguments' => [ 'arguments' => [
'name' => ['help' => __('Setting name'), 'required' => true], 'name' => ['help' => __('Setting name'), 'required' => true],
'value' => ['help' => __('Setting value'), 'required' => true], 'value' => ['help' => __('Setting value')],
], ],
'options' => [ 'options' => [
'force' => [ 'force' => [
@ -72,7 +72,7 @@ class AdminShell extends AppShell
'help' => __('Set if MISP instance is live and accessible for users.'), 'help' => __('Set if MISP instance is live and accessible for users.'),
'parser' => [ 'parser' => [
'arguments' => [ 'arguments' => [
'state' => ['help' => __('Set Live state')], 'state' => ['help' => __('Set Live state (boolean). If not provided, current state will be printed.')],
], ],
], ],
]); ]);
@ -85,6 +85,14 @@ class AdminShell extends AppShell
], ],
], ],
]); ]);
$parser->addSubcommand('isEncryptionKeyValid', [
'help' => __('Check if current encryption key is valid.'),
'parser' => [
'options' => [
'encryptionKey' => ['help' => __('Encryption key to test. If not provided, current key will be used.')],
],
],
]);
$parser->addSubcommand('dumpCurrentDatabaseSchema', [ $parser->addSubcommand('dumpCurrentDatabaseSchema', [
'help' => __('Dump current database schema to JSON file.'), 'help' => __('Dump current database schema to JSON file.'),
]); ]);
@ -109,6 +117,20 @@ class AdminShell extends AppShell
$parser->addSubcommand('configLint', [ $parser->addSubcommand('configLint', [
'help' => __('Check if settings has correct value.'), 'help' => __('Check if settings has correct value.'),
]); ]);
$parser->addSubcommand('createZmqConfig', [
'help' => __('Create config file for ZeroMQ server.'),
]);
$parser->addSubcommand('scanAttachment', [
'help' => __('Scan attachments with AV.'),
'parser' => [
'arguments' => [
'type' => ['help' => __('all, Attribute or ShadowAttribute'), 'required' => true],
'attributeId' => ['help' => __('ID to scan.')],
'jobId' => ['help' => __('Job ID')],
],
],
]);
return $parser; return $parser;
} }
@ -485,32 +507,47 @@ class AdminShell extends AppShell
} }
} }
} }
echo json_encode($result, JSON_PRETTY_PRINT) . PHP_EOL; $this->out($this->json($result));
} }
public function setSetting() public function setSetting()
{ {
list($setting_name, $value) = $this->args; list($settingName) = $this->args;
if ($value === 'false') {
$value = 0; if ($this->params['null'] && isset($this->args[1])) {
} elseif ($value === 'true') { $this->error(__('Trying to set setting to null value, but value was provided.'));
$value = 1; } else if ($this->params['null']) {
}
if ($this->params['null']) {
$value = null; $value = null;
} elseif (isset($this->args[1])) {
$value = $this->args[1];
} else {
$this->error(__('No setting value provided.'));
} }
$cli_user = array('id' => 0, 'email' => 'SYSTEM', 'Organisation' => array('name' => 'SYSTEM'));
if (empty($setting_name) || ($value === null && !$this->params['null'])) { $setting = $this->Server->getSettingData($settingName);
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Set setting'] . PHP_EOL);
}
$setting = $this->Server->getSettingData($setting_name);
if (empty($setting)) { if (empty($setting)) {
$message = 'Invalid setting "' . $setting_name . '". Please make sure that the setting that you are attempting to change exists and if a module parameter, the modules are running.' . PHP_EOL; $message = 'Invalid setting "' . $settingName . '". Please make sure that the setting that you are attempting to change exists and if a module parameter, the modules are running.' . PHP_EOL;
$this->error(__('Setting change rejected.'), $message); $this->error(__('Setting change rejected.'), $message);
} }
$result = $this->Server->serverSettingsEditValue($cli_user, $setting, $value, $this->params['force']);
// Convert value to boolean or to int
if ($value !== null) {
if ($setting['type'] === 'boolean') {
$value = $this->toBoolean($value);
} else if ($setting['type'] === 'numeric') {
if (is_numeric($value)) {
$value = (int)$value;
} elseif ($value === 'true' || $value === 'false') {
$value = $value === 'true' ? 1 : 0; // special case for `debug` setting
} else {
$this->error(__('Setting "%s" change rejected.', $settingName), __('Provided value %s is not a number.', $value));
}
}
}
$result = $this->Server->serverSettingsEditValue('SYSTEM', $setting, $value, $this->params['force']);
if ($result === true) { if ($result === true) {
$this->out(__('Setting "%s" changed to %s', $setting_name, is_string($value) ? '"' . $value . '"' : (string)$value)); $this->out(__('Setting "%s" changed to %s', $settingName, is_string($value) ? '"' . $value . '"' : json_encode($value)));
} else { } else {
$message = __("The setting change was rejected. MISP considers the requested setting value as invalid and would lead to the following error:\n\n\"%s\"\n\nIf you still want to force this change, please supply the --force argument.\n", $result); $message = __("The setting change was rejected. MISP considers the requested setting value as invalid and would lead to the following error:\n\n\"%s\"\n\nIf you still want to force this change, please supply the --force argument.\n", $result);
$this->error(__('Setting change rejected.'), $message); $this->error(__('Setting change rejected.'), $message);
@ -648,6 +685,8 @@ class AdminShell extends AppShell
*/ */
public function change_authkey() public function change_authkey()
{ {
$this->deprecated('cake user change_authkey [user_id]');
if (empty($this->args[0])) { if (empty($this->args[0])) {
echo 'MISP apikey command line tool' . PHP_EOL . 'To assign a new random API key for a user: ' . APP . 'Console/cake Admin change_authkey [user_email]' . PHP_EOL . 'To assign a fixed API key: ' . APP . 'Console/cake Admin change_authkey [user_email] [authkey]' . PHP_EOL; echo 'MISP apikey command line tool' . PHP_EOL . 'To assign a new random API key for a user: ' . APP . 'Console/cake Admin change_authkey [user_email]' . PHP_EOL . 'To assign a fixed API key: ' . APP . 'Console/cake Admin change_authkey [user_email] [authkey]' . PHP_EOL;
die(); die();
@ -787,6 +826,8 @@ class AdminShell extends AppShell
*/ */
public function UserIP() public function UserIP()
{ {
$this->deprecated('cake user user_ips [user_id]');
if (empty($this->args[0])) { if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get IPs for user ID'] . PHP_EOL); die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get IPs for user ID'] . PHP_EOL);
} }
@ -814,6 +855,8 @@ class AdminShell extends AppShell
*/ */
public function IPUser() public function IPUser()
{ {
$this->deprecated('cake user ip_user [ip]');
if (empty($this->args[0])) { if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get user ID for user IP'] . PHP_EOL); die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get user ID for user IP'] . PHP_EOL);
} }
@ -839,8 +882,8 @@ class AdminShell extends AppShell
public function scanAttachment() public function scanAttachment()
{ {
$input = $this->args[0]; $input = $this->args[0];
$attributeId = isset($this->args[1]) ? $this->args[1] : null; $attributeId = $this->args[1] ?? null;
$jobId = isset($this->args[2]) ? $this->args[2] : null; $jobId = $this->args[2] ?? null;
$this->loadModel('AttachmentScan'); $this->loadModel('AttachmentScan');
$result = $this->AttachmentScan->scan($input, $attributeId, $jobId); $result = $this->AttachmentScan->scan($input, $attributeId, $jobId);
@ -951,7 +994,7 @@ class AdminShell extends AppShell
$newStatus = $this->toBoolean($this->args[0]); $newStatus = $this->toBoolean($this->args[0]);
$overallSuccess = false; $overallSuccess = false;
try { try {
$redis = $this->Server->setupRedisWithException(); $redis = RedisTool::init();
if ($newStatus) { if ($newStatus) {
$redis->del('misp:live'); $redis->del('misp:live');
$this->out('Set live status to True in Redis.'); $this->out('Set live status to True in Redis.');
@ -980,7 +1023,7 @@ class AdminShell extends AppShell
} else { } else {
$this->out('Current status:'); $this->out('Current status:');
$this->out('PHP Config file: ' . (Configure::read('MISP.live') ? 'True' : 'False')); $this->out('PHP Config file: ' . (Configure::read('MISP.live') ? 'True' : 'False'));
$newStatus = $this->Server->setupRedisWithException()->get('misp:live'); $newStatus = RedisTool::init()->get('misp:live');
$this->out('Redis: ' . ($newStatus !== '0' ? 'True' : 'False')); $this->out('Redis: ' . ($newStatus !== '0' ? 'True' : 'False'));
} }
} }
@ -1031,6 +1074,27 @@ class AdminShell extends AppShell
$this->out(__('New encryption key "%s" saved into config file.', $new)); $this->out(__('New encryption key "%s" saved into config file.', $new));
} }
public function isEncryptionKeyValid()
{
$encryptionKey = $this->params['encryptionKey'] ?? null;
if ($encryptionKey === null) {
$encryptionKey = Configure::read('Security.encryption_key');
}
if (!$encryptionKey) {
$this->error('No encryption key provided');
}
/** @var SystemSetting $systemSetting */
$systemSetting = ClassRegistry::init('SystemSetting');
try {
$systemSetting->isEncryptionKeyValid($encryptionKey);
$this->Server->isEncryptionKeyValid($encryptionKey);
} catch (Exception $e) {
$this->error($e->getMessage(), __('Probably provided encryption key is invalid'));
}
}
public function redisMemoryUsage() public function redisMemoryUsage()
{ {
$redis = RedisTool::init(); $redis = RedisTool::init();
@ -1240,4 +1304,10 @@ class AdminShell extends AppShell
$this->Job->saveField('message', __('Database truncated: ' . $table)); $this->Job->saveField('message', __('Database truncated: ' . $table));
} }
} }
public function createZmqConfig()
{
$this->Server->getPubSubTool()->createConfigFile();
$this->err("Config file created in " . PubSubTool::SCRIPTS_TMP);
}
} }

15
app/Console/Command/AppShell.php
View File

@ -31,15 +31,13 @@ require_once dirname(__DIR__) . '/../Model/Attribute.php'; // FIXME workaround
*/ */
abstract class AppShell extends Shell abstract class AppShell extends Shell
{ {
public $tasks = array('ConfigLoad');
/** @var BackgroundJobsTool */ /** @var BackgroundJobsTool */
private $BackgroundJobsTool; private $BackgroundJobsTool;
public function initialize() public function initialize()
{ {
$this->ConfigLoad = $this->Tasks->load('ConfigLoad'); $configLoad = $this->Tasks->load('ConfigLoad');
$this->ConfigLoad->execute(); $configLoad->execute();
parent::initialize(); parent::initialize();
} }
@ -84,6 +82,15 @@ abstract class AppShell extends Shell
} }
} }
/**
* @param string $newCommand
* @return void
*/
protected function deprecated($newCommand)
{
$this->err("<warning>Warning: This method is deprecated. Next time please use `$newCommand`.</warning>");
}
/** /**
* @return BackgroundJobsTool * @return BackgroundJobsTool
* @throws Exception * @throws Exception

2
app/Console/Command/AuthkeyShell.php
View File

@ -12,7 +12,7 @@ class AuthkeyShell extends AppShell {
public function main() public function main()
{ {
$this->err('This method is deprecated. Next time please use `cake user change_authkey [user] [authkey]` command.'); $this->deprecated('cake user change_authkey [user] [authkey]');
if (!isset($this->args[0]) || empty($this->args[0])) echo 'MISP authkey reset command line tool.' . PHP_EOL . 'To assign a new authkey for a user:' . PHP_EOL . APP . 'Console/cake Authkey [email] [auth_key | optional]' . PHP_EOL; if (!isset($this->args[0]) || empty($this->args[0])) echo 'MISP authkey reset command line tool.' . PHP_EOL . 'To assign a new authkey for a user:' . PHP_EOL . APP . 'Console/cake Authkey [email] [auth_key | optional]' . PHP_EOL;
else { else {

2
app/Console/Command/BaseurlShell.php
View File

@ -11,7 +11,7 @@ class BaseurlShell extends AppShell {
public function main() public function main()
{ {
$this->err('This method is deprecated. Next time please use `cake admin setSetting MISP.baseurl [baseurl]` command.'); $this->deprecated('cake admin setSetting MISP.baseurl [baseurl]');
$baseurl = $this->args[0]; $baseurl = $this->args[0];
$result = $this->Server->testBaseURL($baseurl); $result = $this->Server->testBaseURL($baseurl);

59
app/Console/Command/EventShell.php
View File

@ -53,10 +53,21 @@ class EventShell extends AppShell
$parser->addSubcommand('mergeTags', [ $parser->addSubcommand('mergeTags', [
'help' => __('Merge tags'), 'help' => __('Merge tags'),
'parser' => [ 'parser' => [
'arguments' => array( 'arguments' => [
'source' => ['help' => __('Source tag ID or name. Source tag will be deleted.'), 'required' => true], 'source' => ['help' => __('Source tag ID or name. Source tag will be deleted.'), 'required' => true],
'destination' => ['help' => __('Destination tag ID or name.'), 'required' => true], 'destination' => ['help' => __('Destination tag ID or name.'), 'required' => true],
) ],
],
]);
$parser->addSubcommand('reportValidationIssuesAttributes', [
'help' => __('Report validation issues on attributes'),
]);
$parser->addSubcommand('normalizeIpAddress', [
'help' => __('Normalize IP address format in old events'),
'parser' => [
'options' => [
'dry-run' => ['help' => __('Just show what changes will be made.'), 'boolean' => true],
],
], ],
]); ]);
return $parser; return $parser;
@ -636,18 +647,28 @@ class EventShell extends AppShell
} }
} }
/** public function reportValidationIssuesAttributes()
* @param int $userId
* @return array
*/
private function getUser($userId)
{ {
$user = $this->User->getAuthUser($userId, true); foreach ($this->Event->Attribute->reportValidationIssuesAttributes() as $validationIssue) {
if (empty($user)) { echo $this->json($validationIssue) . "\n";
$this->error("User with ID $userId does not exist."); }
}
public function normalizeIpAddress()
{
$dryRun = $this->param('dry-run');
$count = 0;
foreach ($this->Event->Attribute->normalizeIpAddress($dryRun) as $attribute) {
$count++;
echo JsonTool::encode($attribute) . "\n";
}
if ($dryRun) {
$this->err(__n("%s attribute to fix", "%s attributes to fix", $count, $count));
} else {
$this->err(__n("%s attribute fixed", "%s attributes fixed", $count, $count));
} }
Configure::write('CurrentUserId', $user['id']); // for audit logging purposes
return $user;
} }
public function generateTopCorrelations() public function generateTopCorrelations()
@ -668,4 +689,18 @@ class EventShell extends AppShell
$this->Job->save($job); $this->Job->save($job);
} }
} }
/**
* @param int $userId
* @return array
*/
private function getUser($userId)
{
$user = $this->User->getAuthUser($userId, true);
if (empty($user)) {
$this->error("User with ID $userId does not exist.");
}
Configure::write('CurrentUserId', $user['id']); // for audit logging purposes
return $user;
}
} }

5
app/Console/Command/LiveShell.php
View File

@ -1,8 +1,9 @@
<?php <?php
/* /**
* Enable/disable misp * Enable/disable misp
* *
* arg0 = [0|1] * arg0 = [0|1]
* @deprecated Use AdminShell::live instead
*/ */
class LiveShell extends AppShell { class LiveShell extends AppShell {
@ -10,6 +11,8 @@ class LiveShell extends AppShell {
public function main() public function main()
{ {
$this->deprecated('cake admin live [0|1]');
$live = $this->args[0]; $live = $this->args[0];
if ($live != 0 && $live != 1) { if ($live != 0 && $live != 1) {
echo 'Invalid parameters. Usage: /var/www/MISP/app/Console/cake Live [0|1]'; echo 'Invalid parameters. Usage: /var/www/MISP/app/Console/cake Live [0|1]';

2
app/Console/Command/PasswordShell.php
View File

@ -12,7 +12,7 @@ class PasswordShell extends AppShell {
public function main() public function main()
{ {
$this->err('This method is deprecated. Next time please use `cake user change_pw [user] [password]` command.'); $this->deprecated('cake user change_pw [user] [password]');
if (!isset($this->args[0]) || empty($this->args[0]) || !isset($this->args[1]) || empty($this->args[1])) echo 'MISP password reset command line tool.' . PHP_EOL . 'To assign a new password for a user:' . PHP_EOL . APP . 'Console/cake Password [email] [password]' . PHP_EOL; if (!isset($this->args[0]) || empty($this->args[0]) || !isset($this->args[1]) || empty($this->args[1])) echo 'MISP password reset command line tool.' . PHP_EOL . 'To assign a new password for a user:' . PHP_EOL . APP . 'Console/cake Password [email] [password]' . PHP_EOL;
else { else {

26
app/Console/Command/StartWorkerShell.php
View File

@ -37,24 +37,32 @@ class StartWorkerShell extends AppShell
public function main() public function main()
{ {
$pid = getmypid();
if ($pid === false) {
throw new RuntimeException("Could not get current process ID");
}
$this->worker = new Worker( $this->worker = new Worker(
[ [
'pid' => getmypid(), 'pid' => $pid,
'queue' => $this->args[0], 'queue' => $this->args[0],
'user' => ProcessTool::whoami(), 'user' => ProcessTool::whoami(),
] ]
); );
$this->maxExecutionTime = (int)$this->params['maxExecutionTime']; $this->maxExecutionTime = (int)$this->params['maxExecutionTime'];
$queue = $this->worker->queue();
$backgroundJobTool = $this->getBackgroundJobsTool();
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - starting to process background jobs..."); CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$queue}] - starting to process background jobs...");
while (true) { while (true) {
$this->checkMaxExecutionTime(); $this->checkMaxExecutionTime();
$job = $this->getBackgroundJobsTool()->dequeue($this->worker->queue()); $job = $backgroundJobTool->dequeue($queue);
if ($job) { if ($job) {
$this->runJob($job); $this->runJob($job);
$backgroundJobTool->removeFromRunning($this->worker, $job);
} }
} }
} }
@ -64,7 +72,7 @@ class StartWorkerShell extends AppShell
*/ */
private function runJob(BackgroundJob $job) private function runJob(BackgroundJob $job)
{ {
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}..."); CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}");
try { try {
$job->setStatus(BackgroundJob::STATUS_RUNNING); $job->setStatus(BackgroundJob::STATUS_RUNNING);
@ -73,12 +81,16 @@ class StartWorkerShell extends AppShell
CakeLog::info("[JOB ID: {$job->id()}] - started command `$command`."); CakeLog::info("[JOB ID: {$job->id()}] - started command `$command`.");
$this->getBackgroundJobsTool()->update($job); $this->getBackgroundJobsTool()->update($job);
$job->run(); $start = microtime(true);
$job->run(function (array $status) use ($job) {
$this->getBackgroundJobsTool()->markAsRunning($this->worker, $job, $status['pid']);
});
$duration = number_format(microtime(true) - $start, 3, '.', '');
if ($job->status() === BackgroundJob::STATUS_COMPLETED) { if ($job->status() === BackgroundJob::STATUS_COMPLETED) {
CakeLog::info("[JOB ID: {$job->id()}] - completed."); CakeLog::info("[JOB ID: {$job->id()}] - successfully completed in $duration seconds.");
} else { } else {
CakeLog::error("[JOB ID: {$job->id()}] - failed with error code {$job->returnCode()}. STDERR: {$job->error()}. STDOUT: {$job->output()}."); CakeLog::error("[JOB ID: {$job->id()}] - failed with error code {$job->returnCode()} after $duration seconds. STDERR: {$job->error()}. STDOUT: {$job->output()}.");
} }
} catch (Exception $exception) { } catch (Exception $exception) {
CakeLog::error("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - job ID: {$job->id()} failed with exception: {$exception->getMessage()}"); CakeLog::error("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - job ID: {$job->id()} failed with exception: {$exception->getMessage()}");

2
app/Console/Command/Task/ConfigLoadTask.php
View File

@ -3,8 +3,6 @@ class ConfigLoadTask extends Shell
{ {
public function execute() public function execute()
{ {
Configure::load('config');
if (Configure::read('MISP.system_setting_db')) { if (Configure::read('MISP.system_setting_db')) {
App::uses('SystemSetting', 'Model'); App::uses('SystemSetting', 'Model');
SystemSetting::setGlobalSetting(); SystemSetting::setGlobalSetting();

6
app/Console/Command/UserInitShell.php
View File

@ -1,7 +1,13 @@
<?php <?php
/**
* @deprecated
*/
class UserInitShell extends AppShell { class UserInitShell extends AppShell {
public $uses = array('User', 'Role', 'Organisation', 'Server', 'ConnectionManager'); public $uses = array('User', 'Role', 'Organisation', 'Server', 'ConnectionManager');
public function main() { public function main() {
$this->deprecated('cake user init');
if (!Configure::read('Security.salt')) { if (!Configure::read('Security.salt')) {
$this->loadModel('Server'); $this->loadModel('Server');
$this->Server->serverSettingsSaveValue('Security.salt', $this->User->generateRandomPassword(32)); $this->Server->serverSettingsSaveValue('Security.salt', $this->User->generateRandomPassword(32));

135
app/Console/Command/UserShell.php
View File

@ -3,10 +3,11 @@
/** /**
* @property User $User * @property User $User
* @property Log $Log * @property Log $Log
* @property UserLoginProfile $UserLoginProfile
*/ */
class UserShell extends AppShell class UserShell extends AppShell
{ {
public $uses = ['User', 'Log']; public $uses = ['User', 'Log', 'UserLoginProfile'];
public function getOptionParser() public function getOptionParser()
{ {
@ -22,16 +23,24 @@ class UserShell extends AppShell
], ],
] ]
]); ]);
$parser->addSubcommand('init', [
'help' => __('Create default role, organisation and user when not exists.'),
]);
$parser->addSubcommand('authkey', [ $parser->addSubcommand('authkey', [
'help' => __('Get information about given authkey.'), 'help' => __('Get information about given authkey.'),
'parser' => [ 'parser' => [
'arguments' => [ 'arguments' => [
'authkey' => ['help' => __('Authentication key. If not provide, it will be read from STDIN.')], 'authkey' => ['help' => __('Authentication key. If not provided, it will be read from STDIN.')],
], ],
] ]
]); ]);
$parser->addSubcommand('authkey_valid', [ $parser->addSubcommand('authkey_valid', [
'help' => __('Check if given authkey by STDIN is valid.'), 'help' => __('Check if given authkey by STDIN is valid.'),
'parser' => [
'options' => [
'disableStdLog' => ['help' => __('Do not show logs in STDOUT or STDERR.'), 'boolean' => true],
],
],
]); ]);
$parser->addSubcommand('block', [ $parser->addSubcommand('block', [
'help' => __('Immediately block user.'), 'help' => __('Immediately block user.'),
@ -104,6 +113,14 @@ class UserShell extends AppShell
], ],
], ],
]); ]);
$parser->addSubcommand('ip_country', [
'help' => __('Get country for given IP address'),
'parser' => [
'arguments' => [
'ip' => ['help' => __('IPv4 or IPv6 address.'), 'required' => true],
]
],
]);
$parser->addSubcommand('require_password_change_for_old_passwords', [ $parser->addSubcommand('require_password_change_for_old_passwords', [
'help' => __('Trigger forced password change on next login for users with an old (older than x days) password.'), 'help' => __('Trigger forced password change on next login for users with an old (older than x days) password.'),
'parser' => [ 'parser' => [
@ -121,7 +138,7 @@ class UserShell extends AppShell
public function list() public function list()
{ {
$userId = isset($this->args[0]) ? $this->args[0] : null; $userId = $this->args[0] ?? null;
if ($userId) { if ($userId) {
$conditions = ['OR' => [ $conditions = ['OR' => [
'User.id' => $userId, 'User.id' => $userId,
@ -163,13 +180,24 @@ class UserShell extends AppShell
} }
} }
public function init()
{
if (!Configure::read('Security.salt')) {
$this->loadModel('Server');
$this->Server->serverSettingsSaveValue('Security.salt', $this->User->generateRandomPassword(32));
}
$authKey = $this->User->init();
if ($authKey === null) {
$this->err('Script aborted: MISP instance already initialised.');
} else {
$this->out($authKey);
}
}
public function authkey() public function authkey()
{ {
if (isset($this->args[0])) { $authkey = $this->args[0] ?? fgets(STDIN);
$authkey = $this->args[0];
} else {
$authkey = fgets(STDIN); // read line from STDIN
}
$authkey = trim($authkey); $authkey = trim($authkey);
if (strlen($authkey) !== 40) { if (strlen($authkey) !== 40) {
$this->error('Authkey has not valid format.'); $this->error('Authkey has not valid format.');
@ -212,28 +240,37 @@ class UserShell extends AppShell
*/ */
public function authkey_valid() public function authkey_valid()
{ {
if ($this->params['disableStdLog']) {
$this->_useLogger(false);
}
$cache = []; $cache = [];
$randomKey = random_bytes(16); $randomKey = random_bytes(16);
do { $advancedAuthKeysEnabled = (bool)Configure::read('Security.advanced_authkeys');
while (true) {
$authkey = fgets(STDIN); // read line from STDIN $authkey = fgets(STDIN); // read line from STDIN
$authkey = trim($authkey); $authkey = trim($authkey);
if (strlen($authkey) !== 40) { if (strlen($authkey) !== 40) {
fwrite(STDOUT, "0\n"); // authkey is not in valid format echo "0\n"; // authkey is not in valid format
$this->log("Authkey in incorrect format provided.", LOG_WARNING); $this->log("Authkey in incorrect format provided, expected 40 chars long string, $authkey provided.", LOG_WARNING);
continue; continue;
} }
$time = time();
// Generate hash from authkey to not store raw authkey in memory // Generate hash from authkey to not store raw authkey in memory
$keyHash = sha1($authkey . $randomKey, true); $keyHash = sha1($authkey . $randomKey, true);
// If authkey is in cache and is fresh, use info from cache
$time = time();
if (isset($cache[$keyHash]) && $cache[$keyHash][1] > $time) { if (isset($cache[$keyHash]) && $cache[$keyHash][1] > $time) {
fwrite(STDOUT, $cache[$keyHash][0] ? "1\n" : "0\n"); echo $cache[$keyHash][0] ? "1\n" : "0\n";
continue; continue;
} }
$user = false; $user = false;
for ($i = 0; $i < 5; $i++) { for ($i = 0; $i < 5; $i++) {
try { try {
if (Configure::read('Security.advanced_authkeys')) { if ($advancedAuthKeysEnabled) {
$user = $this->User->AuthKey->getAuthUserByAuthKey($authkey); $user = $this->User->AuthKey->getAuthUserByAuthKey($authkey);
} else { } else {
$user = $this->User->getAuthUserByAuthkey($authkey); $user = $this->User->getAuthUserByAuthkey($authkey);
@ -251,18 +288,34 @@ class UserShell extends AppShell
} }
} }
$user = (bool)$user;
if (!$user) { if (!$user) {
$start = substr($authkey, 0, 4); $valid = null;
$end = substr($authkey, -4); } else if ($user['disabled']) {
$authKeyToStore = $start . str_repeat('*', 32) . $end; $valid = false;
$this->log("Not valid authkey $authKeyToStore provided.", LOG_WARNING); } else {
$valid = true;
} }
// Cache results for 5 seconds echo $valid ? "1\n" : "0\n";
$cache[$keyHash] = [$user, $time + 5];
fwrite(STDOUT, $user ? "1\n" : "0\n"); if ($valid) {
} while (true); // Cache results for 60 seconds if key is valid
$cache[$keyHash] = [true, $time + 60];
} else {
// Cache results for 5 seconds if key is invalid
$cache[$keyHash] = [false, $time + 5];
$start = substr($authkey, 0, 4);
$end = substr($authkey, -4);
$authKeyForLog = $start . str_repeat('*', 32) . $end;
if ($valid === false) {
$this->log("Authkey $authKeyForLog belongs to user {$user['id']} that is disabled.", LOG_WARNING);
} else {
$this->log("Authkey $authKeyForLog is invalid or expired.", LOG_WARNING);
}
}
}
} }
public function block() public function block()
@ -305,7 +358,7 @@ class UserShell extends AppShell
$conditions = ['User.disabled' => false]; // fetch just not disabled users $conditions = ['User.disabled' => false]; // fetch just not disabled users
$userId = isset($this->args[0]) ? $this->args[0] : null; $userId = $this->args[0] ?? null;
if ($userId) { if ($userId) {
$conditions['OR'] = [ $conditions['OR'] = [
'User.id' => $userId, 'User.id' => $userId,
@ -364,7 +417,7 @@ class UserShell extends AppShell
} }
$user = $this->getUser($userId); $user = $this->getUser($userId);
# validate new authentication key if provided // validate new authentication key if provided
if (!empty($newkey) && (strlen($newkey) != 40 || !ctype_alnum($newkey))) { if (!empty($newkey) && (strlen($newkey) != 40 || !ctype_alnum($newkey))) {
$this->error('The new auth key needs to be 40 characters long and only alphanumeric.'); $this->error('The new auth key needs to be 40 characters long and only alphanumeric.');
} }
@ -399,7 +452,7 @@ class UserShell extends AppShell
$this->out('<warning>Storing user IP addresses is disabled.</warning>'); $this->out('<warning>Storing user IP addresses is disabled.</warning>');
} }
$ips = $this->User->setupRedisWithException()->smembers('misp:user_ip:' . $user['id']); $ips = RedisTool::init()->smembers('misp:user_ip:' . $user['id']);
if ($this->params['json']) { if ($this->params['json']) {
$this->out($this->json($ips)); $this->out($this->json($ips));
@ -422,36 +475,50 @@ class UserShell extends AppShell
$this->out('<warning>Storing user IP addresses is disabled.</warning>'); $this->out('<warning>Storing user IP addresses is disabled.</warning>');
} }
$userId = $this->User->setupRedisWithException()->get('misp:ip_user:' . $ip); $userId = RedisTool::init()->get('misp:ip_user:' . $ip);
if (empty($userId)) { if (empty($userId)) {
$this->out('No hits.'); $this->out('No hits.');
$this->_stop(); $this->_stop();
} }
$user = $this->User->find('first', array( $user = $this->User->find('first', [
'recursive' => -1, 'recursive' => -1,
'conditions' => array('User.id' => $userId), 'conditions' => ['User.id' => $userId],
'fields' => ['id', 'email'], 'fields' => ['id', 'email'],
)); ]);
if (empty($user)) { if (empty($user)) {
$this->error("User with ID $userId doesn't exists anymore."); $this->error("User with ID $userId doesn't exists anymore.");
} }
$ipCountry = $this->UserLoginProfile->countryByIp($ip);
if ($this->params['json']) { if ($this->params['json']) {
$this->out($this->json([ $this->out($this->json([
'ip' => $ip, 'ip' => $ip,
'id' => $user['User']['id'], 'id' => $user['User']['id'],
'email' => $user['User']['email'], 'email' => $user['User']['email'],
'country' => $ipCountry,
])); ]));
} else { } else {
$this->out(sprintf( $this->hr();
'%s==============================%sIP: %s%s==============================%sUser #%s: %s%s==============================%s', $this->out("IP: $ip (country $ipCountry)");
PHP_EOL, PHP_EOL, $ip, PHP_EOL, PHP_EOL, $user['User']['id'], $user['User']['email'], PHP_EOL, PHP_EOL $this->hr();
)); $this->out("User #{$user['User']['id']}: {$user['User']['email']}");
$this->hr();
} }
} }
public function ip_country()
{
list($ip) = $this->args;
if (!filter_var($ip, FILTER_VALIDATE_IP)) {
$this->error("IP `$ip` is not valid IPv4 or IPv6 address");
}
$this->out($this->UserLoginProfile->countryByIp($ip));
}
public function require_password_change_for_old_passwords() public function require_password_change_for_old_passwords()
{ {
list($days) = $this->args; list($days) = $this->args;

119
app/Console/Command/WorkerShell.php Normal file
View File

@ -0,0 +1,119 @@
<?php
declare(strict_types=1);
/**
* @property Job $Job
*/
class WorkerShell extends AppShell
{
public $uses = ['Job'];
public function getOptionParser(): ConsoleOptionParser
{
$parser = parent::getOptionParser();
$parser->addSubcommand('showQueues', [
'help' => __('Show jobs in worker queues'),
]);
$parser->addSubcommand('flushQueue', [
'help' => __('Flush jobs in given queue'),
'parser' => [
'arguments' => [
'queue' => ['help' => __('Queue name'), 'required' => true],
],
],
]);
$parser->addSubcommand('showJobStatus', [
'help' => __('Show job status'),
'parser' => [
'arguments' => [
'job_id' => ['help' => __('Job ID (ID or UUID)'), 'required' => true],
],
],
]);
return $parser;
}
/**
* @throws RedisException
* @throws JsonException
*/
public function showQueues()
{
$tool = $this->getBackgroundJobsTool();
$runningJobs = $tool->runningJobs();
foreach (BackgroundJobsTool::VALID_QUEUES as $queue) {
$this->out("{$queue}:\t{$tool->getQueueSize($queue)}");
$queueJobs = $runningJobs[$queue] ?? [];
foreach ($queueJobs as $jobId => $data) {
$this->out(" - $jobId (" . JsonTool::encode($data) .")");
}
}
}
public function flushQueue()
{
$queue = $this->args[0];
try {
$this->getBackgroundJobsTool()->clearQueue($queue);
} catch (InvalidArgumentException $e) {
$this->error($e->getMessage());
}
}
public function showJobStatus()
{
$processId = $this->args[0];
if (is_numeric($processId)) {
$job = $this->Job->find('first', [
'conditions' => ['Job.id' => $processId],
'recursive' => -1,
]);
if (!$job) {
$this->error('Job not found', "Job with ID {$processId} not found");
}
$this->out($this->json($job['Job']));
$processId = $job['Job']['process_id'];
}
if (!Validation::uuid($processId)) {
$this->error('Job not found', "Job ID must be number or UUID, '$processId' given");
}
$jobStatus = $this->getBackgroundJobsTool()->getJob($processId);
if (!$jobStatus) {
$this->error('Job not found', "Job with UUID {$processId} not found");
}
$jobStatus = $jobStatus->jsonSerialize();
foreach (['createdAt', 'updatedAt'] as $timeField) {
if (isset($jobStatus[$timeField])) {
$jobStatus[$timeField] = date('c', $jobStatus[$timeField]);
}
}
if (isset($jobStatus['status'])) {
$jobStatus['status'] = $this->jobStatusToString($jobStatus['status']);
}
$this->out($this->json($jobStatus));
}
private function jobStatusToString(int $jobStatus)
{
switch ($jobStatus) {
case Job::STATUS_WAITING:
return 'waiting';
case Job::STATUS_RUNNING:
return 'running';
case Job::STATUS_FAILED:
return 'failed';
case Job::STATUS_COMPLETED:
return 'completed';
}
throw new InvalidArgumentException("Invalid job status $jobStatus");
}
}

59
app/Controller/AppController.php
View File

@ -38,8 +38,6 @@ class AppController extends Controller
public $phpmin = '7.2'; public $phpmin = '7.2';
public $phprec = '7.4'; public $phprec = '7.4';
public $phptoonew = '8.0'; public $phptoonew = '8.0';
public $pythonmin = '3.6';
public $pythonrec = '3.7';
private $isApiAuthed = false; private $isApiAuthed = false;
public $baseurl = ''; public $baseurl = '';
@ -232,6 +230,10 @@ class AppController extends Controller
$this->Security->csrfCheck = false; $this->Security->csrfCheck = false;
$loginByAuthKeyResult = $this->__loginByAuthKey(); $loginByAuthKeyResult = $this->__loginByAuthKey();
if ($loginByAuthKeyResult === false || $this->Auth->user() === null) { if ($loginByAuthKeyResult === false || $this->Auth->user() === null) {
if ($this->IndexFilter->isXhr()) {
throw new ForbiddenException('Authentication failed.');
}
if ($loginByAuthKeyResult === null) { if ($loginByAuthKeyResult === null) {
$this->loadModel('Log'); $this->loadModel('Log');
$this->Log->createLogEntry('SYSTEM', 'auth_fail', 'User', 0, "Failed API authentication. No authkey was provided."); $this->Log->createLogEntry('SYSTEM', 'auth_fail', 'User', 0, "Failed API authentication. No authkey was provided.");
@ -601,7 +603,7 @@ class AppController extends Controller
if (!empty($user['allowed_ips'])) { if (!empty($user['allowed_ips'])) {
App::uses('CidrTool', 'Tools'); App::uses('CidrTool', 'Tools');
$cidrTool = new CidrTool($user['allowed_ips']); $cidrTool = new CidrTool($user['allowed_ips']);
$remoteIp = $this->_remoteIp(); $remoteIp = $this->User->_remoteIp();
if ($remoteIp === null) { if ($remoteIp === null) {
$this->Auth->logout(); $this->Auth->logout();
throw new ForbiddenException('Auth key is limited to IP address, but IP address not found'); throw new ForbiddenException('Auth key is limited to IP address, but IP address not found');
@ -694,7 +696,7 @@ class AppController extends Controller
return; return;
} }
$remoteAddress = $this->_remoteIp(); $remoteAddress = $this->User->_remoteIp();
$pipe = $redis->pipeline(); $pipe = $redis->pipeline();
// keep for 30 days // keep for 30 days
@ -737,7 +739,7 @@ class AppController extends Controller
$includeRequestBody = !empty(Configure::read('MISP.log_paranoid_include_post_body')) || $userMonitoringEnabled; $includeRequestBody = !empty(Configure::read('MISP.log_paranoid_include_post_body')) || $userMonitoringEnabled;
/** @var AccessLog $accessLog */ /** @var AccessLog $accessLog */
$accessLog = ClassRegistry::init('AccessLog'); $accessLog = ClassRegistry::init('AccessLog');
$accessLog->logRequest($user, $this->_remoteIp(), $this->request, $includeRequestBody); $accessLog->logRequest($user, $this->User->_remoteIp(), $this->request, $includeRequestBody);
} }
if ( if (
@ -828,29 +830,34 @@ class AppController extends Controller
private function __rateLimitCheck(array $user) private function __rateLimitCheck(array $user)
{ {
$info = array();
$rateLimitCheck = $this->RateLimit->check( $rateLimitCheck = $this->RateLimit->check(
$user, $user,
$this->request->params['controller'], $this->request->params['controller'],
$this->request->action, $this->request->params['action'],
$info,
$this->response->type()
); );
if (!empty($info)) {
$this->RestResponse->setHeader('X-Rate-Limit-Limit', $info['limit']); if ($rateLimitCheck) {
$this->RestResponse->setHeader('X-Rate-Limit-Remaining', $info['remaining']); $headers = [
$this->RestResponse->setHeader('X-Rate-Limit-Reset', $info['reset']); 'X-Rate-Limit-Limit' => $rateLimitCheck['limit'],
'X-Rate-Limit-Remaining' => $rateLimitCheck['remaining'],
'X-Rate-Limit-Reset' => $rateLimitCheck['reset'],
];
if ($rateLimitCheck['exceeded']) {
$response = $this->RestResponse->throwException(
429,
__('Rate limit exceeded.'),
'/' . $this->request->params['controller'] . '/' . $this->request->params['action'],
false,
false,
$headers
);
$response->send();
$this->_stop();
} else {
$this->RestResponse->headers = array_merge($this->RestResponse->headers, $headers);
}
} }
if ($rateLimitCheck !== true) {
$this->response->header('X-Rate-Limit-Limit', $info['limit']);
$this->response->header('X-Rate-Limit-Remaining', $info['remaining']);
$this->response->header('X-Rate-Limit-Reset', $info['reset']);
$this->response->body($rateLimitCheck);
$this->response->statusCode(429);
$this->response->send();
$this->_stop();
}
return true;
} }
public function afterFilter() public function afterFilter()
@ -1143,14 +1150,14 @@ class AppController extends Controller
$headerNamespace = ''; $headerNamespace = '';
} }
if (isset($server[$headerNamespace . $header]) && !empty($server[$headerNamespace . $header])) { if (isset($server[$headerNamespace . $header]) && !empty($server[$headerNamespace . $header])) {
if (Configure::read('Plugin.CustomAuth_only_allow_source') && Configure::read('Plugin.CustomAuth_only_allow_source') !== $this->_remoteIp()) { if (Configure::read('Plugin.CustomAuth_only_allow_source') && Configure::read('Plugin.CustomAuth_only_allow_source') !== $this->User->_remoteIp()) {
$this->Log = ClassRegistry::init('Log'); $this->Log = ClassRegistry::init('Log');
$this->Log->createLogEntry( $this->Log->createLogEntry(
'SYSTEM', 'SYSTEM',
'auth_fail', 'auth_fail',
'User', 'User',
0, 0,
'Failed authentication using external key (' . trim($server[$headerNamespace . $header]) . ') - the user has not arrived from the expected address. Instead the request came from: ' . $this->_remoteIp(), 'Failed authentication using external key (' . trim($server[$headerNamespace . $header]) . ') - the user has not arrived from the expected address. Instead the request came from: ' . $this->User->_remoteIp(),
null); null);
$this->__preAuthException($authName . ' authentication failed. Contact your MISP support for additional information at: ' . Configure::read('MISP.contact')); $this->__preAuthException($authName . ' authentication failed. Contact your MISP support for additional information at: ' . Configure::read('MISP.contact'));
} }
@ -1310,7 +1317,7 @@ class AppController extends Controller
$exception = false; $exception = false;
$filters = $this->_harvestParameters($filterData, $exception, $this->_legacyParams); $filters = $this->_harvestParameters($filterData, $exception, $this->_legacyParams);
if (empty($filters) && $this->request->is('get')) { if (empty($filters) && $this->request->is('get')) {
throw new InvalidArgumentException(__('Restsearch queries using GET and no parameters are not allowed. If you have passed parameters via a JSON body, make sure you use POST requests.')); throw new BadRequestException(__('Restsearch queries using GET and no parameters are not allowed. If you have passed parameters via a JSON body, make sure you use POST requests.'));
} }
if (empty($filters['returnFormat'])) { if (empty($filters['returnFormat'])) {
$filters['returnFormat'] = 'json'; $filters['returnFormat'] = 'json';

2
app/Controller/AttributesController.php
View File

@ -1917,7 +1917,7 @@ class AttributesController extends AppController
public function reportValidationIssuesAttributes($eventId = false) public function reportValidationIssuesAttributes($eventId = false)
{ {
// search for validation problems in the attributes // search for validation problems in the attributes
$this->set('result', $this->Attribute->reportValidationIssuesAttributes($eventId)); $this->set('result', iterator_to_array($this->Attribute->reportValidationIssuesAttributes($eventId)));
} }
public function generateCorrelation() public function generateCorrelation()

26
app/Controller/AuditLogsController.php
View File

@ -134,7 +134,8 @@ class AuditLogsController extends AppController
]); ]);
$this->paginate['conditions'] = $this->__searchConditions($params); $this->paginate['conditions'] = $this->__searchConditions($params);
$acl = $this->__applyAuditACL($this->Auth->user()); $user = $this->Auth->user();
$acl = $this->__applyAuditACL($user);
if ($acl) { if ($acl) {
$this->paginate['conditions']['AND'][] = $acl; $this->paginate['conditions']['AND'][] = $acl;
} }
@ -144,7 +145,7 @@ class AuditLogsController extends AppController
return $this->RestResponse->viewData($list, 'json'); return $this->RestResponse->viewData($list, 'json');
} }
$list = $this->__appendModelLinks($list); $list = $this->__appendModelLinks($user, $list);
foreach ($list as $k => $item) { foreach ($list as $k => $item) {
$list[$k]['AuditLog']['action_human'] = $this->actions[$item['AuditLog']['action']]; $list[$k]['AuditLog']['action_human'] = $this->actions[$item['AuditLog']['action']];
} }
@ -435,10 +436,11 @@ class AuditLogsController extends AppController
/** /**
* Generate link to model view if exists and use has permission to access it. * Generate link to model view if exists and use has permission to access it.
* @param array $user
* @param array $auditLogs * @param array $auditLogs
* @return array * @return array
*/ */
private function __appendModelLinks(array $auditLogs) private function __appendModelLinks(array $user, array $auditLogs)
{ {
$models = []; $models = [];
foreach ($auditLogs as $auditLog) { foreach ($auditLogs as $auditLog) {
@ -449,7 +451,7 @@ class AuditLogsController extends AppController
} }
} }
$eventIds = isset($models['Event']) ? $models['Event'] : []; $eventIds = $models['Event'] ?? [];
if (isset($models['ObjectReference'])) { if (isset($models['ObjectReference'])) {
$this->loadModel('ObjectReference'); $this->loadModel('ObjectReference');
@ -461,11 +463,11 @@ class AuditLogsController extends AppController
if (isset($models['Object']) || isset($objectReferences)) { if (isset($models['Object']) || isset($objectReferences)) {
$objectIds = array_unique(array_merge( $objectIds = array_unique(array_merge(
isset($models['Object']) ? $models['Object'] : [], $models['Object'] ?? [],
isset($objectReferences) ? array_values($objectReferences) : [] isset($objectReferences) ? array_values($objectReferences) : []
)); ));
$this->loadModel('MispObject'); $this->loadModel('MispObject');
$conditions = $this->MispObject->buildConditions($this->Auth->user()); $conditions = $this->MispObject->buildConditions($user);
$conditions['Object.id'] = $objectIds; $conditions['Object.id'] = $objectIds;
$objects = $this->MispObject->find('all', [ $objects = $this->MispObject->find('all', [
'conditions' => $conditions, 'conditions' => $conditions,
@ -473,22 +475,22 @@ class AuditLogsController extends AppController
'fields' => ['Object.id', 'Object.event_id', 'Object.uuid', 'Object.deleted'], 'fields' => ['Object.id', 'Object.event_id', 'Object.uuid', 'Object.deleted'],
]); ]);
$objects = array_column(array_column($objects, 'Object'), null, 'id'); $objects = array_column(array_column($objects, 'Object'), null, 'id');
$eventIds = array_merge($eventIds, array_column($objects, 'event_id')); array_push($eventIds, ...array_column($objects, 'event_id'));
} }
if (isset($models['Attribute'])) { if (isset($models['Attribute'])) {
$this->loadModel('Attribute'); $this->loadModel('Attribute');
$attributes = $this->Attribute->fetchAttributesSimple($this->Auth->user(), [ $attributes = $this->Attribute->fetchAttributesSimple($user, [
'conditions' => ['Attribute.id' => array_unique($models['Attribute'])], 'conditions' => ['Attribute.id' => array_unique($models['Attribute'])],
'fields' => ['Attribute.id', 'Attribute.event_id', 'Attribute.uuid', 'Attribute.deleted'], 'fields' => ['Attribute.id', 'Attribute.event_id', 'Attribute.uuid', 'Attribute.deleted'],
]); ]);
$attributes = array_column(array_column($attributes, 'Attribute'), null, 'id'); $attributes = array_column(array_column($attributes, 'Attribute'), null, 'id');
$eventIds = array_merge($eventIds, array_column($attributes, 'event_id')); array_push($eventIds, ...array_column($attributes, 'event_id'));
} }
if (isset($models['ShadowAttribute'])) { if (isset($models['ShadowAttribute'])) {
$this->loadModel('ShadowAttribute'); $this->loadModel('ShadowAttribute');
$conditions = $this->ShadowAttribute->buildConditions($this->Auth->user()); $conditions = $this->ShadowAttribute->buildConditions($user);
$conditions['AND'][] = ['ShadowAttribute.id' => array_unique($models['ShadowAttribute'])]; $conditions['AND'][] = ['ShadowAttribute.id' => array_unique($models['ShadowAttribute'])];
$shadowAttributes = $this->ShadowAttribute->find('all', [ $shadowAttributes = $this->ShadowAttribute->find('all', [
'conditions' => $conditions, 'conditions' => $conditions,
@ -496,12 +498,12 @@ class AuditLogsController extends AppController
'contain' => ['Event', 'Attribute'], 'contain' => ['Event', 'Attribute'],
]); ]);
$shadowAttributes = array_column(array_column($shadowAttributes, 'ShadowAttribute'), null, 'id'); $shadowAttributes = array_column(array_column($shadowAttributes, 'ShadowAttribute'), null, 'id');
$eventIds = array_merge($eventIds, array_column($shadowAttributes, 'event_id')); array_push($eventIds, ...array_column($shadowAttributes, 'event_id'));
} }
if (!empty($eventIds)) { if (!empty($eventIds)) {
$this->loadModel('Event'); $this->loadModel('Event');
$conditions = $this->Event->createEventConditions($this->Auth->user()); $conditions = $this->Event->createEventConditions($user);
$conditions['Event.id'] = array_unique($eventIds); $conditions['Event.id'] = array_unique($eventIds);
$events = $this->Event->find('list', [ $events = $this->Event->find('list', [
'conditions' => $conditions, 'conditions' => $conditions,

14
app/Controller/Component/IndexFilterComponent.php
View File

@ -8,7 +8,9 @@ class IndexFilterComponent extends Component
{ {
/** @var Controller */ /** @var Controller */
public $Controller; public $Controller;
public $isRest = null;
/** @var bool|null */
private $isRest = null;
// Used for isApiFunction(), a check that returns true if the controller & action combo matches an action that is a non-xml and non-json automation method // Used for isApiFunction(), a check that returns true if the controller & action combo matches an action that is a non-xml and non-json automation method
// This is used to allow authentication via headers for methods not covered by _isRest() - as that only checks for JSON and XML formats // This is used to allow authentication via headers for methods not covered by _isRest() - as that only checks for JSON and XML formats
@ -93,6 +95,11 @@ class IndexFilterComponent extends Component
} }
} }
public function isXhr()
{
return $this->Controller->request->header('X-Requested-With') === 'XMLHttpRequest';
}
public function isJson() public function isJson()
{ {
return $this->Controller->request->header('Accept') === 'application/json' || $this->Controller->RequestHandler->prefers() === 'json'; return $this->Controller->request->header('Accept') === 'application/json' || $this->Controller->RequestHandler->prefers() === 'json';
@ -103,11 +110,6 @@ class IndexFilterComponent extends Component
return $this->Controller->request->header('Accept') === 'text/csv' || $this->Controller->RequestHandler->prefers() === 'csv'; return $this->Controller->request->header('Accept') === 'text/csv' || $this->Controller->RequestHandler->prefers() === 'csv';
} }
public function isXml()
{
}
/** /**
* @param string $controller * @param string $controller
* @param string $action * @param string $action

92
app/Controller/Component/RateLimitComponent.php
View File

@ -12,58 +12,60 @@ class RateLimitComponent extends Component
) )
); );
public $components = array('RestResponse');
/** /**
* @param array $user * @param array $user
* @param string $controller * @param string $controller
* @param string $action * @param string $action
* @param array $info * @return array|null
* @param string $responseType
* @return bool
* @throws RedisException * @throws RedisException
*/ */
public function check(array $user, $controller, $action, &$info = array(), $responseType) public function check(array $user, $controller, $action)
{ {
if (!empty($user['Role']['enforce_rate_limit']) && isset(self::LIMITED_FUNCTIONS[$controller][$action])) { if (!isset(self::LIMITED_FUNCTIONS[$controller][$action])) {
if ($user['Role']['rate_limit_count'] == 0) { return null; // no limit enforced for this controller action
throw new MethodNotAllowedException(__('API searches are not allowed for this user role.'));
}
try {
$redis = RedisTool::init();
} catch (Exception $e) {
return true; // redis is not available, allow access
}
$uuid = Configure::read('MISP.uuid') ?: 'no-uuid';
$keyName = 'misp:' . $uuid . ':rate_limit:' . $user['id'];
$count = $redis->get($keyName);
if ($count !== false && $count >= $user['Role']['rate_limit_count']) {
$info = array(
'limit' => $user['Role']['rate_limit_count'],
'reset' => $redis->ttl($keyName),
'remaining' => $user['Role']['rate_limit_count'] - $count,
);
return $this->RestResponse->throwException(
429,
__('Rate limit exceeded.'),
'/' . $controller . '/' . $action,
$responseType
);
} else {
if ($count === false) {
$redis->setEx($keyName, 900, 1);
} else {
$redis->setEx($keyName, $redis->ttl($keyName), intval($count) + 1);
}
}
$count += 1;
$info = array(
'limit' => $user['Role']['rate_limit_count'],
'reset' => $redis->ttl($keyName),
'remaining' => $user['Role']['rate_limit_count'] - $count
);
} }
return true;
if (empty($user['Role']['enforce_rate_limit'])) {
return null; // no limit enforced for this role
}
$rateLimit = (int)$user['Role']['rate_limit_count'];
if ($rateLimit === 0) {
throw new MethodNotAllowedException(__('API searches are not allowed for this user role.'));
}
try {
$redis = RedisTool::init();
} catch (Exception $e) {
return null; // redis is not available, allow access
}
$uuid = Configure::read('MISP.uuid') ?: 'no-uuid';
$keyName = 'misp:' . $uuid . ':rate_limit:' . $user['id'];
$count = $redis->get($keyName);
if ($count !== false && $count >= $rateLimit) {
return [
'exceeded' => true,
'limit' => $rateLimit,
'reset' => $redis->ttl($keyName),
'remaining' => $rateLimit - $count,
];
}
$newCount = $redis->incr($keyName);
if ($newCount === 1) {
$redis->expire($keyName, 900);
$reset = 900;
} else {
$reset = $redis->ttl($keyName);
}
return [
'exceeded' => false,
'limit' => $rateLimit,
'reset' => $reset,
'remaining' => $rateLimit - $newCount,
];
} }
} }

14
app/Controller/Component/RestResponseComponent.php
View File

@ -517,7 +517,7 @@ class RestResponseComponent extends Component
if ($id) { if ($id) {
$response['id'] = $id; $response['id'] = $id;
} }
return $this->__sendResponse($response, 403, $format); return $this->prepareResponse($response, 403, $format);
} }
/** /**
@ -562,7 +562,7 @@ class RestResponseComponent extends Component
if ($id) { if ($id) {
$response['id'] = $id; $response['id'] = $id;
} }
return $this->__sendResponse($response, 200, $format); return $this->prepareResponse($response, 200, $format);
} }
/** /**
@ -587,7 +587,7 @@ class RestResponseComponent extends Component
* @return CakeResponse * @return CakeResponse
* @throws Exception * @throws Exception
*/ */
private function __sendResponse($response, $code, $format = false, $raw = false, $download = false, $headers = array()) private function prepareResponse($response, $code, $format = false, $raw = false, $download = false, $headers = array())
{ {
App::uses('TmpFileTool', 'Tools'); App::uses('TmpFileTool', 'Tools');
$format = !empty($format) ? strtolower($format) : 'json'; $format = !empty($format) ? strtolower($format) : 'json';
@ -633,7 +633,7 @@ class RestResponseComponent extends Component
} }
// If response is big array, encode items separately to save memory // If response is big array, encode items separately to save memory
if (is_array($response) && count($response) > 10000) { if (is_array($response) && count($response) > 10000 && JsonTool::arrayIsList($response)) {
$output = new TmpFileTool(); $output = new TmpFileTool();
$output->write('['); $output->write('[');
@ -775,7 +775,7 @@ class RestResponseComponent extends Component
if (!empty($errors)) { if (!empty($errors)) {
$data['errors'] = $errors; $data['errors'] = $errors;
} }
return $this->__sendResponse($data, 200, $format, $raw, $download, $headers); return $this->prepareResponse($data, 200, $format, $raw, $download, $headers);
} }
/** /**
@ -807,7 +807,7 @@ class RestResponseComponent extends Component
'message' => $message, 'message' => $message,
'url' => $url 'url' => $url
); );
return $this->__sendResponse($message, $code, $format, $raw, false, $headers); return $this->prepareResponse($message, $code, $format, $raw, false, $headers);
} }
public function setHeader($header, $value) public function setHeader($header, $value)
@ -834,7 +834,7 @@ class RestResponseComponent extends Component
} }
} }
$response['url'] = $this->__generateURL($actionArray, $controller, $params); $response['url'] = $this->__generateURL($actionArray, $controller, $params);
return $this->__sendResponse($response, 200, $format); return $this->prepareResponse($response, 200, $format);
} }
private function __setup() private function __setup()

8
app/Controller/Component/RestSearchComponent.php
View File

@ -152,6 +152,8 @@ class RestSearchComponent extends Component
'category', 'category',
'org', 'org',
'tags', 'tags',
'first_seen',
'last_seen',
'from', 'from',
'to', 'to',
'last', 'last',
@ -184,6 +186,7 @@ class RestSearchComponent extends Component
'context', 'context',
'returnFormat', 'returnFormat',
'id', 'id',
'uuid',
'type', 'type',
'from', 'from',
'to', 'to',
@ -191,7 +194,8 @@ class RestSearchComponent extends Component
'org_id', 'org_id',
'source', 'source',
'includeAttribute', 'includeAttribute',
'includeEvent' 'includeEvent',
'includeUuid',
], ],
'GalaxyCluster' => [ 'GalaxyCluster' => [
'page', 'page',
@ -204,7 +208,7 @@ class RestSearchComponent extends Component
'distribution', 'distribution',
'org', 'org',
'orgc', 'orgc',
'tag', 'tag_name',
'custom', 'custom',
'sgReferenceOnly', 'sgReferenceOnly',
'minimal', 'minimal',

32
app/Controller/EventsController.php
View File

@ -834,33 +834,29 @@ class EventsController extends AppController
} }
if (empty($rules['limit'])) { if (empty($rules['limit'])) {
$events = array(); $events = [];
$i = 1; $i = 1;
$rules['limit'] = 20000; $rules['limit'] = 20000;
while (true) { while (true) {
$rules['page'] = $i; $rules['page'] = $i++;
$temp = $this->Event->find('all', $rules); $temp = $this->Event->find('all', $rules);
$resultCount = count($temp); $resultCount = count($temp);
if ($resultCount !== 0) { if ($resultCount !== 0) {
// this is faster and memory efficient than array_merge array_push($events, ...$temp);
foreach ($temp as $tempEvent) {
$events[] = $tempEvent;
}
} }
if ($resultCount < $rules['limit']) { if ($resultCount < $rules['limit']) {
break; break;
} }
$i++;
} }
unset($temp); unset($temp);
$absolute_total = count($events); $absoluteTotal = count($events);
} else { } else {
$counting_rules = $rules; $counting_rules = $rules;
unset($counting_rules['limit']); unset($counting_rules['limit']);
unset($counting_rules['page']); unset($counting_rules['page']);
$absolute_total = $this->Event->find('count', $counting_rules); $absoluteTotal = $this->Event->find('count', $counting_rules);
$events = $absolute_total === 0 ? [] : $this->Event->find('all', $rules); $events = $absoluteTotal === 0 ? [] : $this->Event->find('all', $rules);
} }
$isCsvResponse = $this->response->type() === 'text/csv'; $isCsvResponse = $this->response->type() === 'text/csv';
@ -979,7 +975,7 @@ class EventsController extends AppController
$events = $export->eventIndex($events); $events = $export->eventIndex($events);
} }
return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, ['X-Result-Count' => $absolute_total]); return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, ['X-Result-Count' => $absoluteTotal]);
} }
private function __indexColumns() private function __indexColumns()
@ -2383,7 +2379,7 @@ class EventsController extends AppController
$results = $this->Event->addMISPExportFile($this->Auth->user(), $data, $isXml, $takeOwnership, $publish); $results = $this->Event->addMISPExportFile($this->Auth->user(), $data, $isXml, $takeOwnership, $publish);
} catch (Exception $e) { } catch (Exception $e) {
$this->log("Exception during processing MISP file import: {$e->getMessage()}"); $this->log("Exception during processing MISP file import: {$e->getMessage()}");
$this->Flash->error(__('Could not process MISP export file. %s.', $e->getMessage())); $this->Flash->error(__('Could not process MISP export file. %s', $e->getMessage()));
$this->redirect(['controller' => 'events', 'action' => 'add_misp_export']); $this->redirect(['controller' => 'events', 'action' => 'add_misp_export']);
} }
} }
@ -3203,7 +3199,7 @@ class EventsController extends AppController
$event = $this->Event->find('first', [ $event = $this->Event->find('first', [
'conditions' => Validation::uuid($id) ? ['Event.uuid' => $id] : ['Event.id' => $id], 'conditions' => Validation::uuid($id) ? ['Event.uuid' => $id] : ['Event.id' => $id],
'recursive' => -1, 'recursive' => -1,
'fields' => ['id', 'info', 'publish_timestamp', 'orgc_id'], 'fields' => ['id', 'info', 'publish_timestamp', 'orgc_id', 'user_id'],
]); ]);
if (empty($event)) { if (empty($event)) {
throw new NotFoundException(__('Invalid event.')); throw new NotFoundException(__('Invalid event.'));
@ -3222,6 +3218,16 @@ class EventsController extends AppController
} }
} }
} }
if (
Configure::read('MISP.block_publishing_for_same_creator', false) &&
$this->Auth->user()['id'] == $event['Event']['user_id']
) {
$message = __('Could not publish the event, the publishing user cannot be the same as the event creator as per this instance\'s configuration.');
if (!$this->_isRest()) {
$this->Flash->error($message);
}
throw new MethodNotAllowedException($message);
}
return $event; return $event;
} }

15
app/Controller/ServersController.php
View File

@ -1073,7 +1073,7 @@ class ServersController extends AppController
); );
$dumpResults = array(); $dumpResults = array();
$tempArray = array(); $tempArray = array();
foreach ($finalSettings as $k => $result) { foreach ($finalSettings as $result) {
if ($result['level'] == 3) { if ($result['level'] == 3) {
$issues['deprecated']++; $issues['deprecated']++;
} }
@ -1105,18 +1105,19 @@ class ServersController extends AppController
$diagnostic_errors = 0; $diagnostic_errors = 0;
App::uses('File', 'Utility'); App::uses('File', 'Utility');
App::uses('Folder', 'Utility'); App::uses('Folder', 'Utility');
if ($tab === 'correlations') { if ($tab === 'correlations') {
$this->loadModel('Correlation'); $this->loadModel('Correlation');
$correlation_metrics = $this->Correlation->collectMetrics(); $correlation_metrics = $this->Correlation->collectMetrics();
$this->set('correlation_metrics', $correlation_metrics); $this->set('correlation_metrics', $correlation_metrics);
} } else if ($tab === 'files') {
if ($tab === 'files') {
if (!empty(Configure::read('Security.disable_instance_file_uploads'))) { if (!empty(Configure::read('Security.disable_instance_file_uploads'))) {
throw new MethodNotAllowedException(__('This functionality is disabled.')); throw new MethodNotAllowedException(__('This functionality is disabled.'));
} }
$files = $this->Server->grabFiles(); $files = $this->Server->grabFiles();
$this->set('files', $files); $this->set('files', $files);
} }
// Only run this check on the diagnostics tab // Only run this check on the diagnostics tab
if ($tab === 'diagnostics' || $tab === 'download' || $this->_isRest()) { if ($tab === 'diagnostics' || $tab === 'download' || $this->_isRest()) {
$php_ini = php_ini_loaded_file(); $php_ini = php_ini_loaded_file();
@ -1279,12 +1280,10 @@ class ServersController extends AppController
$this->set('workerIssueCount', $workerIssueCount); $this->set('workerIssueCount', $workerIssueCount);
$priorityErrorColours = array(0 => 'red', 1 => 'yellow', 2 => 'green'); $priorityErrorColours = array(0 => 'red', 1 => 'yellow', 2 => 'green');
$this->set('priorityErrorColours', $priorityErrorColours); $this->set('priorityErrorColours', $priorityErrorColours);
$this->set('phpversion', phpversion()); $this->set('phpversion', PHP_VERSION);
$this->set('phpmin', $this->phpmin); $this->set('phpmin', $this->phpmin);
$this->set('phprec', $this->phprec); $this->set('phprec', $this->phprec);
$this->set('phptoonew', $this->phptoonew); $this->set('phptoonew', $this->phptoonew);
$this->set('pythonmin', $this->pythonmin);
$this->set('pythonrec', $this->pythonrec);
$this->set('title_for_layout', __('Diagnostics')); $this->set('title_for_layout', __('Diagnostics'));
} }
@ -1863,7 +1862,7 @@ class ServersController extends AppController
} }
if (Configure::read('SimpleBackgroundJobs.enabled')) { if (Configure::read('SimpleBackgroundJobs.enabled')) {
$this->Server->getBackgroundJobsTool()->purgeQueue($worker); $this->Server->getBackgroundJobsTool()->clearQueue($worker);
} else { } else {
// CakeResque // CakeResque
$worker_array = array('cache', 'default', 'email', 'prio'); $worker_array = array('cache', 'default', 'email', 'prio');
@ -2183,7 +2182,7 @@ class ServersController extends AppController
if ($this->_isRest()) { if ($this->_isRest()) {
return $this->RestResponse->saveFailResponse('Servers', 'addFromJson', false, $this->Server->validationErrors, $this->response->type()); return $this->RestResponse->saveFailResponse('Servers', 'addFromJson', false, $this->Server->validationErrors, $this->response->type());
} else { } else {
$this->Flash->error(__('Could not save the server. Error: %s', json_encode($this->Server->validationErrors, true))); $this->Flash->error(__('Could not save the server. Error: %s', json_encode($this->Server->validationErrors)));
$this->redirect(array('action' => 'index')); $this->redirect(array('action' => 'index'));
} }
} }

70
app/Controller/UsersController.php
View File

@ -1244,8 +1244,6 @@ class UsersController extends AppController
// login was successful, do everything that is needed such as logging and more: // login was successful, do everything that is needed such as logging and more:
$this->_postlogin(); $this->_postlogin();
} else { } else {
$dataSourceConfig = ConnectionManager::getDataSource('default')->config;
$dataSource = $dataSourceConfig['datasource'];
// don't display authError before first login attempt // don't display authError before first login attempt
if (str_replace("//", "/", $this->webroot . $this->Session->read('Auth.redirect')) == $this->webroot && $this->Session->read('Message.auth.message') == $this->Auth->authError) { if (str_replace("//", "/", $this->webroot . $this->Session->read('Auth.redirect')) == $this->webroot && $this->Session->read('Message.auth.message') == $this->Auth->authError) {
$this->Session->delete('Message.auth'); $this->Session->delete('Message.auth');
@ -1260,73 +1258,7 @@ class UsersController extends AppController
} }
} }
// $this->User->init();
// Actions needed for the first access, when the database is not populated yet.
//
// populate the DB with the first role (site admin) if it's empty
if (!$this->User->Role->hasAny()) {
$siteAdmin = array('Role' => array(
'id' => 1,
'name' => 'Site Admin',
'permission' => 3,
'perm_add' => 1,
'perm_modify' => 1,
'perm_modify_org' => 1,
'perm_publish' => 1,
'perm_sync' => 1,
'perm_admin' => 1,
'perm_audit' => 1,
'perm_auth' => 1,
'perm_site_admin' => 1,
'perm_regexp_access' => 1,
'perm_sharing_group' => 1,
'perm_template' => 1,
'perm_tagger' => 1,
));
$this->User->Role->save($siteAdmin);
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
if ($dataSource === 'Database/Postgres') {
$sql = "SELECT setval('roles_id_seq', (SELECT MAX(id) FROM roles));";
$this->User->Role->query($sql);
}
}
if (!$this->User->Organisation->hasAny(array('Organisation.local' => true))) {
$this->User->runUpdates();
$date = date('Y-m-d H:i:s');
$org = array('Organisation' => array(
'id' => 1,
'name' => !empty(Configure::read('MISP.org')) ? Configure::read('MISP.org') : 'ADMIN',
'description' => 'Automatically generated admin organisation',
'type' => 'ADMIN',
'uuid' => CakeText::uuid(),
'local' => 1,
'date_created' => $date,
'sector' => '',
'nationality' => ''
));
$this->User->Organisation->save($org);
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
if ($dataSource === 'Database/Postgres') {
$sql = "SELECT setval('organisations_id_seq', (SELECT MAX(id) FROM organisations));";
$this->User->Organisation->query($sql);
}
$org_id = $this->User->Organisation->id;
}
// populate the DB with the first user if it's empty
if (!$this->User->hasAny()) {
if (!isset($org_id)) {
$hostOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.name' => Configure::read('MISP.org'), 'Organisation.local' => true), 'recursive' => -1));
if (!empty($hostOrg)) {
$org_id = $hostOrg['Organisation']['id'];
} else {
$firstOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.local' => true), 'order' => 'Organisation.id ASC'));
$org_id = $firstOrg['Organisation']['id'];
}
}
$this->User->runUpdates();
$this->User->createInitialUser($org_id);
}
} }
} }

191
app/Lib/Export/NidsExport.php
View File

@ -6,10 +6,7 @@ abstract class NidsExport
public $classtype = 'trojan-activity'; public $classtype = 'trojan-activity';
public $format = ""; // suricata (default), snort protected $format; // suricata (default), snort
public $checkWhitelist = true;
public $additional_params = array( public $additional_params = array(
'contain' => array( 'contain' => array(
@ -17,36 +14,31 @@ abstract class NidsExport
'fields' => array('threat_level_id') 'fields' => array('threat_level_id')
) )
), ),
); );
public function handler($data, $options = array()) public function handler($data, $options = array())
{ {
$continue = empty($format);
$this->checkWhitelist = false;
if ($options['scope'] === 'Attribute') { if ($options['scope'] === 'Attribute') {
$this->export( $this->export(
array($data), array($data),
$options['user']['nids_sid'], $options['user']['nids_sid']
$options['returnFormat'],
$continue
); );
} else if ($options['scope'] === 'Event') { } else if ($options['scope'] === 'Event') {
if (!empty($data['EventTag'])) { if (!empty($data['EventTag'])) {
$data['Event']['EventTag'] = $data['EventTag']; $data['Event']['EventTag'] = $data['EventTag'];
} }
if (!empty($data['Attribute'])) { if (!empty($data['Attribute'])) {
$this->__convertFromEventFormat($data['Attribute'], $data, $options, $continue); $this->convertFromEventFormat($data['Attribute'], $data, $options);
} }
if (!empty($data['Object'])) { if (!empty($data['Object'])) {
$this->__convertFromEventFormatObject($data['Object'], $data, $options, $continue); $this->convertFromEventFormatObject($data['Object'], $data, $options);
} }
} }
return ''; return '';
} }
private function __convertFromEventFormat($attributes, $event, $options = array(), $continue = false) { private function convertFromEventFormat($attributes, $event, $options = array())
{
$rearranged = array(); $rearranged = array();
foreach ($attributes as $attribute) { foreach ($attributes as $attribute) {
$attributeTag = array(); $attributeTag = array();
@ -62,15 +54,12 @@ abstract class NidsExport
} }
$this->export( $this->export(
$rearranged, $rearranged,
$options['user']['nids_sid'], $options['user']['nids_sid']
$options['returnFormat'],
$continue
); );
return true; return true;
} }
private function __convertFromEventFormatObject($objects, $event, $options = array(), $continue = false) private function convertFromEventFormatObject($objects, $event, $options = array())
{ {
$rearranged = array(); $rearranged = array();
foreach ($objects as $object) { foreach ($objects as $object) {
@ -93,20 +82,18 @@ abstract class NidsExport
'Event' => $event['Event'] 'Event' => $event['Event']
); );
} else { // In case no custom export exists for the object, the approach falls back to the attribute case } else { // In case no custom export exists for the object, the approach falls back to the attribute case
$this->__convertFromEventFormat($object['Attribute'], $event, $options, $continue); $this->convertFromEventFormat($object['Attribute'], $event, $options);
} }
} }
$this->export( $this->export(
$rearranged, $rearranged,
$options['user']['nids_sid'], $options['user']['nids_sid']
$options['returnFormat'],
$continue
); );
return true; return true;
} }
public function header($options = array()) public function header()
{ {
$this->explain(); $this->explain();
return ''; return '';
@ -122,7 +109,7 @@ abstract class NidsExport
return ''; return '';
} }
public function explain() protected function explain()
{ {
$this->rules[] = '# MISP export of IDS rules - optimized for '.$this->format; $this->rules[] = '# MISP export of IDS rules - optimized for '.$this->format;
$this->rules[] = '#'; $this->rules[] = '#';
@ -136,21 +123,8 @@ abstract class NidsExport
$this->rules[] = '# '; $this->rules[] = '# ';
} }
private $whitelist = null; protected function export($items, $startSid)
public function export($items, $startSid, $format="suricata", $continue = false)
{ {
$this->format = $format;
if ($this->checkWhitelist && !isset($this->Whitelist)) {
$this->Whitelist = ClassRegistry::init('Whitelist');
$this->whitelist = $this->Whitelist->getBlockedValues();
}
// output a short explanation
if (!$continue) {
$this->explain();
}
// generate the rules // generate the rules
foreach ($items as $item) { foreach ($items as $item) {
// retrieve all tags for this item to add them to the msg // retrieve all tags for this item to add them to the msg
@ -180,7 +154,6 @@ abstract class NidsExport
$sid++; $sid++;
if (!empty($item['Attribute']['type'])) { // item is an 'Attribute' if (!empty($item['Attribute']['type'])) { // item is an 'Attribute'
switch ($item['Attribute']['type']) { switch ($item['Attribute']['type']) {
// LATER nids - test all the snort attributes // LATER nids - test all the snort attributes
// LATER nids - add the tag keyword in the rules to capture network traffic // LATER nids - add the tag keyword in the rules to capture network traffic
@ -195,7 +168,7 @@ abstract class NidsExport
break; break;
case 'email': case 'email':
$this->emailSrcRule($ruleFormat, $item['Attribute'], $sid); $this->emailSrcRule($ruleFormat, $item['Attribute'], $sid);
$sid++; $sid++;
$this->emailDstRule($ruleFormat, $item['Attribute'], $sid); $this->emailDstRule($ruleFormat, $item['Attribute'], $sid);
break; break;
case 'email-src': case 'email-src':
@ -228,17 +201,17 @@ abstract class NidsExport
case 'ja3-fingerprint-md5': case 'ja3-fingerprint-md5':
$this->ja3Rule($ruleFormat, $item['Attribute'], $sid); $this->ja3Rule($ruleFormat, $item['Attribute'], $sid);
break; break;
case 'ja3s-fingerprint-md5': // Atribute type doesn't exists yet (2020-12-10) but ready when created. case 'ja3s-fingerprint-md5': // Attribute type doesn't exists yet (2020-12-10) but ready when created.
$this->ja3sRule($ruleFormat, $item['Attribute'], $sid); $this->ja3sRule($ruleFormat, $item['Attribute'], $sid);
break; break;
case 'snort': case 'snort':
$this->snortRule($ruleFormat, $item['Attribute'], $sid, $ruleFormatMsg, $ruleFormatReference); $this->snortRule($item['Attribute'], $sid, $ruleFormatMsg, $ruleFormatReference);
// no break // no break
default: default:
break; break;
} }
} else if(!empty($item['Attribute']['name'])) { // Item is an 'Object' } else if (!empty($item['Attribute']['name'])) { // Item is an 'Object'
switch ($item['Attribute']['name']) { switch ($item['Attribute']['name']) {
case 'network-connection': case 'network-connection':
@ -252,34 +225,30 @@ abstract class NidsExport
} }
} }
} }
return $this->rules;
} }
public function networkConnectionRule($ruleFormat, $object, &$sid) protected function networkConnectionRule($ruleFormat, $object, &$sid)
{ {
$attributes = NidsExport::getObjectAttributes($object); $attributes = NidsExport::getObjectAttributes($object);
if(!array_key_exists('layer4-protocol', $attributes)){ if (!array_key_exists('layer4-protocol', $attributes)) {
$attributes['layer4-protocol'] = 'ip'; // If layer-4 protocol is unknown, we roll-back to layer-3 ('ip') $attributes['layer4-protocol'] = 'ip'; // If layer-4 protocol is unknown, we roll-back to layer-3 ('ip')
} }
if(!array_key_exists('ip-src', $attributes)){ if (!array_key_exists('ip-src', $attributes)) {
$attributes['ip-src'] = '$HOME_NET'; // If ip-src is unknown, we roll-back to $HOME_NET $attributes['ip-src'] = '$HOME_NET'; // If ip-src is unknown, we roll-back to $HOME_NET
} }
if(!array_key_exists('ip-dst', $attributes)){ if (!array_key_exists('ip-dst', $attributes)) {
$attributes['ip-dst'] = '$HOME_NET'; // If ip-dst is unknown, we roll-back to $HOME_NET $attributes['ip-dst'] = '$HOME_NET'; // If ip-dst is unknown, we roll-back to $HOME_NET
} }
if(!array_key_exists('src-port', $attributes)){ if (!array_key_exists('src-port', $attributes)) {
$attributes['src-port'] = 'any'; // If src-port is unknown, we roll-back to 'any' $attributes['src-port'] = 'any'; // If src-port is unknown, we roll-back to 'any'
} }
if(!array_key_exists('dst-port', $attributes)){ if (!array_key_exists('dst-port', $attributes)) {
$attributes['dst-port'] = 'any'; // If dst-port is unknown, we roll-back to 'any' $attributes['dst-port'] = 'any'; // If dst-port is unknown, we roll-back to 'any'
} }
$this->rules[] = sprintf( $this->rules[] = sprintf(
$ruleFormat, $ruleFormat,
false, false,
$attributes['layer4-protocol'], // proto $attributes['layer4-protocol'], // proto
@ -294,12 +263,10 @@ abstract class NidsExport
$sid, // sid $sid, // sid
1 // rev 1 // rev
); );
} }
public function ddosRule($ruleFormat, $object, &$sid) protected function ddosRule($ruleFormat, $object, &$sid)
{ {
$attributes = NidsExport::getObjectAttributes($object); $attributes = NidsExport::getObjectAttributes($object);
if(!array_key_exists('protocol', $attributes)){ if(!array_key_exists('protocol', $attributes)){
@ -318,7 +285,7 @@ abstract class NidsExport
$attributes['dst-port'] = 'any'; // If dst-port is unknown, we roll-back to 'any' $attributes['dst-port'] = 'any'; // If dst-port is unknown, we roll-back to 'any'
} }
$this->rules[] = sprintf( $this->rules[] = sprintf(
$ruleFormat, $ruleFormat,
false, false,
$attributes['protocol'], // proto $attributes['protocol'], // proto
@ -333,12 +300,10 @@ abstract class NidsExport
$sid, // sid $sid, // sid
1 // rev 1 // rev
); );
} }
public static function getObjectAttributes($object) protected static function getObjectAttributes($object)
{ {
$attributes = array(); $attributes = array();
foreach ($object['Attribute'] as $attribute) { foreach ($object['Attribute'] as $attribute) {
@ -348,7 +313,7 @@ abstract class NidsExport
return $attributes; return $attributes;
} }
public function domainIpRule($ruleFormat, $attribute, &$sid) protected function domainIpRule($ruleFormat, $attribute, &$sid)
{ {
$values = explode('|', $attribute['value']); $values = explode('|', $attribute['value']);
$attributeCopy = $attribute; $attributeCopy = $attribute;
@ -361,7 +326,7 @@ abstract class NidsExport
$this->ipSrcRule($ruleFormat, $attributeCopy, $sid); $this->ipSrcRule($ruleFormat, $attributeCopy, $sid);
} }
public function ipDstRule($ruleFormat, $attribute, &$sid) protected function ipDstRule($ruleFormat, $attribute, &$sid)
{ {
$overruled = $this->checkWhitelist($attribute['value']); $overruled = $this->checkWhitelist($attribute['value']);
$ipport = NidsExport::getIpPort($attribute); $ipport = NidsExport::getIpPort($attribute);
@ -382,7 +347,7 @@ abstract class NidsExport
); );
} }
public function ipSrcRule($ruleFormat, $attribute, &$sid) protected function ipSrcRule($ruleFormat, $attribute, &$sid)
{ {
$overruled = $this->checkWhitelist($attribute['value']); $overruled = $this->checkWhitelist($attribute['value']);
$ipport = NidsExport::getIpPort($attribute); $ipport = NidsExport::getIpPort($attribute);
@ -403,7 +368,7 @@ abstract class NidsExport
); );
} }
public function emailSrcRule($ruleFormat, $attribute, &$sid) protected function emailSrcRule($ruleFormat, $attribute, &$sid)
{ {
$overruled = $this->checkWhitelist($attribute['value']); $overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -425,7 +390,7 @@ abstract class NidsExport
); );
} }
public function emailDstRule($ruleFormat, $attribute, &$sid) protected function emailDstRule($ruleFormat, $attribute, &$sid)
{ {
$overruled = $this->checkWhitelist($attribute['value']); $overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -447,7 +412,7 @@ abstract class NidsExport
); );
} }
public function emailSubjectRule($ruleFormat, $attribute, &$sid) protected function emailSubjectRule($ruleFormat, $attribute, &$sid)
{ {
// LATER nids - email-subject rule might not match because of line-wrapping // LATER nids - email-subject rule might not match because of line-wrapping
$overruled = $this->checkWhitelist($attribute['value']); $overruled = $this->checkWhitelist($attribute['value']);
@ -470,7 +435,7 @@ abstract class NidsExport
); );
} }
public function emailAttachmentRule($ruleFormat, $attribute, &$sid) protected function emailAttachmentRule($ruleFormat, $attribute, &$sid)
{ {
// LATER nids - email-attachment rule might not match because of line-wrapping // LATER nids - email-attachment rule might not match because of line-wrapping
$overruled = $this->checkWhitelist($attribute['value']); $overruled = $this->checkWhitelist($attribute['value']);
@ -493,7 +458,7 @@ abstract class NidsExport
); );
} }
public function hostnameRule($ruleFormat, $attribute, &$sid) protected function hostnameRule($ruleFormat, $attribute, &$sid)
{ {
$overruled = $this->checkWhitelist($attribute['value']); $overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -549,7 +514,7 @@ abstract class NidsExport
); );
} }
public function domainRule($ruleFormat, $attribute, &$sid) protected function domainRule($ruleFormat, $attribute, &$sid)
{ {
$overruled = $this->checkWhitelist($attribute['value']); $overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -605,7 +570,7 @@ abstract class NidsExport
); );
} }
public function urlRule($ruleFormat, $attribute, &$sid) protected function urlRule($ruleFormat, $attribute, &$sid)
{ {
// TODO in hindsight, an url should not be excluded given a host or domain name. // TODO in hindsight, an url should not be excluded given a host or domain name.
//$hostpart = parse_url($attribute['value'], PHP_URL_HOST); //$hostpart = parse_url($attribute['value'], PHP_URL_HOST);
@ -630,7 +595,7 @@ abstract class NidsExport
); );
} }
public function userAgentRule($ruleFormat, $attribute, &$sid) protected function userAgentRule($ruleFormat, $attribute, &$sid)
{ {
$overruled = $this->checkWhitelist($attribute['value']); $overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -652,17 +617,17 @@ abstract class NidsExport
); );
} }
public function ja3Rule($ruleFormat, $attribute, &$sid) protected function ja3Rule($ruleFormat, $attribute, &$sid)
{ {
//Empty because Snort doesn't support JA3 Rules //Empty because Snort doesn't support JA3 Rules
} }
public function ja3sRule($ruleFormat, $attribute, &$sid) protected function ja3sRule($ruleFormat, $attribute, &$sid)
{ {
//Empty because Snort doesn't support JA3S Rules //Empty because Snort doesn't support JA3S Rules
} }
public function snortRule($ruleFormat, $attribute, &$sid, $ruleFormatMsg, $ruleFormatReference) protected function snortRule($attribute, &$sid, $ruleFormatMsg, $ruleFormatReference)
{ {
// LATER nids - test using lots of snort rules, some rules don't contain all the necessary to be a valid rule. // LATER nids - test using lots of snort rules, some rules don't contain all the necessary to be a valid rule.
@ -678,46 +643,46 @@ abstract class NidsExport
// tag - '/tag\s*:\s*.+?;/' // tag - '/tag\s*:\s*.+?;/'
$replaceCount = array(); $replaceCount = array();
$tmpRule = preg_replace('/sid\s*:\s*[0-9]+\s*;/', 'sid:' . $sid . ';', $tmpRule, -1, $replaceCount['sid']); $tmpRule = preg_replace('/sid\s*:\s*[0-9]+\s*;/', 'sid:' . $sid . ';', $tmpRule, -1, $replaceCount['sid']);
if (null == $tmpRule) { if (null === $tmpRule) {
return false; return false;
} // don't output the rule on error with the regex } // don't output the rule on error with the regex
$tmpRule = preg_replace('/rev\s*:\s*[0-9]+\s*;/', 'rev:1;', $tmpRule, -1, $replaceCount['rev']); $tmpRule = preg_replace('/rev\s*:\s*[0-9]+\s*;/', 'rev:1;', $tmpRule, -1, $replaceCount['rev']);
if (null == $tmpRule) { if (null === $tmpRule) {
return false; return false;
} // don't output the rule on error with the regex } // don't output the rule on error with the regex
$tmpRule = preg_replace('/classtype:[a-zA-Z_-]+;/', 'classtype:' . $this->classtype . ';', $tmpRule, -1, $replaceCount['classtype']); $tmpRule = preg_replace('/classtype:[a-zA-Z_-]+;/', 'classtype:' . $this->classtype . ';', $tmpRule, -1, $replaceCount['classtype']);
if (null == $tmpRule) { if (null === $tmpRule) {
return false; return false;
} // don't output the rule on error with the regex } // don't output the rule on error with the regex
$tmpRule = preg_replace('/msg\s*:\s*"(.*?)"\s*;/', sprintf($ruleFormatMsg, 'snort-rule | $1') . ';', $tmpRule, -1, $replaceCount['msg']); $tmpRule = preg_replace('/msg\s*:\s*"(.*?)"\s*;/', sprintf($ruleFormatMsg, 'snort-rule | $1') . ';', $tmpRule, -1, $replaceCount['msg']);
if (null == $tmpRule) { if (null === $tmpRule) {
return false; return false;
} // don't output the rule on error with the regex } // don't output the rule on error with the regex
$tmpRule = preg_replace('/reference\s*:\s*.+?;/', $ruleFormatReference . ';', $tmpRule, -1, $replaceCount['reference']); $tmpRule = preg_replace('/reference\s*:\s*.+?;/', $ruleFormatReference . ';', $tmpRule, -1, $replaceCount['reference']);
if (null == $tmpRule) { if (null === $tmpRule) {
return false; return false;
} // don't output the rule on error with the regex } // don't output the rule on error with the regex
$tmpRule = preg_replace('/reference\s*:\s*.+?;/', $ruleFormatReference . ';', $tmpRule, -1, $replaceCount['reference']); $tmpRule = preg_replace('/reference\s*:\s*.+?;/', $ruleFormatReference . ';', $tmpRule, -1, $replaceCount['reference']);
if (null == $tmpRule) { if (null === $tmpRule) {
return false; return false;
} // don't output the rule on error with the regex } // don't output the rule on error with the regex
// FIXME nids - implement priority overwriting // FIXME nids - implement priority overwriting
// some values were not replaced, so we need to add them ourselves, and insert them in the rule // some values were not replaced, so we need to add them ourselves, and insert them in the rule
$extraForRule = ""; $extraForRule = "";
if (0 == $replaceCount['sid']) { if (0 === $replaceCount['sid']) {
$extraForRule .= 'sid:' . $sid . ';'; $extraForRule .= 'sid:' . $sid . ';';
} }
if (0 == $replaceCount['rev']) { if (0 === $replaceCount['rev']) {
$extraForRule .= 'rev:1;'; $extraForRule .= 'rev:1;';
} }
if (0 == $replaceCount['classtype']) { if (0 === $replaceCount['classtype']) {
$extraForRule .= 'classtype:' . $this->classtype . ';'; $extraForRule .= 'classtype:' . $this->classtype . ';';
} }
if (0 == $replaceCount['msg']) { if (0 === $replaceCount['msg']) {
$extraForRule .= $tmpMessage . ';'; $extraForRule .= $ruleFormatMsg . ';';
} }
if (0 == $replaceCount['reference']) { if (0 === $replaceCount['reference']) {
$extraForRule .= $ruleFormatReference . ';'; $extraForRule .= $ruleFormatReference . ';';
} }
$tmpRule = preg_replace('/;\s*\)/', '; ' . $extraForRule . ')', $tmpRule); $tmpRule = preg_replace('/;\s*\)/', '; ' . $extraForRule . ')', $tmpRule);
@ -734,7 +699,7 @@ abstract class NidsExport
* @param string $type the type of dns name - domain (default) or hostname * @param string $type the type of dns name - domain (default) or hostname
* @return string raw snort compatible format of the dns name * @return string raw snort compatible format of the dns name
*/ */
public static function dnsNameToRawFormat($name, $type='domain') protected static function dnsNameToRawFormat($name, $type='domain')
{ {
$rawName = ""; $rawName = "";
if ('hostname' == $type) { if ('hostname' == $type) {
@ -747,7 +712,7 @@ abstract class NidsExport
// count the length of the part, and add |length| before // count the length of the part, and add |length| before
$length = strlen($explodedName); $length = strlen($explodedName);
if ($length > 255) { if ($length > 255) {
log('WARNING: DNS name is too long for RFC: '.$name); CakeLog::notice('WARNING: DNS name is too long for RFC: '.$name);
} }
$hexLength = dechex($length); $hexLength = dechex($length);
if (1 == strlen($hexLength)) { if (1 == strlen($hexLength)) {
@ -768,7 +733,7 @@ abstract class NidsExport
* @param string $name dns name to be converted * @param string $name dns name to be converted
* @return string raw snort compatible format of the dns name * @return string raw snort compatible format of the dns name
*/ */
public static function dnsNameToMSDNSLogFormat($name) protected static function dnsNameToMSDNSLogFormat($name)
{ {
$rawName = ""; $rawName = "";
// in MS DNS log format we can't use (0) to distinguish between hostname and domain (including subdomains) // in MS DNS log format we can't use (0) to distinguish between hostname and domain (including subdomains)
@ -779,7 +744,7 @@ abstract class NidsExport
// count the length of the part, and add |length| before // count the length of the part, and add |length| before
$length = strlen($explodedName); $length = strlen($explodedName);
if ($length > 255) { if ($length > 255) {
log('WARNING: DNS name is too long for RFC: '.$name); CakeLog::notice('WARNING: DNS name is too long for RFC: '.$name);
} }
$hexLength = dechex($length); $hexLength = dechex($length);
$rawName .= '(' . $hexLength . ')' . $explodedName; $rawName .= '(' . $hexLength . ')' . $explodedName;
@ -793,34 +758,32 @@ abstract class NidsExport
/** /**
* Replaces characters that are not allowed in a signature. * Replaces characters that are not allowed in a signature.
* example: " is converted to |22| * example: " is converted to |22|
* @param unknown_type $value * @param string $value
*/ */
public static function replaceIllegalChars($value) protected static function replaceIllegalChars($value)
{ {
$replace_pairs = array( $replace_pairs = array(
'|' => '|7c|', // Needs to stay on top ! '|' => '|7c|', // Needs to stay on top !
'"' => '|22|', '"' => '|22|',
';' => '|3b|', ';' => '|3b|',
':' => '|3a|', ':' => '|3a|',
'\\' => '|5c|', '\\' => '|5c|',
'0x' => '|30 78|' '0x' => '|30 78|'
); );
return strtr($value, $replace_pairs); return strtr($value, $replace_pairs);
} }
public function checkWhitelist($value) /**
* @deprecated
* @param $value
* @return false
*/
protected function checkWhitelist($value)
{ {
if ($this->checkWhitelist && is_array($this->whitelist)) {
foreach ($this->whitelist as $wlitem) {
if (preg_match($wlitem, $value)) {
return true;
}
}
}
return false; return false;
} }
public static function getProtocolPort($protocol, $customPort) protected static function getProtocolPort($protocol, $customPort)
{ {
if ($customPort == null) { if ($customPort == null) {
switch ($protocol) { switch ($protocol) {
@ -840,7 +803,7 @@ abstract class NidsExport
} }
} }
public static function getCustomIP($customIP) protected static function getCustomIP($customIP)
{ {
if (filter_var($customIP, FILTER_VALIDATE_IP)) { if (filter_var($customIP, FILTER_VALIDATE_IP)) {
return $customIP; return $customIP;
@ -853,7 +816,7 @@ abstract class NidsExport
* @param array $attribute * @param array $attribute
* @return array|string[] * @return array|string[]
*/ */
public static function getIpPort($attribute) protected static function getIpPort($attribute)
{ {
if (strpos($attribute['type'], 'port') !== false) { if (strpos($attribute['type'], 'port') !== false) {
return explode('|', $attribute['value']); return explode('|', $attribute['value']);

8
app/Lib/Export/NidsSnortExport.php
View File

@ -4,11 +4,5 @@ App::uses('NidsExport', 'Export');
class NidsSnortExport extends NidsExport class NidsSnortExport extends NidsExport
{ {
public function export($items, $startSid, $format = "suricata", $continue = false) protected $format = 'snort';
{
// set the specific format
$this->format = 'snort';
// call the generic function
return parent::export($items, $startSid, $format, $continue);
}
} }

20
app/Lib/Export/NidsSuricataExport.php
View File

@ -3,16 +3,10 @@ App::uses('NidsExport', 'Export');
class NidsSuricataExport extends NidsExport class NidsSuricataExport extends NidsExport
{ {
public function export($items, $startSid, $format = "suricata", $continue = false) protected $format = "suricata";
{
// set the specific format
$this->format = "suricata";
// call the generic function
return parent::export($items, $startSid, $format, $continue);
}
// below overwrite functions from NidsExport // below overwrite functions from NidsExport
public function hostnameRule($ruleFormat, $attribute, &$sid) protected function hostnameRule($ruleFormat, $attribute, &$sid)
{ {
$overruled = $this->checkWhitelist($attribute['value']); $overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -53,7 +47,7 @@ class NidsSuricataExport extends NidsExport
); );
} }
public function domainRule($ruleFormat, $attribute, &$sid) protected function domainRule($ruleFormat, $attribute, &$sid)
{ {
$overruled = $this->checkWhitelist($attribute['value']); $overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -94,7 +88,7 @@ class NidsSuricataExport extends NidsExport
); );
} }
public function urlRule($ruleFormat, $attribute, &$sid) protected function urlRule($ruleFormat, $attribute, &$sid)
{ {
$createRule = true; $createRule = true;
$overruled = $this->checkWhitelist($attribute['value']); $overruled = $this->checkWhitelist($attribute['value']);
@ -207,7 +201,7 @@ class NidsSuricataExport extends NidsExport
} }
} }
public function userAgentRule($ruleFormat, $attribute, &$sid) protected function userAgentRule($ruleFormat, $attribute, &$sid)
{ {
$overruled = $this->checkWhitelist($attribute['value']); $overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -230,7 +224,7 @@ class NidsSuricataExport extends NidsExport
); );
} }
public function ja3Rule($ruleFormat, $attribute, &$sid) protected function ja3Rule($ruleFormat, $attribute, &$sid)
{ {
$overruled = $this->checkWhitelist($attribute['value']); $overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -253,7 +247,7 @@ class NidsSuricataExport extends NidsExport
} }
// For Future use once JA3S Hash Attribute type is created // For Future use once JA3S Hash Attribute type is created
public function ja3sRule($ruleFormat, $attribute, &$sid) protected function ja3sRule($ruleFormat, $attribute, &$sid)
{ {
$overruled = $this->checkWhitelist($attribute['value']); $overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule

176
app/Lib/Export/RPZExport.php
View File

@ -2,107 +2,104 @@
class RPZExport class RPZExport
{ {
private $__policies = array( const POLICIES = array(
'Local-Data' => array( 'Local-Data' => array(
'explanation' => 'returns the defined alternate location.', 'explanation' => 'returns the defined alternate location.',
'action' => '$walled_garden', 'action' => '$walled_garden',
'setting_id' => 3, 'setting_id' => 3,
), ),
'NXDOMAIN' => array( 'NXDOMAIN' => array(
'explanation' => 'return NXDOMAIN (name does not exist) irrespective of actual result received.', 'explanation' => 'return NXDOMAIN (name does not exist) irrespective of actual result received.',
'action' => '.', 'action' => '.',
'setting_id' => 1, 'setting_id' => 1,
), ),
'NODATA' => array( 'NODATA' => array(
'explanation' => 'returns NODATA (name exists but no answers returned) irrespective of actual result received.', 'explanation' => 'returns NODATA (name exists but no answers returned) irrespective of actual result received.',
'action' => '*.', 'action' => '*.',
'setting_id' => 2, 'setting_id' => 2,
), ),
'DROP' => array( 'DROP' => array(
'explanation' => 'timeout.', 'explanation' => 'timeout.',
'action' => 'rpz-drop.', 'action' => 'rpz-drop.',
'setting_id' => 0, 'setting_id' => 0,
), ),
'PASSTHRU' => array( 'PASSTHRU' => array(
'explanation' => 'lets queries through, but allows for logging the hits (useful for testing).', 'explanation' => 'lets queries through, but allows for logging the hits (useful for testing).',
'action' => 'rpz-passthru.', 'action' => 'rpz-passthru.',
'setting_id' => 4, 'setting_id' => 4,
), ),
'TCP-only' => array( 'TCP-only' => array(
'explanation' => 'force the client to use TCP.', 'explanation' => 'force the client to use TCP.',
'action' => 'rpz-tcp-only.', 'action' => 'rpz-tcp-only.',
'setting_id' => 5, 'setting_id' => 5,
), ),
); );
private $__items = array(); private $items = array();
public $additional_params = array( public $additional_params = array(
'flatten' => 1 'flatten' => 1
); );
private $__rpzSettings = array(); private $rpzSettings = array();
private $__valid_policies = array('NXDOMAIN', 'NODATA', 'DROP', 'Local-Data', 'PASSTHRU', 'TCP-only');
private $__server = null; private $__server = null;
public $validTypes = array( const VALID_TYPES = array(
'ip-src' => array( 'ip-src' => array(
'value' => 'ip' 'value' => 'ip'
), ),
'ip-dst' => array( 'ip-dst' => array(
'value' => 'ip' 'value' => 'ip'
), ),
'domain' => array( 'domain' => array(
'value' => 'domain' 'value' => 'domain'
), ),
'domain|ip' => array( 'domain|ip' => array(
'value1' => 'domain', 'value1' => 'domain',
'value2' => 'ip' 'value2' => 'ip'
), ),
'hostname' => array( 'hostname' => array(
'value' => 'hostname' 'value' => 'hostname'
) )
); );
public function handler($data, $options = array()) public function handler($data, $options = array())
{ {
if ($options['scope'] === 'Attribute') { if ($options['scope'] === 'Attribute') {
return $this->__attributeHandler($data, $options); $this->attributeHandler($data);
} else { } else {
return $this->__eventHandler($data, $options); $this->eventHandler($data);
} }
return '';
} }
private function __eventHandler($event, $options = array()) { private function eventHandler($event)
{
foreach ($event['Attribute'] as $attribute) { foreach ($event['Attribute'] as $attribute) {
if (isset($this->validTypes[$attribute['type']])) { if (isset(self::VALID_TYPES[$attribute['type']])) {
if ($attribute['type'] == 'domain|ip') { if ($attribute['type'] === 'domain|ip') {
$temp = explode('|', $attribute['value']); $temp = explode('|', $attribute['value']);
$attribute['value1'] = $temp[0]; $attribute['value1'] = $temp[0];
$attribute['value2'] = $temp[1]; $attribute['value2'] = $temp[1];
} }
$this->__attributeHandler(array('Attribute' => $attribute, $options)); $this->attributeHandler(array('Attribute' => $attribute));
} }
} }
return '';
} }
private function __attributeHandler($attribute, $options = array()) private function attributeHandler($attribute)
{ {
if (isset($attribute['Attribute'])) { if (isset($attribute['Attribute'])) {
$attribute = $attribute['Attribute']; $attribute = $attribute['Attribute'];
} }
if (isset($this->validTypes[$attribute['type']])) { if (isset(self::VALID_TYPES[$attribute['type']])) {
foreach ($this->validTypes[$attribute['type']] as $field => $mapping) { foreach (self::VALID_TYPES[$attribute['type']] as $field => $mapping) {
// get rid of the in_array check if (!isset($this->items[$mapping][$attribute[$field]])) {
if (empty($this->__items[$mapping]) || !isset($this->__items[$mapping][$attribute[$field]])) { $this->items[$mapping][$attribute[$field]] = true;
$this->__items[$mapping][$attribute[$field]] = true;
} }
} }
} }
return '';
} }
public function header($options = array()) public function header($options = array())
@ -117,16 +114,16 @@ class RPZExport
} }
} }
if (isset($options['filters'][$v])) { if (isset($options['filters'][$v])) {
$this->__rpzSettings[$v] = $options['filters'][$v]; $this->rpzSettings[$v] = $options['filters'][$v];
} else { } else {
$tempSetting = Configure::read('Plugin.RPZ_' . $v); $tempSetting = Configure::read('Plugin.RPZ_' . $v);
if (isset($tempSetting)) { if (isset($tempSetting)) {
$this->__rpzSettings[$v] = Configure::read('Plugin.RPZ_' . $v); $this->rpzSettings[$v] = $tempSetting;
} else { } else {
if (empty($this->__server)) { if (empty($this->__server)) {
$this->__server = ClassRegistry::init('Server'); $this->__server = ClassRegistry::init('Server');
} }
$this->__rpzSettings[$v] = $this->__server->serverSettings['Plugin']['RPZ_' . $v]['value']; $this->rpzSettings[$v] = $this->__server->serverSettings['Plugin']['RPZ_' . $v]['value'];
} }
} }
} }
@ -135,10 +132,7 @@ class RPZExport
public function footer($options = array()) public function footer($options = array())
{ {
foreach ($this->__items as $k => $v) { return $this->export($this->items, $this->rpzSettings);
$this->__items[$k] = array_keys($this->__items[$k]);
}
return $this->export($this->__items, $this->__rpzSettings);
} }
public function separator() public function separator()
@ -146,39 +140,32 @@ class RPZExport
return ''; return '';
} }
public function getPolicyById($id) private function getPolicyById($id)
{ {
foreach ($this->__policies as $k => $v) { foreach (self::POLICIES as $k => $v) {
if ($id == $v['setting_id']) { if ($id === $v['setting_id']) {
return $k; return $k;
} }
} }
return null;
} }
public function getIdByPolicy($policy) private function getIdByPolicy($policy)
{ {
return $this->__policies[$policy]['setting_id']; return self::POLICIES[$policy]['setting_id'];
} }
public function explain($type, $policy) private function explain($type, $policy)
{ {
$explanations = array( $explanations = array(
'ip' => '; The following list of IP addresses will ', 'ip' => '; The following list of IP addresses will ',
'domain' => '; The following domain names and all of their sub-domains will ', 'domain' => '; The following domain names and all of their sub-domains will ',
'hostname' => '; The following hostnames will ' 'hostname' => '; The following hostnames will '
); );
$policy_explanations = array( return $explanations[$type] . self::POLICIES[$policy]['explanation'] . PHP_EOL;
'Local-Data' => 'returns the defined alternate location.',
'NXDOMAIN' => 'return NXDOMAIN (name does not exist) irrespective of actual result received.',
'NODATA' => 'returns NODATA (name exists but no answers returned) irrespective of actual result received.',
'DROP' => 'timeout.',
'PASSTHRU' => 'lets queries through, but allows for logging the hits (useful for testing).',
'TCP-only' => 'force the client to use TCP.',
);
return $explanations[$type] . $this->__policies[$policy]['explanation'] . PHP_EOL;
} }
public function buildHeader($rpzSettings) private function buildHeader(array $rpzSettings)
{ {
$rpzSettings['serial'] = str_replace('$date', date('Ymd'), $rpzSettings['serial']); $rpzSettings['serial'] = str_replace('$date', date('Ymd'), $rpzSettings['serial']);
$rpzSettings['serial'] = str_replace('$time', time(), $rpzSettings['serial']); $rpzSettings['serial'] = str_replace('$time', time(), $rpzSettings['serial']);
@ -196,55 +183,55 @@ class RPZExport
return $header; return $header;
} }
public function export($items, $rpzSettings) private function export(array $items, array $rpzSettings)
{ {
$result = $this->buildHeader($rpzSettings); $result = $this->buildHeader($rpzSettings);
$policy = $this->getPolicyById($rpzSettings['policy']); $policy = $this->getPolicyById($rpzSettings['policy']);
$action = $this->__policies[$policy]['action']; $action = self::POLICIES[$policy]['action'];
if ($policy == 'Local-Data') { if ($policy === 'Local-Data') {
$action = str_replace('$walled_garden', $rpzSettings['walled_garden'], $action); $action = str_replace('$walled_garden', $rpzSettings['walled_garden'], $action);
} }
if (isset($items['ip'])) { if (isset($items['ip'])) {
$result .= $this->explain('ip', $policy); $result .= $this->explain('ip', $policy);
foreach ($items['ip'] as $item) { foreach ($items['ip'] as $item => $foo) {
$result .= $this->__convertIP($item, $action); $result .= $this->convertIp($item, $action);
} }
$result .= PHP_EOL; $result .= PHP_EOL;
} }
if (isset($items['domain'])) { if (isset($items['domain'])) {
$result .= $this->explain('domain', $policy); $result .= $this->explain('domain', $policy);
foreach ($items['domain'] as $item) { foreach ($items['domain'] as $item => $foo) {
$result .= $this->__convertdomain($item, $action); $result .= $this->convertDomain($item, $action);
} }
$result .= PHP_EOL; $result .= PHP_EOL;
} }
if (isset($items['hostname'])) { if (isset($items['hostname'])) {
$result .= $this->explain('hostname', $policy); $result .= $this->explain('hostname', $policy);
foreach ($items['hostname'] as $item) { foreach ($items['hostname'] as $item => $foo) {
$result .= $this->__converthostname($item, $action); $result .= $this->convertHostname($item, $action);
} }
$result .= PHP_EOL; $result .= PHP_EOL;
} }
return $result; return $result;
} }
private function __convertdomain($input, $action) private function convertDomain($input, $action)
{ {
return $input . ' CNAME ' . $action . PHP_EOL . '*.' . $input . ' CNAME ' . $action . PHP_EOL; return $input . ' CNAME ' . $action . PHP_EOL . '*.' . $input . ' CNAME ' . $action . PHP_EOL;
} }
private function __converthostname($input, $action) private function convertHostname($input, $action)
{ {
return $input . ' CNAME ' . $action . PHP_EOL; return $input . ' CNAME ' . $action . PHP_EOL;
} }
private function __convertIP($input, $action) private function convertIp($input, $action)
{ {
$type = filter_var($input, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) ? 'ipv6' : 'ipv4'; $isIpv6 = filter_var($input, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6);
if ($type == 'ipv6') { if ($isIpv6) {
$prefix = '128'; $prefix = '128';
} else { } else {
$prefix = '32'; $prefix = '32';
@ -252,7 +239,8 @@ class RPZExport
if (strpos($input, '/')) { if (strpos($input, '/')) {
list($input, $prefix) = explode('/', $input); list($input, $prefix) = explode('/', $input);
} }
return $prefix . '.' . $this->{'__' . $type}($input) . '.rpz-ip CNAME ' . $action . PHP_EOL; $converted = $isIpv6 ? $this->__ipv6($input) : $this->__ipv4($input);
return $prefix . '.' . $converted . '.rpz-ip CNAME ' . $action . PHP_EOL;
} }
private function __ipv6($input) private function __ipv6($input)

198
app/Lib/Tools/ApcuCacheTool.php Normal file
View File

@ -0,0 +1,198 @@
<?php
declare(strict_types=1);
class ApcuCacheTool implements \Psr\SimpleCache\CacheInterface
{
/** @var string */
private $prefix;
/**
* @param string $prefix
*/
public function __construct(string $prefix)
{
$this->prefix = $prefix;
}
/**
* Fetches a value from the cache.
*
* @param string $key The unique key of this item in the cache.
* @param mixed $default Default value to return if the key does not exist.
*
* @return mixed The value of the item from the cache, or $default in case of cache miss.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if the $key string is not a legal value.
*/
public function get($key, $default = null)
{
$value = \apcu_fetch("$this->prefix:$key", $success);
if ($success) {
return $value;
}
return $default;
}
/**
* Persists data in the cache, uniquely referenced by a key with an optional expiration TTL time.
*
* @param string $key The key of the item to store.
* @param mixed $value The value of the item to store, must be serializable.
* @param null|int|\DateInterval $ttl Optional. The TTL value of this item. If no value is sent and
* the driver supports TTL then the library may set a default value
* for it or let the driver take care of that.
*
* @return bool True on success and false on failure.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if the $key string is not a legal value.
*/
public function set($key, $value, $ttl = null)
{
return \apcu_store("$this->prefix:$key", $value, $this->tllToInt($ttl));
}
/**
* Delete an item from the cache by its unique key.
*
* @param string $key The unique cache key of the item to delete.
*
* @return bool True if the item was successfully removed. False if there was an error.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if the $key string is not a legal value.
*/
public function delete($key)
{
return \apcu_delete("$this->prefix:$key");
}
/**
* Wipes clean the entire cache's keys.
*
* @return bool True on success and false on failure.
*/
public function clear()
{
$iterator = new APCUIterator(
'/^' . preg_quote($this->prefix . ':', '/') . '/',
APC_ITER_NONE
);
return \apcu_delete($iterator);
}
/**
* Obtains multiple cache items by their unique keys.
*
* @param iterable $keys A list of keys that can obtained in a single operation.
* @param mixed $default Default value to return for keys that do not exist.
*
* @return iterable A list of key => value pairs. Cache keys that do not exist or are stale will have $default as value.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if $keys is neither an array nor a Traversable,
* or if any of the $keys are not a legal value.
*/
public function getMultiple($keys, $default = null)
{
$keysToFetch = $this->keysToFetch($keys);
$values = \apcu_fetch($keysToFetch);
foreach ($keysToFetch as $keyToFetch) {
if (!isset($values[$keyToFetch])) {
$values[$keyToFetch] = $default;
}
}
return $values;
}
/**
* Persists a set of key => value pairs in the cache, with an optional TTL.
*
* @param iterable $values A list of key => value pairs for a multiple-set operation.
* @param null|int|\DateInterval $ttl Optional. The TTL value of this item. If no value is sent and
* the driver supports TTL then the library may set a default value
* for it or let the driver take care of that.
*
* @return bool True on success and false on failure.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if $values is neither an array nor a Traversable,
* or if any of the $values are not a legal value.
*/
public function setMultiple($values, $ttl = null)
{
$dataToSave = [];
foreach ($values as $key => $value) {
$dataToSave["$this->prefix:$key"] = $value;
}
return \apcu_store($dataToSave, null, $this->tllToInt($ttl));
}
/**
* Deletes multiple cache items in a single operation.
*
* @param iterable $keys A list of string-based keys to be deleted.
*
* @return bool True if the items were successfully removed. False if there was an error.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if $keys is neither an array nor a Traversable,
* or if any of the $keys are not a legal value.
*/
public function deleteMultiple($keys)
{
$keysToDelete = $this->keysToFetch($keys);
return \apcu_delete($keysToDelete);
}
/**
* Determines whether an item is present in the cache.
*
* NOTE: It is recommended that has() is only to be used for cache warming type purposes
* and not to be used within your live applications operations for get/set, as this method
* is subject to a race condition where your has() will return true and immediately after,
* another script can remove it making the state of your app out of date.
*
* @param string $key The cache item key.
*
* @return bool
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if the $key string is not a legal value.
*/
public function has($key)
{
return \apcu_exists("$this->prefix:$key");
}
/**
* @param iterable $keys
* @return array
*/
private function keysToFetch(iterable $keys): array
{
$keysToFetch = [];
foreach ($keys as $key) {
$keysToFetch[] = "$this->prefix:$key";
}
return $keysToFetch;
}
/**
* @param null|int|\DateInterval $ttl
* @return int
*/
private function tllToInt($ttl = null): int
{
if ($ttl === null) {
return 0;
} elseif (is_int($ttl)) {
return $ttl;
} elseif ($ttl instanceof \DateInterval) {
return $ttl->days * 86400 + $ttl->h * 3600 + $ttl->i * 60 + $ttl->s;
} else {
throw new \Psr\SimpleCache\InvalidArgumentException("Invalid ttl value '$ttl' provided.");
}
}
}

27
app/Lib/Tools/AttributeValidationTool.php
View File

@ -41,7 +41,7 @@ class AttributeValidationTool
switch ($type) { switch ($type) {
case 'ip-src': case 'ip-src':
case 'ip-dst': case 'ip-dst':
return self::compressIpv6($value); return self::normalizeIp($value);
case 'md5': case 'md5':
case 'sha1': case 'sha1':
case 'sha224': case 'sha224':
@ -98,7 +98,7 @@ class AttributeValidationTool
$parts[0] = $punyCode; $parts[0] = $punyCode;
} }
} }
$parts[1] = self::compressIpv6($parts[1]); $parts[1] = self::normalizeIp($parts[1]);
return "$parts[0]|$parts[1]"; return "$parts[0]|$parts[1]";
case 'filename|md5': case 'filename|md5':
case 'filename|sha1': case 'filename|sha1':
@ -175,7 +175,7 @@ class AttributeValidationTool
} else { } else {
return $value; return $value;
} }
return self::compressIpv6($parts[0]) . '|' . $parts[1]; return self::normalizeIp($parts[0]) . '|' . $parts[1];
case 'mac-address': case 'mac-address':
case 'mac-eui-64': case 'mac-eui-64':
$value = str_replace(array('.', ':', '-', ' '), '', strtolower($value)); $value = str_replace(array('.', ':', '-', ' '), '', strtolower($value));
@ -700,11 +700,30 @@ class AttributeValidationTool
* @param string $value * @param string $value
* @return string * @return string
*/ */
private static function compressIpv6($value) private static function normalizeIp($value)
{ {
// If IP is a CIDR
if (strpos($value, '/')) {
list($ip, $range) = explode('/', $value, 2);
// Compress IPv6
if (strpos($ip, ':') && $converted = inet_pton($ip)) {
$ip = inet_ntop($converted);
}
// If IP is in CIDR format, but the network is 32 for IPv4 or 128 for IPv6, normalize to non CIDR type
if (($range === '32' && strpos($value, '.')) || ($range === '128' && strpos($value, ':'))) {
return $ip;
}
return "$ip/$range";
}
// Compress IPv6
if (strpos($value, ':') && $converted = inet_pton($value)) { if (strpos($value, ':') && $converted = inet_pton($value)) {
return inet_ntop($converted); return inet_ntop($converted);
} }
return $value; return $value;
} }

30
app/Lib/Tools/BackgroundJobs/BackgroundJob.php
View File

@ -66,8 +66,9 @@ class BackgroundJob implements JsonSerializable
/** /**
* Run the job command * Run the job command
* @param callable|null $runningCallback
*/ */
public function run(): void public function run(callable $runningCallback = null): void
{ {
$descriptorSpec = [ $descriptorSpec = [
1 => ["pipe", "w"], // stdout 1 => ["pipe", "w"], // stdout
@ -88,7 +89,7 @@ class BackgroundJob implements JsonSerializable
['BACKGROUND_JOB_ID' => $this->id] ['BACKGROUND_JOB_ID' => $this->id]
); );
$this->pool($process, $pipes); $this->pool($process, $pipes, $runningCallback);
if ($this->returnCode === 0 && empty($stderr)) { if ($this->returnCode === 0 && empty($stderr)) {
$this->setStatus(BackgroundJob::STATUS_COMPLETED); $this->setStatus(BackgroundJob::STATUS_COMPLETED);
@ -98,7 +99,13 @@ class BackgroundJob implements JsonSerializable
} }
} }
private function pool($process, array $pipes) /**
* @param resource $process
* @param array $pipes
* @param callable|null $runningCallback
* @return void
*/
private function pool($process, array $pipes, callable $runningCallback = null)
{ {
stream_set_blocking($pipes[1], false); stream_set_blocking($pipes[1], false);
stream_set_blocking($pipes[2], false); stream_set_blocking($pipes[2], false);
@ -106,6 +113,14 @@ class BackgroundJob implements JsonSerializable
$this->output = ''; $this->output = '';
$this->error = ''; $this->error = '';
if ($runningCallback) {
$status = proc_get_status($process);
if ($status === false) {
throw new RuntimeException("Could not get process status");
}
$runningCallback($status);
}
while (true) { while (true) {
$read = [$pipes[1], $pipes[2]]; $read = [$pipes[1], $pipes[2]];
$write = null; $write = null;
@ -118,6 +133,12 @@ class BackgroundJob implements JsonSerializable
$this->error .= stream_get_contents($pipes[2]); $this->error .= stream_get_contents($pipes[2]);
} }
$status = proc_get_status($process); $status = proc_get_status($process);
if ($status === false) {
throw new RuntimeException("Could not get process status");
}
if ($runningCallback) {
$runningCallback($status);
}
if (!$status['running']) { if (!$status['running']) {
// Just in case read rest data from stream // Just in case read rest data from stream
$this->output .= stream_get_contents($pipes[1]); $this->output .= stream_get_contents($pipes[1]);
@ -153,6 +174,9 @@ class BackgroundJob implements JsonSerializable
return ['id', 'command', 'args', 'createdAt', 'updatedAt', 'status', 'output', 'error', 'metadata']; return ['id', 'command', 'args', 'createdAt', 'updatedAt', 'status', 'output', 'error', 'metadata'];
} }
/**
* @return string Background job ID in UUID format
*/
public function id(): string public function id(): string
{ {
return $this->id; return $this->id;

2
app/Lib/Tools/BackgroundJobs/Worker.php
View File

@ -65,7 +65,7 @@ class Worker implements JsonSerializable
]; ];
} }
public function pid(): ?int public function pid(): int
{ {
return $this->pid; return $this->pid;
} }

67
app/Lib/Tools/BackgroundJobsTool.php
View File

@ -91,7 +91,8 @@ class BackgroundJobsTool
]; ];
const JOB_STATUS_PREFIX = 'job_status', const JOB_STATUS_PREFIX = 'job_status',
DATA_CONTENT_PREFIX = 'data_content'; DATA_CONTENT_PREFIX = 'data_content',
RUNNING_JOB_PREFIX = 'running';
/** @var array */ /** @var array */
private $settings; private $settings;
@ -277,6 +278,54 @@ class BackgroundJobsTool
return null; return null;
} }
/**
* @param Worker $worker
* @param BackgroundJob $job
* @param int|null $pid
* @return void
* @throws RedisException
*/
public function markAsRunning(Worker $worker, BackgroundJob $job, $pid = null)
{
$key = self::RUNNING_JOB_PREFIX . ':' . $worker->queue() . ':' . $job->id();
$this->RedisConnection->setex($key, 60, [
'worker_pid' => $worker->pid(),
'process_pid' => $pid,
]);
}
/**
* @param Worker $worker
* @param BackgroundJob $job
* @return void
* @throws RedisException
*/
public function removeFromRunning(Worker $worker, BackgroundJob $job)
{
$key = self::RUNNING_JOB_PREFIX . ':' . $worker->queue() . ':' . $job->id();
$this->RedisConnection->del($key);
}
/**
* Return current running jobs
* @return array
* @throws RedisException
*/
public function runningJobs(): array
{
$pattern = $this->RedisConnection->_prefix(self::RUNNING_JOB_PREFIX . ':*');
$keys = RedisTool::keysByPattern($this->RedisConnection, $pattern);
$jobIds = [];
foreach ($keys as $key) {
$parts = explode(':', $key);
$queue = $parts[2];
$jobId = $parts[3];
$jobIds[$queue][$jobId] = $this->RedisConnection->get(self::RUNNING_JOB_PREFIX . ":$queue:$jobId");
}
return $jobIds;
}
/** /**
* Get the job status. * Get the job status.
* *
@ -500,19 +549,6 @@ class BackgroundJobsTool
$this->getSupervisor()->startProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart); $this->getSupervisor()->startProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart);
} }
/**
* Purge queue
*
* @param string $queue
* @return void
*/
public function purgeQueue(string $queue)
{
$this->validateQueue($queue);
$this->RedisConnection->del($queue);
}
/** /**
* Return Background Jobs status * Return Background Jobs status
* *
@ -728,8 +764,7 @@ class BackgroundJobsTool
* *
* @param integer $pid * @param integer $pid
* @return \Supervisor\Process * @return \Supervisor\Process
* * @throws NotFoundException|Exception
* @throws NotFoundException
*/ */
private function getProcessByPid(int $pid): \Supervisor\Process private function getProcessByPid(int $pid): \Supervisor\Process
{ {

4
app/Lib/Tools/BetterCakeEventManager.php
View File

@ -52,10 +52,10 @@ class BetterCakeEventManager extends CakeEventManager
$result = []; $result = [];
foreach ($priorities as $priority) { foreach ($priorities as $priority) {
if (isset($globalListeners[$priority])) { if (isset($globalListeners[$priority])) {
$result = array_merge($result, $globalListeners[$priority]); array_push($result, ...$globalListeners[$priority]);
} }
if (isset($localListeners[$priority])) { if (isset($localListeners[$priority])) {
$result = array_merge($result, $localListeners[$priority]); array_push($result, ...$localListeners[$priority]);
} }
} }
return $result; return $result;

28
app/Lib/Tools/BetterSecurity.php
View File

@ -7,8 +7,8 @@ class BetterSecurity
/** /**
* @param string $plain * @param string $plain
* @param string $key * @param string $key Encryption key
* @return string * @return string Cipher text with IV and tag
* @throws Exception * @throws Exception
*/ */
public static function encrypt($plain, $key) public static function encrypt($plain, $key)
@ -33,17 +33,17 @@ class BetterSecurity
} }
/** /**
* @param string $cipher * @param string $cipherText Cipher text with IV and tag
* @param string $key * @param string $key Decryption key
* @return string * @return string
* @throws Exception * @throws Exception
*/ */
public static function decrypt($cipher, $key) public static function decrypt($cipherText, $key)
{ {
if (strlen($key) < 32) { if (strlen($key) < 32) {
throw new Exception('Invalid key, key must be at least 256 bits (32 bytes) long.'); throw new Exception('Invalid key, key must be at least 256 bits (32 bytes) long.');
} }
if (empty($cipher)) { if (empty($cipherText)) {
throw new Exception('The data to decrypt cannot be empty.'); throw new Exception('The data to decrypt cannot be empty.');
} }
@ -52,12 +52,18 @@ class BetterSecurity
$ivSize = openssl_cipher_iv_length(self::METHOD); $ivSize = openssl_cipher_iv_length(self::METHOD);
// Split out hmac for comparison if (strlen($cipherText) < $ivSize + self::TAG_SIZE) {
$iv = substr($cipher, 0, $ivSize); $length = strlen($cipherText);
$tag = substr($cipher, $ivSize, self::TAG_SIZE); $minLength = $ivSize + self::TAG_SIZE;
$cipher = substr($cipher, $ivSize + self::TAG_SIZE); throw new Exception("Provided cipher text is too short, $length bytes provided, expected at least $minLength bytes.");
}
$decrypted = openssl_decrypt($cipher, self::METHOD, $key, true, $iv, $tag); // Split out hmac for comparison
$iv = substr($cipherText, 0, $ivSize);
$tag = substr($cipherText, $ivSize, self::TAG_SIZE);
$cipherText = substr($cipherText, $ivSize + self::TAG_SIZE);
$decrypted = openssl_decrypt($cipherText, self::METHOD, $key, OPENSSL_RAW_DATA, $iv, $tag);
if ($decrypted === false) { if ($decrypted === false) {
throw new Exception('Could not decrypt. Maybe invalid encryption key?'); throw new Exception('Could not decrypt. Maybe invalid encryption key?');
} }

11
app/Lib/Tools/ComplexTypeTool.php
View File

@ -308,14 +308,13 @@ class ComplexTypeTool
*/ */
private function parseFreetext($input) private function parseFreetext($input)
{ {
$input = str_replace("\xc2\xa0", ' ', $input); // non breaking space to normal space // convert non breaking space to normal space and all unicode chars from "other" category
$input = preg_replace('/\p{C}+/u', ' ', $input); $input = preg_replace("/\p{C}+|\xc2\xa0/u", ' ', $input);
$iocArray = preg_split("/\r\n|\n|\r|\s|\s+|,|\<|\>|;/", $input); $iocArray = preg_split("/\r\n|\n|\r|\s|\s+|,|<|>|;/", $input);
preg_match_all('/\"([^\"]+)\"/', $input, $matches); preg_match_all('/\"([^\"]+)\"/', $input, $matches);
foreach ($matches[1] as $match) { array_push($iocArray, ...$matches[1]);
$iocArray[] = $match;
}
return $iocArray; return $iocArray;
} }

377
app/Lib/Tools/CurlClient.php Normal file
View File

@ -0,0 +1,377 @@
<?php
App::uses('HttpSocketExtended', 'Tools');
class CurlClient extends HttpSocketExtended
{
/** @var resource */
private $ch;
/** @var int */
private $timeout = 30;
/** @var string|null */
private $caFile;
/** @var string|null */
private $localCert;
/** @var int */
private $cryptoMethod;
/** @var bool */
private $allowSelfSigned;
/** @var bool */
private $verifyPeer;
/** @var bool */
private $compress = true;
/** @var array */
private $proxy = [];
/** @var array */
private $defaultOptions;
/**
* @param array $params
* @noinspection PhpMissingParentConstructorInspection
*/
public function __construct(array $params)
{
if (isset($params['timeout'])) {
$this->timeout = $params['timeout'];
}
if (isset($params['ssl_cafile'])) {
$this->caFile = $params['ssl_cafile'];
}
if (isset($params['ssl_local_cert'])) {
$this->localCert = $params['ssl_local_cert'];
}
if (isset($params['compress'])) {
$this->compress = $params['compress'];
}
if (isset($params['ssl_crypto_method'])) {
$this->cryptoMethod = $this->convertCryptoMethod($params['ssl_crypto_method']);
}
if (isset($params['ssl_allow_self_signed'])) {
$this->allowSelfSigned = $params['ssl_allow_self_signed'];
}
if (isset($params['ssl_verify_peer'])) {
$this->verifyPeer = $params['ssl_verify_peer'];
}
$this->defaultOptions = $this->generateDefaultOptions();
}
/**
* @param string $uri
* @param array $query
* @param array $request
* @return HttpSocketResponseExtended
*/
public function head($uri = null, $query = [], $request = [])
{
return $this->internalRequest('HEAD', $uri, $query, $request);
}
/**
* @param string $uri
* @param array $query
* @param array $request
* @return HttpSocketResponseExtended
*/
public function get($uri = null, $query = [], $request = [])
{
return $this->internalRequest('GET', $uri, $query, $request);
}
/**
* @param string $uri
* @param array $data
* @param array $request
* @return HttpSocketResponseExtended
*/
public function post($uri = null, $data = [], $request = [])
{
return $this->internalRequest('POST', $uri, $data, $request);
}
/**
* @param string $uri
* @param array$data
* @param $request
* @return HttpSocketResponseExtended
*/
public function put($uri = null, $data = [], $request = [])
{
return $this->internalRequest('PUT', $uri, $data, $request);
}
/**
* @param string $uri
* @param array $data
* @param array $request
* @return HttpSocketResponseExtended
*/
public function patch($uri = null, $data = [], $request = [])
{
return $this->internalRequest('PATCH', $uri, $data, $request);
}
/**
* @param string $uri
* @param array $data
* @param array $request
* @return HttpSocketResponseExtended
*/
public function delete($uri = null, $data = array(), $request = array())
{
return $this->internalRequest('DELETE', $uri, $data, $request);
}
public function url($url = null, $uriTemplate = null)
{
throw new Exception('Not implemented');
}
public function request($request = array())
{
throw new Exception('Not implemented');
}
public function setContentResource($resource)
{
throw new Exception('Not implemented');
}
public function getMetaData()
{
return null; // not supported by curl extension
}
/**
* @param string $host
* @param int $port
* @param string $method
* @param string $user
* @param string $pass
* @return void
*/
public function configProxy($host, $port = 3128, $method = null, $user = null, $pass = null)
{
if (empty($host)) {
$this->proxy = [];
return;
}
if (is_array($host)) {
$this->proxy = $host + ['host' => null];
return;
}
$this->proxy = compact('host', 'port', 'method', 'user', 'pass');
}
/**
* @param string $method
* @param string $url
* @param array|string $query
* @param array $request
* @return HttpSocketResponseExtended
*/
private function internalRequest($method, $url, $query, $request)
{
if (empty($url)) {
throw new InvalidArgumentException("No URL provided.");
}
if (!$this->ch) {
// Share handle between requests to allow keep connection alive between requests
$this->ch = curl_init();
if (!$this->ch) {
throw new \RuntimeException("Could not initialize curl");
}
} else {
// Reset options, so we can do another request
curl_reset($this->ch);
}
if (($method === 'GET' || $method === 'HEAD') && !empty($query)) {
$url .= '?' . http_build_query($query, '', '&', PHP_QUERY_RFC3986);
}
$options = $this->defaultOptions;
$options[CURLOPT_URL] = $url;
$options[CURLOPT_CUSTOMREQUEST] = $method;
if (($method === 'POST' || $method === 'DELETE' || $method === 'PUT' || $method === 'PATCH') && !empty($query)) {
$options[CURLOPT_POSTFIELDS] = $query;
}
if (!empty($request['header'])) {
$headers = [];
foreach ($request['header'] as $key => $value) {
if (is_array($value)) {
$value = implode(', ', $value);
}
$headers[] = "$key: $value";
}
$options[CURLOPT_HTTPHEADER] = $headers;
}
// Parse response headers
$responseHeaders = [];
$options[CURLOPT_HEADERFUNCTION] = function ($curl, $header) use (&$responseHeaders){
$len = strlen($header);
$header = explode(':', $header, 2);
if (count($header) < 2) { // ignore invalid headers
return $len;
}
$key = strtolower(trim($header[0]));
$value = trim($header[1]);
if (isset($responseHeaders[$key])) {
$responseHeaders[$key] = array_merge((array)$responseHeaders[$key], [$value]);
} else {
$responseHeaders[$key] = $value;
}
return $len;
};
if (!curl_setopt_array($this->ch, $options)) {
throw new \RuntimeException('curl error: Could not set options');
}
// Download the given URL, and return output
$output = curl_exec($this->ch);
if ($output === false) {
$errorCode = curl_errno($this->ch);
$errorMessage = curl_error($this->ch);
if (!empty($errorMessage)) {
$errorMessage = ": $errorMessage";
}
throw new SocketException("curl error $errorCode '" . curl_strerror($errorCode) . "'" . $errorMessage);
}
$code = curl_getinfo($this->ch, CURLINFO_HTTP_CODE);
return $this->constructResponse($output, $responseHeaders, $code);
}
public function disconnect()
{
if ($this->ch) {
curl_close($this->ch);
$this->ch = null;
}
}
/**
* @param string $body
* @param array $headers
* @param int $code
* @return HttpSocketResponseExtended
*/
private function constructResponse($body, array $headers, $code)
{
if (isset($responseHeaders['content-encoding']) && $responseHeaders['content-encoding'] === 'zstd') {
if (!function_exists('zstd_uncompress')) {
throw new SocketException('Response is zstd encoded, but PHP do not support zstd decoding.');
}
$body = zstd_uncompress($body);
if ($body === false) {
throw new SocketException('Could not decode zstd encoded response.');
}
}
$response = new HttpSocketResponseExtended();
$response->code = $code;
$response->body = $body;
$response->headers = $headers;
return $response;
}
/**
* @param int $cryptoMethod
* @return int
*/
private function convertCryptoMethod($cryptoMethod)
{
switch ($cryptoMethod) {
case STREAM_CRYPTO_METHOD_TLSv1_0_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_1_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
return CURL_SSLVERSION_TLSv1;
case STREAM_CRYPTO_METHOD_TLSv1_1_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
return CURL_SSLVERSION_TLSv1_1;
case STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
return CURL_SSLVERSION_TLSv1_2;
case STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
return CURL_SSLVERSION_TLSv1_3;
default:
throw new InvalidArgumentException("Unsupported crypto method value $cryptoMethod");
}
}
/**
* @return array
*/
private function generateDefaultOptions()
{
$options = [
CURLOPT_FOLLOWLOCATION => true, // Allows to follow redirect
CURLOPT_MAXREDIRS => 10,
CURLOPT_RETURNTRANSFER => true, // Should cURL return or print out the data? (true = return, false = print)
CURLOPT_HEADER => false, // Include header in result?
CURLOPT_TIMEOUT => $this->timeout, // Timeout in seconds
CURLOPT_PROTOCOLS => CURLPROTO_HTTPS | CURLPROTO_HTTP, // be sure that only HTTP and HTTPS protocols are enabled,
];
if ($this->caFile) {
$options[CURLOPT_CAINFO] = $this->caFile;
}
if ($this->localCert) {
$options[CURLOPT_SSLCERT] = $this->localCert;
}
if ($this->cryptoMethod) {
$options[CURLOPT_SSLVERSION] = $this->cryptoMethod;
}
if ($this->compress) {
$options[CURLOPT_ACCEPT_ENCODING] = $this->supportedEncodings();
}
if ($this->allowSelfSigned) {
$options[CURLOPT_SSL_VERIFYPEER] = $this->verifyPeer;
$options[CURLOPT_SSL_VERIFYHOST] = 0;
}
if (!empty($this->proxy)) {
$options[CURLOPT_PROXY] = "{$this->proxy['host']}:{$this->proxy['port']}";
if (!empty($this->proxy['method']) && isset($this->proxy['user'], $this->proxy['pass'])) {
$options[CURLOPT_PROXYUSERPWD] = "{$this->proxy['user']}:{$this->proxy['pass']}";
}
}
return $options;
}
/**
* @return string
*/
private function supportedEncodings()
{
$encodings = [];
// zstd is not supported by curl itself, but add support if PHP zstd extension is installed
if (function_exists('zstd_uncompress')) {
$encodings[] = 'zstd';
}
// brotli and gzip is supported by curl itself if it is compiled with these features
$info = curl_version();
if (defined('CURL_VERSION_BROTLI') && $info['features'] & CURL_VERSION_BROTLI) {
$encodings[] = 'br';
}
if ($info['features'] & CURL_VERSION_LIBZ) {
$encodings[] = 'gzip, deflate';
}
return implode(', ', $encodings);
}
}

65
app/Lib/Tools/GitTool.php
View File

@ -12,11 +12,7 @@ class GitTool
public static function getLatestTags(HttpSocketExtended $HttpSocket) public static function getLatestTags(HttpSocketExtended $HttpSocket)
{ {
$url = 'https://api.github.com/repos/MISP/MISP/tags?per_page=10'; $url = 'https://api.github.com/repos/MISP/MISP/tags?per_page=10';
$response = $HttpSocket->get($url); return self::gitHubRequest($HttpSocket, $url);
if (!$response->isOk()) {
throw new HttpSocketHttpException($response, $url);
}
return $response->json();
} }
/** /**
@ -28,11 +24,7 @@ class GitTool
public static function getLatestCommit(HttpSocketExtended $HttpSocket) public static function getLatestCommit(HttpSocketExtended $HttpSocket)
{ {
$url = 'https://api.github.com/repos/MISP/MISP/commits?per_page=1'; $url = 'https://api.github.com/repos/MISP/MISP/commits?per_page=1';
$response = $HttpSocket->get($url); $data = self::gitHubRequest($HttpSocket, $url);
if (!$response->isOk()) {
throw new HttpSocketHttpException($response, $url);
}
$data = $response->json();
if (!isset($data[0]['sha'])) { if (!isset($data[0]['sha'])) {
throw new Exception("Response do not contains requested data."); throw new Exception("Response do not contains requested data.");
} }
@ -40,20 +32,49 @@ class GitTool
} }
/** /**
* @param HttpSocketExtended $HttpSocket
* @param string $url
* @return array
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
*/
private static function gitHubRequest(HttpSocketExtended $HttpSocket, $url)
{
$response = $HttpSocket->get($url, [], ['header' => ['User-Agent' => 'MISP']]);
if (!$response->isOk()) {
throw new HttpSocketHttpException($response, $url);
}
return $response->json();
}
/**
* Returns current SHA1 hash of current commit
* `git rev-parse HEAD` * `git rev-parse HEAD`
* @param string $repoPath
* @return string * @return string
* @throws Exception * @throws Exception
*/ */
public static function currentCommit() public static function currentCommit($repoPath)
{ {
$head = rtrim(FileAccessTool::readFromFile(ROOT . '/.git/HEAD')); if (is_file($repoPath . '/.git')) {
$fileContent = FileAccessTool::readFromFile($repoPath . '/.git');
if (substr($fileContent, 0, 8) === 'gitdir: ') {
$gitDir = $repoPath . '/' . trim(substr($fileContent, 8)) . '/';
} else {
throw new Exception("$repoPath/.git is file, but contains non expected content $fileContent");
}
} else {
$gitDir = $repoPath . '/.git/';
}
$head = rtrim(FileAccessTool::readFromFile($gitDir . 'HEAD'));
if (substr($head, 0, 5) === 'ref: ') { if (substr($head, 0, 5) === 'ref: ') {
$path = substr($head, 5); $path = substr($head, 5);
return rtrim(FileAccessTool::readFromFile(ROOT . '/.git/' . $path)); return rtrim(FileAccessTool::readFromFile($gitDir . $path));
} else if (strlen($head) === 40) { } else if (strlen($head) === 40) {
return $head; return $head;
} else { } else {
throw new Exception("Invalid head $head"); throw new Exception("Invalid head '$head' in $gitDir/HEAD");
} }
} }
@ -94,30 +115,18 @@ class GitTool
return $output; return $output;
} }
/**
* @param string $submodule Path to Git repo
* @return string|null
*/
public static function submoduleCurrentCommit($submodule)
{
try {
$commit = ProcessTool::execute(['git', 'rev-parse', 'HEAD'], $submodule);
} catch (ProcessException $e) {
return null;
}
return rtrim($commit);
}
/** /**
* @param string $commit * @param string $commit
* @param string|null $submodule Path to Git repo * @param string|null $submodule Path to Git repo
* @return int|null * @return int|null
* @throws Exception
*/ */
public static function commitTimestamp($commit, $submodule = null) public static function commitTimestamp($commit, $submodule = null)
{ {
try { try {
$timestamp = ProcessTool::execute(['git', 'show', '-s', '--pretty=format:%ct', $commit], $submodule); $timestamp = ProcessTool::execute(['git', 'show', '-s', '--pretty=format:%ct', $commit], $submodule);
} catch (ProcessException $e) { } catch (ProcessException $e) {
CakeLog::notice("Could not get Git commit timestamp for $submodule: {$e->getMessage()}");
return null; return null;
} }
return (int)rtrim($timestamp); return (int)rtrim($timestamp);

5
app/Lib/Tools/HttpSocketExtended.php
View File

@ -18,10 +18,15 @@ class HttpSocketHttpException extends Exception
{ {
$this->response = $response; $this->response = $response;
$this->url = $url; $this->url = $url;
$message = "Remote server returns HTTP error code $response->code"; $message = "Remote server returns HTTP error code $response->code";
if ($url) { if ($url) {
$message .= " for URL $url"; $message .= " for URL $url";
} }
if ($response->body) {
$message .= ': ' . substr($response->body, 0, 100);
}
parent::__construct($message, (int)$response->code); parent::__construct($message, (int)$response->code);
} }

2
app/Lib/Tools/JSONConverterTool.php
View File

@ -153,7 +153,7 @@ class JSONConverterTool
return; return;
} }
yield '{"Event":{'; yield '{"Event":{';
$firstKey = key($event['Event']); $firstKey = array_key_first($event['Event']);
foreach ($event['Event'] as $key => $value) { foreach ($event['Event'] as $key => $value) {
if ($key === 'Attribute' || $key === 'Object') { // Encode every object or attribute separately if ($key === 'Attribute' || $key === 'Object') { // Encode every object or attribute separately
yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":["; yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":[";

50
app/Lib/Tools/JsonTool.php
View File

@ -9,10 +9,7 @@ class JsonTool
*/ */
public static function encode($value, $prettyPrint = false) public static function encode($value, $prettyPrint = false)
{ {
$flags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE; $flags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR;
if (defined('JSON_THROW_ON_ERROR')) {
$flags |= JSON_THROW_ON_ERROR; // Throw exception on error if supported
}
if ($prettyPrint) { if ($prettyPrint) {
$flags |= JSON_PRETTY_PRINT; $flags |= JSON_PRETTY_PRINT;
} }
@ -34,16 +31,8 @@ class JsonTool
} catch (SimdJsonException $e) { } catch (SimdJsonException $e) {
throw new JsonException($e->getMessage(), $e->getCode(), $e); throw new JsonException($e->getMessage(), $e->getCode(), $e);
} }
} elseif (defined('JSON_THROW_ON_ERROR')) {
// JSON_THROW_ON_ERROR is supported since PHP 7.3
return json_decode($value, true, 512, JSON_THROW_ON_ERROR);
} else {
$decoded = json_decode($value, true);
if ($decoded === null) {
throw new UnexpectedValueException('Could not parse JSON: ' . json_last_error_msg(), json_last_error());
}
return $decoded;
} }
return json_decode($value, true, 512, JSON_THROW_ON_ERROR);
} }
/** /**
@ -78,4 +67,39 @@ class JsonTool
return false; return false;
} }
} }
/**
* @see https://www.php.net/manual/en/function.array-is-list.php
* @param array $array
* @return bool
*/
public static function arrayIsList(array $array)
{
if (function_exists('array_is_list')) {
return array_is_list($array);
}
$i = -1;
foreach ($array as $k => $v) {
++$i;
if ($k !== $i) {
return false;
}
}
return true;
}
/**
* JSON supports just unicode strings. This helper method converts non unicode chars to Unicode Replacement Character U+FFFD (UTF-8)
* @param string $string
* @return string
*/
public static function escapeNonUnicode($string)
{
if (mb_check_encoding($string, 'UTF-8')) {
return $string; // string is valid unicode
}
return htmlspecialchars_decode(htmlspecialchars($string, ENT_SUBSTITUTE, 'UTF-8'));
}
} }

15
app/Lib/Tools/ProcessTool.php
View File

@ -8,14 +8,14 @@ class ProcessException extends Exception
private $stdout; private $stdout;
/** /**
* @param string|array $command * @param array $command
* @param int $returnCode * @param int $returnCode
* @param string $stderr * @param string $stderr
* @param string $stdout * @param string $stdout
*/ */
public function __construct($command, $returnCode, $stderr, $stdout) public function __construct(array $command, $returnCode, $stderr, $stdout)
{ {
$commandForException = is_array($command) ? implode(' ', $command) : $command; $commandForException = implode(' ', $command);
$message = "Command '$commandForException' finished with error code $returnCode.\nSTDERR: '$stderr'\nSTDOUT: '$stdout'"; $message = "Command '$commandForException' finished with error code $returnCode.\nSTDERR: '$stderr'\nSTDOUT: '$stdout'";
$this->stderr = $stderr; $this->stderr = $stderr;
$this->stdout = $stdout; $this->stdout = $stdout;
@ -56,11 +56,6 @@ class ProcessTool
self::logMessage('Running command ' . implode(' ', $command)); self::logMessage('Running command ' . implode(' ', $command));
} }
// PHP older than 7.4 do not support proc_open with array, so we need to convert values to string manually
if (PHP_VERSION_ID < 70400) {
$command = array_map('escapeshellarg', $command);
$command = implode(' ', $command);
}
$process = proc_open($command, $descriptorSpec, $pipes, $cwd); $process = proc_open($command, $descriptorSpec, $pipes, $cwd);
if (!$process) { if (!$process) {
$commandForException = self::commandFormat($command); $commandForException = self::commandFormat($command);
@ -136,8 +131,8 @@ class ProcessTool
* @param array|string $command * @param array|string $command
* @return string * @return string
*/ */
private static function commandFormat($command) private static function commandFormat(array $command)
{ {
return is_array($command) ? implode(' ', $command) : $command; return implode(' ', $command);
} }
} }

31
app/Lib/Tools/PubSubTool.php
View File

@ -178,8 +178,12 @@ class PubSubTool
public function killService() public function killService()
{ {
$settings = $this->getSetSettings();
if ($settings['supervisor_managed']) {
throw new RuntimeException('ZeroMQ server is managed by supervisor, it is not possible to restart it.');
}
if ($this->checkIfRunning()) { if ($this->checkIfRunning()) {
$settings = $this->getSetSettings();
$redis = $this->createRedisConnection($settings); $redis = $this->createRedisConnection($settings);
$redis->rPush('command', 'kill'); $redis->rPush('command', 'kill');
sleep(1); sleep(1);
@ -213,12 +217,16 @@ class PubSubTool
public function restartServer() public function restartServer()
{ {
$settings = $this->getSetSettings();
if ($settings['supervisor_managed']) {
throw new RuntimeException('ZeroMQ server is managed by supervisor, it is not possible to restart it.');
}
if (!$this->checkIfRunning()) { if (!$this->checkIfRunning()) {
if (!$this->killService()) { if (!$this->killService()) {
return 'Could not kill the previous instance of the ZeroMQ script.'; return 'Could not kill the previous instance of the ZeroMQ script.';
} }
} }
$settings = $this->getSetSettings();
$this->setupPubServer($settings); $this->setupPubServer($settings);
if ($this->checkIfRunning() === false) { if ($this->checkIfRunning() === false) {
return 'Failed starting the ZeroMQ script.'; return 'Failed starting the ZeroMQ script.';
@ -226,12 +234,22 @@ class PubSubTool
return true; return true;
} }
public function createConfigFile()
{
$settings = $this->getSetSettings();
$this->saveSettingToFile($settings);
}
/** /**
* @param array $settings * @param array $settings
* @throws Exception * @throws Exception
*/ */
private function setupPubServer(array $settings) private function setupPubServer(array $settings)
{ {
if ($settings['supervisor_managed']) {
return; // server is managed by supervisor, we don't need to check if is running or start it when not
}
if ($this->checkIfRunning() === false) { if ($this->checkIfRunning() === false) {
if ($this->checkIfRunning(self::OLD_PID_LOCATION)) { if ($this->checkIfRunning(self::OLD_PID_LOCATION)) {
// Old version is running, kill it and start again new one. // Old version is running, kill it and start again new one.
@ -250,6 +268,7 @@ class PubSubTool
* @param string|array $data * @param string|array $data
* @return bool * @return bool
* @throws JsonException * @throws JsonException
* @throws RedisException
*/ */
private function pushToRedis($ns, $data) private function pushToRedis($ns, $data)
{ {
@ -295,9 +314,12 @@ class PubSubTool
FileAccessTool::writeToFile($settingFilePath, JsonTool::encode($settings)); FileAccessTool::writeToFile($settingFilePath, JsonTool::encode($settings));
} }
/**
* @return array
*/
private function getSetSettings() private function getSetSettings()
{ {
$settings = array( $settings = [
'redis_host' => 'localhost', 'redis_host' => 'localhost',
'redis_port' => 6379, 'redis_port' => 6379,
'redis_password' => '', 'redis_password' => '',
@ -307,7 +329,8 @@ class PubSubTool
'port' => '50000', 'port' => '50000',
'username' => null, 'username' => null,
'password' => null, 'password' => null,
); 'supervisor_managed' => false,
];
$pluginConfig = Configure::read('Plugin'); $pluginConfig = Configure::read('Plugin');
foreach ($settings as $key => $setting) { foreach ($settings as $key => $setting) {

23
app/Lib/Tools/RedisTool.php
View File

@ -57,25 +57,34 @@ class RedisTool
/** /**
* @param Redis $redis * @param Redis $redis
* @param string|array $pattern * @param string|array $pattern
* @return int|Redis Number of deleted keys or instance of Redis if used in MULTI mode * @return Generator<string>
* @throws RedisException * @throws RedisException
*/ */
public static function deleteKeysByPattern(Redis $redis, $pattern) public static function keysByPattern(Redis $redis, $pattern)
{ {
if (is_string($pattern)) { if (is_string($pattern)) {
$pattern = [$pattern]; $pattern = [$pattern];
} }
$allKeys = [];
foreach ($pattern as $p) { foreach ($pattern as $p) {
$iterator = null; $iterator = null;
while (false !== ($keys = $redis->scan($iterator, $p, 1000))) { while (false !== ($keys = $redis->scan($iterator, $p, 1000))) {
foreach ($keys as $key) { foreach ($keys as $key) {
$allKeys[] = $key; yield $key;
} }
} }
} }
}
/**
* @param Redis $redis
* @param string|array $pattern
* @return int|Redis Number of deleted keys or instance of Redis if used in MULTI mode
* @throws RedisException
*/
public static function deleteKeysByPattern(Redis $redis, $pattern)
{
$allKeys = iterator_to_array(self::keysByPattern($redis, $pattern));
if (empty($allKeys)) { if (empty($allKeys)) {
return 0; return 0;
} }
@ -153,11 +162,7 @@ class RedisTool
return false; return false;
} }
if (self::$serializer === null) { if ($string[0] === "\x00") {
self::$serializer = Configure::read('MISP.redis_serializer') ?: false;
}
if (self::$serializer === 'igbinary') {
return igbinary_unserialize($string); return igbinary_unserialize($string);
} else { } else {
return JsonTool::decode($string); return JsonTool::decode($string);

8
app/Lib/Tools/ServerSyncTool.php
View File

@ -355,6 +355,14 @@ class ServerSyncTool
return $this->server['Server']['id']; return $this->server['Server']['id'];
} }
/**
* @return string
*/
public function serverName()
{
return $this->server['Server']['name'];
}
/** /**
* @return array * @return array
*/ */

11
app/Lib/Tools/SyncTool.php
View File

@ -1,5 +1,4 @@
<?php <?php
class SyncTool class SyncTool
{ {
@ -84,8 +83,14 @@ class SyncTool
$params['ssl_crypto_method'] = $version; $params['ssl_crypto_method'] = $version;
} }
App::uses('HttpSocketExtended', 'Tools'); if (function_exists('curl_init')) {
$HttpSocket = new HttpSocketExtended($params); App::uses('CurlClient', 'Tools');
$HttpSocket = new CurlClient($params);
} else {
App::uses('HttpSocketExtended', 'Tools');
$HttpSocket = new HttpSocketExtended($params);
}
$proxy = Configure::read('Proxy'); $proxy = Configure::read('Proxy');
if (empty($params['skip_proxy']) && isset($proxy['host']) && !empty($proxy['host'])) { if (empty($params['skip_proxy']) && isset($proxy['host']) && !empty($proxy['host'])) {
$HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']); $HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']);

48
app/Model/AccessLog.php
View File

@ -71,12 +71,6 @@ class AccessLog extends AppModel
{ {
$accessLog = &$this->data['AccessLog']; $accessLog = &$this->data['AccessLog'];
$this->externalLog($accessLog);
if (Configure::read('MISP.log_paranoid_skip_db')) {
return;
}
// Truncate // Truncate
foreach (['request_id', 'user_agent', 'url'] as $field) { foreach (['request_id', 'user_agent', 'url'] as $field) {
if (isset($accessLog[$field]) && strlen($accessLog[$field]) > 255) { if (isset($accessLog[$field]) && strlen($accessLog[$field]) > 255) {
@ -202,7 +196,7 @@ class AccessLog extends AppModel
if ($includeSqlQueries && !empty($sqlLog['log'])) { if ($includeSqlQueries && !empty($sqlLog['log'])) {
foreach ($sqlLog['log'] as &$log) { foreach ($sqlLog['log'] as &$log) {
$log['query'] = $this->escapeNonUnicode($log['query']); $log['query'] = JsonTool::escapeNonUnicode($log['query']);
unset($log['affected']); // affected is the same as numRows unset($log['affected']); // affected is the same as numRows
unset($log['params']); // no need to save for your use case unset($log['params']); // no need to save for your use case
} }
@ -214,6 +208,12 @@ class AccessLog extends AppModel
$data['query_count'] = $queryCount; $data['query_count'] = $queryCount;
$data['duration'] = (int)((microtime(true) - $requestTime->format('U.u')) * 1000); // in milliseconds $data['duration'] = (int)((microtime(true) - $requestTime->format('U.u')) * 1000); // in milliseconds
$this->externalLog($data);
if (Configure::read('MISP.log_paranoid_skip_db')) {
return true; // do not save access log to database
}
try { try {
return $this->save($data, ['atomic' => false]); return $this->save($data, ['atomic' => false]);
} catch (Exception $e) { } catch (Exception $e) {
@ -226,7 +226,7 @@ class AccessLog extends AppModel
* @param array $data * @param array $data
* @return void * @return void
*/ */
public function externalLog(array $data) private function externalLog(array $data)
{ {
if ($this->pubToZmq('audit')) { if ($this->pubToZmq('audit')) {
$this->getPubSubTool()->publish($data, 'audit', 'log'); $this->getPubSubTool()->publish($data, 'audit', 'log');
@ -310,36 +310,4 @@ class AccessLog extends AppModel
} }
return $data; return $data;
} }
/**
* @param $string
* @return string
*/
private function escapeNonUnicode($string)
{
if (json_encode($string, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_LINE_TERMINATORS) !== false) {
return $string; // string is valid unicode
}
if (function_exists('mb_str_split')) {
$result = mb_str_split($string);
} else {
$result = [];
$length = mb_strlen($string);
for ($i = 0; $i < $length; $i++) {
$result[] = mb_substr($string, $i, 1);
}
}
$string = '';
foreach ($result as $char) {
if (strlen($char) === 1 && !preg_match('/[[:print:]]/', $char)) {
$string .= '\x' . bin2hex($char);
} else {
$string .= $char;
}
}
return $string;
}
} }

28
app/Model/AdminSetting.php
View File

@ -90,6 +90,7 @@ class AdminSetting extends AppModel
$time = time(); $time = time();
$this->__deleteScriptTmpFiles($time); $this->__deleteScriptTmpFiles($time);
$this->__deleteTaxiiTmpFiles($time); $this->__deleteTaxiiTmpFiles($time);
$this->__deleteCachedExportFiles($time);
} }
private function __deleteScriptTmpFiles($time) { private function __deleteScriptTmpFiles($time) {
@ -107,6 +108,29 @@ class AdminSetting extends AppModel
} }
} }
private function __deleteCachedExportFiles($time) {
$cache_path = APP . 'tmp/cached_exports';
$cache_dir = new Folder($cache_path);
$cache_data = $cache_dir->read(false, false);
if (!empty($cache_data[0])) {
foreach ($cache_data[0] as $cache_export_dir) {
$tmp_dir = new Folder($cache_path . '/' . $cache_export_dir);
$cache_export_dir_contents = $tmp_dir->read(false, false);
if (!empty(count($cache_export_dir_contents[1]))) {
$files_count = count($cache_export_dir_contents[1]);
$files_removed = 0;
foreach ($cache_export_dir_contents[1] as $tmp_file) {
$tmp_file = new File($cache_path . '/' . $cache_export_dir . '/' . $tmp_file);
if ($time > $tmp_file->lastChange() + 3600) {
$tmp_file->delete();
$files_removed += 1;
}
}
}
}
}
}
private function __deleteTaxiiTmpFiles($time) { private function __deleteTaxiiTmpFiles($time) {
$taxii_path = APP . 'files/scripts/tmp/Taxii'; $taxii_path = APP . 'files/scripts/tmp/Taxii';
$taxii_dir = new Folder($taxii_path); $taxii_dir = new Folder($taxii_path);
@ -114,13 +138,13 @@ class AdminSetting extends AppModel
if (!empty($taxii_contents[0])) { if (!empty($taxii_contents[0])) {
foreach ($taxii_contents[0] as $taxii_temp_dir) { foreach ($taxii_contents[0] as $taxii_temp_dir) {
if (preg_match('/^[a-zA-Z0-9]{12}$/', $taxii_temp_dir)) { if (preg_match('/^[a-zA-Z0-9]{12}$/', $taxii_temp_dir)) {
$tmp_dir = new Folder($taxii_path . $taxii_temp_dir); $tmp_dir = new Folder($taxii_path . '/' .$taxii_temp_dir);
$taxii_temp_dir_contents = $tmp_dir->read(false, false); $taxii_temp_dir_contents = $tmp_dir->read(false, false);
if (!empty(count($taxii_temp_dir_contents[1]))) { if (!empty(count($taxii_temp_dir_contents[1]))) {
$files_count = count($taxii_temp_dir_contents[1]); $files_count = count($taxii_temp_dir_contents[1]);
$files_removed = 0; $files_removed = 0;
foreach ($taxii_temp_dir_contents[1] as $tmp_file) { foreach ($taxii_temp_dir_contents[1] as $tmp_file) {
$tmp_file = new File($taxii_path . $taxii_temp_dir . '/' . $tmp_file); $tmp_file = new File($taxii_path . '/' . $taxii_temp_dir . '/' . $tmp_file);
if ($time > $tmp_file->lastChange() + 3600) { if ($time > $tmp_file->lastChange() + 3600) {
$tmp_file->delete(); $tmp_file->delete();
$files_removed += 1; $files_removed += 1;

3
app/Model/Allowedlist.php
View File

@ -89,6 +89,9 @@ class Allowedlist extends AppModel
if ($isAttributeArray) { if ($isAttributeArray) {
// loop through each attribute and unset the ones that are allowedlisted // loop through each attribute and unset the ones that are allowedlisted
foreach ($data as $k => $attribute) { foreach ($data as $k => $attribute) {
if (empty($attribute['Attribute'])) {
$attribute = ['Attribute' => $attribute];
}
// loop through each allowedlist item and run a preg match against the attribute value. If it matches, unset the attribute // loop through each allowedlist item and run a preg match against the attribute value. If it matches, unset the attribute
foreach ($allowedlists as $wlitem) { foreach ($allowedlists as $wlitem) {
if (preg_match($wlitem, $attribute['Attribute']['value'])) { if (preg_match($wlitem, $attribute['Attribute']['value'])) {

42
app/Model/AppModel.php
View File

@ -2374,9 +2374,9 @@ class AppModel extends Model
} }
// alternative to the build in notempty/notblank validation functions, compatible with cakephp <= 2.6 and cakephp and cakephp >= 2.7 // alternative to the build in notempty/notblank validation functions, compatible with cakephp <= 2.6 and cakephp and cakephp >= 2.7
public function valueNotEmpty($value) public function valueNotEmpty(array $value)
{ {
$field = array_keys($value)[0]; $field = array_key_first($value);
$value = trim($value[$field]); $value = trim($value[$field]);
if (!empty($value)) { if (!empty($value)) {
return true; return true;
@ -2384,27 +2384,27 @@ class AppModel extends Model
return ucfirst($field) . ' cannot be empty.'; return ucfirst($field) . ' cannot be empty.';
} }
public function valueIsJson($value) public function valueIsJson(array $value)
{ {
$value = array_values($value)[0]; $value = current($value);
if (!JsonTool::isValid($value)) { if (!JsonTool::isValid($value)) {
return __('Invalid JSON.'); return __('Invalid JSON.');
} }
return true; return true;
} }
public function valueIsID($value) public function valueIsID(array $value)
{ {
$field = array_keys($value)[0]; $field = array_key_first($value);
if (!is_numeric($value[$field]) || $value[$field] < 0) { if (!is_numeric($value[$field]) || $value[$field] < 0) {
return 'Invalid ' . ucfirst($field) . ' ID'; return 'Invalid ' . ucfirst($field) . ' ID';
} }
return true; return true;
} }
public function stringNotEmpty($value) public function stringNotEmpty(array $value)
{ {
$field = array_keys($value)[0]; $field = array_key_first($value);
$value = trim($value[$field]); $value = trim($value[$field]);
if (!isset($value) || ($value == false && $value !== "0")) { if (!isset($value) || ($value == false && $value !== "0")) {
return ucfirst($field) . ' cannot be empty.'; return ucfirst($field) . ' cannot be empty.';
@ -3267,14 +3267,13 @@ class AppModel extends Model
* Returns MISP version from VERSION.json file as array with major, minor and hotfix keys. * Returns MISP version from VERSION.json file as array with major, minor and hotfix keys.
* *
* @return array * @return array
* @throws JsonException * @throws Exception
*/ */
public function checkMISPVersion() public function checkMISPVersion()
{ {
static $versionArray; static $versionArray;
if ($versionArray === null) { if ($versionArray === null) {
$content = FileAccessTool::readFromFile(ROOT . DS . 'VERSION.json'); $versionArray = FileAccessTool::readJsonFromFile(ROOT . DS . 'VERSION.json', true);
$versionArray = JsonTool::decode($content);
} }
return $versionArray; return $versionArray;
} }
@ -3290,7 +3289,7 @@ class AppModel extends Model
if ($commit === null) { if ($commit === null) {
App::uses('GitTool', 'Tools'); App::uses('GitTool', 'Tools');
try { try {
$commit = GitTool::currentCommit(); $commit = GitTool::currentCommit(ROOT);
} catch (Exception $e) { } catch (Exception $e) {
$this->logException('Could not get current git commit', $e, LOG_NOTICE); $this->logException('Could not get current git commit', $e, LOG_NOTICE);
$commit = false; $commit = false;
@ -3714,7 +3713,7 @@ class AppModel extends Model
if (!$isRule) { if (!$isRule) {
$args = func_get_args(); $args = func_get_args();
$fields = $args[1]; $fields = $args[1];
$or = isset($args[2]) ? $args[2] : true; $or = $args[2] ?? true;
} }
} }
if (!is_array($fields)) { if (!is_array($fields)) {
@ -3859,8 +3858,7 @@ class AppModel extends Model
protected function isMysql() protected function isMysql()
{ {
$dataSource = ConnectionManager::getDataSource('default'); $dataSource = ConnectionManager::getDataSource('default');
$dataSourceName = $dataSource->config['datasource']; return $dataSource instanceof Mysql;
return $dataSourceName === 'Database/Mysql' || $dataSourceName === 'Database/MysqlObserver' || $dataSourceName === 'Database/MysqlExtended' || $dataSource instanceof Mysql;
} }
/** /**
@ -3996,21 +3994,21 @@ class AppModel extends Model
"); ");
} }
public function findOrder($order, $order_model, $valid_order_fields) public function findOrder($order, $orderModel, $validOrderFields)
{ {
if (!is_array($order)) { if (!is_array($order)) {
$order_rules = explode(' ', strtolower($order)); $orderRules = explode(' ', strtolower($order));
$order_field = explode('.', $order_rules[0]); $orderField = explode('.', $orderRules[0]);
$order_field = end($order_field); $orderField = end($orderField);
if (in_array($order_field, $valid_order_fields)) { if (in_array($orderField, $validOrderFields, true)) {
$direction = 'asc'; $direction = 'asc';
if (!empty($order_rules[1]) && trim($order_rules[1]) === 'desc') { if (!empty($orderRules[1]) && trim($orderRules[1]) === 'desc') {
$direction = 'desc'; $direction = 'desc';
} }
} else { } else {
return null; return null;
} }
return $order_model . '.' . $order_field . ' ' . $direction; return $orderModel . '.' . $orderField . ' ' . $direction;
} }
return null; return null;
} }

26
app/Model/AttachmentScan.php
View File

@ -189,6 +189,7 @@ class AttachmentScan extends AppModel
/** @var Job $job */ /** @var Job $job */
$job = ClassRegistry::init('Job'); $job = ClassRegistry::init('Job');
if ($jobId && !$job->exists($jobId)) { if ($jobId && !$job->exists($jobId)) {
$this->log("Job with ID $jobId not found in database", LOG_NOTICE);
$jobId = null; $jobId = null;
} }
@ -252,12 +253,12 @@ class AttachmentScan extends AppModel
$infected = $this->scanAttachment($type, $attribute[$type], $moduleInfo); $infected = $this->scanAttachment($type, $attribute[$type], $moduleInfo);
if ($infected === true) { if ($infected === true) {
$virusFound++; $virusFound++;
$scanned++;
} else if ($infected === false) {
$scanned++;
} }
$scanned++;
} catch (NotFoundException $e) {
// skip if file doesn't exists
} catch (Exception $e) { } catch (Exception $e) {
$this->logException("Could not scan attachment for $type {$attribute['Attribute']['id']}", $e); $this->logException("Could not scan attachment for $type {$attribute['Attribute']['id']}", $e, LOG_WARNING);
$fails++; $fails++;
} }
@ -297,14 +298,14 @@ class AttachmentScan extends AppModel
$job = ClassRegistry::init('Job'); $job = ClassRegistry::init('Job');
$jobId = $job->createJob( $jobId = $job->createJob(
'SYSTEM', 'SYSTEM',
Job::WORKER_DEFAULT, Job::WORKER_PRIO,
'virus_scan', 'virus_scan',
($type === self::TYPE_ATTRIBUTE ? 'Attribute: ' : 'Shadow attribute: ') . $attribute['id'], ($type === self::TYPE_ATTRIBUTE ? 'Attribute: ' : 'Shadow attribute: ') . $attribute['id'],
'Scanning...' 'Scanning...'
); );
$this->getBackgroundJobsTool()->enqueue( $this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE, BackgroundJobsTool::PRIO_QUEUE,
BackgroundJobsTool::CMD_ADMIN, BackgroundJobsTool::CMD_ADMIN,
[ [
'scanAttachment', 'scanAttachment',
@ -319,10 +320,12 @@ class AttachmentScan extends AppModel
} }
/** /**
* Return true if attachment is infected, null if attachment was not scanned and false if attachment is OK
*
* @param string $type * @param string $type
* @param array $attribute * @param array $attribute
* @param array $moduleInfo * @param array $moduleInfo
* @return bool|null Return true if attachment is infected. * @return bool|null
* @throws Exception * @throws Exception
*/ */
private function scanAttachment($type, array $attribute, array $moduleInfo) private function scanAttachment($type, array $attribute, array $moduleInfo)
@ -351,11 +354,10 @@ class AttachmentScan extends AppModel
return false; // empty file is automatically considered as not infected return false; // empty file is automatically considered as not infected
} }
if ($fileSize > 25 * 1024 * 1024) {
/* if ($file->size() > 50 * 1024 * 1024) { $this->log("File '$file->path' is bigger than 25 MB, will be not scanned.", LOG_NOTICE);
$this->log("File '$file->path' is bigger than 50 MB, will be not scanned.", LOG_NOTICE); return null;
return false; }
}*/
$fileContent = $file->read(); $fileContent = $file->read();
if ($fileContent === false) { if ($fileContent === false) {

135
app/Model/Attribute.php
View File

@ -434,7 +434,7 @@ class Attribute extends AppModel
public function afterSave($created, $options = array()) public function afterSave($created, $options = array())
{ {
// Passing event in `parentEvent` field will speed up correlation // Passing event in `parentEvent` field will speed up correlation
$passedEvent = isset($options['parentEvent']) ? $options['parentEvent'] : false; $passedEvent = $options['parentEvent'] ?? false;
$attribute = $this->data['Attribute']; $attribute = $this->data['Attribute'];
@ -545,6 +545,28 @@ class Attribute extends AppModel
return $result; return $result;
} }
/**
* This method is called after all data are successfully saved into database
* @return void
* @throws Exception
*/
private function afterDatabaseSave(array $data)
{
$attribute = $data['Attribute'];
if (isset($attribute['type']) && $this->typeIsAttachment($attribute['type'])) {
$this->loadAttachmentScan()->backgroundScan(AttachmentScan::TYPE_ATTRIBUTE, $attribute);
}
}
public function save($data = null, $validate = true, $fieldList = array())
{
$result = parent::save($data, $validate, $fieldList);
if ($result) {
$this->afterDatabaseSave($result);
}
return $result;
}
public function beforeDelete($cascade = true) public function beforeDelete($cascade = true)
{ {
// delete attachments from the disk // delete attachments from the disk
@ -786,7 +808,7 @@ class Attribute extends AppModel
// check whether the variable is null or datetime // check whether the variable is null or datetime
public function datetimeOrNull($fields) public function datetimeOrNull($fields)
{ {
$seen = array_values($fields)[0]; $seen = current($fields);
if ($seen === null) { if ($seen === null) {
return true; return true;
} }
@ -881,7 +903,6 @@ class Attribute extends AppModel
} }
$result = $this->loadAttachmentTool()->save($attribute['event_id'], $attribute['id'], $attribute['data']); $result = $this->loadAttachmentTool()->save($attribute['event_id'], $attribute['id'], $attribute['data']);
if ($result) { if ($result) {
$this->loadAttachmentScan()->backgroundScan(AttachmentScan::TYPE_ATTRIBUTE, $attribute);
// Clean thumbnail cache // Clean thumbnail cache
if ($this->isImage($attribute) && Configure::read('MISP.thumbnail_in_redis')) { if ($this->isImage($attribute) && Configure::read('MISP.thumbnail_in_redis')) {
$redis = RedisTool::init(); $redis = RedisTool::init();
@ -1224,38 +1245,96 @@ class Attribute extends AppModel
$this->Correlation->purgeCorrelations($eventId); $this->Correlation->purgeCorrelations($eventId);
} }
public function reportValidationIssuesAttributes($eventId) /**
* This method is useful if you want to iterate all attributes sorted by ID
* @param array $conditions
* @param array $fields
* @param bool|string $callbacks
* @return Generator<array>|void
*/
public function fetchAttributesInChunks(array $conditions = [], array $fields = [], $callbacks = true)
{
$query = [
'recursive' => -1,
'conditions' => $conditions,
'limit' => 500,
'order' => ['Attribute.id'],
'fields' => $fields,
'callbacks' => $callbacks,
];
while (true) {
$attributes = $this->find('all', $query);
foreach ($attributes as $attribute) {
yield $attribute;
}
$count = count($attributes);
if ($count < 500) {
return;
}
$lastAttribute = $attributes[$count - 1];
$query['conditions']['Attribute.id >'] = $lastAttribute['Attribute']['id'];
}
}
/**
* @param int|null $eventId
* @return Generator
*/
public function reportValidationIssuesAttributes($eventId = null)
{ {
$conditions = array(); $conditions = array();
if ($eventId && is_numeric($eventId)) { if ($eventId && is_numeric($eventId)) {
$conditions = array('event_id' => $eventId); $conditions = array('event_id' => $eventId);
} }
$attributeIds = $this->find('column', array( $attributes = $this->fetchAttributesInChunks($conditions);
'fields' => array('id'),
'conditions' => $conditions
));
$chunks = array_chunk($attributeIds, 500);
$result = array(); foreach ($attributes as $attribute) {
foreach ($chunks as $chunk) { $this->set($attribute);
$attributes = $this->find('all', array('recursive' => -1, 'conditions' => array('id' => $chunk))); if (!$this->validates()) {
foreach ($attributes as $attribute) { $resultErrors = [];
$this->set($attribute); foreach ($this->validationErrors as $field => $error) {
if (!$this->validates()) { $resultErrors[$field] = ['value' => $attribute['Attribute'][$field], 'error' => $error[0]];
$resultErrors = array();
foreach ($this->validationErrors as $field => $error) {
$resultErrors[$field] = array('value' => $attribute['Attribute'][$field], 'error' => $error[0]);
}
$result[] = [
'id' => $attribute['Attribute']['id'],
'error' => $resultErrors,
'details' => 'Event ID: [' . $attribute['Attribute']['event_id'] . "] - Category: [" . $attribute['Attribute']['category'] . "] - Type: [" . $attribute['Attribute']['type'] . "] - Value: [" . $attribute['Attribute']['value'] . ']',
];
} }
yield [
'id' => $attribute['Attribute']['id'],
'error' => $resultErrors,
'details' => 'Event ID: [' . $attribute['Attribute']['event_id'] . "] - Category: [" . $attribute['Attribute']['category'] . "] - Type: [" . $attribute['Attribute']['type'] . "] - Value: [" . $attribute['Attribute']['value'] . ']',
];
}
}
}
/**
* @param bool $dryRun If true, no changes will be made to
* @return Generator
* @throws Exception
*/
public function normalizeIpAddress($dryRun = false)
{
$attributes = $this->fetchAttributesInChunks([
'Attribute.type' => ['ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'domain|ip'],
]);
foreach ($attributes as $attribute) {
$value = $attribute['Attribute']['value'];
$normalizedValue = AttributeValidationTool::modifyBeforeValidation($attribute['Attribute']['type'], $value);
if ($value !== $normalizedValue) {
if (!$dryRun) {
$attribute['Attribute']['value'] = $normalizedValue;
$this->save($attribute, true, ['value1', 'value2']);
}
yield [
'id' => (int) $attribute['Attribute']['id'],
'event_id' => (int) $attribute['Attribute']['event_id'],
'type' => $attribute['Attribute']['type'],
'value' => $value,
'normalized_value' => $normalizedValue,
];
} }
} }
return $result;
} }
/** /**
@ -1610,6 +1689,7 @@ class Attribute extends AppModel
* @param array $user * @param array $user
* @param array $options * @param array $options
* @param int|false $result_count If false, count is not fetched * @param int|false $result_count If false, count is not fetched
* @param bool $real_count
* @return array * @return array
* @throws Exception * @throws Exception
*/ */
@ -3096,8 +3176,7 @@ class Attribute extends AppModel
$exportTool->additional_params $exportTool->additional_params
); );
} }
ClassRegistry::init('ConnectionManager');
$db = ConnectionManager::getDataSource('default');
$tmpfile = new TmpFileTool(); $tmpfile = new TmpFileTool();
$tmpfile->write($exportTool->header($exportToolParams)); $tmpfile->write($exportTool->header($exportToolParams));
$loop = false; $loop = false;
@ -3673,7 +3752,7 @@ class Attribute extends AppModel
); );
} }
private function findAttributeByValue($attribute) private function findAttributeByValue(array $attribute)
{ {
$type = $attribute['type']; $type = $attribute['type'];
$conditions = [ $conditions = [

2
app/Model/AuthKey.php
View File

@ -207,7 +207,7 @@ class AuthKey extends AppModel
*/ */
private function updateUniqueIp(array $authkey) private function updateUniqueIp(array $authkey)
{ {
if (Configure::read("MISP.disable_seen_ips_authkeys")) { if (PHP_SAPI === 'cli' || Configure::read("MISP.disable_seen_ips_authkeys")) {
return; return;
} }

34
app/Model/Correlation.php
View File

@ -153,7 +153,7 @@ class Correlation extends AppModel
if (!empty($eventIds)) { if (!empty($eventIds)) {
$eventCount = count($eventIds); $eventCount = count($eventIds);
foreach ($eventIds as $j => $currentEventId) { foreach ($eventIds as $j => $currentEventId) {
$attributeCount += $this->__iteratedCorrelation( $attributeCount += $this->iteratedCorrelation(
$jobId, $jobId,
$full, $full,
$attributeId, $attributeId,
@ -179,7 +179,7 @@ class Correlation extends AppModel
* @return int * @return int
* @throws Exception * @throws Exception
*/ */
private function __iteratedCorrelation( private function iteratedCorrelation(
$jobId = false, $jobId = false,
$full = false, $full = false,
$attributeId = null, $attributeId = null,
@ -215,30 +215,14 @@ class Correlation extends AppModel
if ($attributeId) { if ($attributeId) {
$attributeConditions['Attribute.id'] = $attributeId; $attributeConditions['Attribute.id'] = $attributeId;
} }
$query = [
'recursive' => -1, $attributes = $this->Attribute->fetchAttributesInChunks($attributeConditions, $this->getFieldRules(), false);
'conditions' => $attributeConditions,
// fetch just necessary fields to save memory
'fields' => $this->getFieldRules(),
'order' => 'Attribute.id',
'limit' => 5000,
'callbacks' => false, // memory leak fix
];
$attributeCount = 0; $attributeCount = 0;
do { foreach ($attributes as $attribute) {
$attributes = $this->Attribute->find('all', $query); $this->afterSaveCorrelation($attribute['Attribute'], $full, $event);
foreach ($attributes as $attribute) { ++$attributeCount;
$this->afterSaveCorrelation($attribute['Attribute'], $full, $event); }
}
$fetchedAttributes = count($attributes);
unset($attributes);
$attributeCount += $fetchedAttributes;
if ($fetchedAttributes === 5000) { // maximum number of attributes fetched, continue in next loop
$query['conditions']['Attribute.id >'] = $attribute['Attribute']['id'];
} else {
break;
}
} while (true);
// Generating correlations can take long time, so clear caches after each event to refresh them // Generating correlations can take long time, so clear caches after each event to refresh them
$this->cidrListCache = null; $this->cidrListCache = null;

26
app/Model/Datasource/Database/MysqlExtended.php
View File

@ -7,6 +7,14 @@ App::uses('Mysql', 'Model/Datasource/Database');
*/ */
class MysqlExtended extends Mysql class MysqlExtended extends Mysql
{ {
const PDO_MAP = [
'integer' => PDO::PARAM_INT,
'float' => PDO::PARAM_STR,
'boolean' => PDO::PARAM_BOOL,
'string' => PDO::PARAM_STR,
'text' => PDO::PARAM_STR
];
/** /**
* Output MD5 as binary, that is faster and uses less memory * Output MD5 as binary, that is faster and uses less memory
* @param string $value * @param string $value
@ -157,15 +165,9 @@ class MysqlExtended extends Mysql
public function insertMulti($table, $fields, $values) public function insertMulti($table, $fields, $values)
{ {
$table = $this->fullTableName($table); $table = $this->fullTableName($table);
$holder = implode(',', array_fill(0, count($fields), '?')); $holder = substr(str_repeat('?,', count($fields)), 0, -1);
$fields = implode(',', array_map([$this, 'name'], $fields)); $fields = implode(',', array_map([$this, 'name'], $fields));
$pdoMap = [
'integer' => PDO::PARAM_INT,
'float' => PDO::PARAM_STR,
'boolean' => PDO::PARAM_BOOL,
'string' => PDO::PARAM_STR,
'text' => PDO::PARAM_STR
];
$columnMap = []; $columnMap = [];
foreach ($values[key($values)] as $key => $val) { foreach ($values[key($values)] as $key => $val) {
if (is_int($val)) { if (is_int($val)) {
@ -174,21 +176,21 @@ class MysqlExtended extends Mysql
$columnMap[$key] = PDO::PARAM_BOOL; $columnMap[$key] = PDO::PARAM_BOOL;
} else { } else {
$type = $this->introspectType($val); $type = $this->introspectType($val);
$columnMap[$key] = $pdoMap[$type]; $columnMap[$key] = self::PDO_MAP[$type];
} }
} }
$sql = "INSERT INTO $table ($fields) VALUES "; $sql = "INSERT INTO $table ($fields) VALUES ";
$sql .= implode(',', array_fill(0, count($values), "($holder)")); $sql .= substr(str_repeat("($holder),", count($values)), 0, -1);
$statement = $this->_connection->prepare($sql); $statement = $this->_connection->prepare($sql);
$valuesList = array(); $valuesList = array();
$i = 1; $i = 0;
foreach ($values as $value) { foreach ($values as $value) {
foreach ($value as $col => $val) { foreach ($value as $col => $val) {
if ($this->fullDebug) { if ($this->fullDebug) {
$valuesList[] = $val; $valuesList[] = $val;
} }
$statement->bindValue($i++, $val, $columnMap[$col]); $statement->bindValue(++$i, $val, $columnMap[$col]);
} }
} }
$result = $statement->execute(); $result = $statement->execute();

16
app/Model/Event.php
View File

@ -3743,7 +3743,10 @@ class Event extends AppModel
unset($this->Attribute->validate['value']['uniqueValue']); // unset this - we are saving a new event, there are no values to compare against and event_id is not set in the attributes unset($this->Attribute->validate['value']['uniqueValue']); // unset this - we are saving a new event, there are no values to compare against and event_id is not set in the attributes
} }
unset($data['Event']['id']); unset($data['Event']['id']);
if (isset($data['Event']['published']) && $data['Event']['published'] && $user['Role']['perm_publish'] == 0) { if (
(Configure::read('MISP.block_publishing_for_same_creator', false) && !$user['Role']['perm_sync']) ||
(isset($data['Event']['published']) && $data['Event']['published'] && $user['Role']['perm_publish'] == 0)
) {
$data['Event']['published'] = 0; $data['Event']['published'] = 0;
} }
if (isset($data['Event']['uuid'])) { if (isset($data['Event']['uuid'])) {
@ -4059,7 +4062,10 @@ class Event extends AppModel
} else { } else {
return array('error' => 'Event could not be saved: Could not find the local event.'); return array('error' => 'Event could not be saved: Could not find the local event.');
} }
if (!empty($data['Event']['published']) && !$user['Role']['perm_publish']) { if (
(Configure::read('MISP.block_publishing_for_same_creator', false) && !$user['Role']['perm_sync'] && $user['id'] == $existingEvent['Event']['user_id']) ||
(!empty($data['Event']['published']) && !$user['Role']['perm_publish'])
) {
$data['Event']['published'] = 0; $data['Event']['published'] = 0;
} }
if (!isset($data['Event']['published'])) { if (!isset($data['Event']['published'])) {
@ -4190,7 +4196,7 @@ class Event extends AppModel
if ((true != Configure::read('MISP.disablerestalert')) && (empty($server) || empty($server['Server']['publish_without_email']))) { if ((true != Configure::read('MISP.disablerestalert')) && (empty($server) || empty($server['Server']['publish_without_email']))) {
$this->sendAlertEmailRouter($id, $user, $existingEvent['Event']['publish_timestamp']); $this->sendAlertEmailRouter($id, $user, $existingEvent['Event']['publish_timestamp']);
} }
$this->publish($existingEvent['Event']['id']); $this->publish($existingEvent['Event']['id'], $passAlong);
} }
if ($jobId) { if ($jobId) {
$eventLock->deleteBackgroundJobLock($data['Event']['id'], $jobId); $eventLock->deleteBackgroundJobLock($data['Event']['id'], $jobId);
@ -5952,7 +5958,9 @@ class Event extends AppModel
$this->add_original_file($decoded['original'], $originalFile, $created_id, $stixVersion); $this->add_original_file($decoded['original'], $originalFile, $created_id, $stixVersion);
} }
if ($publish && $user['Role']['perm_publish']) { if ($publish && $user['Role']['perm_publish']) {
$this->publish($created_id); if (!Configure::read('MISP.block_publishing_for_same_creator', false) || $user['Role']['perm_sync']) {
$this->publish($created_id);
}
} }
return $created_id; return $created_id;
} else if (is_numeric($result)) { } else if (is_numeric($result)) {

3
app/Model/Feed.php
View File

@ -2062,6 +2062,7 @@ class Feed extends AppModel
$contentType = $response->getHeader('content-type'); $contentType = $response->getHeader('content-type');
if ($contentType === 'application/zip') { if ($contentType === 'application/zip') {
$zipFilePath = FileAccessTool::writeToTempFile($response->body); $zipFilePath = FileAccessTool::writeToTempFile($response->body);
unset($response->body); // cleanup variable to reduce memory usage
try { try {
$response->body = $this->unzipFirstFile($zipFilePath); $response->body = $this->unzipFirstFile($zipFilePath);
@ -2198,7 +2199,7 @@ class Feed extends AppModel
ZipArchive::ER_READ => 'read error', ZipArchive::ER_READ => 'read error',
ZipArchive::ER_SEEK => 'seek error', ZipArchive::ER_SEEK => 'seek error',
]; ];
$message = isset($errorCodes[$result]) ? $errorCodes[$result] : 'error ' . $result; $message = $errorCodes[$result] ?? 'error ' . $result;
throw new Exception("Remote server returns ZIP file, that cannot be open ($message)"); throw new Exception("Remote server returns ZIP file, that cannot be open ($message)");
} }

54
app/Model/FuzzyCorrelateSsdeep.php
View File

@ -9,38 +9,32 @@ class FuzzyCorrelateSsdeep extends AppModel
public function ssdeep_prepare($hash) public function ssdeep_prepare($hash)
{ {
list($block_size, $hash) = explode(':', $hash, 2); list($blockSize, $hash) = explode(':', $hash, 2);
$chars = array(); $uniqueChars = array_unique(str_split($hash), SORT_REGULAR);
for ($i = 0; $i < strlen($hash); $i++) {
if (!in_array($hash[$i], $chars, true)) {
$chars[] = $hash[$i];
}
}
$search = true; $search = true;
while ($search) { while ($search) {
$search = false; $search = false;
foreach ($chars as $c) { foreach ($uniqueChars as $c) {
if (strpos($hash, $c . $c . $c . $c)) { if (strpos($hash, $c . $c . $c . $c)) {
$hash = str_replace($c . $c . $c . $c, $c . $c . $c, $hash); $hash = str_replace($c . $c . $c . $c, $c . $c . $c, $hash);
$search = true; $search = true;
} }
} }
} }
$hash = explode(':', $hash);
$block_data = $hash[0];
$double_block_data = $hash[1];
// (struct.unpack("<Q", base64.b64decode(h[i:i + 7] + "=") + "\x00\x00\x00")[0] for i in range(len(h) - 6)))
$result = array( $hash = explode(':', $hash);
$block_size, list($block_data, $double_block_data) = $hash;
$this->get_all_7_char_chunks($block_data),
$this->get_all_7_char_chunks($double_block_data) return [
); $blockSize,
return $result; $this->getAll7CharChunks($block_data),
$this->getAll7CharChunks($double_block_data)
];
} }
public function get_all_7_char_chunks($hash) private function getAll7CharChunks($hash)
{ {
$results = array(); $results = array();
for ($i = 0; $i < strlen($hash) - 6; $i++) { for ($i = 0; $i < strlen($hash) - 6; $i++) {
@ -56,16 +50,22 @@ class FuzzyCorrelateSsdeep extends AppModel
return $results; return $results;
} }
/**
* @param string $hash
* @param int $attributeId
* @return array
*/
public function query_ssdeep_chunks($hash, $attributeId) public function query_ssdeep_chunks($hash, $attributeId)
{ {
$chunks = $this->ssdeep_prepare($hash); $chunks = $this->ssdeep_prepare($hash);
$bothPartChunks = array_merge($chunks[1], $chunks[2]);
// Original algo from article https://www.virusbulletin.com/virusbulletin/2015/11/optimizing-ssdeep-use-scale // Original algo from article https://www.virusbulletin.com/virusbulletin/2015/11/optimizing-ssdeep-use-scale
// also propose to insert chunk size to database, but current database schema doesn't contain that column. // also propose to insert chunk size to database, but current database schema doesn't contain that column.
// This optimisation can be add in future versions. // This optimisation can be add in future versions.
$result = $this->find('column', array( $result = $this->find('column', array(
'conditions' => array( 'conditions' => array(
'FuzzyCorrelateSsdeep.chunk' => array_merge($chunks[1], $chunks[2]), 'FuzzyCorrelateSsdeep.chunk' => $bothPartChunks,
), ),
'fields' => array('FuzzyCorrelateSsdeep.attribute_id'), 'fields' => array('FuzzyCorrelateSsdeep.attribute_id'),
'unique' => true, 'unique' => true,
@ -73,15 +73,11 @@ class FuzzyCorrelateSsdeep extends AppModel
$toSave = []; $toSave = [];
$attributeId = (int) $attributeId; $attributeId = (int) $attributeId;
foreach (array(1, 2) as $type) { foreach ($bothPartChunks as $chunk) {
foreach ($chunks[$type] as $chunk) { $toSave[] = [$attributeId, $chunk];
$toSave[] = [$attributeId, $chunk];
}
}
if (!empty($toSave)) {
$db = $this->getDataSource();
$db->insertMulti($this->table, ['attribute_id', 'chunk'], $toSave);
} }
$db = $this->getDataSource();
$db->insertMulti($this->table, ['attribute_id', 'chunk'], $toSave);
return $result; return $result;
} }

50
app/Model/Galaxy.php
View File

@ -264,7 +264,7 @@ class Galaxy extends AppModel
$fields = array('galaxy_cluster_id', 'key', 'value'); $fields = array('galaxy_cluster_id', 'key', 'value');
$db->insertMulti('galaxy_elements', $fields, $elements); $db->insertMulti('galaxy_elements', $fields, $elements);
} }
$allRelations = array_merge($allRelations, $relations); array_push($allRelations, ...$relations);
} }
// Save relation as last part when all clusters are created // Save relation as last part when all clusters are created
if (!empty($allRelations)) { if (!empty($allRelations)) {
@ -287,24 +287,42 @@ class Galaxy extends AppModel
if (empty($galaxy['uuid'])) { if (empty($galaxy['uuid'])) {
return false; return false;
} }
$existingGalaxy = $this->find('first', array(
$existingGalaxy = $this->find('first', [
'recursive' => -1, 'recursive' => -1,
'conditions' => array('Galaxy.uuid' => $galaxy['uuid']) 'conditions' => ['Galaxy.uuid' => $galaxy['uuid']],
)); ]);
if (empty($existingGalaxy)) {
if ($user['Role']['perm_site_admin'] || $user['Role']['perm_galaxy_editor']) { unset($galaxy['id']);
$this->create(); if (!empty($existingGalaxy)) {
unset($galaxy['id']); // check if provided galaxy has the same fields as galaxy that are saved in database
$this->save($galaxy); $fieldsToSave = [];
$existingGalaxy = $this->find('first', array( foreach (array_keys(array_intersect_key($existingGalaxy, $galaxy)) as $key) {
'recursive' => -1, if ($existingGalaxy['Galaxy'][$key] != $galaxy[$key]) {
'conditions' => array('Galaxy.id' => $this->id) $fieldsToSave[$key] = $galaxy[$key];
)); }
} else {
return false;
} }
} else {
$fieldsToSave = $galaxy;
} }
return $existingGalaxy;
if (empty($fieldsToSave) && !empty($existingGalaxy)) {
return $existingGalaxy; // galaxy already exists and galaxy fields are the same
}
if (!$user['Role']['perm_site_admin'] && !$user['Role']['perm_galaxy_editor']) {
return false; // user has no permission to modify galaxy
}
if (empty($existingGalaxy)) {
$this->create();
}
$this->save($fieldsToSave);
return $this->find('first', [
'recursive' => -1,
'conditions' => ['Galaxy.id' => $this->id],
]);
} }
/** /**

31
app/Model/GalaxyElement.php
View File

@ -48,37 +48,6 @@ class GalaxyElement extends AppModel
$this->saveMany($tempElements); $this->saveMany($tempElements);
} }
public function update($galaxy_id, $oldClusters, $newClusters)
{
$elementsToSave = array();
// Since we are dealing with flat files as the end all be all content, we are safe to just drop all of the old clusters and recreate them.
foreach ($oldClusters as $oldCluster) {
$this->deleteAll(array('GalaxyElement.galaxy_cluster_id' => $oldCluster['GalaxyCluster']['id']));
}
foreach ($newClusters as $newCluster) {
$tempCluster = array();
foreach ($newCluster as $key => $value) {
// Don't store the reserved fields as elements
if ($key == 'description' || $key == 'value') {
continue;
}
if (is_array($value)) {
foreach ($value as $arrayElement) {
$tempCluster[] = array('key' => $key, 'value' => $arrayElement);
}
} else {
$tempCluster[] = array('key' => $key, 'value' => $value);
}
}
foreach ($tempCluster as $key => $value) {
$tempCluster[$key]['galaxy_cluster_id'] = $oldCluster['GalaxyCluster']['id'];
}
$elementsToSave = array_merge($elementsToSave, $tempCluster);
}
$this->saveMany($elementsToSave);
}
public function captureElements($user, $elements, $clusterId) public function captureElements($user, $elements, $clusterId)
{ {
$tempElements = array(); $tempElements = array();

10
app/Model/MispObject.php
View File

@ -153,6 +153,8 @@ class MispObject extends AppModel
'object_name' => array('function' => 'set_filter_object_name'), 'object_name' => array('function' => 'set_filter_object_name'),
'object_template_uuid' => array('function' => 'set_filter_object_template_uuid'), 'object_template_uuid' => array('function' => 'set_filter_object_template_uuid'),
'object_template_version' => array('function' => 'set_filter_object_template_version'), 'object_template_version' => array('function' => 'set_filter_object_template_version'),
'first_seen' => array('function' => 'set_filter_seen'),
'last_seen' => array('function' => 'set_filter_seen'),
'deleted' => array('function' => 'set_filter_deleted') 'deleted' => array('function' => 'set_filter_deleted')
), ),
'Event' => array( 'Event' => array(
@ -181,8 +183,8 @@ class MispObject extends AppModel
'deleted' => array('function' => 'set_filter_deleted'), 'deleted' => array('function' => 'set_filter_deleted'),
'timestamp' => array('function' => 'set_filter_timestamp'), 'timestamp' => array('function' => 'set_filter_timestamp'),
'attribute_timestamp' => array('function' => 'set_filter_timestamp'), 'attribute_timestamp' => array('function' => 'set_filter_timestamp'),
'first_seen' => array('function' => 'set_filter_seen'), //'first_seen' => array('function' => 'set_filter_seen'),
'last_seen' => array('function' => 'set_filter_seen'), //'last_seen' => array('function' => 'set_filter_seen'),
'to_ids' => array('function' => 'set_filter_to_ids'), 'to_ids' => array('function' => 'set_filter_to_ids'),
'comment' => array('function' => 'set_filter_comment') 'comment' => array('function' => 'set_filter_comment')
) )
@ -1678,7 +1680,9 @@ class MispObject extends AppModel
$results = $this->Sightingdb->attachToObjects($results, $user); $results = $this->Sightingdb->attachToObjects($results, $user);
} }
$params['page'] += 1; $params['page'] += 1;
$results = $this->Allowedlist->removeAllowedlistedFromArray($results, true); foreach ($results as $k => $result) {
$results[$k]['Attribute'] = $this->Allowedlist->removeAllowedlistedFromArray($result['Attribute'], true);
}
$results = array_values($results); $results = array_values($results);
$i = 0; $i = 0;
foreach ($results as $object) { foreach ($results as $object) {

48
app/Model/Module.php
View File

@ -50,6 +50,8 @@ class Module extends AppModel
) )
); );
private $httpSocket = [];
public function validateIPField($value) public function validateIPField($value)
{ {
if (!filter_var($value, FILTER_VALIDATE_IP) === false) { if (!filter_var($value, FILTER_VALIDATE_IP) === false) {
@ -309,16 +311,9 @@ class Module extends AppModel
if (!$serverUrl) { if (!$serverUrl) {
throw new Exception("Module type $moduleFamily is not enabled."); throw new Exception("Module type $moduleFamily is not enabled.");
} }
App::uses('HttpSocketExtended', 'Tools');
$httpSocketSetting = ['timeout' => $timeout]; $httpSocket = $this->initHttpSocket($moduleFamily, $timeout);
$sslSettings = array('ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_verify_peer', 'ssl_cafile');
foreach ($sslSettings as $sslSetting) {
$value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
if ($value && $value !== '') {
$httpSocketSetting[$sslSetting] = $value;
}
}
$httpSocket = new HttpSocketExtended($httpSocketSetting);
$request = []; $request = [];
if ($moduleFamily === 'Cortex') { if ($moduleFamily === 'Cortex') {
if (!empty(Configure::read('Plugin.' . $moduleFamily . '_authkey'))) { if (!empty(Configure::read('Plugin.' . $moduleFamily . '_authkey'))) {
@ -422,4 +417,37 @@ class Module extends AppModel
return false; return false;
} }
/**
* @param string $moduleFamily
* @param int $timeout
* @return HttpSocketExtended|CurlClient
*/
private function initHttpSocket($moduleFamily, $timeout)
{
$unique = "$moduleFamily:$timeout";
if (isset($this->httpSocket[$unique])) {
return $this->httpSocket[$unique];
}
$httpSocketSetting = ['timeout' => $timeout];
$sslSettings = ['ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_cafile'];
foreach ($sslSettings as $sslSetting) {
$value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
if ($value && $value !== '') {
$httpSocketSetting[$sslSetting] = $value;
}
}
if (function_exists('curl_init')) {
App::uses('CurlClient', 'Tools');
$httpSocket = new CurlClient($httpSocketSetting);
} else {
App::uses('HttpSocketExtended', 'Tools');
$httpSocket = new HttpSocketExtended($httpSocketSetting);
}
return $this->httpSocket[$unique] = $httpSocket;
}
} }

17
app/Model/Organisation.php
View File

@ -76,14 +76,28 @@ class Organisation extends AppModel
); );
const ORGANISATION_ASSOCIATIONS = array( const ORGANISATION_ASSOCIATIONS = array(
'AccessLog' => array('table' => 'access_logs', 'fields' => array('org_id')),
'AuditLog' => array('table' => 'audit_logs', 'fields' => array('org_id')),
'Correlation' => array('table' => 'correlations', 'fields' => array('org_id')), 'Correlation' => array('table' => 'correlations', 'fields' => array('org_id')),
'Cerebrate' => array('table' => 'cerebrates', 'fields' => array('org_id')),
'Dashboard' => array('table' => 'dashboards', 'fields' => array('restrict_to_org_id')),
'Event' => array('table' => 'events', 'fields' => array('org_id', 'orgc_id')), 'Event' => array('table' => 'events', 'fields' => array('org_id', 'orgc_id')),
'EventGraph' => array('table' => 'event_graph', 'fields' => array('org_id')),
'Feed' => array('table' => 'feeds', 'fields' => array('orgc_id')),
'GalaxyCluster' => array('table' => 'galaxy_clusters', 'fields' => array('org_id', 'orgc_id')),
'ObjectTemplate' => array('table' => 'object_templates', 'fields' => array('org_id')),
'Job' => array('table' => 'jobs', 'fields' => array('org_id')), 'Job' => array('table' => 'jobs', 'fields' => array('org_id')),
'RestClientHistory' => array('table' => 'rest_client_histories', 'fields' => array('org_id')),
'Server' => array('table' => 'servers', 'fields' => array('org_id', 'remote_org_id')), 'Server' => array('table' => 'servers', 'fields' => array('org_id', 'remote_org_id')),
'ShadowAttribute' => array('table' => 'shadow_attributes', 'fields' => array('org_id', 'event_org_id')), 'ShadowAttribute' => array('table' => 'shadow_attributes', 'fields' => array('org_id', 'event_org_id')),
'SharingGroup' => array('table' => 'sharing_groups', 'fields' => array('org_id')), 'SharingGroup' => array('table' => 'sharing_groups', 'fields' => array('org_id')),
'SharingGroupOrg' => array('table' => 'sharing_group_orgs', 'fields' => array('org_id')), 'SharingGroupOrg' => array('table' => 'sharing_group_orgs', 'fields' => array('org_id')),
'SharingGroupBlueprint' => array('table' => 'sharing_group_blueprints', 'fields' => array('org_id')),
'Sighting' => array('table' => 'sightings', 'fields' => array('org_id')),
'SightingdbOrg' => array('table' => 'sightingdb_orgs', 'fields' => array('org_id')),
'Thread' => array('table' => 'threads', 'fields' => array('org_id')), 'Thread' => array('table' => 'threads', 'fields' => array('org_id')),
'Tag' => array('table' => 'tags', 'fields' => array('org_id')),
'TagCollection' => array('table' => 'tag_collections', 'fields' => array('org_id')),
'User' => array('table' => 'users', 'fields' => array('org_id')) 'User' => array('table' => 'users', 'fields' => array('org_id'))
); );
@ -287,6 +301,9 @@ class Organisation extends AppModel
public function orgMerge($id, $request, $user) public function orgMerge($id, $request, $user)
{ {
$currentOrg = $this->find('first', array('recursive' => -1, 'conditions' => array('Organisation.id' => $id))); $currentOrg = $this->find('first', array('recursive' => -1, 'conditions' => array('Organisation.id' => $id)));
if (isset($currentOrg['Organisation']['restricted_to_domain'])) {
$currentOrg['Organisation']['restricted_to_domain'] = json_encode($currentOrg['Organisation']['restricted_to_domain']);
}
$currentOrgUserCount = $this->User->find('count', array( $currentOrgUserCount = $this->User->find('count', array(
'conditions' => array('User.org_id' => $id) 'conditions' => array('User.org_id' => $id)
)); ));

148
app/Model/Server.php
View File

@ -472,7 +472,21 @@ class Server extends AppModel
return false; return false;
} }
private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, Event $eventModel, $server, $user, $jobId, $force = false, $headers = false, $body = false) /**
* @param array $event
* @param int|string $eventId
* @param array $successes
* @param array $fails
* @param Event $eventModel
* @param array $server
* @param array $user
* @param int $jobId
* @param bool $force
* @param HttpSocketResponseExtended $response
* @return false|void
* @throws Exception
*/
private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, Event $eventModel, $server, $user, $jobId, $force = false, $response)
{ {
// check if the event already exist (using the uuid) // check if the event already exist (using the uuid)
$existingEvent = $eventModel->find('first', [ $existingEvent = $eventModel->find('first', [
@ -485,7 +499,7 @@ class Server extends AppModel
if (!$existingEvent) { if (!$existingEvent) {
// add data for newly imported events // add data for newly imported events
if (isset($event['Event']['protected']) && $event['Event']['protected']) { if (isset($event['Event']['protected']) && $event['Event']['protected']) {
if (!$eventModel->CryptographicKey->validateProtectedEvent($body, $user, $headers['x-pgp-signature'], $event)) { if (!$eventModel->CryptographicKey->validateProtectedEvent($response->body, $user, $response->getHeader('x-pgp-signature'), $event)) {
$fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.'); $fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.');
return false; return false;
} }
@ -505,7 +519,7 @@ class Server extends AppModel
$fails[$eventId] = __('Blocked an edit to an event that was created locally. This can happen if a synchronised event that was created on this instance was modified by an administrator on the remote side.'); $fails[$eventId] = __('Blocked an edit to an event that was created locally. This can happen if a synchronised event that was created on this instance was modified by an administrator on the remote side.');
} else { } else {
if ($existingEvent['Event']['protected']) { if ($existingEvent['Event']['protected']) {
if (!$eventModel->CryptographicKey->validateProtectedEvent($body, $user, $headers['x-pgp-signature'], $existingEvent)) { if (!$eventModel->CryptographicKey->validateProtectedEvent($response->body, $user, $response->getHeader('x-pgp-signature'), $existingEvent)) {
$fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.'); $fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.');
} }
} }
@ -549,12 +563,10 @@ class Server extends AppModel
$params['excludeLocalTags'] = 1; $params['excludeLocalTags'] = 1;
} }
try { try {
$event = $serverSync->fetchEvent($eventId, $params); $response = $serverSync->fetchEvent($eventId, $params);
$headers = $event->headers; $event = $response->json();
$body = $event->body;
$event = $event->json();
} catch (Exception $e) { } catch (Exception $e) {
$this->logException("Failed downloading the event $eventId from remote server {$serverSync->serverId()}", $e); $this->logException("Failed to download the event $eventId from remote server {$serverSync->serverId()} '{$serverSync->serverName()}'", $e);
$fails[$eventId] = __('failed downloading the event'); $fails[$eventId] = __('failed downloading the event');
return false; return false;
} }
@ -568,7 +580,7 @@ class Server extends AppModel
} }
return false; return false;
} }
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $headers, $body); $this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $response);
return true; return true;
} }
@ -2359,23 +2371,21 @@ class Server extends AppModel
return $setting; return $setting;
} }
public function serverSettingsEditValue(array $user, array $setting, $value, $forceSave = false) /**
* @param array|string $user
* @param array $setting
* @param mixed $value
* @param bool $forceSave
* @return mixed|string|true|null
* @throws Exception
*/
public function serverSettingsEditValue($user, array $setting, $value, $forceSave = false)
{ {
if (isset($setting['beforeHook'])) { if (isset($setting['beforeHook'])) {
$beforeResult = call_user_func_array(array($this, $setting['beforeHook']), array($setting['name'], $value)); $beforeResult = $this->{$setting['beforeHook']}($setting['name'], $value);
if ($beforeResult !== true) { if ($beforeResult !== true) {
$this->Log = ClassRegistry::init('Log'); $change = 'There was an issue witch changing ' . $setting['name'] . ' to ' . $value . '. The error message returned is: ' . $beforeResult . 'No changes were made.';
$this->Log->create(); $this->loadLog()->createLogEntry($user, 'serverSettingsEdit', 'Server', 0, 'Server setting issue', $change);
$this->Log->saveOrFailSilently(array(
'org' => $user['Organisation']['name'],
'model' => 'Server',
'model_id' => 0,
'email' => $user['email'],
'action' => 'serverSettingsEdit',
'user_id' => $user['id'],
'title' => 'Server setting issue',
'change' => 'There was an issue witch changing ' . $setting['name'] . ' to ' . $value . '. The error message returned is: ' . $beforeResult . 'No changes were made.',
));
return $beforeResult; return $beforeResult;
} }
} }
@ -2384,7 +2394,7 @@ class Server extends AppModel
if ($setting['type'] === 'boolean') { if ($setting['type'] === 'boolean') {
$value = (bool)$value; $value = (bool)$value;
} else if ($setting['type'] === 'numeric') { } else if ($setting['type'] === 'numeric') {
$value = (int)($value); $value = (int)$value;
} }
if (isset($setting['test'])) { if (isset($setting['test'])) {
if ($setting['test'] instanceof Closure) { if ($setting['test'] instanceof Closure) {
@ -2425,7 +2435,7 @@ class Server extends AppModel
if ($setting['afterHook'] instanceof Closure) { if ($setting['afterHook'] instanceof Closure) {
$afterResult = $setting['afterHook']($setting['name'], $value, $oldValue); $afterResult = $setting['afterHook']($setting['name'], $value, $oldValue);
} else { } else {
$afterResult = call_user_func_array(array($this, $setting['afterHook']), array($setting['name'], $value, $oldValue)); $afterResult = $this->{$setting['afterHook']}($setting['name'], $value, $oldValue);
} }
if ($afterResult !== true) { if ($afterResult !== true) {
$change = 'There was an issue after setting a new setting. The error message returned is: ' . $afterResult; $change = 'There was an issue after setting a new setting. The error message returned is: ' . $afterResult;
@ -2434,9 +2444,8 @@ class Server extends AppModel
} }
} }
return true; return true;
} else {
return __('Something went wrong. MISP tried to save a malformed config file. Setting change reverted.');
} }
return __('Something went wrong. MISP tried to save a malformed config file or you dont have permission to write to config file. Setting change reverted.');
} }
/** /**
@ -2539,10 +2548,10 @@ class Server extends AppModel
'name' => __('Organisation logos'), 'name' => __('Organisation logos'),
'description' => __('The logo used by an organisation on the event index, event view, discussions, proposals, etc. Make sure that the filename is in the org.png format, where org is the case-sensitive organisation name.'), 'description' => __('The logo used by an organisation on the event index, event view, discussions, proposals, etc. Make sure that the filename is in the org.png format, where org is the case-sensitive organisation name.'),
'expected' => array(), 'expected' => array(),
'valid_format' => __('48x48 pixel .png files'), 'valid_format' => __('48x48 pixel .png files or .svg file'),
'path' => APP . 'webroot' . DS . 'img' . DS . 'orgs', 'path' => APP . 'webroot' . DS . 'img' . DS . 'orgs',
'regex' => '.*\.(png|PNG)$', 'regex' => '.*\.(png|svg)$',
'regex_error' => __('Filename must be in the following format: *.png'), 'regex_error' => __('Filename must be in the following format: *.png or *.svg'),
'files' => array(), 'files' => array(),
), ),
'img' => array( 'img' => array(
@ -2578,6 +2587,7 @@ class Server extends AppModel
'read' => $f->isReadable(), 'read' => $f->isReadable(),
'write' => $f->isWritable(), 'write' => $f->isWritable(),
'execute' => $f->isExecutable(), 'execute' => $f->isExecutable(),
'link' => $f->isLink(),
]; ];
} }
} }
@ -4155,12 +4165,13 @@ class Server extends AppModel
private function checkRemoteVersion($HttpSocket) private function checkRemoteVersion($HttpSocket)
{ {
try { try {
$json_decoded_tags = GitTool::getLatestTags($HttpSocket); $tags = GitTool::getLatestTags($HttpSocket);
} catch (Exception $e) { } catch (Exception $e) {
$this->logException('Could not retrieve latest tags from GitHub', $e, LOG_NOTICE);
return false; return false;
} }
// find the latest version tag in the v[major].[minor].[hotfix] format // find the latest version tag in the v[major].[minor].[hotfix] format
foreach ($json_decoded_tags as $tag) { foreach ($tags as $tag) {
if (preg_match('/^v[0-9]+\.[0-9]+\.[0-9]+$/', $tag['name'])) { if (preg_match('/^v[0-9]+\.[0-9]+\.[0-9]+$/', $tag['name'])) {
return $this->checkVersion($tag['name']); return $this->checkVersion($tag['name']);
} }
@ -4182,7 +4193,7 @@ class Server extends AppModel
try { try {
$latestCommit = GitTool::getLatestCommit($HttpSocket); $latestCommit = GitTool::getLatestCommit($HttpSocket);
} catch (Exception $e) { } catch (Exception $e) {
$latestCommit = false; $this->logException('Could not retrieve version from GitHub', $e, LOG_NOTICE);
} }
} }
@ -4202,6 +4213,7 @@ class Server extends AppModel
try { try {
return GitTool::currentBranch(); return GitTool::currentBranch();
} catch (Exception $e) { } catch (Exception $e) {
$this->logException('Could not retrieve current Git branch', $e, LOG_NOTICE);
return false; return false;
} }
} }
@ -4252,38 +4264,38 @@ class Server extends AppModel
'app/files/scripts/misp-opendata', 'app/files/scripts/misp-opendata',
'app/files/scripts/python-maec', 'app/files/scripts/python-maec',
'app/files/scripts/python-stix', 'app/files/scripts/python-stix',
); );
return in_array($submodule, $accepted_submodules_names, true); return in_array($submodule, $accepted_submodules_names, true);
} }
/** /**
* @param string $submodule_name * @param string $submoduleName
* @param string $superproject_submodule_commit_id * @param string $superprojectSubmoduleCommitId
* @return array * @return array
* @throws Exception
*/ */
private function getSubmoduleGitStatus($submodule_name, $superproject_submodule_commit_id) private function getSubmoduleGitStatus($submoduleName, $superprojectSubmoduleCommitId)
{ {
$path = APP . '../' . $submodule_name; $path = APP . '../' . $submoduleName;
$submodule_name = (strpos($submodule_name, '/') >= 0 ? explode('/', $submodule_name) : $submodule_name); $submoduleName = (strpos($submoduleName, '/') >= 0 ? explode('/', $submoduleName) : $submoduleName);
$submodule_name = end($submodule_name); $submoduleName = end($submoduleName);
$submoduleCurrentCommitId = GitTool::submoduleCurrentCommit($path); $submoduleCurrentCommitId = GitTool::currentCommit($path);
$currentTimestamp = GitTool::commitTimestamp($submoduleCurrentCommitId, $path); $currentTimestamp = GitTool::commitTimestamp($submoduleCurrentCommitId, $path);
if ($submoduleCurrentCommitId !== $superproject_submodule_commit_id) { if ($submoduleCurrentCommitId !== $superprojectSubmoduleCommitId) {
$remoteTimestamp = GitTool::commitTimestamp($superproject_submodule_commit_id, $path); $remoteTimestamp = GitTool::commitTimestamp($superprojectSubmoduleCommitId, $path);
} else { } else {
$remoteTimestamp = $currentTimestamp; $remoteTimestamp = $currentTimestamp;
} }
$status = array( $status = array(
'moduleName' => $submodule_name, 'moduleName' => $submoduleName,
'current' => $submoduleCurrentCommitId, 'current' => $submoduleCurrentCommitId,
'currentTimestamp' => $currentTimestamp, 'currentTimestamp' => $currentTimestamp,
'remote' => $superproject_submodule_commit_id, 'remote' => $superprojectSubmoduleCommitId,
'remoteTimestamp' => $remoteTimestamp, 'remoteTimestamp' => $remoteTimestamp,
'upToDate' => '', 'upToDate' => 'error',
'isReadable' => is_readable($path) && is_readable($path . '/.git'), 'isReadable' => is_readable($path) && is_readable($path . '/.git'),
); );
@ -4295,15 +4307,11 @@ class Server extends AppModel
} else { } else {
$status['upToDate'] = 'younger'; $status['upToDate'] = 'younger';
} }
} else {
$status['upToDate'] = 'error';
} }
if ($status['isReadable'] && !empty($status['remoteTimestamp']) && !empty($status['currentTimestamp'])) { if ($status['isReadable'] && !empty($status['remoteTimestamp']) && !empty($status['currentTimestamp'])) {
$date1 = new DateTime(); $date1 = new DateTime("@{$status['remoteTimestamp']}");
$date1->setTimestamp($status['remoteTimestamp']); $date2 = new DateTime("@{$status['currentTimestamp']}");
$date2 = new DateTime();
$date2->setTimestamp($status['currentTimestamp']);
$status['timeDiff'] = $date1->diff($date2); $status['timeDiff'] = $date1->diff($date2);
} else { } else {
$status['upToDate'] = 'error'; $status['upToDate'] = 'error';
@ -4793,11 +4801,11 @@ class Server extends AppModel
$results = [ $results = [
__('User') => $user['User']['email'], __('User') => $user['User']['email'],
__('Role name') => isset($user['Role']['name']) ? $user['Role']['name'] : __('Unknown, outdated instance'), __('Role name') => $user['Role']['name'] ?? __('Unknown, outdated instance'),
__('Sync flag') => isset($user['Role']['perm_sync']) ? ($user['Role']['perm_sync'] ? __('Yes') : __('No')) : __('Unknown, outdated instance'), __('Sync flag') => isset($user['Role']['perm_sync']) ? ($user['Role']['perm_sync'] ? __('Yes') : __('No')) : __('Unknown, outdated instance'),
]; ];
if (isset($response->headers['X-Auth-Key-Expiration'])) { if ($response->getHeader('X-Auth-Key-Expiration')) {
$date = new DateTime($response->headers['X-Auth-Key-Expiration']); $date = new DateTime($response->getHeader('X-Auth-Key-Expiration'));
$results[__('Auth key expiration')] = $date->format('Y-m-d H:i:s'); $results[__('Auth key expiration')] = $date->format('Y-m-d H:i:s');
} }
return $results; return $results;
@ -4935,6 +4943,28 @@ class Server extends AppModel
return $this->saveMany($toSave, ['validate' => false, 'fields' => ['authkey']]); return $this->saveMany($toSave, ['validate' => false, 'fields' => ['authkey']]);
} }
/**
* @param string $encryptionKey
* @return bool
* @throws Exception
*/
public function isEncryptionKeyValid($encryptionKey)
{
$servers = $this->find('list', [
'fields' => ['Server.id', 'Server.authkey'],
]);
foreach ($servers as $id => $authkey) {
if (EncryptedValue::isEncrypted($authkey)) {
try {
BetterSecurity::decrypt(substr($authkey, 2), $encryptionKey);
} catch (Exception $e) {
throw new Exception("Could not decrypt auth key for server #$id", 0, $e);
}
}
}
return true;
}
/** /**
* Return all Attribute and Object types * Return all Attribute and Object types
*/ */
@ -6143,6 +6173,14 @@ class Server extends AppModel
'type' => 'boolean', 'type' => 'boolean',
'null' => true, 'null' => true,
], ],
'block_publishing_for_same_creator' => [
'level' => self::SETTING_OPTIONAL,
'description' => __('Enabling this setting will make MISP block event publishing in the case of the publisher being the same user as the event creator.'),
'value' => false,
'test' => 'testBool',
'type' => 'boolean',
'null' => true,
],
'self_update' => [ 'self_update' => [
'level' => self::SETTING_CRITICAL, 'level' => self::SETTING_CRITICAL,
'description' => __('Enable the GUI button for MISP self-update on the Diagnostics page.'), 'description' => __('Enable the GUI button for MISP self-update on the Diagnostics page.'),

8
app/Model/Sighting.php
View File

@ -1017,16 +1017,16 @@ class Sighting extends AppModel
* @return TmpFileTool * @return TmpFileTool
* @throws Exception * @throws Exception
*/ */
public function restSearch(array $user, $returnFormat, $filters) public function restSearch(array $user, $returnFormat, array $filters)
{ {
$allowedContext = array('event', 'attribute'); $allowedContext = array('event', 'attribute');
// validate context // validate context
if (isset($filters['context']) && !in_array($filters['context'], $allowedContext, true)) { if (isset($filters['context']) && !in_array($filters['context'], $allowedContext, true)) {
throw new MethodNotAllowedException(__('Invalid context %s.', $filters['context'])); throw new BadRequestException(__('Invalid context %s.', $filters['context']));
} }
// ensure that an id or uuid is provided if context is set // ensure that an id or uuid is provided if context is set
if (!empty($filters['context']) && !(isset($filters['id']) || isset($filters['uuid'])) ) { if (!empty($filters['context']) && !(isset($filters['id']) || isset($filters['uuid'])) ) {
throw new MethodNotAllowedException(__('An ID or UUID must be provided if the context is set.')); throw new BadRequestException(__('An ID or UUID must be provided if the context is set.'));
} }
if (!isset($this->validFormats[$returnFormat][1])) { if (!isset($this->validFormats[$returnFormat][1])) {
@ -1396,7 +1396,7 @@ class Sighting extends AppModel
try { try {
$sightings = $serverSync->fetchSightingsForEvents($chunk); $sightings = $serverSync->fetchSightingsForEvents($chunk);
} catch (Exception $e) { } catch (Exception $e) {
$this->logException("Failed downloading the sightings from {$serverSync->server()['Server']['name']}.", $e); $this->logException("Failed to download sightings from {$serverSync->server()['Server']['name']}.", $e);
continue; continue;
} }

30
app/Model/SystemSetting.php
View File

@ -46,7 +46,7 @@ class SystemSetting extends AppModel
{ {
/** @var self $systemSetting */ /** @var self $systemSetting */
$systemSetting = ClassRegistry::init('SystemSetting'); $systemSetting = ClassRegistry::init('SystemSetting');
if (!$systemSetting->databaseExists()) { if (!$systemSetting->tableExists()) {
return; return;
} }
$settings = $systemSetting->getSettings(); $settings = $systemSetting->getSettings();
@ -58,7 +58,7 @@ class SystemSetting extends AppModel
} }
} }
public function databaseExists() private function tableExists()
{ {
$tables = ConnectionManager::getDataSource($this->useDbConfig)->listSources(); $tables = ConnectionManager::getDataSource($this->useDbConfig)->listSources();
return in_array('system_settings', $tables, true); return in_array('system_settings', $tables, true);
@ -154,6 +154,32 @@ class SystemSetting extends AppModel
return $this->saveMany($toSave); return $this->saveMany($toSave);
} }
/**
* Check if provided encryption key is valid for all encrypted settings
* @param string $encryptionKey
* @return bool
* @throws Exception
*/
public function isEncryptionKeyValid($encryptionKey)
{
$settings = $this->find('list', [
'fields' => ['SystemSetting.setting', 'SystemSetting.value'],
]);
foreach ($settings as $setting => $value) {
if (!self::isSensitive($setting)) {
continue;
}
if (EncryptedValue::isEncrypted($value)) {
try {
BetterSecurity::decrypt(substr($value, 2), $encryptionKey);
} catch (Exception $e) {
throw new Exception("Could not decrypt `$setting` setting.", 0, $e);
}
}
}
return true;
}
/** /**
* Sensitive setting are passwords or api keys. * Sensitive setting are passwords or api keys.
* @param string $setting Setting name * @param string $setting Setting name

126
app/Model/User.php
View File

@ -659,21 +659,18 @@ class User extends AppModel
public function getUserById($id) public function getUserById($id)
{ {
if (empty($id)) { if (empty($id)) {
throw new NotFoundException('Invalid user ID.'); throw new InvalidArgumentException('Invalid user ID.');
} }
return $this->find( return $this->find('first', [
'first', 'conditions' => ['User.id' => $id],
array( 'recursive' => -1,
'conditions' => array('User.id' => $id), 'contain' => [
'recursive' => -1, 'Organisation',
'contain' => array( 'Role',
'Organisation', 'Server',
'Role', 'UserSetting',
'Server', ]
'UserSetting', ]);
)
)
);
} }
/** /**
@ -740,7 +737,7 @@ class User extends AppModel
], ],
]); ]);
if (empty($user)) { if (empty($user)) {
return $user; return null;
} }
return $this->rearrangeToAuthForm($user); return $this->rearrangeToAuthForm($user);
} }
@ -861,6 +858,10 @@ class User extends AppModel
return true; return true;
} }
if (!isset($user['User'])) {
throw new InvalidArgumentException("Invalid user model provided.");
}
if ($user['User']['disabled'] || !$this->checkIfUserIsValid($user['User'])) { if ($user['User']['disabled'] || !$this->checkIfUserIsValid($user['User'])) {
return true; return true;
} }
@ -937,6 +938,11 @@ class User extends AppModel
*/ */
public function describeAuthFields() public function describeAuthFields()
{ {
static $fields; // generate array just once
if ($fields) {
return $fields;
}
$fields = $this->schema(); $fields = $this->schema();
// Do not include keys, because they are big and usually not necessary // Do not include keys, because they are big and usually not necessary
unset($fields['gpgkey']); unset($fields['gpgkey']);
@ -1105,13 +1111,18 @@ class User extends AppModel
return $hashed; return $hashed;
} }
public function createInitialUser($org_id) /**
* @param int $orgId
* @return string User auth key
* @throws Exception
*/
public function createInitialUser($orgId)
{ {
$authKey = $this->generateAuthKey(); $authKey = $this->generateAuthKey();
$admin = array('User' => array( $admin = array('User' => array(
'id' => 1, 'id' => 1,
'email' => 'admin@admin.test', 'email' => 'admin@admin.test',
'org_id' => $org_id, 'org_id' => $orgId,
'password' => 'admin', 'password' => 'admin',
'confirm_password' => 'admin', 'confirm_password' => 'admin',
'authkey' => $authKey, 'authkey' => $authKey,
@ -1123,7 +1134,6 @@ class User extends AppModel
$this->validator()->remove('password'); // password is too simple, remove validation $this->validator()->remove('password'); // password is too simple, remove validation
$this->save($admin); $this->save($admin);
if (!empty(Configure::read("Security.advanced_authkeys"))) { if (!empty(Configure::read("Security.advanced_authkeys"))) {
$this->AuthKey = ClassRegistry::init('AuthKey');
$newKey = [ $newKey = [
'authkey' => $authKey, 'authkey' => $authKey,
'user_id' => 1, 'user_id' => 1,
@ -2068,12 +2078,10 @@ class User extends AppModel
return false; return false;
} }
$cutoff = $redis->get('misp:session_destroy:' . $id); list($cutoff, $allcutoff) = $redis->mGet(['misp:session_destroy:' . $id, 'misp:session_destroy:all']);
$allcutoff = $redis->get('misp:session_destroy:all');
if ( if (
empty($cutoff) || empty($cutoff) ||
( (
!empty($cutoff) &&
!empty($allcutoff) && !empty($allcutoff) &&
$allcutoff < $cutoff $allcutoff < $cutoff
) )
@ -2156,7 +2164,7 @@ class User extends AppModel
if (!ctype_alnum($token)) { if (!ctype_alnum($token)) {
return false; return false;
} }
$redis = $this->setupRedis(); $redis = RedisTool::init();
$userId = $redis->get('misp:forgot:' . $token); $userId = $redis->get('misp:forgot:' . $token);
if (empty($userId)) { if (empty($userId)) {
return false; return false;
@ -2167,8 +2175,78 @@ class User extends AppModel
public function purgeForgetToken($token) public function purgeForgetToken($token)
{ {
$redis = $this->setupRedis(); $redis = RedisTool::init();
$userId = $redis->del('misp:forgot:' . $token); $redis->del('misp:forgot:' . $token);
return true; return true;
} }
/**
* Create default Role, Organisation and User
* @return string|null Created user auth key
* @throws Exception
*/
public function init()
{
if (!$this->Role->hasAny()) {
$siteAdmin = ['Role' => [
'id' => 1,
'name' => 'Site Admin',
'permission' => 3,
'perm_add' => 1,
'perm_modify' => 1,
'perm_modify_org' => 1,
'perm_publish' => 1,
'perm_sync' => 1,
'perm_admin' => 1,
'perm_audit' => 1,
'perm_auth' => 1,
'perm_site_admin' => 1,
'perm_regexp_access' => 1,
'perm_sharing_group' => 1,
'perm_template' => 1,
'perm_tagger' => 1,
]];
$this->Role->save($siteAdmin);
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
if (!$this->isMysql()) {
$sql = "SELECT setval('roles_id_seq', (SELECT MAX(id) FROM roles));";
$this->Role->query($sql);
}
}
if (!$this->Organisation->hasAny(['Organisation.local' => true])) {
$this->runUpdates();
$org = ['Organisation' => [
'id' => 1,
'name' => !empty(Configure::read('MISP.org')) ? Configure::read('MISP.org') : 'ADMIN',
'description' => 'Automatically generated admin organisation',
'type' => 'ADMIN',
'date_created' => date('Y-m-d H:i:s'),
'local' => 1,
]];
$this->Organisation->save($org);
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
if (!$this->isMysql()) {
$sql = "SELECT setval('organisations_id_seq', (SELECT MAX(id) FROM organisations));";
$this->Organisation->query($sql);
}
$orgId = $this->Organisation->id;
}
if (!$this->hasAny()) {
if (!isset($orgId)) {
$hostOrg = $this->Organisation->find('first', array('conditions' => array('Organisation.name' => Configure::read('MISP.org'), 'Organisation.local' => true), 'recursive' => -1));
if (!empty($hostOrg)) {
$orgId = $hostOrg['Organisation']['id'];
} else {
$firstOrg = $this->Organisation->find('first', array('conditions' => array('Organisation.local' => true), 'order' => 'Organisation.id ASC'));
$orgId = $firstOrg['Organisation']['id'];
}
}
$this->runUpdates();
return $this->createInitialUser($orgId);
}
return null;
}
} }

79
app/Model/UserLoginProfile.php
View File

@ -36,22 +36,55 @@ class UserLoginProfile extends AppModel
]; ];
const BROWSER_CACHE_DIR = APP . DS . 'tmp' . DS . 'browscap'; const BROWSER_CACHE_DIR = APP . DS . 'tmp' . DS . 'browscap';
const BROWSER_INI_FILE = APP . DS . 'files' . DS . 'browscap'. DS . 'browscap.ini'; // Browscap file managed by MISP - https://browscap.org/stream?q=Lite_PHP_BrowsCapINI const BROWSER_INI_FILE = APP . DS . 'files' . DS . 'browscap'. DS . 'browscap.ini.gz'; // Browscap file managed by MISP - https://browscap.org/stream?q=Lite_PHP_BrowsCapINI
const GEOIP_DB_FILE = APP . DS . 'files' . DS . 'geo-open' . DS . 'GeoOpen-Country.mmdb'; // GeoIP file managed by MISP - https://data.public.lu/en/datasets/geo-open-ip-address-geolocation-per-country-in-mmdb-format/ const GEOIP_DB_FILE = APP . DS . 'files' . DS . 'geo-open' . DS . 'GeoOpen-Country.mmdb'; // GeoIP file managed by MISP - https://data.public.lu/en/datasets/geo-open-ip-address-geolocation-per-country-in-mmdb-format/
private $userProfile; private $userProfile;
private $knownUserProfiles = []; private $knownUserProfiles = [];
private function _buildBrowscapCache() private function browscapGetBrowser()
{ {
$this->log("Browscap - building new cache from browscap.ini file.", LOG_INFO);
$fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR);
$cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache);
$logger = new \Monolog\Logger('name'); $logger = new \Monolog\Logger('name');
$bc = new \BrowscapPHP\BrowscapUpdater($cache, $logger);
$bc->convertFile(UserLoginProfile::BROWSER_INI_FILE); if (function_exists('apcu_fetch')) {
App::uses('ApcuCacheTool', 'Tools');
$cache = new ApcuCacheTool('misp:browscap');
} else {
$fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR);
$cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache);
}
try {
$bc = new \BrowscapPHP\Browscap($cache, $logger);
return $bc->getBrowser();
} catch (\BrowscapPHP\Exception $e) {
$this->log("Browscap - building new cache from browscap.ini file.", LOG_INFO);
$bcUpdater = new \BrowscapPHP\BrowscapUpdater($cache, $logger);
$bcUpdater->convertString(FileAccessTool::readCompressedFile(UserLoginProfile::BROWSER_INI_FILE));
}
$bc = new \BrowscapPHP\Browscap($cache, $logger);
return $bc->getBrowser();
}
/**
* @param string $ip
* @return string|null
*/
public function countryByIp($ip)
{
if (class_exists('GeoIp2\Database\Reader')) {
$geoDbReader = new GeoIp2\Database\Reader(UserLoginProfile::GEOIP_DB_FILE);
try {
$record = $geoDbReader->country($ip);
return $record->country->isoCode;
} catch (InvalidArgumentException $e) {
$this->logException("Could not get country code for IP address", $e, LOG_NOTICE);
return null;
}
}
return null;
} }
public function beforeSave($options = []) public function beforeSave($options = [])
@ -76,16 +109,7 @@ class UserLoginProfile extends AppModel
if (!$this->userProfile) { if (!$this->userProfile) {
// below uses https://github.com/browscap/browscap-php // below uses https://github.com/browscap/browscap-php
if (class_exists('\BrowscapPHP\Browscap')) { if (class_exists('\BrowscapPHP\Browscap')) {
try { $browser = $this->browscapGetBrowser();
$fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR);
$cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache);
$logger = new \Monolog\Logger('name');
$bc = new \BrowscapPHP\Browscap($cache, $logger);
$browser = $bc->getBrowser();
} catch (\BrowscapPHP\Exception $e) {
$this->_buildBrowscapCache();
return $this->_getUserProfile();
}
} else { } else {
// a primitive OS & browser extraction capability // a primitive OS & browser extraction capability
$ua = $_SERVER['HTTP_USER_AGENT'] ?? null; $ua = $_SERVER['HTTP_USER_AGENT'] ?? null;
@ -100,18 +124,7 @@ class UserLoginProfile extends AppModel
$browser->browser = "browser"; $browser->browser = "browser";
} }
$ip = $this->_remoteIp(); $ip = $this->_remoteIp();
if (class_exists('GeoIp2\Database\Reader')) { $country = $this->countryByIp($ip) ?? 'None';
try {
$geoDbReader = new GeoIp2\Database\Reader(UserLoginProfile::GEOIP_DB_FILE);
$record = $geoDbReader->country($ip);
$country = $record->country->isoCode;
} catch (InvalidArgumentException $e) {
$this->logException("Could not get country code for IP address", $e);
$country = 'None';
}
} else {
$country = 'None';
}
$this->userProfile = [ $this->userProfile = [
'user_agent' => $_SERVER['HTTP_USER_AGENT'] ?? null, 'user_agent' => $_SERVER['HTTP_USER_AGENT'] ?? null,
'ip' => $ip, 'ip' => $ip,
@ -247,13 +260,13 @@ class UserLoginProfile extends AppModel
public function emailNewLogin(array $user) public function emailNewLogin(array $user)
{ {
if (!Configure::read('MISP.disable_emailing')) { if (!Configure::read('MISP.disable_emailing')) {
$date_time = date('c'); $user = $this->User->getUserById($user['id']); // fetch in database format
$datetime = date('c'); // ISO 8601 date
$body = new SendEmailTemplate('userloginprofile_newlogin'); $body = new SendEmailTemplate('userloginprofile_newlogin');
$body->set('userLoginProfile', $this->User->UserLoginProfile->_getUserProfile()); $body->set('userLoginProfile', $this->User->UserLoginProfile->_getUserProfile());
$body->set('baseurl', Configure::read('MISP.baseurl')); $body->set('baseurl', Configure::read('MISP.baseurl'));
$body->set('misp_org', Configure::read('MISP.org')); $body->set('misp_org', Configure::read('MISP.org'));
$body->set('date_time', $date_time); $body->set('date_time', $datetime);
// Fetch user that contains also PGP or S/MIME keys for e-mail encryption // Fetch user that contains also PGP or S/MIME keys for e-mail encryption
$this->User->sendEmail($user, $body, false, "[" . Configure::read('MISP.org') . " MISP] New sign in."); $this->User->sendEmail($user, $body, false, "[" . Configure::read('MISP.org') . " MISP] New sign in.");
} }

3
app/Model/Warninglist.php
View File

@ -390,8 +390,7 @@ class Warninglist extends AppModel
$warninglistId = (int)$this->id; $warninglistId = (int)$this->id;
$result = true; $result = true;
$keys = array_keys($list['list']); if (JsonTool::arrayIsList($list['list'])) {
if ($keys === array_keys($keys)) {
foreach (array_chunk($list['list'], 1000) as $chunk) { foreach (array_chunk($list['list'], 1000) as $chunk) {
$valuesToInsert = []; $valuesToInsert = [];
foreach ($chunk as $value) { foreach ($chunk as $value) {

276
app/Plugin/BinaryFileCache/Engine/BinaryFileEngine.php Normal file
View File

@ -0,0 +1,276 @@
<?php
require_once CAKE_CORE_INCLUDE_PATH . '/Cake/Cache/CacheEngine.php';
require_once CAKE_CORE_INCLUDE_PATH . '/Cake/Cache/Engine/FileEngine.php';
/**
* This is faster version of FileEngine cache engine
* - stores file in binary format, so no need to change line endings
* - uses igbinary if supported for faster serialization/deserialization and smaller cache files
* - default file mask is set to 0660, so cache files are not readable by other users
* - optimised file saving and fetching
*/
class BinaryFileEngine extends FileEngine
{
const BINARY_CACHE_TIME_LENGTH = 8;
private $useIgbinary = false;
public function init($settings = [])
{
$settings += [
'engine' => 'BinaryFile',
'path' => CACHE,
'prefix' => 'cake_',
'serialize' => true,
'mask' => 0660,
];
CacheEngine::init($settings);
$this->useIgbinary = function_exists('igbinary_serialize');
if (substr($this->settings['path'], -1) !== DS) {
$this->settings['path'] .= DS;
}
if (!empty($this->_groupPrefix)) {
$this->_groupPrefix = str_replace('_', DS, $this->_groupPrefix);
}
return $this->_active();
}
/**
* @param string $key
* @param mixed $data
* @param int $duration
* @return bool
*/
public function write($key, $data, $duration)
{
if (!$this->_init) {
return false;
}
$fileInfo = $this->cacheFilePath($key);
$resource = $this->createFile($fileInfo);
if (!$resource) {
return false;
}
if (!empty($this->settings['serialize'])) {
if ($this->useIgbinary) {
$data = igbinary_serialize($data);
if ($data === null) {
return false;
}
} else {
$data = serialize($data);
}
}
$expires = pack("q", time() + $duration);
flock($resource, LOCK_EX);
ftruncate($resource, 0);
$result = fwrite($resource, $expires);
if ($result !== self::BINARY_CACHE_TIME_LENGTH) {
$this->handleWriteError($fileInfo);
fclose($resource);
return false;
}
$result = fwrite($resource, $data);
if ($result !== strlen($data)) {
$this->handleWriteError($fileInfo);
fclose($resource);
return false;
}
fclose($resource);
return true;
}
/**
* @param string $key
* @return false|mixed|string
*/
public function read($key)
{
if (!$this->_init) {
return false;
}
$fileInfo = $this->cacheFilePath($key);
$exists = file_exists($fileInfo->getPathname());
if (!$exists) {
return false;
}
$resource = $this->openFile($fileInfo);
if (!$resource) {
return false;
}
$time = time();
flock($resource, LOCK_SH);
$cacheTimeBinary = fread($resource, self::BINARY_CACHE_TIME_LENGTH);
if (!$cacheTimeBinary) {
fclose($resource);
return false;
}
$cacheTime = $this->unpackCacheTime($cacheTimeBinary);
if ($cacheTime < $time || ($time + $this->settings['duration']) < $cacheTime) {
fclose($resource);
return false; // already expired
}
$data = stream_get_contents($resource, null, self::BINARY_CACHE_TIME_LENGTH);
fclose($resource);
if (!empty($this->settings['serialize'])) {
if ($this->useIgbinary) {
$data = igbinary_unserialize($data);
} else {
$data = unserialize($data);
}
}
return $data;
}
/**
* @param string $path
* @param int $now
* @param int $threshold
* @return void
*/
protected function _clearDirectory($path, $now, $threshold)
{
$prefixLength = strlen($this->settings['prefix']);
if (!is_dir($path)) {
return;
}
$dir = dir($path);
if ($dir === false) {
return;
}
while (($entry = $dir->read()) !== false) {
if (substr($entry, 0, $prefixLength) !== $this->settings['prefix']) {
continue;
}
try {
$file = new SplFileObject($path . $entry, 'rb');
} catch (Exception $e) {
continue;
}
if ($threshold) {
$mtime = $file->getMTime();
if ($mtime > $threshold) {
continue;
}
$expires = $this->unpackCacheTime($file->fread(self::BINARY_CACHE_TIME_LENGTH));
if ($expires > $now) {
continue;
}
}
if ($file->isFile()) {
$filePath = $file->getRealPath();
$file = null;
@unlink($filePath);
}
}
}
/**
* @param SplFileInfo $fileInfo
* @return false|resource
*/
private function createFile(SplFileInfo $fileInfo)
{
$exists = file_exists($fileInfo->getPathname());
if (!$exists) {
$resource = $this->openFile($fileInfo, 'cb');
if ($resource && !chmod($fileInfo->getPathname(), (int)$this->settings['mask'])) {
trigger_error(__d(
'cake_dev', 'Could not apply permission mask "%s" on cache file "%s"',
[$fileInfo->getPathname(), $this->settings['mask']]), E_USER_WARNING);
}
return $resource;
}
return $this->openFile($fileInfo, 'cb');
}
/**
* @param SplFileInfo $fileInfo
* @param string $mode
* @return false|resource
*/
private function openFile(SplFileInfo $fileInfo, $mode = 'rb')
{
$resource = fopen($fileInfo->getPathname(), $mode);
if (!$resource) {
trigger_error(__d(
'cake_dev', 'Could not open file %s',
array($fileInfo->getPathname())), E_USER_WARNING);
}
return $resource;
}
/**
* @param string $key
* @return SplFileInfo
*/
private function cacheFilePath(string $key): SplFileInfo
{
$groups = null;
if (!empty($this->_groupPrefix)) {
$groups = vsprintf($this->_groupPrefix, $this->groups());
}
$dir = $this->settings['path'] . $groups;
if (!is_dir($dir)) {
mkdir($dir, 0775, true);
}
$suffix = '.bin';
if ($this->settings['serialize'] && $this->useIgbinary) {
$suffix = '.igbin';
}
return new SplFileInfo($dir . $key . $suffix);
}
/**
* @param SplFileInfo $fileInfo
* @return void
*/
private function handleWriteError(SplFileInfo $fileInfo)
{
unlink($fileInfo->getPathname()); // delete file in case file was just partially written
trigger_error(__d(
'cake_dev', 'Could not write to file %s',
array($fileInfo->getPathname())), E_USER_WARNING);
}
/**
* @param string $cacheTimeBinary
* @return int
*/
private function unpackCacheTime($cacheTimeBinary)
{
if ($cacheTimeBinary === false || strlen($cacheTimeBinary) !== self::BINARY_CACHE_TIME_LENGTH) {
throw new InvalidArgumentException("Invalid cache time in binary format provided '$cacheTimeBinary'");
}
return unpack("q", $cacheTimeBinary)[1];
}
}

2
app/Plugin/EcsLog/Lib/Log/Engine/EcsLog.php
View File

@ -57,7 +57,7 @@ class EcsLog implements CakeLogInterface
'log' => [ 'log' => [
'level' => $type, 'level' => $type,
], ],
'message' => $message, 'message' => JsonTool::escapeNonUnicode($message),
]; ];
static::writeMessage($message); static::writeMessage($message);

5
app/Plugin/OidcAuth/Lib/Oidc.php
View File

@ -28,7 +28,6 @@ class Oidc
$claims = $oidc->getVerifiedClaims(); $claims = $oidc->getVerifiedClaims();
$mispUsername = $claims->email ?? $oidc->requestUserInfo('email'); $mispUsername = $claims->email ?? $oidc->requestUserInfo('email');
if (empty($mispUsername)) { if (empty($mispUsername)) {
$sub = $claims->sub ?? 'UNKNOWN'; $sub = $claims->sub ?? 'UNKNOWN';
throw new Exception("OIDC user $sub doesn't have email address, that is required by MISP."); throw new Exception("OIDC user $sub doesn't have email address, that is required by MISP.");
@ -66,13 +65,13 @@ class Oidc
$roleProperty = $this->getConfig('roles_property', 'roles'); $roleProperty = $this->getConfig('roles_property', 'roles');
$roles = $claims->{$roleProperty} ?? $oidc->requestUserInfo($roleProperty); $roles = $claims->{$roleProperty} ?? $oidc->requestUserInfo($roleProperty);
if ($roles === null) { if ($roles === null) {
$this->log($mispUsername, "Role property `$roleProperty` is missing in claims.", LOG_WARNING); $this->log($mispUsername, "Role property `$roleProperty` is missing in claims, access prohibited.", LOG_WARNING);
return false; return false;
} }
$roleId = $this->getUserRole($roles, $mispUsername); $roleId = $this->getUserRole($roles, $mispUsername);
if ($roleId === null) { if ($roleId === null) {
$this->log($mispUsername, 'No role was assigned.'); $this->log($mispUsername, 'No role was assigned, access prohibited.', LOG_WARNING);
if ($user) { if ($user) {
$this->block($user); $this->block($user);
} }

10
app/Test/AttributeValidationToolTest.php
View File

@ -124,6 +124,16 @@ class AttributeValidationToolTest extends TestCase
]); ]);
} }
public function testRemoveCidrFromIp(): void
{
$this->assertEquals('127.0.0.1', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1/32'));
$this->assertEquals('127.0.0.1/31', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1/31'));
$this->assertEquals('example.com|1234:fd2:5621:1:89::4500', AttributeValidationTool::modifyBeforeValidation('domain|ip', 'example.com|1234:0fd2:5621:0001:0089:0000:0000:4500/128'));
$this->assertEquals('1234:fd2:5621:1:89::4500|80', AttributeValidationTool::modifyBeforeValidation('ip-src|port', '1234:0fd2:5621:0001:0089:0000:0000:4500/128|80'));
$this->assertEquals('1234:fd2:5621:1:89::4500/127|80', AttributeValidationTool::modifyBeforeValidation('ip-src|port', '1234:0fd2:5621:0001:0089:0000:0000:4500/127|80'));
$this->assertEquals('127.0.0.1', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1'));
}
public function testCompressIpv6(): void public function testCompressIpv6(): void
{ {
$this->assertEquals('1234:fd2:5621:1:89::4500', AttributeValidationTool::modifyBeforeValidation('ip-src', '1234:0fd2:5621:0001:0089:0000:0000:4500')); $this->assertEquals('1234:fd2:5621:1:89::4500', AttributeValidationTool::modifyBeforeValidation('ip-src', '1234:0fd2:5621:0001:0089:0000:0000:4500'));

21
app/Test/ComplexTypeToolTest.php
View File

@ -527,10 +527,31 @@ EOT;
public function testCheckFreeTextNonBreakableSpace(): void public function testCheckFreeTextNonBreakableSpace(): void
{ {
$complexTypeTool = new ComplexTypeTool(); $complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText("127.0.0.1\xc2\xa0127.0.0.2"); $results = $complexTypeTool->checkFreeText("127.0.0.1\xc2\xa0127.0.0.2");
$this->assertCount(2, $results); $this->assertCount(2, $results);
$this->assertEquals('127.0.0.1', $results[0]['value']); $this->assertEquals('127.0.0.1', $results[0]['value']);
$this->assertEquals('ip-dst', $results[0]['default_type']); $this->assertEquals('ip-dst', $results[0]['default_type']);
$results = $complexTypeTool->checkFreeText("127.0.0.1\xc2\xa0\xc2\xa0127.0.0.2");
$this->assertCount(2, $results);
$this->assertEquals('127.0.0.1', $results[0]['value']);
$this->assertEquals('ip-dst', $results[0]['default_type']);
}
public function testCheckFreeTextControlCharToSpace(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText("127.0.0.1\x1d127.0.0.2");
$this->assertCount(2, $results);
$this->assertEquals('127.0.0.1', $results[0]['value']);
$this->assertEquals('ip-dst', $results[0]['default_type']);
$results = $complexTypeTool->checkFreeText("127.0.0.1\x1d\x1d127.0.0.2");
$this->assertCount(2, $results);
$this->assertEquals('127.0.0.1', $results[0]['value']);
$this->assertEquals('ip-dst', $results[0]['default_type']);
} }
public function testCheckFreeTextQuoted(): void public function testCheckFreeTextQuoted(): void

8
app/Test/JSONConverterToolTest.php
View File

@ -1,4 +1,5 @@
<?php <?php
require_once __DIR__ . '/../Lib/Tools/JsonTool.php';
require_once __DIR__ . '/../Lib/Tools/JSONConverterTool.php'; require_once __DIR__ . '/../Lib/Tools/JSONConverterTool.php';
use PHPUnit\Framework\TestCase; use PHPUnit\Framework\TestCase;
@ -57,11 +58,6 @@ class JSONConverterToolTest extends TestCase
$jsonNormalWithoutSpaces = preg_replace("/\s+/", "", JSONConverterTool::convert($event)); $jsonNormalWithoutSpaces = preg_replace("/\s+/", "", JSONConverterTool::convert($event));
$this->assertEquals($jsonNormalWithoutSpaces, $jsonStreamWithoutSpaces); $this->assertEquals($jsonNormalWithoutSpaces, $jsonStreamWithoutSpaces);
if (defined('JSON_THROW_ON_ERROR')) { $this->assertTrue(JsonTool::isValid($json));
json_decode($json, true, 512, JSON_THROW_ON_ERROR);
$this->assertTrue(true);
} else {
$this->assertNotNull(json_decode($json));
}
} }
} }

4
app/View/AuthKeys/view.ctp
View File

@ -3,12 +3,12 @@ $keyUsageCsv = null;
if (isset($keyUsage)) { if (isset($keyUsage)) {
$todayString = date('Y-m-d'); $todayString = date('Y-m-d');
$today = strtotime($todayString); $today = strtotime($todayString);
$startDate = key($keyUsage); // oldest date for sparkline $startDate = array_key_first($keyUsage); // oldest date for sparkline
$startDate = strtotime($startDate) - (3600 * 24 * 3); $startDate = strtotime($startDate) - (3600 * 24 * 3);
$keyUsageCsv = 'Date,Close\n'; $keyUsageCsv = 'Date,Close\n';
for ($date = $startDate; $date <= $today; $date += (3600 * 24)) { for ($date = $startDate; $date <= $today; $date += (3600 * 24)) {
$dateAsString = date('Y-m-d', $date); $dateAsString = date('Y-m-d', $date);
$keyUsageCsv .= $dateAsString . ',' . (isset($keyUsage[$dateAsString]) ? $keyUsage[$dateAsString] : 0) . '\n'; $keyUsageCsv .= $dateAsString . ',' . ($keyUsage[$dateAsString] ?? '0') . '\n';
} }
} else { } else {
$lastUsed = null; $lastUsed = null;

12
app/View/Elements/global_menu.ctp
View File

@ -277,11 +277,6 @@
'url' => $baseurl . '/servers/createSync', 'url' => $baseurl . '/servers/createSync',
'requirement' => $isAclSync && !$isSiteAdmin 'requirement' => $isAclSync && !$isSiteAdmin
), ),
array(
'text' => __('Import Server Settings'),
'url' => $baseurl . '/servers/import',
'requirement' => $this->Acl->canAccess('servers', 'import'),
),
array( array(
'text' => __('Remote Servers'), 'text' => __('Remote Servers'),
'url' => $baseurl . '/servers/index', 'url' => $baseurl . '/servers/index',
@ -292,11 +287,6 @@
'url' => $baseurl . '/feeds/index', 'url' => $baseurl . '/feeds/index',
'requirement' => $this->Acl->canAccess('feeds', 'index'), 'requirement' => $this->Acl->canAccess('feeds', 'index'),
), ),
array(
'text' => __('Search Feed Caches'),
'url' => $baseurl . '/feeds/searchCaches',
'requirement' => $this->Acl->canAccess('feeds', 'searchCaches'),
),
array( array(
'text' => __('SightingDB'), 'text' => __('SightingDB'),
'url' => $baseurl . '/sightingdb/index', 'url' => $baseurl . '/sightingdb/index',
@ -313,7 +303,7 @@
'requirement' => $this->Acl->canAccess('cerebrates', 'index'), 'requirement' => $this->Acl->canAccess('cerebrates', 'index'),
), ),
array( array(
'text' => __('List Taxii Servers'), 'text' => __('TAXII Servers'),
'url' => $baseurl . '/TaxiiServers/index', 'url' => $baseurl . '/TaxiiServers/index',
'requirement' => $this->Acl->canAccess('taxiiServers', 'index'), 'requirement' => $this->Acl->canAccess('taxiiServers', 'index'),
), ),

33
app/View/Elements/healthElements/files.ctp
View File

@ -1,3 +1,10 @@
<?php
$humanReadableFilesize = function ($bytes, $dec = 2) {
$size = ['B', 'kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
$factor = floor((strlen($bytes) - 1) / 3);
return sprintf("%.{$dec}f&nbsp;%s", ($bytes / (1024 ** $factor)), $size[$factor]);
};
?>
<div style="border:1px solid #dddddd; margin-top:1px; width:100%; padding:10px"> <div style="border:1px solid #dddddd; margin-top:1px; width:100%; padding:10px">
<p><?php echo __('Below you will find a list of the uploaded files based on type.');?></p> <p><?php echo __('Below you will find a list of the uploaded files based on type.');?></p>
<?php <?php
@ -5,13 +12,13 @@
?> ?>
<h3><?php echo h($file['name']); ?></h3> <h3><?php echo h($file['name']); ?></h3>
<div> <div>
<b><?php echo __('Description');?></b>: <?php echo $file['description']; ?><br /> <b><?php echo __('Description');?></b>: <?php echo $file['description']; ?><br>
<b><?php echo __('Expected Format');?></b>: <?php echo h($file['valid_format']);?><br /> <b><?php echo __('Expected Format');?></b>: <?php echo h($file['valid_format']);?><br>
<b><?php echo __('Path');?></b>: <?php echo h($file['path']);?><br /> <b><?php echo __('Path');?></b>: <?php echo h($file['path']);?><br>
<?php <?php
if (!empty($file['expected'])): if (!empty($file['expected'])):
?> ?>
<b><?php echo __('Files set for each relevant setting');?>:</b><br /> <b><?php echo __('Files set for each relevant setting');?>:</b><br>
<ul> <ul>
<?php foreach ($file['expected'] as $expectedKey => $expectedValue): <?php foreach ($file['expected'] as $expectedKey => $expectedValue):
$colour = 'red'; $colour = 'red';
@ -24,7 +31,7 @@
endif; endif;
?> ?>
</div> </div>
<table class="table table-striped table-hover table-condensed" style="width:600px;"> <table class="table table-striped table-hover table-condensed" style="width:700px">
<tr> <tr>
<th><?php echo __('Filename');?></th> <th><?php echo __('Filename');?></th>
<th><?php echo __('Used by');?></th> <th><?php echo __('Used by');?></th>
@ -35,19 +42,10 @@
<?php <?php
foreach ($file['files'] as $f): foreach ($file['files'] as $f):
$permission = ""; $permission = "";
if ($f['link']) $permission .= "l";
if ($f['read']) $permission .= "r"; if ($f['read']) $permission .= "r";
if ($f['write']) $permission .= "w"; if ($f['write']) $permission .= "w";
if ($f['execute']) $permission .= "x"; if ($f['execute']) $permission .= "x";
$sizeUnit = "B";
if (($f['filesize'] / 1024) > 1) {
$f['filesize'] = $f['filesize'] / 1024;
$sizeUnit = "KB";
if (($f['filesize'] / 1024) > 1) {
$f['filesize'] = $f['filesize'] / 1024;
$sizeUnit = "MB";
}
$f['filesize'] = round($f['filesize'], 1);
}
?> ?>
<tr> <tr>
<td><?php echo h($f['filename']);?></td> <td><?php echo h($f['filename']);?></td>
@ -55,7 +53,7 @@
<?php <?php
if ($k != 'orgs'): if ($k != 'orgs'):
foreach ($file['expected'] as $ek => $ev): foreach ($file['expected'] as $ek => $ev):
if ($f['filename'] == $ev) echo h($ek) . "<br />"; if ($f['filename'] == $ev) echo h($ek) . "<br>";
endforeach; endforeach;
else: else:
echo __('N/A'); echo __('N/A');
@ -63,7 +61,7 @@
?> ?>
</td> </td>
<td width="75px;"> <td width="75px;">
<?php echo h($f['filesize']) . ' ' . $sizeUnit;?> <?= $humanReadableFilesize($f['filesize'], 1) ?>
</td> </td>
<td class="short"> <td class="short">
<?php echo $permission;?> <?php echo $permission;?>
@ -93,5 +91,4 @@
echo $this->Form->end(); echo $this->Form->end();
endforeach; endforeach;
?> ?>
</div> </div>

8
app/View/Pages/administration.ctp
View File

@ -1,11 +1,3 @@
<?php
if (!$isSiteAdmin) exit();
?>
<div class="actions">
<ol class="nav nav-list">
</ol>
</div>
<div class="index"> <div class="index">
<h2><?php echo __('Administrative actions');?></h2> <h2><?php echo __('Administrative actions');?></h2>
<ul> <ul>

4
app/View/Users/totp_new.ctp
View File

@ -1,7 +1,7 @@
<?php echo $this->Flash->render(); ?> <?php echo $this->Flash->render(); ?>
<?php <?php
$detailsHtml = __("To enable TOTP for your account, scan the following QR code with your TOTP application and validate the token.");; $detailsHtml = __("To enable TOTP for your account, scan the following QR code with your TOTP application (for example Google authenticator or KeepassXC) and validate the token.");;
$secretHtml = __("Alternatively you can enter the following secret in your TOTP application: ") . "<pre>" . $secret . "</pre>"; $secretHtml = __("Alternatively you can enter the following secret in your TOTP application. This can be particularly handy in case you don't have a supported application in your working environment. Once the verification is done you'll also get 50 \"paper-based\" login tokens so you don't have to use a TOTP application each time: ") . "<pre>" . $secret . "</pre>";
echo $this->element('/genericElements/Form/genericForm', array( echo $this->element('/genericElements/Form/genericForm', array(
"form" => $this->Form, "form" => $this->Form,

1
app/composer.json
View File

@ -36,6 +36,7 @@
"ext-rdkafka": "Required for publishing events to Kafka broker", "ext-rdkafka": "Required for publishing events to Kafka broker",
"ext-apcu": "To cache data in memory instead of file system", "ext-apcu": "To cache data in memory instead of file system",
"ext-simdjson": "To decode JSON structures faster", "ext-simdjson": "To decode JSON structures faster",
"ext-curl": "For faster remote requests",
"elasticsearch/elasticsearch": "For logging to elasticsearch", "elasticsearch/elasticsearch": "For logging to elasticsearch",
"aws/aws-sdk-php": "To upload samples to S3", "aws/aws-sdk-php": "To upload samples to S3",
"jakub-onderka/openid-connect-php": "For OIDC authentication", "jakub-onderka/openid-connect-php": "For OIDC authentication",

35702
app/files/browscap/browscap.ini
View File

File diff suppressed because it is too large Load Diff

BIN
app/files/browscap/browscap.ini.gz Normal file
View File

Binary file not shown.

79
app/files/scripts/mispzmq/mispzmq.py
View File

@ -4,15 +4,18 @@ from zmq.auth.thread import ThreadAuthenticator
from zmq.utils.monitor import recv_monitor_message from zmq.utils.monitor import recv_monitor_message
import sys import sys
import redis import redis
import json
import os import os
import time import time
import threading import threading
import logging import logging
import typing
import argparse
from pathlib import Path from pathlib import Path
try:
logging.basicConfig(level=logging.INFO, format="%(asctime)s:%(levelname)s:%(name)s:%(message)s") import orjson as json
except ImportError:
import json
def check_pid(pid): def check_pid(pid):
@ -55,10 +58,11 @@ class MispZmq:
socket = None socket = None
pidfile = None pidfile = None
r: redis.StrictRedis redis: redis.StrictRedis
namespace: str namespace: str
def __init__(self): def __init__(self, debug=False):
logging.basicConfig(level=logging.DEBUG if debug else logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
self._logger = logging.getLogger() self._logger = logging.getLogger()
self.tmp_location = Path(__file__).parent.parent / "tmp" self.tmp_location = Path(__file__).parent.parent / "tmp"
@ -67,7 +71,7 @@ class MispZmq:
with open(self.pidfile.as_posix()) as f: with open(self.pidfile.as_posix()) as f:
pid = f.read() pid = f.read()
if check_pid(pid): if check_pid(pid):
raise Exception("mispzmq already running on PID {}".format(pid)) raise Exception(f"mispzmq already running on PID {pid}")
else: else:
# Cleanup # Cleanup
self.pidfile.unlink() self.pidfile.unlink()
@ -77,17 +81,18 @@ class MispZmq:
raise Exception("The settings file is missing.") raise Exception("The settings file is missing.")
def _setup(self): def _setup(self):
with open((self.tmp_location / "mispzmq_settings.json").as_posix()) as settings_file: with open((self.tmp_location / "mispzmq_settings.json").as_posix(), 'rb') as settings_file:
self.settings = json.load(settings_file) self.settings = json.loads(settings_file.read())
self.namespace = self.settings["redis_namespace"] self.namespace = self.settings["redis_namespace"]
# Check if TLS is being used with Redis host # Check if TLS is being used with Redis host
redis_host = self.settings["redis_host"] redis_host = self.settings["redis_host"]
redis_ssl = redis_host.startswith("tls://") redis_ssl = redis_host.startswith("tls://")
if redis_host.startswith("tls://"): if redis_host.startswith("tls://"):
redis_host = redis_host[6:] redis_host = redis_host[6:]
self.r = redis.StrictRedis(host=redis_host, db=self.settings["redis_database"], self.redis = redis.StrictRedis(host=redis_host, db=self.settings["redis_database"],
password=self.settings["redis_password"], port=self.settings["redis_port"], password=self.settings["redis_password"], port=self.settings["redis_port"],
decode_responses=True, ssl=redis_ssl) ssl=redis_ssl)
self.timestamp_settings = time.time() self.timestamp_settings = time.time()
self._logger.debug("Connected to Redis {}:{}/{}".format(self.settings["redis_host"], self.settings["redis_port"], self._logger.debug("Connected to Redis {}:{}/{}".format(self.settings["redis_host"], self.settings["redis_port"],
self.settings["redis_database"])) self.settings["redis_database"]))
@ -122,34 +127,38 @@ class MispZmq:
self.socket.disable_monitor() self.socket.disable_monitor()
self.monitor_thread = None self.monitor_thread = None
def _handle_command(self, command): def _handle_command(self, command: bytes):
if command == "kill": if command == b"kill":
self._logger.info("Kill command received, shutting down.") self._logger.info("Kill command received, shutting down.")
self.clean() self.clean()
sys.exit() sys.exit()
elif command == "reload": elif command == b"reload":
self._logger.info("Reload command received, reloading settings from file.") self._logger.info("Reload command received, reloading settings from file.")
self._setup() self._setup()
self._setup_zmq() self._setup_zmq()
elif command == "status": elif command == b"status":
self._logger.info("Status command received, responding with latest stats.") self._logger.info("Status command received, responding with latest stats.")
self.r.delete("{}:status".format(self.namespace)) self.redis.delete(f"{self.namespace}:status")
self.r.lpush("{}:status".format(self.namespace), self.redis.lpush(f"{self.namespace}:status",
json.dumps({"timestamp": time.time(), json.dumps({"timestamp": time.time(),
"timestampSettings": self.timestamp_settings, "timestampSettings": self.timestamp_settings,
"publishCount": self.publish_count, "publishCount": self.publish_count,
"messageCount": self.message_count})) "messageCount": self.message_count}))
else: else:
self._logger.warning("Received invalid command '{}'.".format(command)) self._logger.warning(f"Received invalid command '{command}'.")
def _create_pid_file(self): def _create_pid_file(self):
with open(self.pidfile.as_posix(), "w") as f: with open(self.pidfile.as_posix(), "w") as f:
f.write(str(os.getpid())) f.write(str(os.getpid()))
def _pub_message(self, topic, data): def _pub_message(self, topic: bytes, data: typing.Union[str, bytes]):
self.socket.send_string("{} {}".format(topic, data)) data_to_send = bytearray()
data_to_send.extend(topic)
data_to_send.extend(b" ")
data_to_send.extend(data.encode("utf-8") if isinstance(data, str) else data)
self.socket.send(bytes(data_to_send))
def clean(self): def clean(self):
if self.monitor_thread: if self.monitor_thread:
@ -179,12 +188,14 @@ class MispZmq:
"misp_json_tag", "misp_json_warninglist", "misp_json_workflow" "misp_json_tag", "misp_json_warninglist", "misp_json_workflow"
] ]
lists = ["{}:command".format(self.namespace)] lists = [f"{self.namespace}:command"]
for topic in topics: for topic in topics:
lists.append("{}:data:{}".format(self.namespace, topic)) lists.append(f"{self.namespace}:data:{topic}")
key_prefix = f"{self.namespace}:".encode("utf-8")
while True: while True:
data = self.r.blpop(lists, timeout=10) data = self.redis.blpop(lists, timeout=10)
if data is None: if data is None:
# redis timeout expired # redis timeout expired
@ -195,26 +206,30 @@ class MispZmq:
"status": status_array[status_entry], "status": status_array[status_entry],
"uptime": current_time - int(self.timestamp_settings) "uptime": current_time - int(self.timestamp_settings)
} }
self._pub_message("misp_json_self", json.dumps(status_message)) self._pub_message(b"misp_json_self", json.dumps(status_message))
self._logger.debug("No message received for 10 seconds, sending ZMQ status message.") self._logger.debug("No message received from Redis for 10 seconds, sending ZMQ status message.")
else: else:
key, value = data key, value = data
key = key.replace("{}:".format(self.namespace), "") key = key.replace(key_prefix, b"")
if key == "command": if key == b"command":
self._handle_command(value) self._handle_command(value)
elif key.startswith("data:"): elif key.startswith(b"data:"):
topic = key.split(":")[1] topic = key.split(b":", 1)[1]
self._logger.debug("Received data for topic '{}', sending to ZMQ.".format(topic)) self._logger.debug("Received data for topic %s, sending to ZMQ.", topic)
self._pub_message(topic, value) self._pub_message(topic, value)
self.message_count += 1 self.message_count += 1
if topic == "misp_json": if topic == b"misp_json":
self.publish_count += 1 self.publish_count += 1
else: else:
self._logger.warning("Received invalid message '{}'.".format(key)) self._logger.warning("Received invalid message type %s.", key)
if __name__ == "__main__": if __name__ == "__main__":
mzq = MispZmq() arg_parser = argparse.ArgumentParser(description="MISP ZeroMQ PUB server")
arg_parser.add_argument("--debug", action="store_true", help="Enable debugging messages")
parsed = arg_parser.parse_args()
mzq = MispZmq(parsed.debug)
try: try:
mzq.main() mzq.main()
except KeyboardInterrupt: except KeyboardInterrupt:

29
app/files/scripts/mispzmq/mispzmqclient.py Normal file
View File

@ -0,0 +1,29 @@
#!/usr/bin/env python3
import sys
import zmq
import argparse
def main(port: int):
context = zmq.Context()
print("Connecting to MISP ZeroMQ server…", file=sys.stderr)
socket = context.socket(zmq.SUB)
socket.connect(f"tcp://localhost:{port}")
socket.setsockopt(zmq.SUBSCRIBE, b"misp_")
print(f"Connected to tcp://localhost:{port}", file=sys.stderr)
while True:
message = socket.recv()
print(message)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Example Python client for MISP ZMQ")
parser.add_argument("--port", default=50000, type=int)
parsed = parser.parse_args()
try:
main(parsed.port)
except KeyboardInterrupt:
pass

2
travis/database.php → build/database.php
View File

@ -60,7 +60,7 @@
class DATABASE_CONFIG { class DATABASE_CONFIG {
public $default = array( public $default = array(
'datasource' => 'Database/Mysql', 'datasource' => 'Database/MysqlExtended',
'persistent' => false, 'persistent' => false,
'host' => '127.0.0.1', 'host' => '127.0.0.1',
'login' => 'misp', 'login' => 'misp',

6
travis/email.php → build/email.php
View File

@ -1,7 +1,7 @@
<?php <?php
class EmailConfig { class EmailConfig {
public $default = array( public $default = [
'transport' => 'Debug', 'transport' => 'Debug',
'log' => true 'log' => true,
); ];
} }

0
travis/gpg → build/gpg
View File

15
build/travis-ci-apache
View File

@ -1,15 +0,0 @@
<VirtualHost misp.local>
ServerAdmin me@me.local
ServerName misp.local
DocumentRoot %TRAVIS_BUILD_DIR%/app/webroot
<Directory %TRAVIS_BUILD_DIR%/app/webroot>
Options -Indexes
AllowOverride all
Require all granted
</Directory>
LogLevel warn
ErrorLog /var/log/apache2/misp.local_error.log
CustomLog /var/log/apache2/misp.local_access.log combined
ServerSignature Off
</VirtualHost>

5
tests/build-test.sh
View File

@ -9,16 +9,15 @@ python ./../app/files/scripts/mispzmq/mispzmqtest.py
python ./../app/files/scripts/generate_file_objects.py -c | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if len([i for i in data.values() if i is not False]) == 0 else 1)' python ./../app/files/scripts/generate_file_objects.py -c | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if len([i for i in data.values() if i is not False]) == 0 else 1)'
# Try to extract data from file # Try to extract data from file
python ./../app/files/scripts/generate_file_objects.py -p /bin/ls
python ./../app/files/scripts/generate_file_objects.py -p /bin/ls | python3 -c 'import sys, json; data = json.load(sys.stdin); sys.exit(0 if "objects" in data else 1)' python ./../app/files/scripts/generate_file_objects.py -p /bin/ls | python3 -c 'import sys, json; data = json.load(sys.stdin); sys.exit(0 if "objects" in data else 1)'
# Test converting stix1 to MISP format # Test converting stix1 to MISP format
curl https://stixproject.github.io/documentation/idioms/c2-indicator/indicator-for-c2-ip-address.xml > ./../app/files/scripts/tmp/test-stix1.xml curl -sS --compressed https://stixproject.github.io/documentation/idioms/c2-indicator/indicator-for-c2-ip-address.xml > ./../app/files/scripts/tmp/test-stix1.xml
python ./../app/files/scripts/stix2misp.py test-stix1.xml 1 1 ./../app/files/scripts/synonymsToTagNames.json | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if data["success"] == 1 else 1)' python ./../app/files/scripts/stix2misp.py test-stix1.xml 1 1 ./../app/files/scripts/synonymsToTagNames.json | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if data["success"] == 1 else 1)'
rm -f ./../app/files/scripts/tmp/{test-stix1.xml,test-stix1.xml.json} rm -f ./../app/files/scripts/tmp/{test-stix1.xml,test-stix1.xml.json}
# Test converting stix2 to MISP format # Test converting stix2 to MISP format
curl https://raw.githubusercontent.com/oasis-open/cti-stix2-json-schemas/master/examples/indicator-for-c2-ip-address.json > ./../app/files/scripts/tmp/test-stix2.json curl -sS --compressed https://raw.githubusercontent.com/oasis-open/cti-stix2-json-schemas/master/examples/indicator-for-c2-ip-address.json > ./../app/files/scripts/tmp/test-stix2.json
python ./../app/files/scripts/stix2/stix2misp.py -i ./../app/files/scripts/tmp/test-stix2.json --distribution 1 | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if data["success"] == 1 else 1)' python ./../app/files/scripts/stix2/stix2misp.py -i ./../app/files/scripts/tmp/test-stix2.json --distribution 1 | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if data["success"] == 1 else 1)'
rm -f ./../app/files/scripts/tmp/{test-stix2.json,test-stix2.json.stix2} rm -f ./../app/files/scripts/tmp/{test-stix2.json,test-stix2.json.stix2}

15
tests/curl_tests.sh
View File

@ -1,15 +0,0 @@
#!/usr/bin/env bash
set -e
set -x
AUTH="$1"
# Check if user is logged
curl -i -H "Authorization: $AUTH" -H "Accept: application/json" -X GET http://misp.local/servers/getVersion
curl -i -H "Accept: application/json" -H "content-type: application/json" -H "Authorization: $AUTH" --data "@event.json" -X POST http://misp.local/events
curl -H "Authorization: $AUTH" -X GET http://misp.local/events/csv/download/1/ignore:1 | sed -e 's/^M//g' | cut -d, -f2 --complement | sort > 1.csv
cat 1.csv
cut -d, -f2 --complement event.csv | sort > compare.csv
diff compare.csv 1.csv
curl -i -H "Accept: application/json" -H "content-type: application/json" -H "Authorization: $AUTH" -X POST http://misp.local/events/delete/1

8
tests/curl_tests_GH.sh
View File

@ -7,13 +7,13 @@ AUTH="$1"
HOST="$2" HOST="$2"
# Check if user is logged # Check if user is logged
curl -i -H "Authorization: $AUTH" -H "Accept: application/json" -X GET http://${HOST}/servers/getVersion curl -sS -i -H "Authorization: $AUTH" -H "Accept: application/json" -X GET http://${HOST}/servers/getVersion
curl -i -H "Accept: application/json" -H "content-type: application/json" -H "Authorization: $AUTH" --data "@event.json" -X POST http://${HOST}/events > /dev/null curl -sS -i -H "Accept: application/json" -H "content-type: application/json" -H "Authorization: $AUTH" --data "@event.json" -X POST http://${HOST}/events > /dev/null
curl -H "Authorization: $AUTH" -X GET http://${HOST}/events/csv/download/1/ignore:1 | sed -e 's/^M//g' | cut -d, -f2 --complement | sort > 1.csv curl -sS -H "Authorization: $AUTH" -X GET http://${HOST}/events/csv/download/1/ignore:1 | sed -e 's/^M//g' | cut -d, -f2 --complement | sort > 1.csv
cat 1.csv cat 1.csv
cut -d, -f2 --complement event.csv | sort > compare.csv cut -d, -f2 --complement event.csv | sort > compare.csv
diff compare.csv 1.csv diff compare.csv 1.csv
# Test alert email generating # Test alert email generating
sudo -u www-data ../app/Console/cake Event testEventNotificationEmail 1 1 > /dev/null sudo -u www-data ../app/Console/cake Event testEventNotificationEmail 1 1 > /dev/null
# Delete created event # Delete created event
curl -i -H "Accept: application/json" -H "content-type: application/json" -H "Authorization: $AUTH" -X POST http://${HOST}/events/delete/1 curl -sS -i -H "Accept: application/json" -H "content-type: application/json" -H "Authorization: $AUTH" -X POST http://${HOST}/events/delete/1

9
tests/modify_config.php
View File

@ -10,9 +10,10 @@ if (!isset($argv[2])) {
if (!in_array($argv[1], ['modify', 'replace'], true)) { if (!in_array($argv[1], ['modify', 'replace'], true)) {
fail(1, "Invalid argument '{$argv[1]}', it must be 'modify' or 'replace'."); fail(1, "Invalid argument '{$argv[1]}', it must be 'modify' or 'replace'.");
} }
$newConfig = json_decode($argv[2], true); try {
if ($newConfig === null) { $newConfig = json_decode($argv[2], true, JSON_THROW_ON_ERROR);
fail(2, "Could not decode new config, it is not JSON: " . json_last_error_msg()); } catch (Exception $e) {
fail(2, "Could not decode new config, it is not JSON: " . $e->getMessage());
} }
if (!is_array($newConfig)) { if (!is_array($newConfig)) {
fail(2, "Provided new config is not array, `" . gettype($newConfig) . "` given."); fail(2, "Provided new config is not array, `" . gettype($newConfig) . "` given.");
@ -41,4 +42,4 @@ if ($argv[1] === 'modify') {
file_put_contents($configFile, "<?php\n\$config = " . var_export($merged, true) . ';', LOCK_EX); file_put_contents($configFile, "<?php\n\$config = " . var_export($merged, true) . ';', LOCK_EX);
// Returns config file before modification // Returns config file before modification
echo json_encode($config); echo json_encode($config, JSON_THROW_ON_ERROR | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);

99
tests/testlive_comprehensive_local.py
View File

@ -2,6 +2,7 @@
import os import os
import json import json
import uuid import uuid
import logging
import inspect import inspect
import subprocess import subprocess
import unittest import unittest
@ -11,21 +12,19 @@ from xml.etree import ElementTree as ET
from io import BytesIO from io import BytesIO
import urllib3 # type: ignore import urllib3 # type: ignore
from datetime import datetime, timedelta from datetime import datetime, timedelta
from typing import Union
import logging
logging.disable(logging.CRITICAL)
logger = logging.getLogger('pymisp')
from pymisp import PyMISP, MISPOrganisation, MISPUser, MISPRole, MISPSharingGroup, MISPEvent, MISPLog, MISPSighting, Distribution, ThreatLevel, Analysis, MISPEventReport, MISPServerError from pymisp import PyMISP, MISPOrganisation, MISPUser, MISPRole, MISPSharingGroup, MISPEvent, MISPLog, MISPSighting, Distribution, ThreatLevel, Analysis, MISPEventReport, MISPServerError
from pymisp.tools import DomainIPObject from pymisp.tools import DomainIPObject
from pymisp.api import get_uuid_or_id_from_abstract_misp
logging.disable(logging.CRITICAL)
logger = logging.getLogger('pymisp')
urllib3.disable_warnings()
# Load access information for env variables # Load access information for env variables
url = "http://" + os.environ["HOST"] url = "http://" + os.environ["HOST"]
key = os.environ["AUTH"] key = os.environ["AUTH"]
urllib3.disable_warnings()
def create_simple_event() -> MISPEvent: def create_simple_event() -> MISPEvent:
caller_name = inspect.stack()[1].function caller_name = inspect.stack()[1].function
@ -52,13 +51,19 @@ def request(pymisp: PyMISP, request_type: str, url: str, data: dict = {}) -> dic
return pymisp._check_json_response(response) return pymisp._check_json_response(response)
def publish_immediately(pymisp: PyMISP, event: Union[MISPEvent, int, str, uuid.UUID], with_email: bool = False):
event_id = get_uuid_or_id_from_abstract_misp(event)
action = "alert" if with_email else "publish"
return check_response(request(pymisp, 'POST', f'events/{action}/{event_id}/disable_background_processing:1'))
class MISPSetting: class MISPSetting:
def __init__(self, admin_connector: PyMISP, new_setting: dict): def __init__(self, admin_connector: PyMISP, new_setting: dict):
self.admin_connector = admin_connector self.admin_connector = admin_connector
self.new_setting = new_setting self.new_setting = new_setting
def __enter__(self): def __enter__(self):
self.original = self.__run("modify", json.dumps(self.new_setting)) self.original = self.__run("modify", json.dumps(self.new_setting).encode("utf-8"))
# Try to reset config cache # Try to reset config cache
self.admin_connector.get_server_setting("MISP.live") self.admin_connector.get_server_setting("MISP.live")
@ -68,12 +73,12 @@ class MISPSetting:
self.admin_connector.get_server_setting("MISP.live") self.admin_connector.get_server_setting("MISP.live")
@staticmethod @staticmethod
def __run(command: str, data: str) -> str: def __run(command: str, data: bytes) -> bytes:
dir_path = os.path.dirname(os.path.realpath(__file__)) dir_path = os.path.dirname(os.path.realpath(__file__))
r = subprocess.run(["php", dir_path + "/modify_config.php", command, data], stdout=subprocess.PIPE, stderr=subprocess.PIPE) r = subprocess.run(["php", dir_path + "/modify_config.php", command, data], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if r.returncode != 0: if r.returncode != 0:
raise Exception([r.returncode, r.stdout, r.stderr]) raise Exception([r.returncode, r.stdout, r.stderr])
return r.stdout.decode("utf-8") return r.stdout
class TestComprehensive(unittest.TestCase): class TestComprehensive(unittest.TestCase):
@ -465,8 +470,6 @@ class TestComprehensive(unittest.TestCase):
check_response(self.admin_misp_connector.delete_event(event)) check_response(self.admin_misp_connector.delete_event(event))
def test_publish_alert_filter(self): def test_publish_alert_filter(self):
check_response(self.admin_misp_connector.set_server_setting('MISP.background_jobs', 0, force=True))
first = create_simple_event() first = create_simple_event()
first.add_tag('test_publish_filter') first.add_tag('test_publish_filter')
first.threat_level_id = ThreatLevel.medium first.threat_level_id = ThreatLevel.medium
@ -499,7 +502,7 @@ class TestComprehensive(unittest.TestCase):
# Publish events # Publish events
for event in (first, second, third, four): for event in (first, second, third, four):
check_response(self.admin_misp_connector.publish(event, alert=True)) publish_immediately(self.admin_misp_connector, event, with_email=True)
# Email notification should be send just to first event # Email notification should be send just to first event
mail_logs = self.admin_misp_connector.search_logs(model='User', action='email') mail_logs = self.admin_misp_connector.search_logs(model='User', action='email')
@ -516,8 +519,6 @@ class TestComprehensive(unittest.TestCase):
check_response(self.admin_misp_connector.update_user(self.admin_misp_connector._current_user)) check_response(self.admin_misp_connector.update_user(self.admin_misp_connector._current_user))
# Delete filter # Delete filter
self.admin_misp_connector.delete_user_setting('publish_alert_filter') self.admin_misp_connector.delete_user_setting('publish_alert_filter')
# Reenable background jobs
check_response(self.admin_misp_connector.set_server_setting('MISP.background_jobs', 1, force=True))
# Delete events # Delete events
for event in (first, second, third, four): for event in (first, second, third, four):
check_response(self.admin_misp_connector.delete_event(event)) check_response(self.admin_misp_connector.delete_event(event))
@ -740,20 +741,27 @@ class TestComprehensive(unittest.TestCase):
event.add_attribute("ip-src", "1.2.4.5", to_ids=True) event.add_attribute("ip-src", "1.2.4.5", to_ids=True)
event = check_response(self.admin_misp_connector.add_event(event)) event = check_response(self.admin_misp_connector.add_event(event))
result = self._search({'returnFormat': "openioc", 'eventid': event.id, "published": [0, 1]}) result = self._search_event({'returnFormat': "openioc", 'eventid': event.id, "published": [0, 1]})
ET.fromstring(result) # check if result is valid XML ET.fromstring(result) # check if result is valid XML
self.assertTrue("1.2.4.5" in result, result) self.assertTrue("1.2.4.5" in result, result)
result = self._search({'returnFormat': "yara", 'eventid': event.id, "published": [0, 1]}) result = self._search_event({'returnFormat': "yara", 'eventid': event.id, "published": [0, 1]})
self.assertTrue("1.2.4.5" in result, result) self.assertTrue("1.2.4.5" in result, result)
self.assertTrue("GENERATED" in result, result) self.assertTrue("GENERATED" in result, result)
self.assertTrue("AS-IS" in result, result) self.assertTrue("AS-IS" in result, result)
result = self._search({'returnFormat': "yara-json", 'eventid': event.id, "published": [0, 1]}) result = self._search_event({'returnFormat': "yara-json", 'eventid': event.id, "published": [0, 1]})
self.assertIn("generated", result) self.assertIn("generated", result)
self.assertEqual(len(result["generated"]), 1, result) self.assertEqual(len(result["generated"]), 1, result)
self.assertIn("as-is", result) self.assertIn("as-is", result)
# RPZ
result = self._search_event({'returnFormat': "rpz", 'eventid': event.id, "published": [0, 1]})
self.assertTrue("32.5.4.2.1" in result, result)
result = self._search_attribute({'returnFormat': "rpz", 'eventid': event.id, "published": [0, 1]})
self.assertTrue("32.5.4.2.1" in result, result)
check_response(self.admin_misp_connector.delete_event(event)) check_response(self.admin_misp_connector.delete_event(event))
def test_event_report_empty_name(self): def test_event_report_empty_name(self):
@ -916,16 +924,18 @@ class TestComprehensive(unittest.TestCase):
def test_search_snort_suricata(self): def test_search_snort_suricata(self):
event = create_simple_event() event = create_simple_event()
event.add_attribute('ip-src', '8.8.8.8', to_ids=True) event.add_attribute('ip-src', '8.8.8.8', to_ids=True)
event = self.user_misp_connector.add_event(event) event.add_attribute('snort', 'alert tcp 192.168.1.0/24 any -> 131.171.127.1 25 (content: "hacking"; msg: "malicious packet"; sid:2000001;)', to_ids=True)
check_response(event) # Snort rule without msg, test for #9515
event.add_attribute('snort', 'alert tcp 192.168.1.0/24 any -> 131.171.127.1 25 (content: "hacking"; sid:2000001;)', to_ids=True)
event = check_response(self.user_misp_connector.add_event(event))
self.admin_misp_connector.publish(event, alert=False) publish_immediately(self.admin_misp_connector, event)
time.sleep(6)
snort = self._search({'returnFormat': 'snort', 'eventid': event.id}) snort = self._search_event({'returnFormat': 'snort', 'eventid': event.id})
self.assertIsInstance(snort, str) self.assertIsInstance(snort, str)
self.assertIn('8.8.8.8', snort) self.assertIn('8.8.8.8', snort)
suricata = self._search({'returnFormat': 'suricata', 'eventid': event.id}) suricata = self._search_event({'returnFormat': 'suricata', 'eventid': event.id})
self.assertIsInstance(suricata, str) self.assertIsInstance(suricata, str)
self.assertIn('8.8.8.8', suricata) self.assertIn('8.8.8.8', suricata)
@ -962,7 +972,38 @@ class TestComprehensive(unittest.TestCase):
self.admin_misp_connector.delete_event(event) self.admin_misp_connector.delete_event(event)
def _search(self, query: dict): def test_restsearch_sightings(self):
# Create test event
event = create_simple_event()
event = self.admin_misp_connector.add_event(event)
check_response(event)
# Add sighting
sighting = MISPSighting()
sighting.value = 'test'
sighting.source = 'Testcases'
sighting.type = '1'
response = self.admin_misp_connector.add_sighting(sighting, event.attributes[0])
check_response(response)
self.assertEqual(response.source, 'Testcases')
# Try to find sighting by event UUID, this is the same type of request when doing sync
search_result = self._search_sighting('event', {
'returnFormat': 'json',
'last': 0,
'includeUuid': True,
'uuid': [event.uuid],
})
self.assertEqual(len(search_result), 1, search_result)
sighting = search_result[0]["Sighting"]
self.assertIn("attribute_uuid", sighting)
self.assertIn("event_uuid", sighting)
self.assertEqual(sighting["event_uuid"], event.uuid, search_result)
self.admin_misp_connector.delete_event(event)
def _search_event(self, query: dict):
response = self.admin_misp_connector._prepare_request('POST', 'events/restSearch', data=query) response = self.admin_misp_connector._prepare_request('POST', 'events/restSearch', data=query)
response = self.admin_misp_connector._check_response(response) response = self.admin_misp_connector._check_response(response)
check_response(response) check_response(response)
@ -974,6 +1015,12 @@ class TestComprehensive(unittest.TestCase):
check_response(response) check_response(response)
return response return response
def _search_sighting(self, context: str, query: dict):
response = self.admin_misp_connector._prepare_request('POST', f'sightings/restSearch/{context}', data=query)
response = self.admin_misp_connector._check_response(response)
check_response(response)
return response
class TestLastPwChange(unittest.TestCase): class TestLastPwChange(unittest.TestCase):
@classmethod @classmethod

24
tools/misp-delegation/src/misp-delegation.py
View File

@ -33,8 +33,14 @@ def main():
global unpublish_event_on_remote global unpublish_event_on_remote
parser = argparse.ArgumentParser(Path(__file__).name) parser = argparse.ArgumentParser(Path(__file__).name)
parser.add_argument('-c', '--config', default='config.json', help='The JSON config file to use') parser.add_argument('-c', '--config', default='config.json', help='The JSON config file to use')
parser.add_argument('-l', '--log', default='INFO', help='The loglevel to use.', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR',])
options = parser.parse_args() options = parser.parse_args()
numeric_log_level = getattr(logging, options.log.upper(), None)
if not isinstance(numeric_log_level, int):
raise ValueError('Invalid log level: %s' % numeric_log_level)
logger.setLevel(numeric_log_level)
config = {} config = {}
config_file_path = Path(__file__).parent / options.config config_file_path = Path(__file__).parent / options.config
with open(config_file_path, 'r') as f: with open(config_file_path, 'r') as f:
@ -74,7 +80,7 @@ def main():
return 1 return 1
# Collect events from source # Collect events from source
logger.debug('Collecting events from source...') logger.debug('Collecting *published* events from source...')
try: try:
events_on_source = collect_events_from_source(source_instance, filters, incremental_sync) events_on_source = collect_events_from_source(source_instance, filters, incremental_sync)
except Exception as err: except Exception as err:
@ -185,9 +191,10 @@ def update_event_for_push(event: dict) -> dict:
elif attribute['distribution'] == 2: elif attribute['distribution'] == 2:
event['Attribute'][i]['distribution'] = 1 event['Attribute'][i]['distribution'] = 1
for t, tag in enumerate(attribute['Tag']): if 'Tag' in attribute:
if tag['local']: for t, tag in enumerate(attribute['Tag']):
event['Attribute'][i]['Tag'].pop(t) if tag['local']:
event['Attribute'][i]['Tag'].pop(t)
# Downgrade distribution for Objects and their Attributes # Downgrade distribution for Objects and their Attributes
for i, object in enumerate(event['Object'][:]): for i, object in enumerate(event['Object'][:]):
@ -201,9 +208,10 @@ def update_event_for_push(event: dict) -> dict:
elif attribute['distribution'] == 2: elif attribute['distribution'] == 2:
event['Object'][i]['Attribute'][j]['distribution'] = 1 event['Object'][i]['Attribute'][j]['distribution'] = 1
for t, tag in enumerate(attribute['Tag']): if 'Tag' in attribute:
if tag['local']: for t, tag in enumerate(attribute['Tag']):
event['Object'][i]['Attribute'][j]['Tag'].pop(t) if tag['local']:
event['Object'][i]['Attribute'][j]['Tag'].pop(t)
# Downgrade distribution for EventReport # Downgrade distribution for EventReport
for i, report in enumerate(event['EventReport'][:]): for i, report in enumerate(event['EventReport'][:]):
@ -222,7 +230,7 @@ def collect_events_from_source(source_instance: MISPInstance, filters: dict, inc
} }
last_timestamp = get_last_sync_timestamp() last_timestamp = get_last_sync_timestamp()
if incremental_sync and last_timestamp is not None: if incremental_sync and last_timestamp is not None:
logger.debug('Using timestamp from last synchronisation %s (%s)', last_timestamp, datetime.fromtimestamp(last_timestamp)) logger.info('Using timestamp from last synchronisation %s (%s)', last_timestamp, datetime.fromtimestamp(last_timestamp))
sync_filters['timestamp'] = last_timestamp # type: ignore sync_filters['timestamp'] = last_timestamp # type: ignore
sync_filters.update(filters) sync_filters.update(filters)
events = source_instance.POST('/events/index', payload=sync_filters) events = source_instance.POST('/events/index', payload=sync_filters)

154
travis/bootstrap.php
View File

@ -1,154 +0,0 @@
<?php
/**
* This file is loaded automatically by the app/webroot/index.php file after core.php
*
* This file should load/create any application wide configuration settings, such as
* Caching, Logging, loading additional configuration files.
*
* You should also use this file to include any files that provide global functions/constants
* that your application uses.
*/
/**
* Cache Engine Configuration
* Default settings provided below
*
* File storage engine.
*
* Cache::config('default', array(
* 'engine' => 'File', //[required]
* 'duration'=> 3600, //[optional]
* 'probability'=> 100, //[optional]
* 'path' => CACHE, //[optional] use system tmp directory - remember to use absolute path
* 'prefix' => 'cake_', //[optional] prefix every cache file with this string
* 'lock' => false, //[optional] use file locking
* 'serialize' => true, // [optional]
* 'mask' => 0666, // [optional] permission mask to use when creating cache files
* ));
*
* APC (http://pecl.php.net/package/APC)
*
* Cache::config('default', array(
* 'engine' => 'Apc', //[required]
* 'duration'=> 3600, //[optional]
* 'probability'=> 100, //[optional]
* 'prefix' => Inflector::slug(APP_DIR) . '_', //[optional] prefix every cache file with this string
* ));
*
* Xcache (http://xcache.lighttpd.net/)
*
* Cache::config('default', array(
* 'engine' => 'Xcache', //[required]
* 'duration'=> 3600, //[optional]
* 'probability'=> 100, //[optional]
* 'prefix' => Inflector::slug(APP_DIR) . '_', //[optional] prefix every cache file with this string
* 'user' => 'user', //user from xcache.admin.user settings
* 'password' => 'password', //plaintext password (xcache.admin.pass)
* ));
*
* Memcache (http://memcached.org/)
*
* Cache::config('default', array(
* 'engine' => 'Memcache', //[required]
* 'duration'=> 3600, //[optional]
* 'probability'=> 100, //[optional]
* 'prefix' => Inflector::slug(APP_DIR) . '_', //[optional] prefix every cache file with this string
* 'servers' => array(
* '127.0.0.1:11211' // localhost, default port 11211
* ), //[optional]
* 'persistent' => true, // [optional] set this to false for non-persistent connections
* 'compress' => false, // [optional] compress data in Memcache (slower, but uses less memory)
* ));
*
* Wincache (http://php.net/wincache)
*
* Cache::config('default', array(
* 'engine' => 'Wincache', //[required]
* 'duration'=> 3600, //[optional]
* 'probability'=> 100, //[optional]
* 'prefix' => Inflector::slug(APP_DIR) . '_', //[optional] prefix every cache file with this string
* ));
*
* Redis (http://http://redis.io/)
*
* Cache::config('default', array(
* 'engine' => 'Redis', //[required]
* 'duration'=> 3600, //[optional]
* 'probability'=> 100, //[optional]
* 'prefix' => Inflector::slug(APP_DIR) . '_', //[optional] prefix every cache file with this string
* 'server' => '127.0.0.1' // localhost
* 'port' => 6379 // default port 6379
* 'timeout' => 0 // timeout in seconds, 0 = unlimited
* 'persistent' => true, // [optional] set this to false for non-persistent connections
* ));
*/
Cache::config('default', array('engine' => 'File'));
Configure::load('config');
if (!Configure::read('MISP.baseurl')) {
if (!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off' || $_SERVER['SERVER_PORT'] == 443) {
Configure::write('MISP.baseurl', sprintf('https://%s:%d', $_SERVER['SERVER_ADDR'], $_SERVER['SERVER_PORT']));
} else {
Configure::write('MISP.baseurl', sprintf('http://%s:%d', $_SERVER['SERVER_ADDR'], $_SERVER['SERVER_PORT']));
}
}
/**
* Plugins need to be loaded manually, you can either load them one by one or all of them in a single call
* Uncomment one of the lines below, as you need. make sure you read the documentation on CakePlugin to use more
* advanced ways of loading plugins
*
* CakePlugin::loadAll(); // Loads all plugins at once
* CakePlugin::load('DebugKit'); //Loads a single plugin named DebugKit
*
*/
CakePlugin::load('SysLog');
CakePlugin::load('Assets'); // having Logable
CakePlugin::load('SysLogLogable');
/**
* Uncomment the following line to enable client SSL certificate authentication.
* It's also necessary to configure the plugin — for more information, please read app/Plugin/CertAuth/reame.md
*/
// CakePlugin::load('CertAuth');
/**
* You can attach event listeners to the request lifecyle as Dispatcher Filter . By Default CakePHP bundles two filters:
*
* - AssetDispatcher filter will serve your asset files (css, images, js, etc) from your themes and plugins
* - CacheDispatcher filter will read the Cache.check configure variable and try to serve cached content generated from controllers
*
* Feel free to remove or add filters as you see fit for your application. A few examples:
*
* Configure::write('Dispatcher.filters', array(
* 'MyCacheFilter', // will use MyCacheFilter class from the Routing/Filter package in your app.
* 'MyPlugin.MyFilter', // will use MyFilter class from the Routing/Filter package in MyPlugin plugin.
* array('callable' => $aFunction, 'on' => 'before', 'priority' => 9), // A valid PHP callback type to be called on beforeDispatch
* array('callable' => $anotherMethod, 'on' => 'after'), // A valid PHP callback type to be called on afterDispatch
*
* ));
*/
Configure::write('Dispatcher.filters', array(
'AssetDispatcher',
'CacheDispatcher'
));
/**
* Configures default file logging options
*/
App::uses('CakeLog', 'Log');
CakeLog::config('debug', array(
'engine' => 'FileLog',
'types' => array('notice', 'info', 'debug'),
'file' => 'debug',
));
CakeLog::config('error', array(
'engine' => 'FileLog',
'types' => array('warning', 'error', 'critical', 'alert', 'emergency'),
'file' => 'error',
));
CakePlugin::loadAll(array(
'CakeResque' => array('bootstrap' => true)
));

81
travis/config.php
View File

@ -1,81 +0,0 @@
<?php
$config = array (
'debug' => 1,
'Security' =>
array (
'level' => 'medium',
'salt' => 'Rooraenietu8Eeyo<Qu2eeNfterd-dd+',
'cipherSeed' => '',
//'auth'=>array('CertAuth.Certificate'), // additional authentication methods
),
'MISP' =>
array (
'baseurl' => 'http://misp.local',
'footerpart1' => 'Powered by MISP',
'footerpart2' => '&copy; Belgian Defense CERT & NCIRC',
'org' => 'ORGNAME',
'showorg' => true,
'background_jobs' => false,
'email' => 'email@address.com',
'contact' => 'email@address.com',
'cveurl' => 'http://web.nvd.nist.gov/view/vuln/detail?vulnId=',
'disablerestalert' => false,
'default_event_distribution' => '0',
'default_attribute_distribution' => 'event',
'tagging' => true,
'full_tags_on_event_index' => true,
'footer_logo' => '',
'take_ownership_xml_import' => false,
'unpublishedprivate' => false,
),
'GnuPG' =>
array (
'onlyencrypted' => false,
'email' => '',
'homedir' => '',
'password' => '',
'bodyonlyencrypted' => false,
),
'Proxy' =>
array (
'host' => '',
'port' => '',
'method' => '',
'user' => '',
'password' => '',
),
'SecureAuth' =>
array (
'amount' => 5,
'expire' => 300,
),
// Uncomment the following to enable client SSL certificate authentication
/*
'CertAuth' =>
array(
'ca' => array( 'FIRST.Org' ), // allowed CAs
'caId' => 'O', // which attribute will be used to verify the CA
'userModel' => 'User', // name of the User class to check if user exists
'userModelKey' => 'nids_sid', // User field that will be used for querying
'map' => array( // maps client certificate attributes to User properties
'O' => 'org',
'emailAddress'=>'email',
),
'syncUser' => true, // should the User be synchronized with an external REST API
'userDefaults'=> array( // default user attributes, only used when creating new users
'role_id' => 4,
),
'restApi' => array( // API parameters
'url' => 'https://example.com/data/users', // URL to query
'headers' => array(), // additional headers, used for authentication
'param' => array( 'email' => 'email'), // query parameters to add to the URL, mapped to USer properties
'map' => array( // maps REST result to the User properties
'uid' => 'nids_sid',
'team' => 'org',
'email' => 'email',
'pgp_public'=> 'gpgkey',
),
),
),
*/
);

285
travis/core.php
View File

@ -1,285 +0,0 @@
<?php
/**
* This is core configuration file.
*
* Use it to configure core behavior of Cake.
*
* PHP 5
*
* CakePHP(tm) : Rapid Development Framework (http://cakephp.org)
* Copyright 2005-2012, Cake Software Foundation, Inc. (http://cakefoundation.org)
*
* Licensed under The MIT License
* Redistributions of files must retain the above copyright notice.
*
* @copyright Copyright 2005-2012, Cake Software Foundation, Inc. (http://cakefoundation.org)
* @link http://cakephp.org CakePHP(tm) Project
* @package app.Config
* @since CakePHP(tm) v 0.2.9
* @license MIT License (http://www.opensource.org/licenses/mit-license.php)
*/
/**
* CakePHP Debug Level:
*
* Production Mode:
* 0: No error messages, errors, or warnings shown. Flash messages redirect.
*
* Development Mode:
* 1: Errors and warnings shown, model caches refreshed, flash messages halted.
* 2: As in 1, but also with full debug messages and SQL output.
*
* In production mode, flash messages redirect after a time interval.
* In development mode, you need to click the flash message to continue.
*/
Configure::write('debug', 2); // 0 = for production, 2 = full debug mode
/**
* Configure the Error handler used to handle errors for your application. By default
* ErrorHandler::handleError() is used. It will display errors using Debugger, when debug > 0
* and log errors with CakeLog when debug = 0.
*
* Options:
*
* - `handler` - callback - The callback to handle errors. You can set this to any callable type,
* including anonymous functions.
* - `level` - int - The level of errors you are interested in capturing.
* - `trace` - boolean - Include stack traces for errors in log files.
*
* @see ErrorHandler for more information on error handling and configuration.
*/
Configure::write('Error', array(
'handler' => 'ErrorHandler::handleError',
'level' => E_ALL & ~E_DEPRECATED,
'trace' => true
));
/**
* Configure the Exception handler used for uncaught exceptions. By default,
* ErrorHandler::handleException() is used. It will display a HTML page for the exception, and
* while debug > 0, framework errors like Missing Controller will be displayed. When debug = 0,
* framework errors will be coerced into generic HTTP errors.
*
* Options:
*
* - `handler` - callback - The callback to handle exceptions. You can set this to any callback type,
* including anonymous functions.
* - `renderer` - string - The class responsible for rendering uncaught exceptions. If you choose a custom class you
* should place the file for that class in app/Lib/Error. This class needs to implement a render method.
* - `log` - boolean - Should Exceptions be logged?
*
* @see ErrorHandler for more information on exception handling and configuration.
*/
Configure::write('Exception', array(
'handler' => 'ErrorHandler::handleException',
'renderer' => 'ExceptionRenderer',
'log' => true,
'skipLog' => array(
'NotFoundException',
)
));
/**
* Application wide charset encoding
*/
Configure::write('App.encoding', 'UTF-8');
/**
* To configure CakePHP *not* to use mod_rewrite and to
* use CakePHP pretty URLs, remove these .htaccess
* files:
*
* /.htaccess
* /app/.htaccess
* /app/webroot/.htaccess
*
* And uncomment the App.baseUrl below:
*/
//Configure::write('App.baseUrl', env('SCRIPT_NAME'));
/**
* Uncomment the define below to use CakePHP prefix routes.
*
* The value of the define determines the names of the routes
* and their associated controller actions:
*
* Set to an array of prefixes you want to use in your application. Use for
* admin or other prefixed routes.
*
* Routing.prefixes = array('admin', 'manager');
*
* Enables:
* `admin_index()` and `/admin/controller/index`
* `manager_index()` and `/manager/controller/index`
*
*/
Configure::write('Routing.prefixes', array('admin'));
/**
* Turn off all caching application-wide.
*
*/
Configure::write('Cache.disable', false);
/**
* Enable cache checking.
*
* If set to true, for view caching you must still use the controller
* public $cacheAction inside your controllers to define caching settings.
* You can either set it controller-wide by setting public $cacheAction = true,
* or in each action using $this->cacheAction = true.
*
*/
//Configure::write('Cache.check', true);
/**
* Defines the default error type when using the log() function. Used for
* differentiating error logging and debugging. Currently PHP supports LOG_DEBUG.
*/
define('LOG_ERROR', LOG_ERR);
/**
* Session configuration.
*
* Contains an array of settings to use for session configuration. The defaults key is
* used to define a default preset to use for sessions, any settings declared here will override
* the settings of the default config.
*
* ## Options
*
* - `Session.cookie` - The name of the cookie to use. Defaults to 'CAKEPHP'
* - `Session.timeout` - The number of minutes you want sessions to live for. This timeout is handled by CakePHP
* - `Session.cookieTimeout` - The number of minutes you want session cookies to live for.
* - `Session.checkAgent` - Do you want the user agent to be checked when starting sessions? You might want to set the
* value to false, when dealing with older versions of IE, Chrome Frame or certain web-browsing devices and AJAX
* - `Session.defaults` - The default configuration set to use as a basis for your session.
* There are four builtins: php, cake, cache, database.
* - `Session.handler` - Can be used to enable a custom session handler. Expects an array of callables,
* that can be used with `session_save_handler`. Using this option will automatically add `session.save_handler`
* to the ini array.
* - `Session.autoRegenerate` - Enabling this setting, turns on automatic renewal of sessions, and
* sessionids that change frequently. See CakeSession::$requestCountdown.
* - `Session.ini` - An associative array of additional ini values to set.
*
* The built in defaults are:
*
* - 'php' - Uses settings defined in your php.ini.
* - 'cake' - Saves session files in CakePHP's /tmp directory.
* - 'database' - Uses CakePHP's database sessions.
* - 'cache' - Use the Cache class to save sessions.
*
* To define a custom session handler, save it at /app/Model/Datasource/Session/<name>.php.
* Make sure the class implements `CakeSessionHandlerInterface` and set Session.handler to <name>
*
* To use database sessions, run the app/Config/Schema/sessions.php schema using
* the cake shell command: cake schema create Sessions
*
*/
Configure::write('Session', array(
'timeout' => 60, // Session timeout, default is 1 hour
'defaults' => 'php'
));
/**
* The level of CakePHP security.
*/
Configure::write('Security.level', 'medium');
/**
* A random string used in security hashing methods.
*/
Configure::write('Security.salt', 'Rooraenietu8Eeyo<Qu2eeNfterd-dd+');
/**
* A random numeric string (digits only) used to encrypt/decrypt strings.
*/
Configure::write('Security.cipherSeed', '395786739573056621429506834955');
/**
* Apply timestamps with the last modified time to static assets (js, css, images).
* Will append a querystring parameter containing the time the file was modified. This is
* useful for invalidating browser caches.
*
* Set to `true` to apply timestamps when debug > 0. Set to 'force' to always enable
* timestamping regardless of debug value.
*/
//Configure::write('Asset.timestamp', true);
/**
* Compress CSS output by removing comments, whitespace, repeating tags, etc.
* This requires a/var/cache directory to be writable by the web server for caching.
* and /vendors/csspp/csspp.php
*
* To use, prefix the CSS link URL with '/ccss/' instead of '/css/' or use HtmlHelper::css().
*/
//Configure::write('Asset.filter.css', 'css.php');
/**
* Plug in your own custom JavaScript compressor by dropping a script in your webroot to handle the
* output, and setting the config below to the name of the script.
*
* To use, prefix your JavaScript link URLs with '/cjs/' instead of '/js/' or use JavaScriptHelper::link().
*/
//Configure::write('Asset.filter.js', 'custom_javascript_output_filter.php');
/**
* The classname and database used in CakePHP's
* access control lists.
*/
Configure::write('Acl.classname', 'DbAcl');
Configure::write('Acl.database', 'default');
/**
* Uncomment this line and correct your server timezone to fix
* any date & time related errors.
*/
//date_default_timezone_set('UTC');
/**
* Pick the caching engine to use. If APC is enabled use it.
* If running via cli - apc is disabled by default. ensure it's available and enabled in this case
*
* Note: 'default' and other application caches should be configured in app/Config/bootstrap.php.
* Please check the comments in boostrap.php for more info on the cache engines available
* and their setttings.
*/
$engine = 'File';
if (function_exists('apcu_dec') && (PHP_SAPI !== 'cli' || ini_get('apc.enable_cli'))) {
require_once APP . 'Plugin/ApcuCache/Engine/ApcuEngine.php'; // it is not possible to use plugin
$engine = 'Apcu';
}
// In development mode, caches should expire quickly.
$duration = '+999 days';
if (Configure::read('debug') >= 1) {
$duration = '+10 seconds';
}
// Prefix each application on the same server with a different string, to avoid Memcache and APC conflicts.
$prefix = 'myapp_';
/**
* Configure the cache used for general framework caching. Path information,
* object listings, and translation cache files are stored with this configuration.
*/
Cache::config('_cake_core_', array(
'engine' => $engine,
'prefix' => $prefix . 'cake_core_',
'path' => CACHE . 'persistent' . DS,
'serialize' => ($engine === 'File'),
'duration' => $duration
));
/**
* Configure the cache for model and datasource caches. This cache configuration
* is used to store schema descriptions, and table listings in connections.
*/
Cache::config('_cake_model_', array(
'engine' => $engine,
'prefix' => $prefix . 'cake_model_',
'path' => CACHE . 'models' . DS,
'serialize' => ($engine === 'File'),
'duration' => $duration
));
//require_once dirname(__DIR__) . '/Vendor/autoload.php';