MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'develop' of github.com:MISP/MISP into develop

pull/9538/head
Alexandre Dulaunoy 2024-01-31 13:34:26 +01:00
parent 7b49980b5c 9425c99894
commit 4be80d39a8
No known key found for this signature in database
GPG Key ID: 09E2CD4944E6CBCD
98 changed files with 2866 additions and 37744 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

381
.github/workflows/main.yml vendored
View File

@ -6,9 +6,9 @@ name: misp
# events but only for the 2.4 and develop branches
on:
push:
branches: [ 2.4, develop, misp-stix, taxii ]
branches: [ '2.4', develop, misp-stix, taxii ]
pull_request:
branches: [ 2.4, develop, misp-stix ]
branches: [ '2.4', develop, misp-stix ]
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
@ -62,186 +62,178 @@ jobs:
php_version: ${{ matrix.php }}
run: |
sudo apt-get -y update
# Repo is missing for unknown reason
LC_ALL=C.UTF-8 sudo apt-add-repository ppa:ondrej/php -y
if [[ $php_version == "7.2" ]]; then
# hotfix due to: https://bugs.php.net/bug.php?id=81640 TODO: remove after libpcre2-8-0:10.36 gets to stable channel
sudo apt-get --fix-broken install
fi
sudo apt-get -y install curl python3 python3-pip python3-virtualenv apache2 libapache2-mod-php$php_version
# Runs a set of commands using the runners shell
- name: Install deps
run: |
sudo chown $USER:www-data $HOME/.composer
pushd app
sudo -H -u $USER composer config --no-plugins allow-plugins.composer/installers true
sudo -H -u $USER composer install --no-progress
popd
cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php
# Set perms
sudo chown -R $USER:www-data `pwd`
sudo chmod -R 775 `pwd`
sudo chmod -R g+ws `pwd`/app/tmp
sudo chmod -R g+ws `pwd`/app/tmp/cache
sudo chmod -R g+ws `pwd`/app/tmp/cache/persistent
sudo chmod -R g+ws `pwd`/app/tmp/cache/models
sudo chmod -R g+ws `pwd`/app/tmp/logs
sudo chmod -R g+ws `pwd`/app/files
sudo chmod -R g+ws `pwd`/app/files/scripts/tmp
sudo chown -R $USER:www-data `pwd`
# Resque perms
sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp
sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp
# install MySQL
sudo chmod -R 777 `pwd`/INSTALL
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';"
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant usage on *.* to misp@'%' identified by 'blah';"
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant all privileges on misp.* to misp@'%';"
mysql -h 127.0.0.1 --port 3306 -u misp -pblah misp < INSTALL/MYSQL.sql
# configure apache virtual hosts
sudo chmod -R 777 `pwd`/build
sudo mkdir -p /etc/apache2/sites-available
sudo cp -f build/github-action-ci-apache /etc/apache2/sites-available/misp.conf
sudo sed -e "s?%GITHUB_WORKSPACE%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.conf
sudo sed -e "s?%HOST%?${HOST}?g" --in-place /etc/apache2/sites-available/misp.conf
sudo a2dissite 000-default
sudo a2ensite misp.conf
cat /etc/apache2/sites-enabled/misp.conf
sudo a2enmod rewrite
sudo systemctl restart apache2
# MISP configuration
sudo chmod -R 777 `pwd`/travis
sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php
sudo cp travis/database.php app/Config/database.php
sudo cp app/Config/core.default.php app/Config/core.php
sudo cp app/Config/config.default.php app/Config/config.php
sudo cp travis/email.php app/Config/email.php
# Ensure the perms
sudo chown -R $USER:www-data `pwd`/app/Config
sudo chmod -R 777 `pwd`/app/Config
# GPG setup
sudo mkdir `pwd`/.gnupg
# /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS
sudo cp -a /dev/urandom /dev/random
sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/travis/gpg
sudo gpg --list-secret-keys --homedir `pwd`/.gnupg
# change perms
sudo chown -R $USER:www-data `pwd`
sudo chown -R www-data:www-data `pwd`/.gnupg
sudo chmod -R 700 `pwd`/.gnupg
sudo usermod -a -G www-data $USER
sudo chmod -R 777 `pwd`/app/Plugin/CakeResque/tmp/
# Ensure the perms of config files
sudo chown -R $USER:www-data `pwd`/app/Config
sudo chmod -R 777 `pwd`/app/Config
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.server_settings_skip_backup_rotate" 1'
sudo chown -R $USER:www-data `pwd`/app/Config
sudo chmod -R 777 `pwd`/app/Config
sudo chown $USER:www-data $HOME/.composer
pushd app
composer config --no-plugins allow-plugins.composer/installers true
composer install --no-progress
popd
cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php
# Set perms
sudo chown -R $USER:www-data `pwd`
sudo chmod -R 775 `pwd`
sudo chmod -R g+ws `pwd`/app/tmp
sudo chmod -R g+ws `pwd`/app/tmp/cache
sudo chmod -R g+ws `pwd`/app/tmp/cache/persistent
sudo chmod -R g+ws `pwd`/app/tmp/cache/models
sudo chmod -R g+ws `pwd`/app/tmp/logs
sudo chmod -R g+ws `pwd`/app/files
sudo chmod -R g+ws `pwd`/app/files/scripts/tmp
sudo chown -R $USER:www-data `pwd`
# Resque perms
sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp
sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp
# Fill database with basic MISP schema
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';"
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant usage on *.* to misp@'%' identified by 'blah';"
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant all privileges on misp.* to misp@'%';"
mysql -h 127.0.0.1 --port 3306 -u misp -pblah misp < INSTALL/MYSQL.sql
# configure apache virtual hosts
sudo mkdir -p /etc/apache2/sites-available
sudo cp -f build/github-action-ci-apache /etc/apache2/sites-available/misp.conf
sudo sed -e "s?%GITHUB_WORKSPACE%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.conf
sudo sed -e "s?%HOST%?${HOST}?g" --in-place /etc/apache2/sites-available/misp.conf
sudo a2dissite 000-default
sudo a2ensite misp.conf
cat /etc/apache2/sites-enabled/misp.conf
sudo a2enmod rewrite
sudo systemctl start --no-block apache2
# MISP configuration
sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php
sudo cp build/database.php app/Config/database.php
sudo cp app/Config/core.default.php app/Config/core.php
sudo cp app/Config/config.default.php app/Config/config.php
sudo cp build/email.php app/Config/email.php
# GPG setup
sudo mkdir `pwd`/.gnupg
# /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS
sudo cp -a /dev/urandom /dev/random
sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/build/gpg
sudo gpg --list-secret-keys --homedir `pwd`/.gnupg
# change perms
sudo chown -R $USER:www-data `pwd`
sudo chown -R www-data:www-data `pwd`/.gnupg
sudo chmod -R 700 `pwd`/.gnupg
sudo usermod -a -G www-data $USER
sudo chmod -R 777 `pwd`/app/Plugin/CakeResque/tmp/
# Ensure the perms of config files
sudo chown -R $USER:www-data `pwd`/app/Config
sudo chmod -R 777 `pwd`/app/Config
app/Console/cake Admin setSetting "MISP.server_settings_skip_backup_rotate" 1
sudo chown -R $USER:www-data `pwd`/app/Config
sudo chmod -R 777 `pwd`/app/Config
# fix perms (?)
namei -m /home/runner/work
sudo chmod +x /home/runner/work
sudo chmod +x /home/runner
sudo chmod +x /home
sudo chmod +x /
- name: DB Update
run: |
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.osuser" $USER'
sudo -E su $USER -c 'app/Console/cake Admin runUpdates'
sudo -E su $USER -c 'app/Console/cake Admin schemaDiagnostics'
- name: Configure MISP
run: |
sudo -u $USER app/Console/cake userInit -q | sudo tee ./key.txt
echo "AUTH=`cat key.txt`" >> $GITHUB_ENV
sudo -u $USER app/Console/cake Admin setSetting "Session.autoRegenerate" 0
sudo -u $USER app/Console/cake Admin setSetting "Session.timeout" 600
sudo -u $USER app/Console/cake Admin setSetting "Session.cookieTimeout" 3600
sudo -u $USER app/Console/cake Admin setSetting "MISP.host_org_id" 1
sudo -u $USER app/Console/cake Admin setSetting "MISP.email" "info@admin.test"
sudo -u $USER app/Console/cake Admin setSetting "MISP.disable_emailing" false
sudo -u $USER app/Console/cake Admin setSetting --force "debug" true
sudo -u $USER app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false
sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1"
sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_port" 6379
sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_database" 13
sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_password" ""
sudo -u $USER app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test"
sudo -u $USER app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg"
sudo -u $USER app/Console/cake Admin setSetting "GnuPG.password" "travistest"
sudo -u $USER app/Console/cake Admin setSetting "MISP.download_gpg_from_homedir" 1
- name: Configure ZMQ
run: |
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_host" "127.0.0.1"
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_port" 6379
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_database" 1
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_password" ""
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" 1
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_audit_notifications_enable" 1
- name: Update Galaxies
run: sudo -E su $USER -c 'app/Console/cake Admin updateGalaxies'
- name: Update Taxonomies
run: sudo -E su $USER -c 'app/Console/cake Admin updateTaxonomies'
- name: Update Warninglists
run: sudo -E su $USER -c 'app/Console/cake Admin updateWarningLists --verbose'
- name: Update Noticelists
run: sudo -E su $USER -c 'app/Console/cake Admin updateNoticeLists'
- name: Update Object Templates
run: sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates 1'
- name: Turn MISP live
run: sudo -E su $USER -c 'app/Console/cake Live 1'
- name: Check if Redis is ready
run: sudo -E su $USER -c 'app/Console/cake Admin redisReady'
- name: Start workers
run: |
sudo chmod +x app/Console/worker/start.sh
sudo -u www-data 'app/Console/worker/start.sh'
# fix perms (?)
namei -m /home/runner/work
sudo chmod +x /home/runner/work
sudo chmod +x /home/runner
sudo chmod +x /home
sudo chmod +x /
- name: Python setup
run: |
sudo chmod 777 ./key.txt
sudo chmod -R 777 ./tests
# Start workers
# Dirty install python stuff
python3 -m virtualenv -p python3 ./venv
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python"'
. ./venv/bin/activate
export PYTHONPATH=$PYTHONPATH:./app/files/scripts
pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara pytest
deactivate
# Dirty install python stuff
python3 -m virtualenv -p python3 ./venv
app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python"
. ./venv/bin/activate
export PYTHONPATH=$PYTHONPATH:./app/files/scripts
pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara pytest
deactivate
- name: DB Update
run: |
app/Console/cake Admin setSetting "MISP.osuser" $USER
app/Console/cake Admin runUpdates
app/Console/cake Admin schemaDiagnostics
- name: Configure MISP
run: |
app/Console/cake User init | sudo tee ./key.txt
echo "AUTH=`cat key.txt`" >> $GITHUB_ENV
app/Console/cake Admin setSetting "Session.autoRegenerate" 0
app/Console/cake Admin setSetting "Session.timeout" 600
app/Console/cake Admin setSetting "Session.cookieTimeout" 3600
app/Console/cake Admin setSetting "MISP.host_org_id" 1
app/Console/cake Admin setSetting "MISP.email" "info@admin.test"
app/Console/cake Admin setSetting "MISP.disable_emailing" false
app/Console/cake Admin setSetting --force "debug" true
app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false
app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1"
app/Console/cake Admin setSetting "MISP.redis_port" 6379
app/Console/cake Admin setSetting "MISP.redis_database" 13
app/Console/cake Admin setSetting "MISP.redis_password" ""
app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test"
app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg"
app/Console/cake Admin setSetting "GnuPG.password" "travistest"
app/Console/cake Admin setSetting "MISP.download_gpg_from_homedir" 1
app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_host" "127.0.0.1"
app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_port" 6379
app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_database" 1
app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_password" ""
app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" 1
app/Console/cake Admin setSetting "Plugin.ZeroMQ_audit_notifications_enable" 1
- name: Update Galaxies
run: app/Console/cake Admin updateGalaxies
- name: Update Taxonomies
run: app/Console/cake Admin updateTaxonomies
- name: Update Warninglists
run: app/Console/cake Admin updateWarningLists --verbose
- name: Update Noticelists
run: app/Console/cake Admin updateNoticeLists
- name: Update Object Templates
run: app/Console/cake Admin updateObjectTemplates 1
- name: Turn MISP live
run: app/Console/cake Admin live 1
- name: Check if Redis is ready
run: app/Console/cake Admin redisReady
- name: Start workers
run: |
sudo chmod +x app/Console/worker/start.sh
sudo -u www-data 'app/Console/worker/start.sh'
- name: Test if apache is working
run: |
sudo systemctl status apache2 --no-pager -l
sudo apache2ctl -S
curl http://${HOST}
sudo chmod -R 777 PyMISP
pushd PyMISP
echo 'url = "http://'${HOST}'"' >> tests/keys.py
echo 'key = "'${AUTH}'"' >> tests/keys.py
cat tests/keys.py
popd
. ./venv/bin/activate
pushd tests
bash ./build-test.sh
popd
deactivate
sudo systemctl status apache2 --no-pager -l
sudo apache2ctl -S
curl -sS http://${HOST}
- name: Check if dependencies working as expected
run: |
sudo chmod -R 777 PyMISP
pushd PyMISP
echo 'url = "http://'${HOST}'"' >> tests/keys.py
echo 'key = "'${AUTH}'"' >> tests/keys.py
cat tests/keys.py
popd
. ./venv/bin/activate
pushd tests
bash ./build-test.sh
popd
deactivate
- name: Run PHP tests
run: |
./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
sudo -u www-data ./app/Vendor/bin/phpunit app/Test/
./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ -e php,ctp app/
sudo -u www-data ./app/Vendor/bin/phpunit app/Test/
- name: Clone test files
uses: actions/checkout@v4
@ -249,31 +241,30 @@ jobs:
repository: viper-framework/viper-test-files
path: PyMISP/tests/viper-test-files
- name: Run tests
run: |
pushd tests
./curl_tests_GH.sh $AUTH $HOST
popd
sudo chmod -R g+ws `pwd`/app/tmp/logs
. ./venv/bin/activate
pushd PyMISP
cp tests/keys.py .
python -m pytest -v --durations=0 tests/test_mispevent.py
python -m pytest -v --durations=0 tests/testlive_comprehensive.py
popd
python tests/testlive_security.py -v
python tests/testlive_sync.py
python tests/testlive_comprehensive_local.py -v
cp PyMISP/tests/keys.py PyMISP/examples/events/
pushd PyMISP/examples/events/
python ./create_massive_dummy_events.py -l 5 -a 30
popd
pip install jsonschema
python tools/misp-feed/validate.py
deactivate
pushd tests
./curl_tests_GH.sh $AUTH $HOST
popd
sudo chmod -R g+ws `pwd`/app/tmp/logs
. ./venv/bin/activate
pushd PyMISP
cp tests/keys.py .
python -m pytest -v --durations=0 tests/test_mispevent.py
python -m pytest -v --durations=0 tests/testlive_comprehensive.py
popd
python tests/testlive_security.py -v
python tests/testlive_sync.py
python tests/testlive_comprehensive_local.py -v
cp PyMISP/tests/keys.py PyMISP/examples/events/
pushd PyMISP/examples/events/
python ./create_massive_dummy_events.py -l 5 -a 30
popd
pip install jsonschema
python tools/misp-feed/validate.py
deactivate
- name: Check requirements.txt
run: python tests/check_requirements.py
@ -282,13 +273,13 @@ jobs:
if: ${{ always() }}
# update logs_test.sh when adding more logsources here
run: |
tail -n +1 `pwd`/app/tmp/logs/*
tail -n +1 /var/log/apache2/*.log
tail -n +1 `pwd`/app/tmp/logs/*
tail -n +1 /var/log/apache2/*.log
sudo -u $USER app/Console/cake Log export /tmp/logs.json.gz --without-changes
zcat /tmp/logs.json.gz
app/Console/cake Log export /tmp/logs.json.gz --without-changes
zcat /tmp/logs.json.gz
- name: Errors in Logs
if: ${{ always() }}
run: |
./tests/logs_tests.sh
./tests/logs_tests.sh

195
.travis.yml
View File

@ -1,195 +0,0 @@
language: php
php:
- 7.2
- 7.3
- 7.4
- nightly
services:
- redis
sudo: required
dist: bionic
addons:
mariadb: '10.2'
hosts:
- misp.local
- localhost
before_install:
- git config --global user.name "TravisCI"
- export PATH="$HOME/.local/bin:$PATH"
install:
- date
- sudo apt-get -y update
# Install haveged, because Travis lacks entropy.
- sudo apt-get -y install haveged python3 python3-venv python3-pip python3-dev python3-nose python3-redis python3-lxml python3-dateutil python3-msgpack libxml2-dev libzmq3-dev zlib1g-dev apache2 curl php-mysql php-dev php-cli libapache2-mod-php libfuzzy-dev php-mbstring libonig4 php-json php-xml php-opcache php-readline php-redis php-gnupg php-gd
- sudo pip3 install --upgrade pip setuptools requests
- sudo pip3 install --upgrade -r requirements.txt
- sudo pip3 install --upgrade -r requirements-dev.txt
- pip3 install --user poetry
- phpenv rehash
- sudo mkdir $HOME/.composer ; sudo chown $USER:www-data $HOME/.composer
- pushd app
- sudo -H -u $USER php composer.phar install --no-progress
- sudo phpenmod redis
- sudo phpenmod gnupg
- popd
- cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php
# Set perms
- sudo chown -R $USER:www-data `pwd`
- sudo chmod -R 775 `pwd`
- sudo chmod -R g+ws `pwd`/app/tmp
- sudo chmod -R g+ws `pwd`/app/tmp/cache
- sudo chmod -R g+ws `pwd`/app/tmp/cache/persistent
- sudo chmod -R g+ws `pwd`/app/tmp/cache/models
- sudo chmod -R g+ws `pwd`/app/tmp/logs
- sudo chmod -R g+ws `pwd`/app/files
- sudo chmod -R g+ws `pwd`/app/files/scripts/tmp
- sudo chown -R $USER:www-data `pwd`
# Resque perms
- sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp
- sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp
# install MySQL
- sudo chmod -R 777 `pwd`/INSTALL
- mysql -u root -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';"
- mysql -u root -e 'create database misp;'
- mysql -u root -e "grant usage on *.* to misp@localhost identified by 'blah'";
- mysql -u root -e "grant all privileges on misp.* to misp@localhost;"
- mysql -u misp -pblah misp < INSTALL/MYSQL.sql
# configure apache virtual hosts
- sudo chmod -R 777 `pwd`/build
- sudo mkdir -p /etc/apache2/sites-available
- sudo cp -f build/travis-ci-apache /etc/apache2/sites-available/misp.local.conf
- sudo sed -e "s?%TRAVIS_BUILD_DIR%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.local.conf
- sudo a2dissite 000-default
- sudo a2ensite misp.local.conf
- sudo a2enmod rewrite
- sudo service apache2 restart
# MISP configuration
- sudo chmod -R 777 `pwd`/travis
- sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php
- sudo cp travis/database.php app/Config/database.php
- sudo cp app/Config/core.default.php app/Config/core.php
- sudo cp app/Config/config.default.php app/Config/config.php
- sudo cp travis/email.php app/Config/email.php
# Ensure the perms
- sudo chown -R $USER:www-data `pwd`/app/Config
- sudo chmod -R 770 `pwd`/app/Config
# GPG setup
- sudo mkdir `pwd`/.gnupg
# /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS
- sudo cp -a /dev/urandom /dev/random
- sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/travis/gpg
- sudo gpg --list-secret-keys --homedir `pwd`/.gnupg
# change perms
- sudo chown -R $USER:www-data `pwd`
- sudo chmod +x /home/travis/build
- sudo chmod +x /home/travis
- sudo chmod +x /home
- sudo chmod -R 770 `pwd`/.gnupg
# Get authkey
- sudo usermod -a -G www-data $USER
- sudo -E su $USER -c 'app/Console/cake Admin runUpdates'
- sudo -E su $USER -c 'app/Console/cake userInit -q | sudo tee ./key.txt'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.autoRegenerate" 0'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.timeout" 600'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.cookieTimeout" 3600'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.host_org_id" 1'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.email" "info@admin.test"'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.disable_emailing" false'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "debug" true'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1"'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_port" 6379'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_database" 13'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_password" ""'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test"'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg"'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.password" "travistest"'
- sudo -E su $USER -c 'app/Console/cake Admin updateGalaxies'
- sudo -E su $USER -c 'app/Console/cake Admin updateTaxonomies'
- sudo -E su $USER -c 'app/Console/cake Admin updateWarningLists'
- sudo -E su $USER -c 'app/Console/cake Admin updateNoticeLists'
- sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates 1'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" true'
- sudo -E su $USER -c 'app/Console/cake Live 1'
- sudo chmod 777 ./key.txt
- sudo chmod -R 777 ./tests
# Start workers
- sudo chmod +x app/Console/worker/start.sh
- sudo -E su $USER -c 'app/Console/worker/start.sh &'
- sleep 10
# Dirty install python stuff
- virtualenv -p python3.6 ./venv
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$TRAVIS_BUILD_DIR/venv/bin/python"'
- . ./venv/bin/activate
- pushd cti-python-stix2
- pip install .
- popd
- pushd PyMISP
- pip install .[fileobjects]
- popd
- pip install stix zmq redis plyara
- deactivate
before_script:
- curl http://misp.local
- AUTH=`cat key.txt`
- sudo chmod -R 777 PyMISP
- pushd PyMISP
- echo 'url = "http://misp.local"' >> tests/keys.py
- echo 'key = "'${AUTH}'"' >> tests/keys.py
- cat tests/keys.py
- popd
script:
- ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
- ./app/Vendor/bin/phpunit app/Test/ComplexTypeToolTest.php
- ./app/Vendor/bin/phpunit app/Test/JSONConverterToolTest.php
# Ensure the perms
- sudo chown -R $USER:www-data `pwd`/app/Config
- sudo chmod -R 770 `pwd`/app/Config
- pushd tests
- ./curl_tests.sh $AUTH
- popd
- pushd PyMISP
- git submodule init
- git submodule update
- travis_retry poetry install -E fileobjects -E openioc -E virustotal -E docs -E pdfexport
- poetry run python tests/testlive_comprehensive.py
- poetry run python tests/test_mispevent.py
- popd
- cp PyMISP/tests/keys.py PyMISP/examples/events/
- pushd PyMISP/examples/events/
- poetry run python ./create_massive_dummy_events.py -l 5 -a 30
- popd
- python3 tools/misp-feed/validate.py
after_failure:
- curl http://misp.local
- cat /etc/apache2/sites-available/misp.local.conf
- sudo tail -n +1 `pwd`/app/tmp/logs/*
- sudo ls -l /var/log/apache2
- sudo cat /var/log/apache2/error.log
- sudo cat /var/log/apache2/misp.local_error.log
- sudo cat /var/log/apache2/misp.local_access.log
- pwd
notifications:
webhooks:
urls:
- https://webhooks.gitter.im/e/05e30284086a8e948d31
on_success: change # options: [always|never|change] default: always
on_failure: always # options: [always|never|change] default: always
on_start: never # options: [always|never|change] default: always
after_success:
- sudo tail -n +1 `pwd`/app/tmp/logs/*
- coveralls
- coverage report
- coverage xml
- codecov

116
app/Console/Command/AdminShell.php
View File

@ -46,11 +46,11 @@ class AdminShell extends AppShell
'help' => __('Update the JSON definition of taxonomies.'),
));
$parser->addSubcommand('setSetting', [
'help' => __('Set setting in PHP config file.'),
'help' => __('Set setting in MISP config'),
'parser' => [
'arguments' => [
'name' => ['help' => __('Setting name'), 'required' => true],
'value' => ['help' => __('Setting value'), 'required' => true],
'value' => ['help' => __('Setting value')],
],
'options' => [
'force' => [
@ -72,7 +72,7 @@ class AdminShell extends AppShell
'help' => __('Set if MISP instance is live and accessible for users.'),
'parser' => [
'arguments' => [
'state' => ['help' => __('Set Live state')],
'state' => ['help' => __('Set Live state (boolean). If not provided, current state will be printed.')],
],
],
]);
@ -85,6 +85,14 @@ class AdminShell extends AppShell
],
],
]);
$parser->addSubcommand('isEncryptionKeyValid', [
'help' => __('Check if current encryption key is valid.'),
'parser' => [
'options' => [
'encryptionKey' => ['help' => __('Encryption key to test. If not provided, current key will be used.')],
],
],
]);
$parser->addSubcommand('dumpCurrentDatabaseSchema', [
'help' => __('Dump current database schema to JSON file.'),
]);
@ -109,6 +117,20 @@ class AdminShell extends AppShell
$parser->addSubcommand('configLint', [
'help' => __('Check if settings has correct value.'),
]);
$parser->addSubcommand('createZmqConfig', [
'help' => __('Create config file for ZeroMQ server.'),
]);
$parser->addSubcommand('scanAttachment', [
'help' => __('Scan attachments with AV.'),
'parser' => [
'arguments' => [
'type' => ['help' => __('all, Attribute or ShadowAttribute'), 'required' => true],
'attributeId' => ['help' => __('ID to scan.')],
'jobId' => ['help' => __('Job ID')],
],
],
]);
return $parser;
}
@ -485,32 +507,47 @@ class AdminShell extends AppShell
}
}
}
echo json_encode($result, JSON_PRETTY_PRINT) . PHP_EOL;
$this->out($this->json($result));
}
public function setSetting()
{
list($setting_name, $value) = $this->args;
if ($value === 'false') {
$value = 0;
} elseif ($value === 'true') {
$value = 1;
}
if ($this->params['null']) {
list($settingName) = $this->args;
if ($this->params['null'] && isset($this->args[1])) {
$this->error(__('Trying to set setting to null value, but value was provided.'));
} else if ($this->params['null']) {
$value = null;
} elseif (isset($this->args[1])) {
$value = $this->args[1];
} else {
$this->error(__('No setting value provided.'));
}
$cli_user = array('id' => 0, 'email' => 'SYSTEM', 'Organisation' => array('name' => 'SYSTEM'));
if (empty($setting_name) || ($value === null && !$this->params['null'])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Set setting'] . PHP_EOL);
}
$setting = $this->Server->getSettingData($setting_name);
$setting = $this->Server->getSettingData($settingName);
if (empty($setting)) {
$message = 'Invalid setting "' . $setting_name . '". Please make sure that the setting that you are attempting to change exists and if a module parameter, the modules are running.' . PHP_EOL;
$message = 'Invalid setting "' . $settingName . '". Please make sure that the setting that you are attempting to change exists and if a module parameter, the modules are running.' . PHP_EOL;
$this->error(__('Setting change rejected.'), $message);
}
$result = $this->Server->serverSettingsEditValue($cli_user, $setting, $value, $this->params['force']);
// Convert value to boolean or to int
if ($value !== null) {
if ($setting['type'] === 'boolean') {
$value = $this->toBoolean($value);
} else if ($setting['type'] === 'numeric') {
if (is_numeric($value)) {
$value = (int)$value;
} elseif ($value === 'true' || $value === 'false') {
$value = $value === 'true' ? 1 : 0; // special case for `debug` setting
} else {
$this->error(__('Setting "%s" change rejected.', $settingName), __('Provided value %s is not a number.', $value));
}
}
}
$result = $this->Server->serverSettingsEditValue('SYSTEM', $setting, $value, $this->params['force']);
if ($result === true) {
$this->out(__('Setting "%s" changed to %s', $setting_name, is_string($value) ? '"' . $value . '"' : (string)$value));
$this->out(__('Setting "%s" changed to %s', $settingName, is_string($value) ? '"' . $value . '"' : json_encode($value)));
} else {
$message = __("The setting change was rejected. MISP considers the requested setting value as invalid and would lead to the following error:\n\n\"%s\"\n\nIf you still want to force this change, please supply the --force argument.\n", $result);
$this->error(__('Setting change rejected.'), $message);
@ -648,6 +685,8 @@ class AdminShell extends AppShell
*/
public function change_authkey()
{
$this->deprecated('cake user change_authkey [user_id]');
if (empty($this->args[0])) {
echo 'MISP apikey command line tool' . PHP_EOL . 'To assign a new random API key for a user: ' . APP . 'Console/cake Admin change_authkey [user_email]' . PHP_EOL . 'To assign a fixed API key: ' . APP . 'Console/cake Admin change_authkey [user_email] [authkey]' . PHP_EOL;
die();
@ -787,6 +826,8 @@ class AdminShell extends AppShell
*/
public function UserIP()
{
$this->deprecated('cake user user_ips [user_id]');
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get IPs for user ID'] . PHP_EOL);
}
@ -814,6 +855,8 @@ class AdminShell extends AppShell
*/
public function IPUser()
{
$this->deprecated('cake user ip_user [ip]');
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get user ID for user IP'] . PHP_EOL);
}
@ -839,8 +882,8 @@ class AdminShell extends AppShell
public function scanAttachment()
{
$input = $this->args[0];
$attributeId = isset($this->args[1]) ? $this->args[1] : null;
$jobId = isset($this->args[2]) ? $this->args[2] : null;
$attributeId = $this->args[1] ?? null;
$jobId = $this->args[2] ?? null;
$this->loadModel('AttachmentScan');
$result = $this->AttachmentScan->scan($input, $attributeId, $jobId);
@ -951,7 +994,7 @@ class AdminShell extends AppShell
$newStatus = $this->toBoolean($this->args[0]);
$overallSuccess = false;
try {
$redis = $this->Server->setupRedisWithException();
$redis = RedisTool::init();
if ($newStatus) {
$redis->del('misp:live');
$this->out('Set live status to True in Redis.');
@ -980,7 +1023,7 @@ class AdminShell extends AppShell
} else {
$this->out('Current status:');
$this->out('PHP Config file: ' . (Configure::read('MISP.live') ? 'True' : 'False'));
$newStatus = $this->Server->setupRedisWithException()->get('misp:live');
$newStatus = RedisTool::init()->get('misp:live');
$this->out('Redis: ' . ($newStatus !== '0' ? 'True' : 'False'));
}
}
@ -1031,6 +1074,27 @@ class AdminShell extends AppShell
$this->out(__('New encryption key "%s" saved into config file.', $new));
}
public function isEncryptionKeyValid()
{
$encryptionKey = $this->params['encryptionKey'] ?? null;
if ($encryptionKey === null) {
$encryptionKey = Configure::read('Security.encryption_key');
}
if (!$encryptionKey) {
$this->error('No encryption key provided');
}
/** @var SystemSetting $systemSetting */
$systemSetting = ClassRegistry::init('SystemSetting');
try {
$systemSetting->isEncryptionKeyValid($encryptionKey);
$this->Server->isEncryptionKeyValid($encryptionKey);
} catch (Exception $e) {
$this->error($e->getMessage(), __('Probably provided encryption key is invalid'));
}
}
public function redisMemoryUsage()
{
$redis = RedisTool::init();
@ -1240,4 +1304,10 @@ class AdminShell extends AppShell
$this->Job->saveField('message', __('Database truncated: ' . $table));
}
}
public function createZmqConfig()
{
$this->Server->getPubSubTool()->createConfigFile();
$this->err("Config file created in " . PubSubTool::SCRIPTS_TMP);
}
}

15
app/Console/Command/AppShell.php
View File

@ -31,15 +31,13 @@ require_once dirname(__DIR__) . '/../Model/Attribute.php'; // FIXME workaround
*/
abstract class AppShell extends Shell
{
public $tasks = array('ConfigLoad');
/** @var BackgroundJobsTool */
private $BackgroundJobsTool;
public function initialize()
{
$this->ConfigLoad = $this->Tasks->load('ConfigLoad');
$this->ConfigLoad->execute();
$configLoad = $this->Tasks->load('ConfigLoad');
$configLoad->execute();
parent::initialize();
}
@ -84,6 +82,15 @@ abstract class AppShell extends Shell
}
}
/**
* @param string $newCommand
* @return void
*/
protected function deprecated($newCommand)
{
$this->err("<warning>Warning: This method is deprecated. Next time please use `$newCommand`.</warning>");
}
/**
* @return BackgroundJobsTool
* @throws Exception

2
app/Console/Command/AuthkeyShell.php
View File

@ -12,7 +12,7 @@ class AuthkeyShell extends AppShell {
public function main()
{
$this->err('This method is deprecated. Next time please use `cake user change_authkey [user] [authkey]` command.');
$this->deprecated('cake user change_authkey [user] [authkey]');
if (!isset($this->args[0]) || empty($this->args[0])) echo 'MISP authkey reset command line tool.' . PHP_EOL . 'To assign a new authkey for a user:' . PHP_EOL . APP . 'Console/cake Authkey [email] [auth_key | optional]' . PHP_EOL;
else {

2
app/Console/Command/BaseurlShell.php
View File

@ -11,7 +11,7 @@ class BaseurlShell extends AppShell {
public function main()
{
$this->err('This method is deprecated. Next time please use `cake admin setSetting MISP.baseurl [baseurl]` command.');
$this->deprecated('cake admin setSetting MISP.baseurl [baseurl]');
$baseurl = $this->args[0];
$result = $this->Server->testBaseURL($baseurl);

59
app/Console/Command/EventShell.php
View File

@ -53,10 +53,21 @@ class EventShell extends AppShell
$parser->addSubcommand('mergeTags', [
'help' => __('Merge tags'),
'parser' => [
'arguments' => array(
'arguments' => [
'source' => ['help' => __('Source tag ID or name. Source tag will be deleted.'), 'required' => true],
'destination' => ['help' => __('Destination tag ID or name.'), 'required' => true],
)
],
],
]);
$parser->addSubcommand('reportValidationIssuesAttributes', [
'help' => __('Report validation issues on attributes'),
]);
$parser->addSubcommand('normalizeIpAddress', [
'help' => __('Normalize IP address format in old events'),
'parser' => [
'options' => [
'dry-run' => ['help' => __('Just show what changes will be made.'), 'boolean' => true],
],
],
]);
return $parser;
@ -636,18 +647,28 @@ class EventShell extends AppShell
}
}
/**
* @param int $userId
* @return array
*/
private function getUser($userId)
public function reportValidationIssuesAttributes()
{
$user = $this->User->getAuthUser($userId, true);
if (empty($user)) {
$this->error("User with ID $userId does not exist.");
foreach ($this->Event->Attribute->reportValidationIssuesAttributes() as $validationIssue) {
echo $this->json($validationIssue) . "\n";
}
}
public function normalizeIpAddress()
{
$dryRun = $this->param('dry-run');
$count = 0;
foreach ($this->Event->Attribute->normalizeIpAddress($dryRun) as $attribute) {
$count++;
echo JsonTool::encode($attribute) . "\n";
}
if ($dryRun) {
$this->err(__n("%s attribute to fix", "%s attributes to fix", $count, $count));
} else {
$this->err(__n("%s attribute fixed", "%s attributes fixed", $count, $count));
}
Configure::write('CurrentUserId', $user['id']); // for audit logging purposes
return $user;
}
public function generateTopCorrelations()
@ -668,4 +689,18 @@ class EventShell extends AppShell
$this->Job->save($job);
}
}
/**
* @param int $userId
* @return array
*/
private function getUser($userId)
{
$user = $this->User->getAuthUser($userId, true);
if (empty($user)) {
$this->error("User with ID $userId does not exist.");
}
Configure::write('CurrentUserId', $user['id']); // for audit logging purposes
return $user;
}
}

5
app/Console/Command/LiveShell.php
View File

@ -1,8 +1,9 @@
<?php
/*
/**
* Enable/disable misp
*
* arg0 = [0|1]
* @deprecated Use AdminShell::live instead
*/
class LiveShell extends AppShell {
@ -10,6 +11,8 @@ class LiveShell extends AppShell {
public function main()
{
$this->deprecated('cake admin live [0|1]');
$live = $this->args[0];
if ($live != 0 && $live != 1) {
echo 'Invalid parameters. Usage: /var/www/MISP/app/Console/cake Live [0|1]';

2
app/Console/Command/PasswordShell.php
View File

@ -12,7 +12,7 @@ class PasswordShell extends AppShell {
public function main()
{
$this->err('This method is deprecated. Next time please use `cake user change_pw [user] [password]` command.');
$this->deprecated('cake user change_pw [user] [password]');
if (!isset($this->args[0]) || empty($this->args[0]) || !isset($this->args[1]) || empty($this->args[1])) echo 'MISP password reset command line tool.' . PHP_EOL . 'To assign a new password for a user:' . PHP_EOL . APP . 'Console/cake Password [email] [password]' . PHP_EOL;
else {

26
app/Console/Command/StartWorkerShell.php
View File

@ -37,24 +37,32 @@ class StartWorkerShell extends AppShell
public function main()
{
$pid = getmypid();
if ($pid === false) {
throw new RuntimeException("Could not get current process ID");
}
$this->worker = new Worker(
[
'pid' => getmypid(),
'pid' => $pid,
'queue' => $this->args[0],
'user' => ProcessTool::whoami(),
]
);
$this->maxExecutionTime = (int)$this->params['maxExecutionTime'];
$queue = $this->worker->queue();
$backgroundJobTool = $this->getBackgroundJobsTool();
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - starting to process background jobs...");
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$queue}] - starting to process background jobs...");
while (true) {
$this->checkMaxExecutionTime();
$job = $this->getBackgroundJobsTool()->dequeue($this->worker->queue());
$job = $backgroundJobTool->dequeue($queue);
if ($job) {
$this->runJob($job);
$backgroundJobTool->removeFromRunning($this->worker, $job);
}
}
}
@ -64,7 +72,7 @@ class StartWorkerShell extends AppShell
*/
private function runJob(BackgroundJob $job)
{
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}...");
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}");
try {
$job->setStatus(BackgroundJob::STATUS_RUNNING);
@ -73,12 +81,16 @@ class StartWorkerShell extends AppShell
CakeLog::info("[JOB ID: {$job->id()}] - started command `$command`.");
$this->getBackgroundJobsTool()->update($job);
$job->run();
$start = microtime(true);
$job->run(function (array $status) use ($job) {
$this->getBackgroundJobsTool()->markAsRunning($this->worker, $job, $status['pid']);
});
$duration = number_format(microtime(true) - $start, 3, '.', '');
if ($job->status() === BackgroundJob::STATUS_COMPLETED) {
CakeLog::info("[JOB ID: {$job->id()}] - completed.");
CakeLog::info("[JOB ID: {$job->id()}] - successfully completed in $duration seconds.");
} else {
CakeLog::error("[JOB ID: {$job->id()}] - failed with error code {$job->returnCode()}. STDERR: {$job->error()}. STDOUT: {$job->output()}.");
CakeLog::error("[JOB ID: {$job->id()}] - failed with error code {$job->returnCode()} after $duration seconds. STDERR: {$job->error()}. STDOUT: {$job->output()}.");
}
} catch (Exception $exception) {
CakeLog::error("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - job ID: {$job->id()} failed with exception: {$exception->getMessage()}");

2
app/Console/Command/Task/ConfigLoadTask.php
View File

@ -3,8 +3,6 @@ class ConfigLoadTask extends Shell
{
public function execute()
{
Configure::load('config');
if (Configure::read('MISP.system_setting_db')) {
App::uses('SystemSetting', 'Model');
SystemSetting::setGlobalSetting();

6
app/Console/Command/UserInitShell.php
View File

@ -1,7 +1,13 @@
<?php
/**
* @deprecated
*/
class UserInitShell extends AppShell {
public $uses = array('User', 'Role', 'Organisation', 'Server', 'ConnectionManager');
public function main() {
$this->deprecated('cake user init');
if (!Configure::read('Security.salt')) {
$this->loadModel('Server');
$this->Server->serverSettingsSaveValue('Security.salt', $this->User->generateRandomPassword(32));

135
app/Console/Command/UserShell.php
View File

@ -3,10 +3,11 @@
/**
* @property User $User
* @property Log $Log
* @property UserLoginProfile $UserLoginProfile
*/
class UserShell extends AppShell
{
public $uses = ['User', 'Log'];
public $uses = ['User', 'Log', 'UserLoginProfile'];
public function getOptionParser()
{
@ -22,16 +23,24 @@ class UserShell extends AppShell
],
]
]);
$parser->addSubcommand('init', [
'help' => __('Create default role, organisation and user when not exists.'),
]);
$parser->addSubcommand('authkey', [
'help' => __('Get information about given authkey.'),
'parser' => [
'arguments' => [
'authkey' => ['help' => __('Authentication key. If not provide, it will be read from STDIN.')],
'authkey' => ['help' => __('Authentication key. If not provided, it will be read from STDIN.')],
],
]
]);
$parser->addSubcommand('authkey_valid', [
'help' => __('Check if given authkey by STDIN is valid.'),
'parser' => [
'options' => [
'disableStdLog' => ['help' => __('Do not show logs in STDOUT or STDERR.'), 'boolean' => true],
],
],
]);
$parser->addSubcommand('block', [
'help' => __('Immediately block user.'),
@ -104,6 +113,14 @@ class UserShell extends AppShell
],
],
]);
$parser->addSubcommand('ip_country', [
'help' => __('Get country for given IP address'),
'parser' => [
'arguments' => [
'ip' => ['help' => __('IPv4 or IPv6 address.'), 'required' => true],
]
],
]);
$parser->addSubcommand('require_password_change_for_old_passwords', [
'help' => __('Trigger forced password change on next login for users with an old (older than x days) password.'),
'parser' => [
@ -121,7 +138,7 @@ class UserShell extends AppShell
public function list()
{
$userId = isset($this->args[0]) ? $this->args[0] : null;
$userId = $this->args[0] ?? null;
if ($userId) {
$conditions = ['OR' => [
'User.id' => $userId,
@ -163,13 +180,24 @@ class UserShell extends AppShell
}
}
public function init()
{
if (!Configure::read('Security.salt')) {
$this->loadModel('Server');
$this->Server->serverSettingsSaveValue('Security.salt', $this->User->generateRandomPassword(32));
}
$authKey = $this->User->init();
if ($authKey === null) {
$this->err('Script aborted: MISP instance already initialised.');
} else {
$this->out($authKey);
}
}
public function authkey()
{
if (isset($this->args[0])) {
$authkey = $this->args[0];
} else {
$authkey = fgets(STDIN); // read line from STDIN
}
$authkey = $this->args[0] ?? fgets(STDIN);
$authkey = trim($authkey);
if (strlen($authkey) !== 40) {
$this->error('Authkey has not valid format.');
@ -212,28 +240,37 @@ class UserShell extends AppShell
*/
public function authkey_valid()
{
if ($this->params['disableStdLog']) {
$this->_useLogger(false);
}
$cache = [];
$randomKey = random_bytes(16);
do {
$advancedAuthKeysEnabled = (bool)Configure::read('Security.advanced_authkeys');
while (true) {
$authkey = fgets(STDIN); // read line from STDIN
$authkey = trim($authkey);
if (strlen($authkey) !== 40) {
fwrite(STDOUT, "0\n"); // authkey is not in valid format
$this->log("Authkey in incorrect format provided.", LOG_WARNING);
echo "0\n"; // authkey is not in valid format
$this->log("Authkey in incorrect format provided, expected 40 chars long string, $authkey provided.", LOG_WARNING);
continue;
}
$time = time();
// Generate hash from authkey to not store raw authkey in memory
$keyHash = sha1($authkey . $randomKey, true);
// If authkey is in cache and is fresh, use info from cache
$time = time();
if (isset($cache[$keyHash]) && $cache[$keyHash][1] > $time) {
fwrite(STDOUT, $cache[$keyHash][0] ? "1\n" : "0\n");
echo $cache[$keyHash][0] ? "1\n" : "0\n";
continue;
}
$user = false;
for ($i = 0; $i < 5; $i++) {
try {
if (Configure::read('Security.advanced_authkeys')) {
if ($advancedAuthKeysEnabled) {
$user = $this->User->AuthKey->getAuthUserByAuthKey($authkey);
} else {
$user = $this->User->getAuthUserByAuthkey($authkey);
@ -251,18 +288,34 @@ class UserShell extends AppShell
}
}
$user = (bool)$user;
if (!$user) {
$start = substr($authkey, 0, 4);
$end = substr($authkey, -4);
$authKeyToStore = $start . str_repeat('*', 32) . $end;
$this->log("Not valid authkey $authKeyToStore provided.", LOG_WARNING);
$valid = null;
} else if ($user['disabled']) {
$valid = false;
} else {
$valid = true;
}
// Cache results for 5 seconds
$cache[$keyHash] = [$user, $time + 5];
fwrite(STDOUT, $user ? "1\n" : "0\n");
} while (true);
echo $valid ? "1\n" : "0\n";
if ($valid) {
// Cache results for 60 seconds if key is valid
$cache[$keyHash] = [true, $time + 60];
} else {
// Cache results for 5 seconds if key is invalid
$cache[$keyHash] = [false, $time + 5];
$start = substr($authkey, 0, 4);
$end = substr($authkey, -4);
$authKeyForLog = $start . str_repeat('*', 32) . $end;
if ($valid === false) {
$this->log("Authkey $authKeyForLog belongs to user {$user['id']} that is disabled.", LOG_WARNING);
} else {
$this->log("Authkey $authKeyForLog is invalid or expired.", LOG_WARNING);
}
}
}
}
public function block()
@ -305,7 +358,7 @@ class UserShell extends AppShell
$conditions = ['User.disabled' => false]; // fetch just not disabled users
$userId = isset($this->args[0]) ? $this->args[0] : null;
$userId = $this->args[0] ?? null;
if ($userId) {
$conditions['OR'] = [
'User.id' => $userId,
@ -364,7 +417,7 @@ class UserShell extends AppShell
}
$user = $this->getUser($userId);
# validate new authentication key if provided
// validate new authentication key if provided
if (!empty($newkey) && (strlen($newkey) != 40 || !ctype_alnum($newkey))) {
$this->error('The new auth key needs to be 40 characters long and only alphanumeric.');
}
@ -399,7 +452,7 @@ class UserShell extends AppShell
$this->out('<warning>Storing user IP addresses is disabled.</warning>');
}
$ips = $this->User->setupRedisWithException()->smembers('misp:user_ip:' . $user['id']);
$ips = RedisTool::init()->smembers('misp:user_ip:' . $user['id']);
if ($this->params['json']) {
$this->out($this->json($ips));
@ -422,36 +475,50 @@ class UserShell extends AppShell
$this->out('<warning>Storing user IP addresses is disabled.</warning>');
}
$userId = $this->User->setupRedisWithException()->get('misp:ip_user:' . $ip);
$userId = RedisTool::init()->get('misp:ip_user:' . $ip);
if (empty($userId)) {
$this->out('No hits.');
$this->_stop();
}
$user = $this->User->find('first', array(
$user = $this->User->find('first', [
'recursive' => -1,
'conditions' => array('User.id' => $userId),
'conditions' => ['User.id' => $userId],
'fields' => ['id', 'email'],
));
]);
if (empty($user)) {
$this->error("User with ID $userId doesn't exists anymore.");
}
$ipCountry = $this->UserLoginProfile->countryByIp($ip);
if ($this->params['json']) {
$this->out($this->json([
'ip' => $ip,
'id' => $user['User']['id'],
'email' => $user['User']['email'],
'country' => $ipCountry,
]));
} else {
$this->out(sprintf(
'%s==============================%sIP: %s%s==============================%sUser #%s: %s%s==============================%s',
PHP_EOL, PHP_EOL, $ip, PHP_EOL, PHP_EOL, $user['User']['id'], $user['User']['email'], PHP_EOL, PHP_EOL
));
$this->hr();
$this->out("IP: $ip (country $ipCountry)");
$this->hr();
$this->out("User #{$user['User']['id']}: {$user['User']['email']}");
$this->hr();
}
}
public function ip_country()
{
list($ip) = $this->args;
if (!filter_var($ip, FILTER_VALIDATE_IP)) {
$this->error("IP `$ip` is not valid IPv4 or IPv6 address");
}
$this->out($this->UserLoginProfile->countryByIp($ip));
}
public function require_password_change_for_old_passwords()
{
list($days) = $this->args;

119
app/Console/Command/WorkerShell.php Normal file
View File

@ -0,0 +1,119 @@
<?php
declare(strict_types=1);
/**
* @property Job $Job
*/
class WorkerShell extends AppShell
{
public $uses = ['Job'];
public function getOptionParser(): ConsoleOptionParser
{
$parser = parent::getOptionParser();
$parser->addSubcommand('showQueues', [
'help' => __('Show jobs in worker queues'),
]);
$parser->addSubcommand('flushQueue', [
'help' => __('Flush jobs in given queue'),
'parser' => [
'arguments' => [
'queue' => ['help' => __('Queue name'), 'required' => true],
],
],
]);
$parser->addSubcommand('showJobStatus', [
'help' => __('Show job status'),
'parser' => [
'arguments' => [
'job_id' => ['help' => __('Job ID (ID or UUID)'), 'required' => true],
],
],
]);
return $parser;
}
/**
* @throws RedisException
* @throws JsonException
*/
public function showQueues()
{
$tool = $this->getBackgroundJobsTool();
$runningJobs = $tool->runningJobs();
foreach (BackgroundJobsTool::VALID_QUEUES as $queue) {
$this->out("{$queue}:\t{$tool->getQueueSize($queue)}");
$queueJobs = $runningJobs[$queue] ?? [];
foreach ($queueJobs as $jobId => $data) {
$this->out(" - $jobId (" . JsonTool::encode($data) .")");
}
}
}
public function flushQueue()
{
$queue = $this->args[0];
try {
$this->getBackgroundJobsTool()->clearQueue($queue);
} catch (InvalidArgumentException $e) {
$this->error($e->getMessage());
}
}
public function showJobStatus()
{
$processId = $this->args[0];
if (is_numeric($processId)) {
$job = $this->Job->find('first', [
'conditions' => ['Job.id' => $processId],
'recursive' => -1,
]);
if (!$job) {
$this->error('Job not found', "Job with ID {$processId} not found");
}
$this->out($this->json($job['Job']));
$processId = $job['Job']['process_id'];
}
if (!Validation::uuid($processId)) {
$this->error('Job not found', "Job ID must be number or UUID, '$processId' given");
}
$jobStatus = $this->getBackgroundJobsTool()->getJob($processId);
if (!$jobStatus) {
$this->error('Job not found', "Job with UUID {$processId} not found");
}
$jobStatus = $jobStatus->jsonSerialize();
foreach (['createdAt', 'updatedAt'] as $timeField) {
if (isset($jobStatus[$timeField])) {
$jobStatus[$timeField] = date('c', $jobStatus[$timeField]);
}
}
if (isset($jobStatus['status'])) {
$jobStatus['status'] = $this->jobStatusToString($jobStatus['status']);
}
$this->out($this->json($jobStatus));
}
private function jobStatusToString(int $jobStatus)
{
switch ($jobStatus) {
case Job::STATUS_WAITING:
return 'waiting';
case Job::STATUS_RUNNING:
return 'running';
case Job::STATUS_FAILED:
return 'failed';
case Job::STATUS_COMPLETED:
return 'completed';
}
throw new InvalidArgumentException("Invalid job status $jobStatus");
}
}

59
app/Controller/AppController.php
View File

@ -38,8 +38,6 @@ class AppController extends Controller
public $phpmin = '7.2';
public $phprec = '7.4';
public $phptoonew = '8.0';
public $pythonmin = '3.6';
public $pythonrec = '3.7';
private $isApiAuthed = false;
public $baseurl = '';
@ -232,6 +230,10 @@ class AppController extends Controller
$this->Security->csrfCheck = false;
$loginByAuthKeyResult = $this->__loginByAuthKey();
if ($loginByAuthKeyResult === false || $this->Auth->user() === null) {
if ($this->IndexFilter->isXhr()) {
throw new ForbiddenException('Authentication failed.');
}
if ($loginByAuthKeyResult === null) {
$this->loadModel('Log');
$this->Log->createLogEntry('SYSTEM', 'auth_fail', 'User', 0, "Failed API authentication. No authkey was provided.");
@ -601,7 +603,7 @@ class AppController extends Controller
if (!empty($user['allowed_ips'])) {
App::uses('CidrTool', 'Tools');
$cidrTool = new CidrTool($user['allowed_ips']);
$remoteIp = $this->_remoteIp();
$remoteIp = $this->User->_remoteIp();
if ($remoteIp === null) {
$this->Auth->logout();
throw new ForbiddenException('Auth key is limited to IP address, but IP address not found');
@ -694,7 +696,7 @@ class AppController extends Controller
return;
}
$remoteAddress = $this->_remoteIp();
$remoteAddress = $this->User->_remoteIp();
$pipe = $redis->pipeline();
// keep for 30 days
@ -737,7 +739,7 @@ class AppController extends Controller
$includeRequestBody = !empty(Configure::read('MISP.log_paranoid_include_post_body')) || $userMonitoringEnabled;
/** @var AccessLog $accessLog */
$accessLog = ClassRegistry::init('AccessLog');
$accessLog->logRequest($user, $this->_remoteIp(), $this->request, $includeRequestBody);
$accessLog->logRequest($user, $this->User->_remoteIp(), $this->request, $includeRequestBody);
}
if (
@ -828,29 +830,34 @@ class AppController extends Controller
private function __rateLimitCheck(array $user)
{
$info = array();
$rateLimitCheck = $this->RateLimit->check(
$user,
$this->request->params['controller'],
$this->request->action,
$info,
$this->response->type()
$this->request->params['action'],
);
if (!empty($info)) {
$this->RestResponse->setHeader('X-Rate-Limit-Limit', $info['limit']);
$this->RestResponse->setHeader('X-Rate-Limit-Remaining', $info['remaining']);
$this->RestResponse->setHeader('X-Rate-Limit-Reset', $info['reset']);
if ($rateLimitCheck) {
$headers = [
'X-Rate-Limit-Limit' => $rateLimitCheck['limit'],
'X-Rate-Limit-Remaining' => $rateLimitCheck['remaining'],
'X-Rate-Limit-Reset' => $rateLimitCheck['reset'],
];
if ($rateLimitCheck['exceeded']) {
$response = $this->RestResponse->throwException(
429,
__('Rate limit exceeded.'),
'/' . $this->request->params['controller'] . '/' . $this->request->params['action'],
false,
false,
$headers
);
$response->send();
$this->_stop();
} else {
$this->RestResponse->headers = array_merge($this->RestResponse->headers, $headers);
}
}
if ($rateLimitCheck !== true) {
$this->response->header('X-Rate-Limit-Limit', $info['limit']);
$this->response->header('X-Rate-Limit-Remaining', $info['remaining']);
$this->response->header('X-Rate-Limit-Reset', $info['reset']);
$this->response->body($rateLimitCheck);
$this->response->statusCode(429);
$this->response->send();
$this->_stop();
}
return true;
}
public function afterFilter()
@ -1143,14 +1150,14 @@ class AppController extends Controller
$headerNamespace = '';
}
if (isset($server[$headerNamespace . $header]) && !empty($server[$headerNamespace . $header])) {
if (Configure::read('Plugin.CustomAuth_only_allow_source') && Configure::read('Plugin.CustomAuth_only_allow_source') !== $this->_remoteIp()) {
if (Configure::read('Plugin.CustomAuth_only_allow_source') && Configure::read('Plugin.CustomAuth_only_allow_source') !== $this->User->_remoteIp()) {
$this->Log = ClassRegistry::init('Log');
$this->Log->createLogEntry(
'SYSTEM',
'auth_fail',
'User',
0,
'Failed authentication using external key (' . trim($server[$headerNamespace . $header]) . ') - the user has not arrived from the expected address. Instead the request came from: ' . $this->_remoteIp(),
'Failed authentication using external key (' . trim($server[$headerNamespace . $header]) . ') - the user has not arrived from the expected address. Instead the request came from: ' . $this->User->_remoteIp(),
null);
$this->__preAuthException($authName . ' authentication failed. Contact your MISP support for additional information at: ' . Configure::read('MISP.contact'));
}
@ -1310,7 +1317,7 @@ class AppController extends Controller
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception, $this->_legacyParams);
if (empty($filters) && $this->request->is('get')) {
throw new InvalidArgumentException(__('Restsearch queries using GET and no parameters are not allowed. If you have passed parameters via a JSON body, make sure you use POST requests.'));
throw new BadRequestException(__('Restsearch queries using GET and no parameters are not allowed. If you have passed parameters via a JSON body, make sure you use POST requests.'));
}
if (empty($filters['returnFormat'])) {
$filters['returnFormat'] = 'json';

2
app/Controller/AttributesController.php
View File

@ -1917,7 +1917,7 @@ class AttributesController extends AppController
public function reportValidationIssuesAttributes($eventId = false)
{
// search for validation problems in the attributes
$this->set('result', $this->Attribute->reportValidationIssuesAttributes($eventId));
$this->set('result', iterator_to_array($this->Attribute->reportValidationIssuesAttributes($eventId)));
}
public function generateCorrelation()

26
app/Controller/AuditLogsController.php
View File

@ -134,7 +134,8 @@ class AuditLogsController extends AppController
]);
$this->paginate['conditions'] = $this->__searchConditions($params);
$acl = $this->__applyAuditACL($this->Auth->user());
$user = $this->Auth->user();
$acl = $this->__applyAuditACL($user);
if ($acl) {
$this->paginate['conditions']['AND'][] = $acl;
}
@ -144,7 +145,7 @@ class AuditLogsController extends AppController
return $this->RestResponse->viewData($list, 'json');
}
$list = $this->__appendModelLinks($list);
$list = $this->__appendModelLinks($user, $list);
foreach ($list as $k => $item) {
$list[$k]['AuditLog']['action_human'] = $this->actions[$item['AuditLog']['action']];
}
@ -435,10 +436,11 @@ class AuditLogsController extends AppController
/**
* Generate link to model view if exists and use has permission to access it.
* @param array $user
* @param array $auditLogs
* @return array
*/
private function __appendModelLinks(array $auditLogs)
private function __appendModelLinks(array $user, array $auditLogs)
{
$models = [];
foreach ($auditLogs as $auditLog) {
@ -449,7 +451,7 @@ class AuditLogsController extends AppController
}
}
$eventIds = isset($models['Event']) ? $models['Event'] : [];
$eventIds = $models['Event'] ?? [];
if (isset($models['ObjectReference'])) {
$this->loadModel('ObjectReference');
@ -461,11 +463,11 @@ class AuditLogsController extends AppController
if (isset($models['Object']) || isset($objectReferences)) {
$objectIds = array_unique(array_merge(
isset($models['Object']) ? $models['Object'] : [],
$models['Object'] ?? [],
isset($objectReferences) ? array_values($objectReferences) : []
));
$this->loadModel('MispObject');
$conditions = $this->MispObject->buildConditions($this->Auth->user());
$conditions = $this->MispObject->buildConditions($user);
$conditions['Object.id'] = $objectIds;
$objects = $this->MispObject->find('all', [
'conditions' => $conditions,
@ -473,22 +475,22 @@ class AuditLogsController extends AppController
'fields' => ['Object.id', 'Object.event_id', 'Object.uuid', 'Object.deleted'],
]);
$objects = array_column(array_column($objects, 'Object'), null, 'id');
$eventIds = array_merge($eventIds, array_column($objects, 'event_id'));
array_push($eventIds, ...array_column($objects, 'event_id'));
}
if (isset($models['Attribute'])) {
$this->loadModel('Attribute');
$attributes = $this->Attribute->fetchAttributesSimple($this->Auth->user(), [
$attributes = $this->Attribute->fetchAttributesSimple($user, [
'conditions' => ['Attribute.id' => array_unique($models['Attribute'])],
'fields' => ['Attribute.id', 'Attribute.event_id', 'Attribute.uuid', 'Attribute.deleted'],
]);
$attributes = array_column(array_column($attributes, 'Attribute'), null, 'id');
$eventIds = array_merge($eventIds, array_column($attributes, 'event_id'));
array_push($eventIds, ...array_column($attributes, 'event_id'));
}
if (isset($models['ShadowAttribute'])) {
$this->loadModel('ShadowAttribute');
$conditions = $this->ShadowAttribute->buildConditions($this->Auth->user());
$conditions = $this->ShadowAttribute->buildConditions($user);
$conditions['AND'][] = ['ShadowAttribute.id' => array_unique($models['ShadowAttribute'])];
$shadowAttributes = $this->ShadowAttribute->find('all', [
'conditions' => $conditions,
@ -496,12 +498,12 @@ class AuditLogsController extends AppController
'contain' => ['Event', 'Attribute'],
]);
$shadowAttributes = array_column(array_column($shadowAttributes, 'ShadowAttribute'), null, 'id');
$eventIds = array_merge($eventIds, array_column($shadowAttributes, 'event_id'));
array_push($eventIds, ...array_column($shadowAttributes, 'event_id'));
}
if (!empty($eventIds)) {
$this->loadModel('Event');
$conditions = $this->Event->createEventConditions($this->Auth->user());
$conditions = $this->Event->createEventConditions($user);
$conditions['Event.id'] = array_unique($eventIds);
$events = $this->Event->find('list', [
'conditions' => $conditions,

14
app/Controller/Component/IndexFilterComponent.php
View File

@ -8,7 +8,9 @@ class IndexFilterComponent extends Component
{
/** @var Controller */
public $Controller;
public $isRest = null;
/** @var bool|null */
private $isRest = null;
// Used for isApiFunction(), a check that returns true if the controller & action combo matches an action that is a non-xml and non-json automation method
// This is used to allow authentication via headers for methods not covered by _isRest() - as that only checks for JSON and XML formats
@ -93,6 +95,11 @@ class IndexFilterComponent extends Component
}
}
public function isXhr()
{
return $this->Controller->request->header('X-Requested-With') === 'XMLHttpRequest';
}
public function isJson()
{
return $this->Controller->request->header('Accept') === 'application/json' || $this->Controller->RequestHandler->prefers() === 'json';
@ -103,11 +110,6 @@ class IndexFilterComponent extends Component
return $this->Controller->request->header('Accept') === 'text/csv' || $this->Controller->RequestHandler->prefers() === 'csv';
}
public function isXml()
{
}
/**
* @param string $controller
* @param string $action

92
app/Controller/Component/RateLimitComponent.php
View File

@ -12,58 +12,60 @@ class RateLimitComponent extends Component
)
);
public $components = array('RestResponse');
/**
* @param array $user
* @param string $controller
* @param string $action
* @param array $info
* @param string $responseType
* @return bool
* @return array|null
* @throws RedisException
*/
public function check(array $user, $controller, $action, &$info = array(), $responseType)
public function check(array $user, $controller, $action)
{
if (!empty($user['Role']['enforce_rate_limit']) && isset(self::LIMITED_FUNCTIONS[$controller][$action])) {
if ($user['Role']['rate_limit_count'] == 0) {
throw new MethodNotAllowedException(__('API searches are not allowed for this user role.'));
}
try {
$redis = RedisTool::init();
} catch (Exception $e) {
return true; // redis is not available, allow access
}
$uuid = Configure::read('MISP.uuid') ?: 'no-uuid';
$keyName = 'misp:' . $uuid . ':rate_limit:' . $user['id'];
$count = $redis->get($keyName);
if ($count !== false && $count >= $user['Role']['rate_limit_count']) {
$info = array(
'limit' => $user['Role']['rate_limit_count'],
'reset' => $redis->ttl($keyName),
'remaining' => $user['Role']['rate_limit_count'] - $count,
);
return $this->RestResponse->throwException(
429,
__('Rate limit exceeded.'),
'/' . $controller . '/' . $action,
$responseType
);
} else {
if ($count === false) {
$redis->setEx($keyName, 900, 1);
} else {
$redis->setEx($keyName, $redis->ttl($keyName), intval($count) + 1);
}
}
$count += 1;
$info = array(
'limit' => $user['Role']['rate_limit_count'],
'reset' => $redis->ttl($keyName),
'remaining' => $user['Role']['rate_limit_count'] - $count
);
if (!isset(self::LIMITED_FUNCTIONS[$controller][$action])) {
return null; // no limit enforced for this controller action
}
return true;
if (empty($user['Role']['enforce_rate_limit'])) {
return null; // no limit enforced for this role
}
$rateLimit = (int)$user['Role']['rate_limit_count'];
if ($rateLimit === 0) {
throw new MethodNotAllowedException(__('API searches are not allowed for this user role.'));
}
try {
$redis = RedisTool::init();
} catch (Exception $e) {
return null; // redis is not available, allow access
}
$uuid = Configure::read('MISP.uuid') ?: 'no-uuid';
$keyName = 'misp:' . $uuid . ':rate_limit:' . $user['id'];
$count = $redis->get($keyName);
if ($count !== false && $count >= $rateLimit) {
return [
'exceeded' => true,
'limit' => $rateLimit,
'reset' => $redis->ttl($keyName),
'remaining' => $rateLimit - $count,
];
}
$newCount = $redis->incr($keyName);
if ($newCount === 1) {
$redis->expire($keyName, 900);
$reset = 900;
} else {
$reset = $redis->ttl($keyName);
}
return [
'exceeded' => false,
'limit' => $rateLimit,
'reset' => $reset,
'remaining' => $rateLimit - $newCount,
];
}
}

14
app/Controller/Component/RestResponseComponent.php
View File

@ -517,7 +517,7 @@ class RestResponseComponent extends Component
if ($id) {
$response['id'] = $id;
}
return $this->__sendResponse($response, 403, $format);
return $this->prepareResponse($response, 403, $format);
}
/**
@ -562,7 +562,7 @@ class RestResponseComponent extends Component
if ($id) {
$response['id'] = $id;
}
return $this->__sendResponse($response, 200, $format);
return $this->prepareResponse($response, 200, $format);
}
/**
@ -587,7 +587,7 @@ class RestResponseComponent extends Component
* @return CakeResponse
* @throws Exception
*/
private function __sendResponse($response, $code, $format = false, $raw = false, $download = false, $headers = array())
private function prepareResponse($response, $code, $format = false, $raw = false, $download = false, $headers = array())
{
App::uses('TmpFileTool', 'Tools');
$format = !empty($format) ? strtolower($format) : 'json';
@ -633,7 +633,7 @@ class RestResponseComponent extends Component
}
// If response is big array, encode items separately to save memory
if (is_array($response) && count($response) > 10000) {
if (is_array($response) && count($response) > 10000 && JsonTool::arrayIsList($response)) {
$output = new TmpFileTool();
$output->write('[');
@ -775,7 +775,7 @@ class RestResponseComponent extends Component
if (!empty($errors)) {
$data['errors'] = $errors;
}
return $this->__sendResponse($data, 200, $format, $raw, $download, $headers);
return $this->prepareResponse($data, 200, $format, $raw, $download, $headers);
}
/**
@ -807,7 +807,7 @@ class RestResponseComponent extends Component
'message' => $message,
'url' => $url
);
return $this->__sendResponse($message, $code, $format, $raw, false, $headers);
return $this->prepareResponse($message, $code, $format, $raw, false, $headers);
}
public function setHeader($header, $value)
@ -834,7 +834,7 @@ class RestResponseComponent extends Component
}
}
$response['url'] = $this->__generateURL($actionArray, $controller, $params);
return $this->__sendResponse($response, 200, $format);
return $this->prepareResponse($response, 200, $format);
}
private function __setup()

8
app/Controller/Component/RestSearchComponent.php
View File

@ -152,6 +152,8 @@ class RestSearchComponent extends Component
'category',
'org',
'tags',
'first_seen',
'last_seen',
'from',
'to',
'last',
@ -184,6 +186,7 @@ class RestSearchComponent extends Component
'context',
'returnFormat',
'id',
'uuid',
'type',
'from',
'to',
@ -191,7 +194,8 @@ class RestSearchComponent extends Component
'org_id',
'source',
'includeAttribute',
'includeEvent'
'includeEvent',
'includeUuid',
],
'GalaxyCluster' => [
'page',
@ -204,7 +208,7 @@ class RestSearchComponent extends Component
'distribution',
'org',
'orgc',
'tag',
'tag_name',
'custom',
'sgReferenceOnly',
'minimal',

32
app/Controller/EventsController.php
View File

@ -834,33 +834,29 @@ class EventsController extends AppController
}
if (empty($rules['limit'])) {
$events = array();
$events = [];
$i = 1;
$rules['limit'] = 20000;
while (true) {
$rules['page'] = $i;
$rules['page'] = $i++;
$temp = $this->Event->find('all', $rules);
$resultCount = count($temp);
if ($resultCount !== 0) {
// this is faster and memory efficient than array_merge
foreach ($temp as $tempEvent) {
$events[] = $tempEvent;
}
array_push($events, ...$temp);
}
if ($resultCount < $rules['limit']) {
break;
}
$i++;
}
unset($temp);
$absolute_total = count($events);
$absoluteTotal = count($events);
} else {
$counting_rules = $rules;
unset($counting_rules['limit']);
unset($counting_rules['page']);
$absolute_total = $this->Event->find('count', $counting_rules);
$absoluteTotal = $this->Event->find('count', $counting_rules);
$events = $absolute_total === 0 ? [] : $this->Event->find('all', $rules);
$events = $absoluteTotal === 0 ? [] : $this->Event->find('all', $rules);
}
$isCsvResponse = $this->response->type() === 'text/csv';
@ -979,7 +975,7 @@ class EventsController extends AppController
$events = $export->eventIndex($events);
}
return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, ['X-Result-Count' => $absolute_total]);
return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, ['X-Result-Count' => $absoluteTotal]);
}
private function __indexColumns()
@ -2383,7 +2379,7 @@ class EventsController extends AppController
$results = $this->Event->addMISPExportFile($this->Auth->user(), $data, $isXml, $takeOwnership, $publish);
} catch (Exception $e) {
$this->log("Exception during processing MISP file import: {$e->getMessage()}");
$this->Flash->error(__('Could not process MISP export file. %s.', $e->getMessage()));
$this->Flash->error(__('Could not process MISP export file. %s', $e->getMessage()));
$this->redirect(['controller' => 'events', 'action' => 'add_misp_export']);
}
}
@ -3203,7 +3199,7 @@ class EventsController extends AppController
$event = $this->Event->find('first', [
'conditions' => Validation::uuid($id) ? ['Event.uuid' => $id] : ['Event.id' => $id],
'recursive' => -1,
'fields' => ['id', 'info', 'publish_timestamp', 'orgc_id'],
'fields' => ['id', 'info', 'publish_timestamp', 'orgc_id', 'user_id'],
]);
if (empty($event)) {
throw new NotFoundException(__('Invalid event.'));
@ -3222,6 +3218,16 @@ class EventsController extends AppController
}
}
}
if (
Configure::read('MISP.block_publishing_for_same_creator', false) &&
$this->Auth->user()['id'] == $event['Event']['user_id']
) {
$message = __('Could not publish the event, the publishing user cannot be the same as the event creator as per this instance\'s configuration.');
if (!$this->_isRest()) {
$this->Flash->error($message);
}
throw new MethodNotAllowedException($message);
}
return $event;
}

15
app/Controller/ServersController.php
View File

@ -1073,7 +1073,7 @@ class ServersController extends AppController
);
$dumpResults = array();
$tempArray = array();
foreach ($finalSettings as $k => $result) {
foreach ($finalSettings as $result) {
if ($result['level'] == 3) {
$issues['deprecated']++;
}
@ -1105,18 +1105,19 @@ class ServersController extends AppController
$diagnostic_errors = 0;
App::uses('File', 'Utility');
App::uses('Folder', 'Utility');
if ($tab === 'correlations') {
$this->loadModel('Correlation');
$correlation_metrics = $this->Correlation->collectMetrics();
$this->set('correlation_metrics', $correlation_metrics);
}
if ($tab === 'files') {
} else if ($tab === 'files') {
if (!empty(Configure::read('Security.disable_instance_file_uploads'))) {
throw new MethodNotAllowedException(__('This functionality is disabled.'));
}
$files = $this->Server->grabFiles();
$this->set('files', $files);
}
// Only run this check on the diagnostics tab
if ($tab === 'diagnostics' || $tab === 'download' || $this->_isRest()) {
$php_ini = php_ini_loaded_file();
@ -1279,12 +1280,10 @@ class ServersController extends AppController
$this->set('workerIssueCount', $workerIssueCount);
$priorityErrorColours = array(0 => 'red', 1 => 'yellow', 2 => 'green');
$this->set('priorityErrorColours', $priorityErrorColours);
$this->set('phpversion', phpversion());
$this->set('phpversion', PHP_VERSION);
$this->set('phpmin', $this->phpmin);
$this->set('phprec', $this->phprec);
$this->set('phptoonew', $this->phptoonew);
$this->set('pythonmin', $this->pythonmin);
$this->set('pythonrec', $this->pythonrec);
$this->set('title_for_layout', __('Diagnostics'));
}
@ -1863,7 +1862,7 @@ class ServersController extends AppController
}
if (Configure::read('SimpleBackgroundJobs.enabled')) {
$this->Server->getBackgroundJobsTool()->purgeQueue($worker);
$this->Server->getBackgroundJobsTool()->clearQueue($worker);
} else {
// CakeResque
$worker_array = array('cache', 'default', 'email', 'prio');
@ -2183,7 +2182,7 @@ class ServersController extends AppController
if ($this->_isRest()) {
return $this->RestResponse->saveFailResponse('Servers', 'addFromJson', false, $this->Server->validationErrors, $this->response->type());
} else {
$this->Flash->error(__('Could not save the server. Error: %s', json_encode($this->Server->validationErrors, true)));
$this->Flash->error(__('Could not save the server. Error: %s', json_encode($this->Server->validationErrors)));
$this->redirect(array('action' => 'index'));
}
}

70
app/Controller/UsersController.php
View File

@ -1244,8 +1244,6 @@ class UsersController extends AppController
// login was successful, do everything that is needed such as logging and more:
$this->_postlogin();
} else {
$dataSourceConfig = ConnectionManager::getDataSource('default')->config;
$dataSource = $dataSourceConfig['datasource'];
// don't display authError before first login attempt
if (str_replace("//", "/", $this->webroot . $this->Session->read('Auth.redirect')) == $this->webroot && $this->Session->read('Message.auth.message') == $this->Auth->authError) {
$this->Session->delete('Message.auth');
@ -1260,73 +1258,7 @@ class UsersController extends AppController
}
}
//
// Actions needed for the first access, when the database is not populated yet.
//
// populate the DB with the first role (site admin) if it's empty
if (!$this->User->Role->hasAny()) {
$siteAdmin = array('Role' => array(
'id' => 1,
'name' => 'Site Admin',
'permission' => 3,
'perm_add' => 1,
'perm_modify' => 1,
'perm_modify_org' => 1,
'perm_publish' => 1,
'perm_sync' => 1,
'perm_admin' => 1,
'perm_audit' => 1,
'perm_auth' => 1,
'perm_site_admin' => 1,
'perm_regexp_access' => 1,
'perm_sharing_group' => 1,
'perm_template' => 1,
'perm_tagger' => 1,
));
$this->User->Role->save($siteAdmin);
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
if ($dataSource === 'Database/Postgres') {
$sql = "SELECT setval('roles_id_seq', (SELECT MAX(id) FROM roles));";
$this->User->Role->query($sql);
}
}
if (!$this->User->Organisation->hasAny(array('Organisation.local' => true))) {
$this->User->runUpdates();
$date = date('Y-m-d H:i:s');
$org = array('Organisation' => array(
'id' => 1,
'name' => !empty(Configure::read('MISP.org')) ? Configure::read('MISP.org') : 'ADMIN',
'description' => 'Automatically generated admin organisation',
'type' => 'ADMIN',
'uuid' => CakeText::uuid(),
'local' => 1,
'date_created' => $date,
'sector' => '',
'nationality' => ''
));
$this->User->Organisation->save($org);
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
if ($dataSource === 'Database/Postgres') {
$sql = "SELECT setval('organisations_id_seq', (SELECT MAX(id) FROM organisations));";
$this->User->Organisation->query($sql);
}
$org_id = $this->User->Organisation->id;
}
// populate the DB with the first user if it's empty
if (!$this->User->hasAny()) {
if (!isset($org_id)) {
$hostOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.name' => Configure::read('MISP.org'), 'Organisation.local' => true), 'recursive' => -1));
if (!empty($hostOrg)) {
$org_id = $hostOrg['Organisation']['id'];
} else {
$firstOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.local' => true), 'order' => 'Organisation.id ASC'));
$org_id = $firstOrg['Organisation']['id'];
}
}
$this->User->runUpdates();
$this->User->createInitialUser($org_id);
}
$this->User->init();
}
}

191
app/Lib/Export/NidsExport.php
View File

@ -6,10 +6,7 @@ abstract class NidsExport
public $classtype = 'trojan-activity';
public $format = ""; // suricata (default), snort
public $checkWhitelist = true;
protected $format; // suricata (default), snort
public $additional_params = array(
'contain' => array(
@ -17,36 +14,31 @@ abstract class NidsExport
'fields' => array('threat_level_id')
)
),
);
public function handler($data, $options = array())
{
$continue = empty($format);
$this->checkWhitelist = false;
if ($options['scope'] === 'Attribute') {
$this->export(
array($data),
$options['user']['nids_sid'],
$options['returnFormat'],
$continue
$options['user']['nids_sid']
);
} else if ($options['scope'] === 'Event') {
if (!empty($data['EventTag'])) {
$data['Event']['EventTag'] = $data['EventTag'];
}
if (!empty($data['Attribute'])) {
$this->__convertFromEventFormat($data['Attribute'], $data, $options, $continue);
$this->convertFromEventFormat($data['Attribute'], $data, $options);
}
if (!empty($data['Object'])) {
$this->__convertFromEventFormatObject($data['Object'], $data, $options, $continue);
$this->convertFromEventFormatObject($data['Object'], $data, $options);
}
}
return '';
}
private function __convertFromEventFormat($attributes, $event, $options = array(), $continue = false) {
private function convertFromEventFormat($attributes, $event, $options = array())
{
$rearranged = array();
foreach ($attributes as $attribute) {
$attributeTag = array();
@ -62,15 +54,12 @@ abstract class NidsExport
}
$this->export(
$rearranged,
$options['user']['nids_sid'],
$options['returnFormat'],
$continue
$options['user']['nids_sid']
);
return true;
}
private function __convertFromEventFormatObject($objects, $event, $options = array(), $continue = false)
private function convertFromEventFormatObject($objects, $event, $options = array())
{
$rearranged = array();
foreach ($objects as $object) {
@ -93,20 +82,18 @@ abstract class NidsExport
'Event' => $event['Event']
);
} else { // In case no custom export exists for the object, the approach falls back to the attribute case
$this->__convertFromEventFormat($object['Attribute'], $event, $options, $continue);
$this->convertFromEventFormat($object['Attribute'], $event, $options);
}
}
$this->export(
$rearranged,
$options['user']['nids_sid'],
$options['returnFormat'],
$continue
$options['user']['nids_sid']
);
return true;
}
public function header($options = array())
public function header()
{
$this->explain();
return '';
@ -122,7 +109,7 @@ abstract class NidsExport
return '';
}
public function explain()
protected function explain()
{
$this->rules[] = '# MISP export of IDS rules - optimized for '.$this->format;
$this->rules[] = '#';
@ -136,21 +123,8 @@ abstract class NidsExport
$this->rules[] = '# ';
}
private $whitelist = null;
public function export($items, $startSid, $format="suricata", $continue = false)
protected function export($items, $startSid)
{
$this->format = $format;
if ($this->checkWhitelist && !isset($this->Whitelist)) {
$this->Whitelist = ClassRegistry::init('Whitelist');
$this->whitelist = $this->Whitelist->getBlockedValues();
}
// output a short explanation
if (!$continue) {
$this->explain();
}
// generate the rules
foreach ($items as $item) {
// retrieve all tags for this item to add them to the msg
@ -180,7 +154,6 @@ abstract class NidsExport
$sid++;
if (!empty($item['Attribute']['type'])) { // item is an 'Attribute'
switch ($item['Attribute']['type']) {
// LATER nids - test all the snort attributes
// LATER nids - add the tag keyword in the rules to capture network traffic
@ -195,7 +168,7 @@ abstract class NidsExport
break;
case 'email':
$this->emailSrcRule($ruleFormat, $item['Attribute'], $sid);
$sid++;
$sid++;
$this->emailDstRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'email-src':
@ -228,17 +201,17 @@ abstract class NidsExport
case 'ja3-fingerprint-md5':
$this->ja3Rule($ruleFormat, $item['Attribute'], $sid);
break;
case 'ja3s-fingerprint-md5': // Atribute type doesn't exists yet (2020-12-10) but ready when created.
case 'ja3s-fingerprint-md5': // Attribute type doesn't exists yet (2020-12-10) but ready when created.
$this->ja3sRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'snort':
$this->snortRule($ruleFormat, $item['Attribute'], $sid, $ruleFormatMsg, $ruleFormatReference);
$this->snortRule($item['Attribute'], $sid, $ruleFormatMsg, $ruleFormatReference);
// no break
default:
break;
}
} else if(!empty($item['Attribute']['name'])) { // Item is an 'Object'
} else if (!empty($item['Attribute']['name'])) { // Item is an 'Object'
switch ($item['Attribute']['name']) {
case 'network-connection':
@ -252,34 +225,30 @@ abstract class NidsExport
}
}
}
return $this->rules;
}
public function networkConnectionRule($ruleFormat, $object, &$sid)
protected function networkConnectionRule($ruleFormat, $object, &$sid)
{
$attributes = NidsExport::getObjectAttributes($object);
if(!array_key_exists('layer4-protocol', $attributes)){
if (!array_key_exists('layer4-protocol', $attributes)) {
$attributes['layer4-protocol'] = 'ip'; // If layer-4 protocol is unknown, we roll-back to layer-3 ('ip')
}
if(!array_key_exists('ip-src', $attributes)){
if (!array_key_exists('ip-src', $attributes)) {
$attributes['ip-src'] = '$HOME_NET'; // If ip-src is unknown, we roll-back to $HOME_NET
}
if(!array_key_exists('ip-dst', $attributes)){
if (!array_key_exists('ip-dst', $attributes)) {
$attributes['ip-dst'] = '$HOME_NET'; // If ip-dst is unknown, we roll-back to $HOME_NET
}
if(!array_key_exists('src-port', $attributes)){
if (!array_key_exists('src-port', $attributes)) {
$attributes['src-port'] = 'any'; // If src-port is unknown, we roll-back to 'any'
}
if(!array_key_exists('dst-port', $attributes)){
if (!array_key_exists('dst-port', $attributes)) {
$attributes['dst-port'] = 'any'; // If dst-port is unknown, we roll-back to 'any'
}
$this->rules[] = sprintf(
$this->rules[] = sprintf(
$ruleFormat,
false,
$attributes['layer4-protocol'], // proto
@ -294,12 +263,10 @@ abstract class NidsExport
$sid, // sid
1 // rev
);
}
public function ddosRule($ruleFormat, $object, &$sid)
protected function ddosRule($ruleFormat, $object, &$sid)
{
$attributes = NidsExport::getObjectAttributes($object);
if(!array_key_exists('protocol', $attributes)){
@ -318,7 +285,7 @@ abstract class NidsExport
$attributes['dst-port'] = 'any'; // If dst-port is unknown, we roll-back to 'any'
}
$this->rules[] = sprintf(
$this->rules[] = sprintf(
$ruleFormat,
false,
$attributes['protocol'], // proto
@ -333,12 +300,10 @@ abstract class NidsExport
$sid, // sid
1 // rev
);
}
public static function getObjectAttributes($object)
protected static function getObjectAttributes($object)
{
$attributes = array();
foreach ($object['Attribute'] as $attribute) {
@ -348,7 +313,7 @@ abstract class NidsExport
return $attributes;
}
public function domainIpRule($ruleFormat, $attribute, &$sid)
protected function domainIpRule($ruleFormat, $attribute, &$sid)
{
$values = explode('|', $attribute['value']);
$attributeCopy = $attribute;
@ -361,7 +326,7 @@ abstract class NidsExport
$this->ipSrcRule($ruleFormat, $attributeCopy, $sid);
}
public function ipDstRule($ruleFormat, $attribute, &$sid)
protected function ipDstRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$ipport = NidsExport::getIpPort($attribute);
@ -382,7 +347,7 @@ abstract class NidsExport
);
}
public function ipSrcRule($ruleFormat, $attribute, &$sid)
protected function ipSrcRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$ipport = NidsExport::getIpPort($attribute);
@ -403,7 +368,7 @@ abstract class NidsExport
);
}
public function emailSrcRule($ruleFormat, $attribute, &$sid)
protected function emailSrcRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -425,7 +390,7 @@ abstract class NidsExport
);
}
public function emailDstRule($ruleFormat, $attribute, &$sid)
protected function emailDstRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -447,7 +412,7 @@ abstract class NidsExport
);
}
public function emailSubjectRule($ruleFormat, $attribute, &$sid)
protected function emailSubjectRule($ruleFormat, $attribute, &$sid)
{
// LATER nids - email-subject rule might not match because of line-wrapping
$overruled = $this->checkWhitelist($attribute['value']);
@ -470,7 +435,7 @@ abstract class NidsExport
);
}
public function emailAttachmentRule($ruleFormat, $attribute, &$sid)
protected function emailAttachmentRule($ruleFormat, $attribute, &$sid)
{
// LATER nids - email-attachment rule might not match because of line-wrapping
$overruled = $this->checkWhitelist($attribute['value']);
@ -493,7 +458,7 @@ abstract class NidsExport
);
}
public function hostnameRule($ruleFormat, $attribute, &$sid)
protected function hostnameRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -549,7 +514,7 @@ abstract class NidsExport
);
}
public function domainRule($ruleFormat, $attribute, &$sid)
protected function domainRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -605,7 +570,7 @@ abstract class NidsExport
);
}
public function urlRule($ruleFormat, $attribute, &$sid)
protected function urlRule($ruleFormat, $attribute, &$sid)
{
// TODO in hindsight, an url should not be excluded given a host or domain name.
//$hostpart = parse_url($attribute['value'], PHP_URL_HOST);
@ -630,7 +595,7 @@ abstract class NidsExport
);
}
public function userAgentRule($ruleFormat, $attribute, &$sid)
protected function userAgentRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -652,17 +617,17 @@ abstract class NidsExport
);
}
public function ja3Rule($ruleFormat, $attribute, &$sid)
protected function ja3Rule($ruleFormat, $attribute, &$sid)
{
//Empty because Snort doesn't support JA3 Rules
}
public function ja3sRule($ruleFormat, $attribute, &$sid)
protected function ja3sRule($ruleFormat, $attribute, &$sid)
{
//Empty because Snort doesn't support JA3S Rules
}
public function snortRule($ruleFormat, $attribute, &$sid, $ruleFormatMsg, $ruleFormatReference)
protected function snortRule($attribute, &$sid, $ruleFormatMsg, $ruleFormatReference)
{
// LATER nids - test using lots of snort rules, some rules don't contain all the necessary to be a valid rule.
@ -678,46 +643,46 @@ abstract class NidsExport
// tag - '/tag\s*:\s*.+?;/'
$replaceCount = array();
$tmpRule = preg_replace('/sid\s*:\s*[0-9]+\s*;/', 'sid:' . $sid . ';', $tmpRule, -1, $replaceCount['sid']);
if (null == $tmpRule) {
if (null === $tmpRule) {
return false;
} // don't output the rule on error with the regex
$tmpRule = preg_replace('/rev\s*:\s*[0-9]+\s*;/', 'rev:1;', $tmpRule, -1, $replaceCount['rev']);
if (null == $tmpRule) {
if (null === $tmpRule) {
return false;
} // don't output the rule on error with the regex
$tmpRule = preg_replace('/classtype:[a-zA-Z_-]+;/', 'classtype:' . $this->classtype . ';', $tmpRule, -1, $replaceCount['classtype']);
if (null == $tmpRule) {
if (null === $tmpRule) {
return false;
} // don't output the rule on error with the regex
$tmpRule = preg_replace('/msg\s*:\s*"(.*?)"\s*;/', sprintf($ruleFormatMsg, 'snort-rule | $1') . ';', $tmpRule, -1, $replaceCount['msg']);
if (null == $tmpRule) {
if (null === $tmpRule) {
return false;
} // don't output the rule on error with the regex
$tmpRule = preg_replace('/reference\s*:\s*.+?;/', $ruleFormatReference . ';', $tmpRule, -1, $replaceCount['reference']);
if (null == $tmpRule) {
if (null === $tmpRule) {
return false;
} // don't output the rule on error with the regex
$tmpRule = preg_replace('/reference\s*:\s*.+?;/', $ruleFormatReference . ';', $tmpRule, -1, $replaceCount['reference']);
if (null == $tmpRule) {
if (null === $tmpRule) {
return false;
} // don't output the rule on error with the regex
// FIXME nids - implement priority overwriting
// some values were not replaced, so we need to add them ourselves, and insert them in the rule
$extraForRule = "";
if (0 == $replaceCount['sid']) {
if (0 === $replaceCount['sid']) {
$extraForRule .= 'sid:' . $sid . ';';
}
if (0 == $replaceCount['rev']) {
if (0 === $replaceCount['rev']) {
$extraForRule .= 'rev:1;';
}
if (0 == $replaceCount['classtype']) {
if (0 === $replaceCount['classtype']) {
$extraForRule .= 'classtype:' . $this->classtype . ';';
}
if (0 == $replaceCount['msg']) {
$extraForRule .= $tmpMessage . ';';
if (0 === $replaceCount['msg']) {
$extraForRule .= $ruleFormatMsg . ';';
}
if (0 == $replaceCount['reference']) {
if (0 === $replaceCount['reference']) {
$extraForRule .= $ruleFormatReference . ';';
}
$tmpRule = preg_replace('/;\s*\)/', '; ' . $extraForRule . ')', $tmpRule);
@ -734,7 +699,7 @@ abstract class NidsExport
* @param string $type the type of dns name - domain (default) or hostname
* @return string raw snort compatible format of the dns name
*/
public static function dnsNameToRawFormat($name, $type='domain')
protected static function dnsNameToRawFormat($name, $type='domain')
{
$rawName = "";
if ('hostname' == $type) {
@ -747,7 +712,7 @@ abstract class NidsExport
// count the length of the part, and add |length| before
$length = strlen($explodedName);
if ($length > 255) {
log('WARNING: DNS name is too long for RFC: '.$name);
CakeLog::notice('WARNING: DNS name is too long for RFC: '.$name);
}
$hexLength = dechex($length);
if (1 == strlen($hexLength)) {
@ -768,7 +733,7 @@ abstract class NidsExport
* @param string $name dns name to be converted
* @return string raw snort compatible format of the dns name
*/
public static function dnsNameToMSDNSLogFormat($name)
protected static function dnsNameToMSDNSLogFormat($name)
{
$rawName = "";
// in MS DNS log format we can't use (0) to distinguish between hostname and domain (including subdomains)
@ -779,7 +744,7 @@ abstract class NidsExport
// count the length of the part, and add |length| before
$length = strlen($explodedName);
if ($length > 255) {
log('WARNING: DNS name is too long for RFC: '.$name);
CakeLog::notice('WARNING: DNS name is too long for RFC: '.$name);
}
$hexLength = dechex($length);
$rawName .= '(' . $hexLength . ')' . $explodedName;
@ -793,34 +758,32 @@ abstract class NidsExport
/**
* Replaces characters that are not allowed in a signature.
* example: " is converted to |22|
* @param unknown_type $value
* @param string $value
*/
public static function replaceIllegalChars($value)
protected static function replaceIllegalChars($value)
{
$replace_pairs = array(
'|' => '|7c|', // Needs to stay on top !
'"' => '|22|',
';' => '|3b|',
':' => '|3a|',
'\\' => '|5c|',
'0x' => '|30 78|'
);
'|' => '|7c|', // Needs to stay on top !
'"' => '|22|',
';' => '|3b|',
':' => '|3a|',
'\\' => '|5c|',
'0x' => '|30 78|'
);
return strtr($value, $replace_pairs);
}
public function checkWhitelist($value)
/**
* @deprecated
* @param $value
* @return false
*/
protected function checkWhitelist($value)
{
if ($this->checkWhitelist && is_array($this->whitelist)) {
foreach ($this->whitelist as $wlitem) {
if (preg_match($wlitem, $value)) {
return true;
}
}
}
return false;
}
public static function getProtocolPort($protocol, $customPort)
protected static function getProtocolPort($protocol, $customPort)
{
if ($customPort == null) {
switch ($protocol) {
@ -840,7 +803,7 @@ abstract class NidsExport
}
}
public static function getCustomIP($customIP)
protected static function getCustomIP($customIP)
{
if (filter_var($customIP, FILTER_VALIDATE_IP)) {
return $customIP;
@ -853,7 +816,7 @@ abstract class NidsExport
* @param array $attribute
* @return array|string[]
*/
public static function getIpPort($attribute)
protected static function getIpPort($attribute)
{
if (strpos($attribute['type'], 'port') !== false) {
return explode('|', $attribute['value']);

8
app/Lib/Export/NidsSnortExport.php
View File

@ -4,11 +4,5 @@ App::uses('NidsExport', 'Export');
class NidsSnortExport extends NidsExport
{
public function export($items, $startSid, $format = "suricata", $continue = false)
{
// set the specific format
$this->format = 'snort';
// call the generic function
return parent::export($items, $startSid, $format, $continue);
}
protected $format = 'snort';
}

20
app/Lib/Export/NidsSuricataExport.php
View File

@ -3,16 +3,10 @@ App::uses('NidsExport', 'Export');
class NidsSuricataExport extends NidsExport
{
public function export($items, $startSid, $format = "suricata", $continue = false)
{
// set the specific format
$this->format = "suricata";
// call the generic function
return parent::export($items, $startSid, $format, $continue);
}
protected $format = "suricata";
// below overwrite functions from NidsExport
public function hostnameRule($ruleFormat, $attribute, &$sid)
protected function hostnameRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -53,7 +47,7 @@ class NidsSuricataExport extends NidsExport
);
}
public function domainRule($ruleFormat, $attribute, &$sid)
protected function domainRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -94,7 +88,7 @@ class NidsSuricataExport extends NidsExport
);
}
public function urlRule($ruleFormat, $attribute, &$sid)
protected function urlRule($ruleFormat, $attribute, &$sid)
{
$createRule = true;
$overruled = $this->checkWhitelist($attribute['value']);
@ -207,7 +201,7 @@ class NidsSuricataExport extends NidsExport
}
}
public function userAgentRule($ruleFormat, $attribute, &$sid)
protected function userAgentRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -230,7 +224,7 @@ class NidsSuricataExport extends NidsExport
);
}
public function ja3Rule($ruleFormat, $attribute, &$sid)
protected function ja3Rule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -253,7 +247,7 @@ class NidsSuricataExport extends NidsExport
}
// For Future use once JA3S Hash Attribute type is created
public function ja3sRule($ruleFormat, $attribute, &$sid)
protected function ja3sRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule

176
app/Lib/Export/RPZExport.php
View File

@ -2,107 +2,104 @@
class RPZExport
{
private $__policies = array(
'Local-Data' => array(
'explanation' => 'returns the defined alternate location.',
'action' => '$walled_garden',
'setting_id' => 3,
),
'NXDOMAIN' => array(
'explanation' => 'return NXDOMAIN (name does not exist) irrespective of actual result received.',
'action' => '.',
'setting_id' => 1,
),
'NODATA' => array(
'explanation' => 'returns NODATA (name exists but no answers returned) irrespective of actual result received.',
'action' => '*.',
'setting_id' => 2,
),
'DROP' => array(
'explanation' => 'timeout.',
'action' => 'rpz-drop.',
'setting_id' => 0,
),
'PASSTHRU' => array(
'explanation' => 'lets queries through, but allows for logging the hits (useful for testing).',
'action' => 'rpz-passthru.',
'setting_id' => 4,
),
'TCP-only' => array(
'explanation' => 'force the client to use TCP.',
'action' => 'rpz-tcp-only.',
'setting_id' => 5,
),
const POLICIES = array(
'Local-Data' => array(
'explanation' => 'returns the defined alternate location.',
'action' => '$walled_garden',
'setting_id' => 3,
),
'NXDOMAIN' => array(
'explanation' => 'return NXDOMAIN (name does not exist) irrespective of actual result received.',
'action' => '.',
'setting_id' => 1,
),
'NODATA' => array(
'explanation' => 'returns NODATA (name exists but no answers returned) irrespective of actual result received.',
'action' => '*.',
'setting_id' => 2,
),
'DROP' => array(
'explanation' => 'timeout.',
'action' => 'rpz-drop.',
'setting_id' => 0,
),
'PASSTHRU' => array(
'explanation' => 'lets queries through, but allows for logging the hits (useful for testing).',
'action' => 'rpz-passthru.',
'setting_id' => 4,
),
'TCP-only' => array(
'explanation' => 'force the client to use TCP.',
'action' => 'rpz-tcp-only.',
'setting_id' => 5,
),
);
private $__items = array();
private $items = array();
public $additional_params = array(
'flatten' => 1
);
private $__rpzSettings = array();
private $__valid_policies = array('NXDOMAIN', 'NODATA', 'DROP', 'Local-Data', 'PASSTHRU', 'TCP-only');
private $rpzSettings = array();
private $__server = null;
public $validTypes = array(
const VALID_TYPES = array(
'ip-src' => array(
'value' => 'ip'
'value' => 'ip'
),
'ip-dst' => array(
'value' => 'ip'
'value' => 'ip'
),
'domain' => array(
'value' => 'domain'
'value' => 'domain'
),
'domain|ip' => array(
'value1' => 'domain',
'value2' => 'ip'
'value1' => 'domain',
'value2' => 'ip'
),
'hostname' => array(
'value' => 'hostname'
'value' => 'hostname'
)
);
public function handler($data, $options = array())
{
if ($options['scope'] === 'Attribute') {
return $this->__attributeHandler($data, $options);
$this->attributeHandler($data);
} else {
return $this->__eventHandler($data, $options);
$this->eventHandler($data);
}
return '';
}
private function __eventHandler($event, $options = array()) {
private function eventHandler($event)
{
foreach ($event['Attribute'] as $attribute) {
if (isset($this->validTypes[$attribute['type']])) {
if ($attribute['type'] == 'domain|ip') {
if (isset(self::VALID_TYPES[$attribute['type']])) {
if ($attribute['type'] === 'domain|ip') {
$temp = explode('|', $attribute['value']);
$attribute['value1'] = $temp[0];
$attribute['value2'] = $temp[1];
}
$this->__attributeHandler(array('Attribute' => $attribute, $options));
$this->attributeHandler(array('Attribute' => $attribute));
}
}
return '';
}
private function __attributeHandler($attribute, $options = array())
private function attributeHandler($attribute)
{
if (isset($attribute['Attribute'])) {
$attribute = $attribute['Attribute'];
}
if (isset($this->validTypes[$attribute['type']])) {
foreach ($this->validTypes[$attribute['type']] as $field => $mapping) {
// get rid of the in_array check
if (empty($this->__items[$mapping]) || !isset($this->__items[$mapping][$attribute[$field]])) {
$this->__items[$mapping][$attribute[$field]] = true;
if (isset(self::VALID_TYPES[$attribute['type']])) {
foreach (self::VALID_TYPES[$attribute['type']] as $field => $mapping) {
if (!isset($this->items[$mapping][$attribute[$field]])) {
$this->items[$mapping][$attribute[$field]] = true;
}
}
}
return '';
}
public function header($options = array())
@ -117,16 +114,16 @@ class RPZExport
}
}
if (isset($options['filters'][$v])) {
$this->__rpzSettings[$v] = $options['filters'][$v];
$this->rpzSettings[$v] = $options['filters'][$v];
} else {
$tempSetting = Configure::read('Plugin.RPZ_' . $v);
if (isset($tempSetting)) {
$this->__rpzSettings[$v] = Configure::read('Plugin.RPZ_' . $v);
$this->rpzSettings[$v] = $tempSetting;
} else {
if (empty($this->__server)) {
$this->__server = ClassRegistry::init('Server');
}
$this->__rpzSettings[$v] = $this->__server->serverSettings['Plugin']['RPZ_' . $v]['value'];
$this->rpzSettings[$v] = $this->__server->serverSettings['Plugin']['RPZ_' . $v]['value'];
}
}
}
@ -135,10 +132,7 @@ class RPZExport
public function footer($options = array())
{
foreach ($this->__items as $k => $v) {
$this->__items[$k] = array_keys($this->__items[$k]);
}
return $this->export($this->__items, $this->__rpzSettings);
return $this->export($this->items, $this->rpzSettings);
}
public function separator()
@ -146,39 +140,32 @@ class RPZExport
return '';
}
public function getPolicyById($id)
private function getPolicyById($id)
{
foreach ($this->__policies as $k => $v) {
if ($id == $v['setting_id']) {
foreach (self::POLICIES as $k => $v) {
if ($id === $v['setting_id']) {
return $k;
}
}
return null;
}
public function getIdByPolicy($policy)
private function getIdByPolicy($policy)
{
return $this->__policies[$policy]['setting_id'];
return self::POLICIES[$policy]['setting_id'];
}
public function explain($type, $policy)
private function explain($type, $policy)
{
$explanations = array(
'ip' => '; The following list of IP addresses will ',
'domain' => '; The following domain names and all of their sub-domains will ',
'hostname' => '; The following hostnames will '
);
$policy_explanations = array(
'Local-Data' => 'returns the defined alternate location.',
'NXDOMAIN' => 'return NXDOMAIN (name does not exist) irrespective of actual result received.',
'NODATA' => 'returns NODATA (name exists but no answers returned) irrespective of actual result received.',
'DROP' => 'timeout.',
'PASSTHRU' => 'lets queries through, but allows for logging the hits (useful for testing).',
'TCP-only' => 'force the client to use TCP.',
);
return $explanations[$type] . $this->__policies[$policy]['explanation'] . PHP_EOL;
return $explanations[$type] . self::POLICIES[$policy]['explanation'] . PHP_EOL;
}
public function buildHeader($rpzSettings)
private function buildHeader(array $rpzSettings)
{
$rpzSettings['serial'] = str_replace('$date', date('Ymd'), $rpzSettings['serial']);
$rpzSettings['serial'] = str_replace('$time', time(), $rpzSettings['serial']);
@ -196,55 +183,55 @@ class RPZExport
return $header;
}
public function export($items, $rpzSettings)
private function export(array $items, array $rpzSettings)
{
$result = $this->buildHeader($rpzSettings);
$policy = $this->getPolicyById($rpzSettings['policy']);
$action = $this->__policies[$policy]['action'];
if ($policy == 'Local-Data') {
$action = self::POLICIES[$policy]['action'];
if ($policy === 'Local-Data') {
$action = str_replace('$walled_garden', $rpzSettings['walled_garden'], $action);
}
if (isset($items['ip'])) {
$result .= $this->explain('ip', $policy);
foreach ($items['ip'] as $item) {
$result .= $this->__convertIP($item, $action);
foreach ($items['ip'] as $item => $foo) {
$result .= $this->convertIp($item, $action);
}
$result .= PHP_EOL;
}
if (isset($items['domain'])) {
$result .= $this->explain('domain', $policy);
foreach ($items['domain'] as $item) {
$result .= $this->__convertdomain($item, $action);
foreach ($items['domain'] as $item => $foo) {
$result .= $this->convertDomain($item, $action);
}
$result .= PHP_EOL;
}
if (isset($items['hostname'])) {
$result .= $this->explain('hostname', $policy);
foreach ($items['hostname'] as $item) {
$result .= $this->__converthostname($item, $action);
foreach ($items['hostname'] as $item => $foo) {
$result .= $this->convertHostname($item, $action);
}
$result .= PHP_EOL;
}
return $result;
}
private function __convertdomain($input, $action)
private function convertDomain($input, $action)
{
return $input . ' CNAME ' . $action . PHP_EOL . '*.' . $input . ' CNAME ' . $action . PHP_EOL;
}
private function __converthostname($input, $action)
private function convertHostname($input, $action)
{
return $input . ' CNAME ' . $action . PHP_EOL;
}
private function __convertIP($input, $action)
private function convertIp($input, $action)
{
$type = filter_var($input, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) ? 'ipv6' : 'ipv4';
if ($type == 'ipv6') {
$isIpv6 = filter_var($input, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6);
if ($isIpv6) {
$prefix = '128';
} else {
$prefix = '32';
@ -252,7 +239,8 @@ class RPZExport
if (strpos($input, '/')) {
list($input, $prefix) = explode('/', $input);
}
return $prefix . '.' . $this->{'__' . $type}($input) . '.rpz-ip CNAME ' . $action . PHP_EOL;
$converted = $isIpv6 ? $this->__ipv6($input) : $this->__ipv4($input);
return $prefix . '.' . $converted . '.rpz-ip CNAME ' . $action . PHP_EOL;
}
private function __ipv6($input)

198
app/Lib/Tools/ApcuCacheTool.php Normal file
View File

@ -0,0 +1,198 @@
<?php
declare(strict_types=1);
class ApcuCacheTool implements \Psr\SimpleCache\CacheInterface
{
/** @var string */
private $prefix;
/**
* @param string $prefix
*/
public function __construct(string $prefix)
{
$this->prefix = $prefix;
}
/**
* Fetches a value from the cache.
*
* @param string $key The unique key of this item in the cache.
* @param mixed $default Default value to return if the key does not exist.
*
* @return mixed The value of the item from the cache, or $default in case of cache miss.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if the $key string is not a legal value.
*/
public function get($key, $default = null)
{
$value = \apcu_fetch("$this->prefix:$key", $success);
if ($success) {
return $value;
}
return $default;
}
/**
* Persists data in the cache, uniquely referenced by a key with an optional expiration TTL time.
*
* @param string $key The key of the item to store.
* @param mixed $value The value of the item to store, must be serializable.
* @param null|int|\DateInterval $ttl Optional. The TTL value of this item. If no value is sent and
* the driver supports TTL then the library may set a default value
* for it or let the driver take care of that.
*
* @return bool True on success and false on failure.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if the $key string is not a legal value.
*/
public function set($key, $value, $ttl = null)
{
return \apcu_store("$this->prefix:$key", $value, $this->tllToInt($ttl));
}
/**
* Delete an item from the cache by its unique key.
*
* @param string $key The unique cache key of the item to delete.
*
* @return bool True if the item was successfully removed. False if there was an error.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if the $key string is not a legal value.
*/
public function delete($key)
{
return \apcu_delete("$this->prefix:$key");
}
/**
* Wipes clean the entire cache's keys.
*
* @return bool True on success and false on failure.
*/
public function clear()
{
$iterator = new APCUIterator(
'/^' . preg_quote($this->prefix . ':', '/') . '/',
APC_ITER_NONE
);
return \apcu_delete($iterator);
}
/**
* Obtains multiple cache items by their unique keys.
*
* @param iterable $keys A list of keys that can obtained in a single operation.
* @param mixed $default Default value to return for keys that do not exist.
*
* @return iterable A list of key => value pairs. Cache keys that do not exist or are stale will have $default as value.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if $keys is neither an array nor a Traversable,
* or if any of the $keys are not a legal value.
*/
public function getMultiple($keys, $default = null)
{
$keysToFetch = $this->keysToFetch($keys);
$values = \apcu_fetch($keysToFetch);
foreach ($keysToFetch as $keyToFetch) {
if (!isset($values[$keyToFetch])) {
$values[$keyToFetch] = $default;
}
}
return $values;
}
/**
* Persists a set of key => value pairs in the cache, with an optional TTL.
*
* @param iterable $values A list of key => value pairs for a multiple-set operation.
* @param null|int|\DateInterval $ttl Optional. The TTL value of this item. If no value is sent and
* the driver supports TTL then the library may set a default value
* for it or let the driver take care of that.
*
* @return bool True on success and false on failure.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if $values is neither an array nor a Traversable,
* or if any of the $values are not a legal value.
*/
public function setMultiple($values, $ttl = null)
{
$dataToSave = [];
foreach ($values as $key => $value) {
$dataToSave["$this->prefix:$key"] = $value;
}
return \apcu_store($dataToSave, null, $this->tllToInt($ttl));
}
/**
* Deletes multiple cache items in a single operation.
*
* @param iterable $keys A list of string-based keys to be deleted.
*
* @return bool True if the items were successfully removed. False if there was an error.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if $keys is neither an array nor a Traversable,
* or if any of the $keys are not a legal value.
*/
public function deleteMultiple($keys)
{
$keysToDelete = $this->keysToFetch($keys);
return \apcu_delete($keysToDelete);
}
/**
* Determines whether an item is present in the cache.
*
* NOTE: It is recommended that has() is only to be used for cache warming type purposes
* and not to be used within your live applications operations for get/set, as this method
* is subject to a race condition where your has() will return true and immediately after,
* another script can remove it making the state of your app out of date.
*
* @param string $key The cache item key.
*
* @return bool
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if the $key string is not a legal value.
*/
public function has($key)
{
return \apcu_exists("$this->prefix:$key");
}
/**
* @param iterable $keys
* @return array
*/
private function keysToFetch(iterable $keys): array
{
$keysToFetch = [];
foreach ($keys as $key) {
$keysToFetch[] = "$this->prefix:$key";
}
return $keysToFetch;
}
/**
* @param null|int|\DateInterval $ttl
* @return int
*/
private function tllToInt($ttl = null): int
{
if ($ttl === null) {
return 0;
} elseif (is_int($ttl)) {
return $ttl;
} elseif ($ttl instanceof \DateInterval) {
return $ttl->days * 86400 + $ttl->h * 3600 + $ttl->i * 60 + $ttl->s;
} else {
throw new \Psr\SimpleCache\InvalidArgumentException("Invalid ttl value '$ttl' provided.");
}
}
}

27
app/Lib/Tools/AttributeValidationTool.php
View File

@ -41,7 +41,7 @@ class AttributeValidationTool
switch ($type) {
case 'ip-src':
case 'ip-dst':
return self::compressIpv6($value);
return self::normalizeIp($value);
case 'md5':
case 'sha1':
case 'sha224':
@ -98,7 +98,7 @@ class AttributeValidationTool
$parts[0] = $punyCode;
}
}
$parts[1] = self::compressIpv6($parts[1]);
$parts[1] = self::normalizeIp($parts[1]);
return "$parts[0]|$parts[1]";
case 'filename|md5':
case 'filename|sha1':
@ -175,7 +175,7 @@ class AttributeValidationTool
} else {
return $value;
}
return self::compressIpv6($parts[0]) . '|' . $parts[1];
return self::normalizeIp($parts[0]) . '|' . $parts[1];
case 'mac-address':
case 'mac-eui-64':
$value = str_replace(array('.', ':', '-', ' '), '', strtolower($value));
@ -700,11 +700,30 @@ class AttributeValidationTool
* @param string $value
* @return string
*/
private static function compressIpv6($value)
private static function normalizeIp($value)
{
// If IP is a CIDR
if (strpos($value, '/')) {
list($ip, $range) = explode('/', $value, 2);
// Compress IPv6
if (strpos($ip, ':') && $converted = inet_pton($ip)) {
$ip = inet_ntop($converted);
}
// If IP is in CIDR format, but the network is 32 for IPv4 or 128 for IPv6, normalize to non CIDR type
if (($range === '32' && strpos($value, '.')) || ($range === '128' && strpos($value, ':'))) {
return $ip;
}
return "$ip/$range";
}
// Compress IPv6
if (strpos($value, ':') && $converted = inet_pton($value)) {
return inet_ntop($converted);
}
return $value;
}

30
app/Lib/Tools/BackgroundJobs/BackgroundJob.php
View File

@ -66,8 +66,9 @@ class BackgroundJob implements JsonSerializable
/**
* Run the job command
* @param callable|null $runningCallback
*/
public function run(): void
public function run(callable $runningCallback = null): void
{
$descriptorSpec = [
1 => ["pipe", "w"], // stdout
@ -88,7 +89,7 @@ class BackgroundJob implements JsonSerializable
['BACKGROUND_JOB_ID' => $this->id]
);
$this->pool($process, $pipes);
$this->pool($process, $pipes, $runningCallback);
if ($this->returnCode === 0 && empty($stderr)) {
$this->setStatus(BackgroundJob::STATUS_COMPLETED);
@ -98,7 +99,13 @@ class BackgroundJob implements JsonSerializable
}
}
private function pool($process, array $pipes)
/**
* @param resource $process
* @param array $pipes
* @param callable|null $runningCallback
* @return void
*/
private function pool($process, array $pipes, callable $runningCallback = null)
{
stream_set_blocking($pipes[1], false);
stream_set_blocking($pipes[2], false);
@ -106,6 +113,14 @@ class BackgroundJob implements JsonSerializable
$this->output = '';
$this->error = '';
if ($runningCallback) {
$status = proc_get_status($process);
if ($status === false) {
throw new RuntimeException("Could not get process status");
}
$runningCallback($status);
}
while (true) {
$read = [$pipes[1], $pipes[2]];
$write = null;
@ -118,6 +133,12 @@ class BackgroundJob implements JsonSerializable
$this->error .= stream_get_contents($pipes[2]);
}
$status = proc_get_status($process);
if ($status === false) {
throw new RuntimeException("Could not get process status");
}
if ($runningCallback) {
$runningCallback($status);
}
if (!$status['running']) {
// Just in case read rest data from stream
$this->output .= stream_get_contents($pipes[1]);
@ -153,6 +174,9 @@ class BackgroundJob implements JsonSerializable
return ['id', 'command', 'args', 'createdAt', 'updatedAt', 'status', 'output', 'error', 'metadata'];
}
/**
* @return string Background job ID in UUID format
*/
public function id(): string
{
return $this->id;

2
app/Lib/Tools/BackgroundJobs/Worker.php
View File

@ -65,7 +65,7 @@ class Worker implements JsonSerializable
];
}
public function pid(): ?int
public function pid(): int
{
return $this->pid;
}

67
app/Lib/Tools/BackgroundJobsTool.php
View File

@ -91,7 +91,8 @@ class BackgroundJobsTool
];
const JOB_STATUS_PREFIX = 'job_status',
DATA_CONTENT_PREFIX = 'data_content';
DATA_CONTENT_PREFIX = 'data_content',
RUNNING_JOB_PREFIX = 'running';
/** @var array */
private $settings;
@ -277,6 +278,54 @@ class BackgroundJobsTool
return null;
}
/**
* @param Worker $worker
* @param BackgroundJob $job
* @param int|null $pid
* @return void
* @throws RedisException
*/
public function markAsRunning(Worker $worker, BackgroundJob $job, $pid = null)
{
$key = self::RUNNING_JOB_PREFIX . ':' . $worker->queue() . ':' . $job->id();
$this->RedisConnection->setex($key, 60, [
'worker_pid' => $worker->pid(),
'process_pid' => $pid,
]);
}
/**
* @param Worker $worker
* @param BackgroundJob $job
* @return void
* @throws RedisException
*/
public function removeFromRunning(Worker $worker, BackgroundJob $job)
{
$key = self::RUNNING_JOB_PREFIX . ':' . $worker->queue() . ':' . $job->id();
$this->RedisConnection->del($key);
}
/**
* Return current running jobs
* @return array
* @throws RedisException
*/
public function runningJobs(): array
{
$pattern = $this->RedisConnection->_prefix(self::RUNNING_JOB_PREFIX . ':*');
$keys = RedisTool::keysByPattern($this->RedisConnection, $pattern);
$jobIds = [];
foreach ($keys as $key) {
$parts = explode(':', $key);
$queue = $parts[2];
$jobId = $parts[3];
$jobIds[$queue][$jobId] = $this->RedisConnection->get(self::RUNNING_JOB_PREFIX . ":$queue:$jobId");
}
return $jobIds;
}
/**
* Get the job status.
*
@ -500,19 +549,6 @@ class BackgroundJobsTool
$this->getSupervisor()->startProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart);
}
/**
* Purge queue
*
* @param string $queue
* @return void
*/
public function purgeQueue(string $queue)
{
$this->validateQueue($queue);
$this->RedisConnection->del($queue);
}
/**
* Return Background Jobs status
*
@ -728,8 +764,7 @@ class BackgroundJobsTool
*
* @param integer $pid
* @return \Supervisor\Process
*
* @throws NotFoundException
* @throws NotFoundException|Exception
*/
private function getProcessByPid(int $pid): \Supervisor\Process
{

4
app/Lib/Tools/BetterCakeEventManager.php
View File

@ -52,10 +52,10 @@ class BetterCakeEventManager extends CakeEventManager
$result = [];
foreach ($priorities as $priority) {
if (isset($globalListeners[$priority])) {
$result = array_merge($result, $globalListeners[$priority]);
array_push($result, ...$globalListeners[$priority]);
}
if (isset($localListeners[$priority])) {
$result = array_merge($result, $localListeners[$priority]);
array_push($result, ...$localListeners[$priority]);
}
}
return $result;

28
app/Lib/Tools/BetterSecurity.php
View File

@ -7,8 +7,8 @@ class BetterSecurity
/**
* @param string $plain
* @param string $key
* @return string
* @param string $key Encryption key
* @return string Cipher text with IV and tag
* @throws Exception
*/
public static function encrypt($plain, $key)
@ -33,17 +33,17 @@ class BetterSecurity
}
/**
* @param string $cipher
* @param string $key
* @param string $cipherText Cipher text with IV and tag
* @param string $key Decryption key
* @return string
* @throws Exception
*/
public static function decrypt($cipher, $key)
public static function decrypt($cipherText, $key)
{
if (strlen($key) < 32) {
throw new Exception('Invalid key, key must be at least 256 bits (32 bytes) long.');
}
if (empty($cipher)) {
if (empty($cipherText)) {
throw new Exception('The data to decrypt cannot be empty.');
}
@ -52,12 +52,18 @@ class BetterSecurity
$ivSize = openssl_cipher_iv_length(self::METHOD);
// Split out hmac for comparison
$iv = substr($cipher, 0, $ivSize);
$tag = substr($cipher, $ivSize, self::TAG_SIZE);
$cipher = substr($cipher, $ivSize + self::TAG_SIZE);
if (strlen($cipherText) < $ivSize + self::TAG_SIZE) {
$length = strlen($cipherText);
$minLength = $ivSize + self::TAG_SIZE;
throw new Exception("Provided cipher text is too short, $length bytes provided, expected at least $minLength bytes.");
}
$decrypted = openssl_decrypt($cipher, self::METHOD, $key, true, $iv, $tag);
// Split out hmac for comparison
$iv = substr($cipherText, 0, $ivSize);
$tag = substr($cipherText, $ivSize, self::TAG_SIZE);
$cipherText = substr($cipherText, $ivSize + self::TAG_SIZE);
$decrypted = openssl_decrypt($cipherText, self::METHOD, $key, OPENSSL_RAW_DATA, $iv, $tag);
if ($decrypted === false) {
throw new Exception('Could not decrypt. Maybe invalid encryption key?');
}

11
app/Lib/Tools/ComplexTypeTool.php
View File

@ -308,14 +308,13 @@ class ComplexTypeTool
*/
private function parseFreetext($input)
{
$input = str_replace("\xc2\xa0", ' ', $input); // non breaking space to normal space
$input = preg_replace('/\p{C}+/u', ' ', $input);
$iocArray = preg_split("/\r\n|\n|\r|\s|\s+|,|\<|\>|;/", $input);
// convert non breaking space to normal space and all unicode chars from "other" category
$input = preg_replace("/\p{C}+|\xc2\xa0/u", ' ', $input);
$iocArray = preg_split("/\r\n|\n|\r|\s|\s+|,|<|>|;/", $input);
preg_match_all('/\"([^\"]+)\"/', $input, $matches);
foreach ($matches[1] as $match) {
$iocArray[] = $match;
}
array_push($iocArray, ...$matches[1]);
return $iocArray;
}

377
app/Lib/Tools/CurlClient.php Normal file
View File

@ -0,0 +1,377 @@
<?php
App::uses('HttpSocketExtended', 'Tools');
class CurlClient extends HttpSocketExtended
{
/** @var resource */
private $ch;
/** @var int */
private $timeout = 30;
/** @var string|null */
private $caFile;
/** @var string|null */
private $localCert;
/** @var int */
private $cryptoMethod;
/** @var bool */
private $allowSelfSigned;
/** @var bool */
private $verifyPeer;
/** @var bool */
private $compress = true;
/** @var array */
private $proxy = [];
/** @var array */
private $defaultOptions;
/**
* @param array $params
* @noinspection PhpMissingParentConstructorInspection
*/
public function __construct(array $params)
{
if (isset($params['timeout'])) {
$this->timeout = $params['timeout'];
}
if (isset($params['ssl_cafile'])) {
$this->caFile = $params['ssl_cafile'];
}
if (isset($params['ssl_local_cert'])) {
$this->localCert = $params['ssl_local_cert'];
}
if (isset($params['compress'])) {
$this->compress = $params['compress'];
}
if (isset($params['ssl_crypto_method'])) {
$this->cryptoMethod = $this->convertCryptoMethod($params['ssl_crypto_method']);
}
if (isset($params['ssl_allow_self_signed'])) {
$this->allowSelfSigned = $params['ssl_allow_self_signed'];
}
if (isset($params['ssl_verify_peer'])) {
$this->verifyPeer = $params['ssl_verify_peer'];
}
$this->defaultOptions = $this->generateDefaultOptions();
}
/**
* @param string $uri
* @param array $query
* @param array $request
* @return HttpSocketResponseExtended
*/
public function head($uri = null, $query = [], $request = [])
{
return $this->internalRequest('HEAD', $uri, $query, $request);
}
/**
* @param string $uri
* @param array $query
* @param array $request
* @return HttpSocketResponseExtended
*/
public function get($uri = null, $query = [], $request = [])
{
return $this->internalRequest('GET', $uri, $query, $request);
}
/**
* @param string $uri
* @param array $data
* @param array $request
* @return HttpSocketResponseExtended
*/
public function post($uri = null, $data = [], $request = [])
{
return $this->internalRequest('POST', $uri, $data, $request);
}
/**
* @param string $uri
* @param array$data
* @param $request
* @return HttpSocketResponseExtended
*/
public function put($uri = null, $data = [], $request = [])
{
return $this->internalRequest('PUT', $uri, $data, $request);
}
/**
* @param string $uri
* @param array $data
* @param array $request
* @return HttpSocketResponseExtended
*/
public function patch($uri = null, $data = [], $request = [])
{
return $this->internalRequest('PATCH', $uri, $data, $request);
}
/**
* @param string $uri
* @param array $data
* @param array $request
* @return HttpSocketResponseExtended
*/
public function delete($uri = null, $data = array(), $request = array())
{
return $this->internalRequest('DELETE', $uri, $data, $request);
}
public function url($url = null, $uriTemplate = null)
{
throw new Exception('Not implemented');
}
public function request($request = array())
{
throw new Exception('Not implemented');
}
public function setContentResource($resource)
{
throw new Exception('Not implemented');
}
public function getMetaData()
{
return null; // not supported by curl extension
}
/**
* @param string $host
* @param int $port
* @param string $method
* @param string $user
* @param string $pass
* @return void
*/
public function configProxy($host, $port = 3128, $method = null, $user = null, $pass = null)
{
if (empty($host)) {
$this->proxy = [];
return;
}
if (is_array($host)) {
$this->proxy = $host + ['host' => null];
return;
}
$this->proxy = compact('host', 'port', 'method', 'user', 'pass');
}
/**
* @param string $method
* @param string $url
* @param array|string $query
* @param array $request
* @return HttpSocketResponseExtended
*/
private function internalRequest($method, $url, $query, $request)
{
if (empty($url)) {
throw new InvalidArgumentException("No URL provided.");
}
if (!$this->ch) {
// Share handle between requests to allow keep connection alive between requests
$this->ch = curl_init();
if (!$this->ch) {
throw new \RuntimeException("Could not initialize curl");
}
} else {
// Reset options, so we can do another request
curl_reset($this->ch);
}
if (($method === 'GET' || $method === 'HEAD') && !empty($query)) {
$url .= '?' . http_build_query($query, '', '&', PHP_QUERY_RFC3986);
}
$options = $this->defaultOptions;
$options[CURLOPT_URL] = $url;
$options[CURLOPT_CUSTOMREQUEST] = $method;
if (($method === 'POST' || $method === 'DELETE' || $method === 'PUT' || $method === 'PATCH') && !empty($query)) {
$options[CURLOPT_POSTFIELDS] = $query;
}
if (!empty($request['header'])) {
$headers = [];
foreach ($request['header'] as $key => $value) {
if (is_array($value)) {
$value = implode(', ', $value);
}
$headers[] = "$key: $value";
}
$options[CURLOPT_HTTPHEADER] = $headers;
}
// Parse response headers
$responseHeaders = [];
$options[CURLOPT_HEADERFUNCTION] = function ($curl, $header) use (&$responseHeaders){
$len = strlen($header);
$header = explode(':', $header, 2);
if (count($header) < 2) { // ignore invalid headers
return $len;
}
$key = strtolower(trim($header[0]));
$value = trim($header[1]);
if (isset($responseHeaders[$key])) {
$responseHeaders[$key] = array_merge((array)$responseHeaders[$key], [$value]);
} else {
$responseHeaders[$key] = $value;
}
return $len;
};
if (!curl_setopt_array($this->ch, $options)) {
throw new \RuntimeException('curl error: Could not set options');
}
// Download the given URL, and return output
$output = curl_exec($this->ch);
if ($output === false) {
$errorCode = curl_errno($this->ch);
$errorMessage = curl_error($this->ch);
if (!empty($errorMessage)) {
$errorMessage = ": $errorMessage";
}
throw new SocketException("curl error $errorCode '" . curl_strerror($errorCode) . "'" . $errorMessage);
}
$code = curl_getinfo($this->ch, CURLINFO_HTTP_CODE);
return $this->constructResponse($output, $responseHeaders, $code);
}
public function disconnect()
{
if ($this->ch) {
curl_close($this->ch);
$this->ch = null;
}
}
/**
* @param string $body
* @param array $headers
* @param int $code
* @return HttpSocketResponseExtended
*/
private function constructResponse($body, array $headers, $code)
{
if (isset($responseHeaders['content-encoding']) && $responseHeaders['content-encoding'] === 'zstd') {
if (!function_exists('zstd_uncompress')) {
throw new SocketException('Response is zstd encoded, but PHP do not support zstd decoding.');
}
$body = zstd_uncompress($body);
if ($body === false) {
throw new SocketException('Could not decode zstd encoded response.');
}
}
$response = new HttpSocketResponseExtended();
$response->code = $code;
$response->body = $body;
$response->headers = $headers;
return $response;
}
/**
* @param int $cryptoMethod
* @return int
*/
private function convertCryptoMethod($cryptoMethod)
{
switch ($cryptoMethod) {
case STREAM_CRYPTO_METHOD_TLSv1_0_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_1_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
return CURL_SSLVERSION_TLSv1;
case STREAM_CRYPTO_METHOD_TLSv1_1_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
return CURL_SSLVERSION_TLSv1_1;
case STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
return CURL_SSLVERSION_TLSv1_2;
case STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
return CURL_SSLVERSION_TLSv1_3;
default:
throw new InvalidArgumentException("Unsupported crypto method value $cryptoMethod");
}
}
/**
* @return array
*/
private function generateDefaultOptions()
{
$options = [
CURLOPT_FOLLOWLOCATION => true, // Allows to follow redirect
CURLOPT_MAXREDIRS => 10,
CURLOPT_RETURNTRANSFER => true, // Should cURL return or print out the data? (true = return, false = print)
CURLOPT_HEADER => false, // Include header in result?
CURLOPT_TIMEOUT => $this->timeout, // Timeout in seconds
CURLOPT_PROTOCOLS => CURLPROTO_HTTPS | CURLPROTO_HTTP, // be sure that only HTTP and HTTPS protocols are enabled,
];
if ($this->caFile) {
$options[CURLOPT_CAINFO] = $this->caFile;
}
if ($this->localCert) {
$options[CURLOPT_SSLCERT] = $this->localCert;
}
if ($this->cryptoMethod) {
$options[CURLOPT_SSLVERSION] = $this->cryptoMethod;
}
if ($this->compress) {
$options[CURLOPT_ACCEPT_ENCODING] = $this->supportedEncodings();
}
if ($this->allowSelfSigned) {
$options[CURLOPT_SSL_VERIFYPEER] = $this->verifyPeer;
$options[CURLOPT_SSL_VERIFYHOST] = 0;
}
if (!empty($this->proxy)) {
$options[CURLOPT_PROXY] = "{$this->proxy['host']}:{$this->proxy['port']}";
if (!empty($this->proxy['method']) && isset($this->proxy['user'], $this->proxy['pass'])) {
$options[CURLOPT_PROXYUSERPWD] = "{$this->proxy['user']}:{$this->proxy['pass']}";
}
}
return $options;
}
/**
* @return string
*/
private function supportedEncodings()
{
$encodings = [];
// zstd is not supported by curl itself, but add support if PHP zstd extension is installed
if (function_exists('zstd_uncompress')) {
$encodings[] = 'zstd';
}
// brotli and gzip is supported by curl itself if it is compiled with these features
$info = curl_version();
if (defined('CURL_VERSION_BROTLI') && $info['features'] & CURL_VERSION_BROTLI) {
$encodings[] = 'br';
}
if ($info['features'] & CURL_VERSION_LIBZ) {
$encodings[] = 'gzip, deflate';
}
return implode(', ', $encodings);
}
}

65
app/Lib/Tools/GitTool.php
View File

@ -12,11 +12,7 @@ class GitTool
public static function getLatestTags(HttpSocketExtended $HttpSocket)
{
$url = 'https://api.github.com/repos/MISP/MISP/tags?per_page=10';
$response = $HttpSocket->get($url);
if (!$response->isOk()) {
throw new HttpSocketHttpException($response, $url);
}
return $response->json();
return self::gitHubRequest($HttpSocket, $url);
}
/**
@ -28,11 +24,7 @@ class GitTool
public static function getLatestCommit(HttpSocketExtended $HttpSocket)
{
$url = 'https://api.github.com/repos/MISP/MISP/commits?per_page=1';
$response = $HttpSocket->get($url);
if (!$response->isOk()) {
throw new HttpSocketHttpException($response, $url);
}
$data = $response->json();
$data = self::gitHubRequest($HttpSocket, $url);
if (!isset($data[0]['sha'])) {
throw new Exception("Response do not contains requested data.");
}
@ -40,20 +32,49 @@ class GitTool
}
/**
* @param HttpSocketExtended $HttpSocket
* @param string $url
* @return array
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
*/
private static function gitHubRequest(HttpSocketExtended $HttpSocket, $url)
{
$response = $HttpSocket->get($url, [], ['header' => ['User-Agent' => 'MISP']]);
if (!$response->isOk()) {
throw new HttpSocketHttpException($response, $url);
}
return $response->json();
}
/**
* Returns current SHA1 hash of current commit
* `git rev-parse HEAD`
* @param string $repoPath
* @return string
* @throws Exception
*/
public static function currentCommit()
public static function currentCommit($repoPath)
{
$head = rtrim(FileAccessTool::readFromFile(ROOT . '/.git/HEAD'));
if (is_file($repoPath . '/.git')) {
$fileContent = FileAccessTool::readFromFile($repoPath . '/.git');
if (substr($fileContent, 0, 8) === 'gitdir: ') {
$gitDir = $repoPath . '/' . trim(substr($fileContent, 8)) . '/';
} else {
throw new Exception("$repoPath/.git is file, but contains non expected content $fileContent");
}
} else {
$gitDir = $repoPath . '/.git/';
}
$head = rtrim(FileAccessTool::readFromFile($gitDir . 'HEAD'));
if (substr($head, 0, 5) === 'ref: ') {
$path = substr($head, 5);
return rtrim(FileAccessTool::readFromFile(ROOT . '/.git/' . $path));
return rtrim(FileAccessTool::readFromFile($gitDir . $path));
} else if (strlen($head) === 40) {
return $head;
} else {
throw new Exception("Invalid head $head");
throw new Exception("Invalid head '$head' in $gitDir/HEAD");
}
}
@ -94,30 +115,18 @@ class GitTool
return $output;
}
/**
* @param string $submodule Path to Git repo
* @return string|null
*/
public static function submoduleCurrentCommit($submodule)
{
try {
$commit = ProcessTool::execute(['git', 'rev-parse', 'HEAD'], $submodule);
} catch (ProcessException $e) {
return null;
}
return rtrim($commit);
}
/**
* @param string $commit
* @param string|null $submodule Path to Git repo
* @return int|null
* @throws Exception
*/
public static function commitTimestamp($commit, $submodule = null)
{
try {
$timestamp = ProcessTool::execute(['git', 'show', '-s', '--pretty=format:%ct', $commit], $submodule);
} catch (ProcessException $e) {
CakeLog::notice("Could not get Git commit timestamp for $submodule: {$e->getMessage()}");
return null;
}
return (int)rtrim($timestamp);

5
app/Lib/Tools/HttpSocketExtended.php
View File

@ -18,10 +18,15 @@ class HttpSocketHttpException extends Exception
{
$this->response = $response;
$this->url = $url;
$message = "Remote server returns HTTP error code $response->code";
if ($url) {
$message .= " for URL $url";
}
if ($response->body) {
$message .= ': ' . substr($response->body, 0, 100);
}
parent::__construct($message, (int)$response->code);
}

2
app/Lib/Tools/JSONConverterTool.php
View File

@ -153,7 +153,7 @@ class JSONConverterTool
return;
}
yield '{"Event":{';
$firstKey = key($event['Event']);
$firstKey = array_key_first($event['Event']);
foreach ($event['Event'] as $key => $value) {
if ($key === 'Attribute' || $key === 'Object') { // Encode every object or attribute separately
yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":[";

50
app/Lib/Tools/JsonTool.php
View File

@ -9,10 +9,7 @@ class JsonTool
*/
public static function encode($value, $prettyPrint = false)
{
$flags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE;
if (defined('JSON_THROW_ON_ERROR')) {
$flags |= JSON_THROW_ON_ERROR; // Throw exception on error if supported
}
$flags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR;
if ($prettyPrint) {
$flags |= JSON_PRETTY_PRINT;
}
@ -34,16 +31,8 @@ class JsonTool
} catch (SimdJsonException $e) {
throw new JsonException($e->getMessage(), $e->getCode(), $e);
}
} elseif (defined('JSON_THROW_ON_ERROR')) {
// JSON_THROW_ON_ERROR is supported since PHP 7.3
return json_decode($value, true, 512, JSON_THROW_ON_ERROR);
} else {
$decoded = json_decode($value, true);
if ($decoded === null) {
throw new UnexpectedValueException('Could not parse JSON: ' . json_last_error_msg(), json_last_error());
}
return $decoded;
}
return json_decode($value, true, 512, JSON_THROW_ON_ERROR);
}
/**
@ -78,4 +67,39 @@ class JsonTool
return false;
}
}
/**
* @see https://www.php.net/manual/en/function.array-is-list.php
* @param array $array
* @return bool
*/
public static function arrayIsList(array $array)
{
if (function_exists('array_is_list')) {
return array_is_list($array);
}
$i = -1;
foreach ($array as $k => $v) {
++$i;
if ($k !== $i) {
return false;
}
}
return true;
}
/**
* JSON supports just unicode strings. This helper method converts non unicode chars to Unicode Replacement Character U+FFFD (UTF-8)
* @param string $string
* @return string
*/
public static function escapeNonUnicode($string)
{
if (mb_check_encoding($string, 'UTF-8')) {
return $string; // string is valid unicode
}
return htmlspecialchars_decode(htmlspecialchars($string, ENT_SUBSTITUTE, 'UTF-8'));
}
}

15
app/Lib/Tools/ProcessTool.php
View File

@ -8,14 +8,14 @@ class ProcessException extends Exception
private $stdout;
/**
* @param string|array $command
* @param array $command
* @param int $returnCode
* @param string $stderr
* @param string $stdout
*/
public function __construct($command, $returnCode, $stderr, $stdout)
public function __construct(array $command, $returnCode, $stderr, $stdout)
{
$commandForException = is_array($command) ? implode(' ', $command) : $command;
$commandForException = implode(' ', $command);
$message = "Command '$commandForException' finished with error code $returnCode.\nSTDERR: '$stderr'\nSTDOUT: '$stdout'";
$this->stderr = $stderr;
$this->stdout = $stdout;
@ -56,11 +56,6 @@ class ProcessTool
self::logMessage('Running command ' . implode(' ', $command));
}
// PHP older than 7.4 do not support proc_open with array, so we need to convert values to string manually
if (PHP_VERSION_ID < 70400) {
$command = array_map('escapeshellarg', $command);
$command = implode(' ', $command);
}
$process = proc_open($command, $descriptorSpec, $pipes, $cwd);
if (!$process) {
$commandForException = self::commandFormat($command);
@ -136,8 +131,8 @@ class ProcessTool
* @param array|string $command
* @return string
*/
private static function commandFormat($command)
private static function commandFormat(array $command)
{
return is_array($command) ? implode(' ', $command) : $command;
return implode(' ', $command);
}
}

31
app/Lib/Tools/PubSubTool.php
View File

@ -178,8 +178,12 @@ class PubSubTool
public function killService()
{
$settings = $this->getSetSettings();
if ($settings['supervisor_managed']) {
throw new RuntimeException('ZeroMQ server is managed by supervisor, it is not possible to restart it.');
}
if ($this->checkIfRunning()) {
$settings = $this->getSetSettings();
$redis = $this->createRedisConnection($settings);
$redis->rPush('command', 'kill');
sleep(1);
@ -213,12 +217,16 @@ class PubSubTool
public function restartServer()
{
$settings = $this->getSetSettings();
if ($settings['supervisor_managed']) {
throw new RuntimeException('ZeroMQ server is managed by supervisor, it is not possible to restart it.');
}
if (!$this->checkIfRunning()) {
if (!$this->killService()) {
return 'Could not kill the previous instance of the ZeroMQ script.';
}
}
$settings = $this->getSetSettings();
$this->setupPubServer($settings);
if ($this->checkIfRunning() === false) {
return 'Failed starting the ZeroMQ script.';
@ -226,12 +234,22 @@ class PubSubTool
return true;
}
public function createConfigFile()
{
$settings = $this->getSetSettings();
$this->saveSettingToFile($settings);
}
/**
* @param array $settings
* @throws Exception
*/
private function setupPubServer(array $settings)
{
if ($settings['supervisor_managed']) {
return; // server is managed by supervisor, we don't need to check if is running or start it when not
}
if ($this->checkIfRunning() === false) {
if ($this->checkIfRunning(self::OLD_PID_LOCATION)) {
// Old version is running, kill it and start again new one.
@ -250,6 +268,7 @@ class PubSubTool
* @param string|array $data
* @return bool
* @throws JsonException
* @throws RedisException
*/
private function pushToRedis($ns, $data)
{
@ -295,9 +314,12 @@ class PubSubTool
FileAccessTool::writeToFile($settingFilePath, JsonTool::encode($settings));
}
/**
* @return array
*/
private function getSetSettings()
{
$settings = array(
$settings = [
'redis_host' => 'localhost',
'redis_port' => 6379,
'redis_password' => '',
@ -307,7 +329,8 @@ class PubSubTool
'port' => '50000',
'username' => null,
'password' => null,
);
'supervisor_managed' => false,
];
$pluginConfig = Configure::read('Plugin');
foreach ($settings as $key => $setting) {

23
app/Lib/Tools/RedisTool.php
View File

@ -57,25 +57,34 @@ class RedisTool
/**
* @param Redis $redis
* @param string|array $pattern
* @return int|Redis Number of deleted keys or instance of Redis if used in MULTI mode
* @return Generator<string>
* @throws RedisException
*/
public static function deleteKeysByPattern(Redis $redis, $pattern)
public static function keysByPattern(Redis $redis, $pattern)
{
if (is_string($pattern)) {
$pattern = [$pattern];
}
$allKeys = [];
foreach ($pattern as $p) {
$iterator = null;
while (false !== ($keys = $redis->scan($iterator, $p, 1000))) {
foreach ($keys as $key) {
$allKeys[] = $key;
yield $key;
}
}
}
}
/**
* @param Redis $redis
* @param string|array $pattern
* @return int|Redis Number of deleted keys or instance of Redis if used in MULTI mode
* @throws RedisException
*/
public static function deleteKeysByPattern(Redis $redis, $pattern)
{
$allKeys = iterator_to_array(self::keysByPattern($redis, $pattern));
if (empty($allKeys)) {
return 0;
}
@ -153,11 +162,7 @@ class RedisTool
return false;
}
if (self::$serializer === null) {
self::$serializer = Configure::read('MISP.redis_serializer') ?: false;
}
if (self::$serializer === 'igbinary') {
if ($string[0] === "\x00") {
return igbinary_unserialize($string);
} else {
return JsonTool::decode($string);

8
app/Lib/Tools/ServerSyncTool.php
View File

@ -355,6 +355,14 @@ class ServerSyncTool
return $this->server['Server']['id'];
}
/**
* @return string
*/
public function serverName()
{
return $this->server['Server']['name'];
}
/**
* @return array
*/

11
app/Lib/Tools/SyncTool.php
View File

@ -1,5 +1,4 @@
<?php
class SyncTool
{
@ -84,8 +83,14 @@ class SyncTool
$params['ssl_crypto_method'] = $version;
}
App::uses('HttpSocketExtended', 'Tools');
$HttpSocket = new HttpSocketExtended($params);
if (function_exists('curl_init')) {
App::uses('CurlClient', 'Tools');
$HttpSocket = new CurlClient($params);
} else {
App::uses('HttpSocketExtended', 'Tools');
$HttpSocket = new HttpSocketExtended($params);
}
$proxy = Configure::read('Proxy');
if (empty($params['skip_proxy']) && isset($proxy['host']) && !empty($proxy['host'])) {
$HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']);

48
app/Model/AccessLog.php
View File

@ -71,12 +71,6 @@ class AccessLog extends AppModel
{
$accessLog = &$this->data['AccessLog'];
$this->externalLog($accessLog);
if (Configure::read('MISP.log_paranoid_skip_db')) {
return;
}
// Truncate
foreach (['request_id', 'user_agent', 'url'] as $field) {
if (isset($accessLog[$field]) && strlen($accessLog[$field]) > 255) {
@ -202,7 +196,7 @@ class AccessLog extends AppModel
if ($includeSqlQueries && !empty($sqlLog['log'])) {
foreach ($sqlLog['log'] as &$log) {
$log['query'] = $this->escapeNonUnicode($log['query']);
$log['query'] = JsonTool::escapeNonUnicode($log['query']);
unset($log['affected']); // affected is the same as numRows
unset($log['params']); // no need to save for your use case
}
@ -214,6 +208,12 @@ class AccessLog extends AppModel
$data['query_count'] = $queryCount;
$data['duration'] = (int)((microtime(true) - $requestTime->format('U.u')) * 1000); // in milliseconds
$this->externalLog($data);
if (Configure::read('MISP.log_paranoid_skip_db')) {
return true; // do not save access log to database
}
try {
return $this->save($data, ['atomic' => false]);
} catch (Exception $e) {
@ -226,7 +226,7 @@ class AccessLog extends AppModel
* @param array $data
* @return void
*/
public function externalLog(array $data)
private function externalLog(array $data)
{
if ($this->pubToZmq('audit')) {
$this->getPubSubTool()->publish($data, 'audit', 'log');
@ -310,36 +310,4 @@ class AccessLog extends AppModel
}
return $data;
}
/**
* @param $string
* @return string
*/
private function escapeNonUnicode($string)
{
if (json_encode($string, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_LINE_TERMINATORS) !== false) {
return $string; // string is valid unicode
}
if (function_exists('mb_str_split')) {
$result = mb_str_split($string);
} else {
$result = [];
$length = mb_strlen($string);
for ($i = 0; $i < $length; $i++) {
$result[] = mb_substr($string, $i, 1);
}
}
$string = '';
foreach ($result as $char) {
if (strlen($char) === 1 && !preg_match('/[[:print:]]/', $char)) {
$string .= '\x' . bin2hex($char);
} else {
$string .= $char;
}
}
return $string;
}
}

28
app/Model/AdminSetting.php
View File

@ -90,6 +90,7 @@ class AdminSetting extends AppModel
$time = time();
$this->__deleteScriptTmpFiles($time);
$this->__deleteTaxiiTmpFiles($time);
$this->__deleteCachedExportFiles($time);
}
private function __deleteScriptTmpFiles($time) {
@ -107,6 +108,29 @@ class AdminSetting extends AppModel
}
}
private function __deleteCachedExportFiles($time) {
$cache_path = APP . 'tmp/cached_exports';
$cache_dir = new Folder($cache_path);
$cache_data = $cache_dir->read(false, false);
if (!empty($cache_data[0])) {
foreach ($cache_data[0] as $cache_export_dir) {
$tmp_dir = new Folder($cache_path . '/' . $cache_export_dir);
$cache_export_dir_contents = $tmp_dir->read(false, false);
if (!empty(count($cache_export_dir_contents[1]))) {
$files_count = count($cache_export_dir_contents[1]);
$files_removed = 0;
foreach ($cache_export_dir_contents[1] as $tmp_file) {
$tmp_file = new File($cache_path . '/' . $cache_export_dir . '/' . $tmp_file);
if ($time > $tmp_file->lastChange() + 3600) {
$tmp_file->delete();
$files_removed += 1;
}
}
}
}
}
}
private function __deleteTaxiiTmpFiles($time) {
$taxii_path = APP . 'files/scripts/tmp/Taxii';
$taxii_dir = new Folder($taxii_path);
@ -114,13 +138,13 @@ class AdminSetting extends AppModel
if (!empty($taxii_contents[0])) {
foreach ($taxii_contents[0] as $taxii_temp_dir) {
if (preg_match('/^[a-zA-Z0-9]{12}$/', $taxii_temp_dir)) {
$tmp_dir = new Folder($taxii_path . $taxii_temp_dir);
$tmp_dir = new Folder($taxii_path . '/' .$taxii_temp_dir);
$taxii_temp_dir_contents = $tmp_dir->read(false, false);
if (!empty(count($taxii_temp_dir_contents[1]))) {
$files_count = count($taxii_temp_dir_contents[1]);
$files_removed = 0;
foreach ($taxii_temp_dir_contents[1] as $tmp_file) {
$tmp_file = new File($taxii_path . $taxii_temp_dir . '/' . $tmp_file);
$tmp_file = new File($taxii_path . '/' . $taxii_temp_dir . '/' . $tmp_file);
if ($time > $tmp_file->lastChange() + 3600) {
$tmp_file->delete();
$files_removed += 1;

3
app/Model/Allowedlist.php
View File

@ -89,6 +89,9 @@ class Allowedlist extends AppModel
if ($isAttributeArray) {
// loop through each attribute and unset the ones that are allowedlisted
foreach ($data as $k => $attribute) {
if (empty($attribute['Attribute'])) {
$attribute = ['Attribute' => $attribute];
}
// loop through each allowedlist item and run a preg match against the attribute value. If it matches, unset the attribute
foreach ($allowedlists as $wlitem) {
if (preg_match($wlitem, $attribute['Attribute']['value'])) {

42
app/Model/AppModel.php
View File

@ -2374,9 +2374,9 @@ class AppModel extends Model
}
// alternative to the build in notempty/notblank validation functions, compatible with cakephp <= 2.6 and cakephp and cakephp >= 2.7
public function valueNotEmpty($value)
public function valueNotEmpty(array $value)
{
$field = array_keys($value)[0];
$field = array_key_first($value);
$value = trim($value[$field]);
if (!empty($value)) {
return true;
@ -2384,27 +2384,27 @@ class AppModel extends Model
return ucfirst($field) . ' cannot be empty.';
}
public function valueIsJson($value)
public function valueIsJson(array $value)
{
$value = array_values($value)[0];
$value = current($value);
if (!JsonTool::isValid($value)) {
return __('Invalid JSON.');
}
return true;
}
public function valueIsID($value)
public function valueIsID(array $value)
{
$field = array_keys($value)[0];
$field = array_key_first($value);
if (!is_numeric($value[$field]) || $value[$field] < 0) {
return 'Invalid ' . ucfirst($field) . ' ID';
}
return true;
}
public function stringNotEmpty($value)
public function stringNotEmpty(array $value)
{
$field = array_keys($value)[0];
$field = array_key_first($value);
$value = trim($value[$field]);
if (!isset($value) || ($value == false && $value !== "0")) {
return ucfirst($field) . ' cannot be empty.';
@ -3267,14 +3267,13 @@ class AppModel extends Model
* Returns MISP version from VERSION.json file as array with major, minor and hotfix keys.
*
* @return array
* @throws JsonException
* @throws Exception
*/
public function checkMISPVersion()
{
static $versionArray;
if ($versionArray === null) {
$content = FileAccessTool::readFromFile(ROOT . DS . 'VERSION.json');
$versionArray = JsonTool::decode($content);
$versionArray = FileAccessTool::readJsonFromFile(ROOT . DS . 'VERSION.json', true);
}
return $versionArray;
}
@ -3290,7 +3289,7 @@ class AppModel extends Model
if ($commit === null) {
App::uses('GitTool', 'Tools');
try {
$commit = GitTool::currentCommit();
$commit = GitTool::currentCommit(ROOT);
} catch (Exception $e) {
$this->logException('Could not get current git commit', $e, LOG_NOTICE);
$commit = false;
@ -3714,7 +3713,7 @@ class AppModel extends Model
if (!$isRule) {
$args = func_get_args();
$fields = $args[1];
$or = isset($args[2]) ? $args[2] : true;
$or = $args[2] ?? true;
}
}
if (!is_array($fields)) {
@ -3859,8 +3858,7 @@ class AppModel extends Model
protected function isMysql()
{
$dataSource = ConnectionManager::getDataSource('default');
$dataSourceName = $dataSource->config['datasource'];
return $dataSourceName === 'Database/Mysql' || $dataSourceName === 'Database/MysqlObserver' || $dataSourceName === 'Database/MysqlExtended' || $dataSource instanceof Mysql;
return $dataSource instanceof Mysql;
}
/**
@ -3996,21 +3994,21 @@ class AppModel extends Model
");
}
public function findOrder($order, $order_model, $valid_order_fields)
public function findOrder($order, $orderModel, $validOrderFields)
{
if (!is_array($order)) {
$order_rules = explode(' ', strtolower($order));
$order_field = explode('.', $order_rules[0]);
$order_field = end($order_field);
if (in_array($order_field, $valid_order_fields)) {
$orderRules = explode(' ', strtolower($order));
$orderField = explode('.', $orderRules[0]);
$orderField = end($orderField);
if (in_array($orderField, $validOrderFields, true)) {
$direction = 'asc';
if (!empty($order_rules[1]) && trim($order_rules[1]) === 'desc') {
if (!empty($orderRules[1]) && trim($orderRules[1]) === 'desc') {
$direction = 'desc';
}
} else {
return null;
}
return $order_model . '.' . $order_field . ' ' . $direction;
return $orderModel . '.' . $orderField . ' ' . $direction;
}
return null;
}

26
app/Model/AttachmentScan.php
View File

@ -189,6 +189,7 @@ class AttachmentScan extends AppModel
/** @var Job $job */
$job = ClassRegistry::init('Job');
if ($jobId && !$job->exists($jobId)) {
$this->log("Job with ID $jobId not found in database", LOG_NOTICE);
$jobId = null;
}
@ -252,12 +253,12 @@ class AttachmentScan extends AppModel
$infected = $this->scanAttachment($type, $attribute[$type], $moduleInfo);
if ($infected === true) {
$virusFound++;
$scanned++;
} else if ($infected === false) {
$scanned++;
}
$scanned++;
} catch (NotFoundException $e) {
// skip if file doesn't exists
} catch (Exception $e) {
$this->logException("Could not scan attachment for $type {$attribute['Attribute']['id']}", $e);
$this->logException("Could not scan attachment for $type {$attribute['Attribute']['id']}", $e, LOG_WARNING);
$fails++;
}
@ -297,14 +298,14 @@ class AttachmentScan extends AppModel
$job = ClassRegistry::init('Job');
$jobId = $job->createJob(
'SYSTEM',
Job::WORKER_DEFAULT,
Job::WORKER_PRIO,
'virus_scan',
($type === self::TYPE_ATTRIBUTE ? 'Attribute: ' : 'Shadow attribute: ') . $attribute['id'],
'Scanning...'
);
$this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::PRIO_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[
'scanAttachment',
@ -319,10 +320,12 @@ class AttachmentScan extends AppModel
}
/**
* Return true if attachment is infected, null if attachment was not scanned and false if attachment is OK
*
* @param string $type
* @param array $attribute
* @param array $moduleInfo
* @return bool|null Return true if attachment is infected.
* @return bool|null
* @throws Exception
*/
private function scanAttachment($type, array $attribute, array $moduleInfo)
@ -351,11 +354,10 @@ class AttachmentScan extends AppModel
return false; // empty file is automatically considered as not infected
}
/* if ($file->size() > 50 * 1024 * 1024) {
$this->log("File '$file->path' is bigger than 50 MB, will be not scanned.", LOG_NOTICE);
return false;
}*/
if ($fileSize > 25 * 1024 * 1024) {
$this->log("File '$file->path' is bigger than 25 MB, will be not scanned.", LOG_NOTICE);
return null;
}
$fileContent = $file->read();
if ($fileContent === false) {

135
app/Model/Attribute.php
View File

@ -434,7 +434,7 @@ class Attribute extends AppModel
public function afterSave($created, $options = array())
{
// Passing event in `parentEvent` field will speed up correlation
$passedEvent = isset($options['parentEvent']) ? $options['parentEvent'] : false;
$passedEvent = $options['parentEvent'] ?? false;
$attribute = $this->data['Attribute'];
@ -545,6 +545,28 @@ class Attribute extends AppModel
return $result;
}
/**
* This method is called after all data are successfully saved into database
* @return void
* @throws Exception
*/
private function afterDatabaseSave(array $data)
{
$attribute = $data['Attribute'];
if (isset($attribute['type']) && $this->typeIsAttachment($attribute['type'])) {
$this->loadAttachmentScan()->backgroundScan(AttachmentScan::TYPE_ATTRIBUTE, $attribute);
}
}
public function save($data = null, $validate = true, $fieldList = array())
{
$result = parent::save($data, $validate, $fieldList);
if ($result) {
$this->afterDatabaseSave($result);
}
return $result;
}
public function beforeDelete($cascade = true)
{
// delete attachments from the disk
@ -786,7 +808,7 @@ class Attribute extends AppModel
// check whether the variable is null or datetime
public function datetimeOrNull($fields)
{
$seen = array_values($fields)[0];
$seen = current($fields);
if ($seen === null) {
return true;
}
@ -881,7 +903,6 @@ class Attribute extends AppModel
}
$result = $this->loadAttachmentTool()->save($attribute['event_id'], $attribute['id'], $attribute['data']);
if ($result) {
$this->loadAttachmentScan()->backgroundScan(AttachmentScan::TYPE_ATTRIBUTE, $attribute);
// Clean thumbnail cache
if ($this->isImage($attribute) && Configure::read('MISP.thumbnail_in_redis')) {
$redis = RedisTool::init();
@ -1224,38 +1245,96 @@ class Attribute extends AppModel
$this->Correlation->purgeCorrelations($eventId);
}
public function reportValidationIssuesAttributes($eventId)
/**
* This method is useful if you want to iterate all attributes sorted by ID
* @param array $conditions
* @param array $fields
* @param bool|string $callbacks
* @return Generator<array>|void
*/
public function fetchAttributesInChunks(array $conditions = [], array $fields = [], $callbacks = true)
{
$query = [
'recursive' => -1,
'conditions' => $conditions,
'limit' => 500,
'order' => ['Attribute.id'],
'fields' => $fields,
'callbacks' => $callbacks,
];
while (true) {
$attributes = $this->find('all', $query);
foreach ($attributes as $attribute) {
yield $attribute;
}
$count = count($attributes);
if ($count < 500) {
return;
}
$lastAttribute = $attributes[$count - 1];
$query['conditions']['Attribute.id >'] = $lastAttribute['Attribute']['id'];
}
}
/**
* @param int|null $eventId
* @return Generator
*/
public function reportValidationIssuesAttributes($eventId = null)
{
$conditions = array();
if ($eventId && is_numeric($eventId)) {
$conditions = array('event_id' => $eventId);
}
$attributeIds = $this->find('column', array(
'fields' => array('id'),
'conditions' => $conditions
));
$chunks = array_chunk($attributeIds, 500);
$attributes = $this->fetchAttributesInChunks($conditions);
$result = array();
foreach ($chunks as $chunk) {
$attributes = $this->find('all', array('recursive' => -1, 'conditions' => array('id' => $chunk)));
foreach ($attributes as $attribute) {
$this->set($attribute);
if (!$this->validates()) {
$resultErrors = array();
foreach ($this->validationErrors as $field => $error) {
$resultErrors[$field] = array('value' => $attribute['Attribute'][$field], 'error' => $error[0]);
}
$result[] = [
'id' => $attribute['Attribute']['id'],
'error' => $resultErrors,
'details' => 'Event ID: [' . $attribute['Attribute']['event_id'] . "] - Category: [" . $attribute['Attribute']['category'] . "] - Type: [" . $attribute['Attribute']['type'] . "] - Value: [" . $attribute['Attribute']['value'] . ']',
];
foreach ($attributes as $attribute) {
$this->set($attribute);
if (!$this->validates()) {
$resultErrors = [];
foreach ($this->validationErrors as $field => $error) {
$resultErrors[$field] = ['value' => $attribute['Attribute'][$field], 'error' => $error[0]];
}
yield [
'id' => $attribute['Attribute']['id'],
'error' => $resultErrors,
'details' => 'Event ID: [' . $attribute['Attribute']['event_id'] . "] - Category: [" . $attribute['Attribute']['category'] . "] - Type: [" . $attribute['Attribute']['type'] . "] - Value: [" . $attribute['Attribute']['value'] . ']',
];
}
}
}
/**
* @param bool $dryRun If true, no changes will be made to
* @return Generator
* @throws Exception
*/
public function normalizeIpAddress($dryRun = false)
{
$attributes = $this->fetchAttributesInChunks([
'Attribute.type' => ['ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'domain|ip'],
]);
foreach ($attributes as $attribute) {
$value = $attribute['Attribute']['value'];
$normalizedValue = AttributeValidationTool::modifyBeforeValidation($attribute['Attribute']['type'], $value);
if ($value !== $normalizedValue) {
if (!$dryRun) {
$attribute['Attribute']['value'] = $normalizedValue;
$this->save($attribute, true, ['value1', 'value2']);
}
yield [
'id' => (int) $attribute['Attribute']['id'],
'event_id' => (int) $attribute['Attribute']['event_id'],
'type' => $attribute['Attribute']['type'],
'value' => $value,
'normalized_value' => $normalizedValue,
];
}
}
return $result;
}
/**
@ -1610,6 +1689,7 @@ class Attribute extends AppModel
* @param array $user
* @param array $options
* @param int|false $result_count If false, count is not fetched
* @param bool $real_count
* @return array
* @throws Exception
*/
@ -3096,8 +3176,7 @@ class Attribute extends AppModel
$exportTool->additional_params
);
}
ClassRegistry::init('ConnectionManager');
$db = ConnectionManager::getDataSource('default');
$tmpfile = new TmpFileTool();
$tmpfile->write($exportTool->header($exportToolParams));
$loop = false;
@ -3673,7 +3752,7 @@ class Attribute extends AppModel
);
}
private function findAttributeByValue($attribute)
private function findAttributeByValue(array $attribute)
{
$type = $attribute['type'];
$conditions = [

2
app/Model/AuthKey.php
View File

@ -207,7 +207,7 @@ class AuthKey extends AppModel
*/
private function updateUniqueIp(array $authkey)
{
if (Configure::read("MISP.disable_seen_ips_authkeys")) {
if (PHP_SAPI === 'cli' || Configure::read("MISP.disable_seen_ips_authkeys")) {
return;
}

34
app/Model/Correlation.php
View File

@ -153,7 +153,7 @@ class Correlation extends AppModel
if (!empty($eventIds)) {
$eventCount = count($eventIds);
foreach ($eventIds as $j => $currentEventId) {
$attributeCount += $this->__iteratedCorrelation(
$attributeCount += $this->iteratedCorrelation(
$jobId,
$full,
$attributeId,
@ -179,7 +179,7 @@ class Correlation extends AppModel
* @return int
* @throws Exception
*/
private function __iteratedCorrelation(
private function iteratedCorrelation(
$jobId = false,
$full = false,
$attributeId = null,
@ -215,30 +215,14 @@ class Correlation extends AppModel
if ($attributeId) {
$attributeConditions['Attribute.id'] = $attributeId;
}
$query = [
'recursive' => -1,
'conditions' => $attributeConditions,
// fetch just necessary fields to save memory
'fields' => $this->getFieldRules(),
'order' => 'Attribute.id',
'limit' => 5000,
'callbacks' => false, // memory leak fix
];
$attributes = $this->Attribute->fetchAttributesInChunks($attributeConditions, $this->getFieldRules(), false);
$attributeCount = 0;
do {
$attributes = $this->Attribute->find('all', $query);
foreach ($attributes as $attribute) {
$this->afterSaveCorrelation($attribute['Attribute'], $full, $event);
}
$fetchedAttributes = count($attributes);
unset($attributes);
$attributeCount += $fetchedAttributes;
if ($fetchedAttributes === 5000) { // maximum number of attributes fetched, continue in next loop
$query['conditions']['Attribute.id >'] = $attribute['Attribute']['id'];
} else {
break;
}
} while (true);
foreach ($attributes as $attribute) {
$this->afterSaveCorrelation($attribute['Attribute'], $full, $event);
++$attributeCount;
}
// Generating correlations can take long time, so clear caches after each event to refresh them
$this->cidrListCache = null;

26
app/Model/Datasource/Database/MysqlExtended.php
View File

@ -7,6 +7,14 @@ App::uses('Mysql', 'Model/Datasource/Database');
*/
class MysqlExtended extends Mysql
{
const PDO_MAP = [
'integer' => PDO::PARAM_INT,
'float' => PDO::PARAM_STR,
'boolean' => PDO::PARAM_BOOL,
'string' => PDO::PARAM_STR,
'text' => PDO::PARAM_STR
];
/**
* Output MD5 as binary, that is faster and uses less memory
* @param string $value
@ -157,15 +165,9 @@ class MysqlExtended extends Mysql
public function insertMulti($table, $fields, $values)
{
$table = $this->fullTableName($table);
$holder = implode(',', array_fill(0, count($fields), '?'));
$holder = substr(str_repeat('?,', count($fields)), 0, -1);
$fields = implode(',', array_map([$this, 'name'], $fields));
$pdoMap = [
'integer' => PDO::PARAM_INT,
'float' => PDO::PARAM_STR,
'boolean' => PDO::PARAM_BOOL,
'string' => PDO::PARAM_STR,
'text' => PDO::PARAM_STR
];
$columnMap = [];
foreach ($values[key($values)] as $key => $val) {
if (is_int($val)) {
@ -174,21 +176,21 @@ class MysqlExtended extends Mysql
$columnMap[$key] = PDO::PARAM_BOOL;
} else {
$type = $this->introspectType($val);
$columnMap[$key] = $pdoMap[$type];
$columnMap[$key] = self::PDO_MAP[$type];
}
}
$sql = "INSERT INTO $table ($fields) VALUES ";
$sql .= implode(',', array_fill(0, count($values), "($holder)"));
$sql .= substr(str_repeat("($holder),", count($values)), 0, -1);
$statement = $this->_connection->prepare($sql);
$valuesList = array();
$i = 1;
$i = 0;
foreach ($values as $value) {
foreach ($value as $col => $val) {
if ($this->fullDebug) {
$valuesList[] = $val;
}
$statement->bindValue($i++, $val, $columnMap[$col]);
$statement->bindValue(++$i, $val, $columnMap[$col]);
}
}
$result = $statement->execute();

16
app/Model/Event.php
View File

@ -3743,7 +3743,10 @@ class Event extends AppModel
unset($this->Attribute->validate['value']['uniqueValue']); // unset this - we are saving a new event, there are no values to compare against and event_id is not set in the attributes
}
unset($data['Event']['id']);
if (isset($data['Event']['published']) && $data['Event']['published'] && $user['Role']['perm_publish'] == 0) {
if (
(Configure::read('MISP.block_publishing_for_same_creator', false) && !$user['Role']['perm_sync']) ||
(isset($data['Event']['published']) && $data['Event']['published'] && $user['Role']['perm_publish'] == 0)
) {
$data['Event']['published'] = 0;
}
if (isset($data['Event']['uuid'])) {
@ -4059,7 +4062,10 @@ class Event extends AppModel
} else {
return array('error' => 'Event could not be saved: Could not find the local event.');
}
if (!empty($data['Event']['published']) && !$user['Role']['perm_publish']) {
if (
(Configure::read('MISP.block_publishing_for_same_creator', false) && !$user['Role']['perm_sync'] && $user['id'] == $existingEvent['Event']['user_id']) ||
(!empty($data['Event']['published']) && !$user['Role']['perm_publish'])
) {
$data['Event']['published'] = 0;
}
if (!isset($data['Event']['published'])) {
@ -4190,7 +4196,7 @@ class Event extends AppModel
if ((true != Configure::read('MISP.disablerestalert')) && (empty($server) || empty($server['Server']['publish_without_email']))) {
$this->sendAlertEmailRouter($id, $user, $existingEvent['Event']['publish_timestamp']);
}
$this->publish($existingEvent['Event']['id']);
$this->publish($existingEvent['Event']['id'], $passAlong);
}
if ($jobId) {
$eventLock->deleteBackgroundJobLock($data['Event']['id'], $jobId);
@ -5952,7 +5958,9 @@ class Event extends AppModel
$this->add_original_file($decoded['original'], $originalFile, $created_id, $stixVersion);
}
if ($publish && $user['Role']['perm_publish']) {
$this->publish($created_id);
if (!Configure::read('MISP.block_publishing_for_same_creator', false) || $user['Role']['perm_sync']) {
$this->publish($created_id);
}
}
return $created_id;
} else if (is_numeric($result)) {

3
app/Model/Feed.php
View File

@ -2062,6 +2062,7 @@ class Feed extends AppModel
$contentType = $response->getHeader('content-type');
if ($contentType === 'application/zip') {
$zipFilePath = FileAccessTool::writeToTempFile($response->body);
unset($response->body); // cleanup variable to reduce memory usage
try {
$response->body = $this->unzipFirstFile($zipFilePath);
@ -2198,7 +2199,7 @@ class Feed extends AppModel
ZipArchive::ER_READ => 'read error',
ZipArchive::ER_SEEK => 'seek error',
];
$message = isset($errorCodes[$result]) ? $errorCodes[$result] : 'error ' . $result;
$message = $errorCodes[$result] ?? 'error ' . $result;
throw new Exception("Remote server returns ZIP file, that cannot be open ($message)");
}

54
app/Model/FuzzyCorrelateSsdeep.php
View File

@ -9,38 +9,32 @@ class FuzzyCorrelateSsdeep extends AppModel
public function ssdeep_prepare($hash)
{
list($block_size, $hash) = explode(':', $hash, 2);
$chars = array();
for ($i = 0; $i < strlen($hash); $i++) {
if (!in_array($hash[$i], $chars, true)) {
$chars[] = $hash[$i];
}
}
list($blockSize, $hash) = explode(':', $hash, 2);
$uniqueChars = array_unique(str_split($hash), SORT_REGULAR);
$search = true;
while ($search) {
$search = false;
foreach ($chars as $c) {
foreach ($uniqueChars as $c) {
if (strpos($hash, $c . $c . $c . $c)) {
$hash = str_replace($c . $c . $c . $c, $c . $c . $c, $hash);
$search = true;
}
}
}
$hash = explode(':', $hash);
$block_data = $hash[0];
$double_block_data = $hash[1];
// (struct.unpack("<Q", base64.b64decode(h[i:i + 7] + "=") + "\x00\x00\x00")[0] for i in range(len(h) - 6)))
$result = array(
$block_size,
$this->get_all_7_char_chunks($block_data),
$this->get_all_7_char_chunks($double_block_data)
);
return $result;
$hash = explode(':', $hash);
list($block_data, $double_block_data) = $hash;
return [
$blockSize,
$this->getAll7CharChunks($block_data),
$this->getAll7CharChunks($double_block_data)
];
}
public function get_all_7_char_chunks($hash)
private function getAll7CharChunks($hash)
{
$results = array();
for ($i = 0; $i < strlen($hash) - 6; $i++) {
@ -56,16 +50,22 @@ class FuzzyCorrelateSsdeep extends AppModel
return $results;
}
/**
* @param string $hash
* @param int $attributeId
* @return array
*/
public function query_ssdeep_chunks($hash, $attributeId)
{
$chunks = $this->ssdeep_prepare($hash);
$bothPartChunks = array_merge($chunks[1], $chunks[2]);
// Original algo from article https://www.virusbulletin.com/virusbulletin/2015/11/optimizing-ssdeep-use-scale
// also propose to insert chunk size to database, but current database schema doesn't contain that column.
// This optimisation can be add in future versions.
$result = $this->find('column', array(
'conditions' => array(
'FuzzyCorrelateSsdeep.chunk' => array_merge($chunks[1], $chunks[2]),
'FuzzyCorrelateSsdeep.chunk' => $bothPartChunks,
),
'fields' => array('FuzzyCorrelateSsdeep.attribute_id'),
'unique' => true,
@ -73,15 +73,11 @@ class FuzzyCorrelateSsdeep extends AppModel
$toSave = [];
$attributeId = (int) $attributeId;
foreach (array(1, 2) as $type) {
foreach ($chunks[$type] as $chunk) {
$toSave[] = [$attributeId, $chunk];
}
}
if (!empty($toSave)) {
$db = $this->getDataSource();
$db->insertMulti($this->table, ['attribute_id', 'chunk'], $toSave);
foreach ($bothPartChunks as $chunk) {
$toSave[] = [$attributeId, $chunk];
}
$db = $this->getDataSource();
$db->insertMulti($this->table, ['attribute_id', 'chunk'], $toSave);
return $result;
}

50
app/Model/Galaxy.php
View File

@ -264,7 +264,7 @@ class Galaxy extends AppModel
$fields = array('galaxy_cluster_id', 'key', 'value');
$db->insertMulti('galaxy_elements', $fields, $elements);
}
$allRelations = array_merge($allRelations, $relations);
array_push($allRelations, ...$relations);
}
// Save relation as last part when all clusters are created
if (!empty($allRelations)) {
@ -287,24 +287,42 @@ class Galaxy extends AppModel
if (empty($galaxy['uuid'])) {
return false;
}
$existingGalaxy = $this->find('first', array(
$existingGalaxy = $this->find('first', [
'recursive' => -1,
'conditions' => array('Galaxy.uuid' => $galaxy['uuid'])
));
if (empty($existingGalaxy)) {
if ($user['Role']['perm_site_admin'] || $user['Role']['perm_galaxy_editor']) {
$this->create();
unset($galaxy['id']);
$this->save($galaxy);
$existingGalaxy = $this->find('first', array(
'recursive' => -1,
'conditions' => array('Galaxy.id' => $this->id)
));
} else {
return false;
'conditions' => ['Galaxy.uuid' => $galaxy['uuid']],
]);
unset($galaxy['id']);
if (!empty($existingGalaxy)) {
// check if provided galaxy has the same fields as galaxy that are saved in database
$fieldsToSave = [];
foreach (array_keys(array_intersect_key($existingGalaxy, $galaxy)) as $key) {
if ($existingGalaxy['Galaxy'][$key] != $galaxy[$key]) {
$fieldsToSave[$key] = $galaxy[$key];
}
}
} else {
$fieldsToSave = $galaxy;
}
return $existingGalaxy;
if (empty($fieldsToSave) && !empty($existingGalaxy)) {
return $existingGalaxy; // galaxy already exists and galaxy fields are the same
}
if (!$user['Role']['perm_site_admin'] && !$user['Role']['perm_galaxy_editor']) {
return false; // user has no permission to modify galaxy
}
if (empty($existingGalaxy)) {
$this->create();
}
$this->save($fieldsToSave);
return $this->find('first', [
'recursive' => -1,
'conditions' => ['Galaxy.id' => $this->id],
]);
}
/**

31
app/Model/GalaxyElement.php
View File

@ -48,37 +48,6 @@ class GalaxyElement extends AppModel
$this->saveMany($tempElements);
}
public function update($galaxy_id, $oldClusters, $newClusters)
{
$elementsToSave = array();
// Since we are dealing with flat files as the end all be all content, we are safe to just drop all of the old clusters and recreate them.
foreach ($oldClusters as $oldCluster) {
$this->deleteAll(array('GalaxyElement.galaxy_cluster_id' => $oldCluster['GalaxyCluster']['id']));
}
foreach ($newClusters as $newCluster) {
$tempCluster = array();
foreach ($newCluster as $key => $value) {
// Don't store the reserved fields as elements
if ($key == 'description' || $key == 'value') {
continue;
}
if (is_array($value)) {
foreach ($value as $arrayElement) {
$tempCluster[] = array('key' => $key, 'value' => $arrayElement);
}
} else {
$tempCluster[] = array('key' => $key, 'value' => $value);
}
}
foreach ($tempCluster as $key => $value) {
$tempCluster[$key]['galaxy_cluster_id'] = $oldCluster['GalaxyCluster']['id'];
}
$elementsToSave = array_merge($elementsToSave, $tempCluster);
}
$this->saveMany($elementsToSave);
}
public function captureElements($user, $elements, $clusterId)
{
$tempElements = array();

10
app/Model/MispObject.php
View File

@ -153,6 +153,8 @@ class MispObject extends AppModel
'object_name' => array('function' => 'set_filter_object_name'),
'object_template_uuid' => array('function' => 'set_filter_object_template_uuid'),
'object_template_version' => array('function' => 'set_filter_object_template_version'),
'first_seen' => array('function' => 'set_filter_seen'),
'last_seen' => array('function' => 'set_filter_seen'),
'deleted' => array('function' => 'set_filter_deleted')
),
'Event' => array(
@ -181,8 +183,8 @@ class MispObject extends AppModel
'deleted' => array('function' => 'set_filter_deleted'),
'timestamp' => array('function' => 'set_filter_timestamp'),
'attribute_timestamp' => array('function' => 'set_filter_timestamp'),
'first_seen' => array('function' => 'set_filter_seen'),
'last_seen' => array('function' => 'set_filter_seen'),
//'first_seen' => array('function' => 'set_filter_seen'),
//'last_seen' => array('function' => 'set_filter_seen'),
'to_ids' => array('function' => 'set_filter_to_ids'),
'comment' => array('function' => 'set_filter_comment')
)
@ -1678,7 +1680,9 @@ class MispObject extends AppModel
$results = $this->Sightingdb->attachToObjects($results, $user);
}
$params['page'] += 1;
$results = $this->Allowedlist->removeAllowedlistedFromArray($results, true);
foreach ($results as $k => $result) {
$results[$k]['Attribute'] = $this->Allowedlist->removeAllowedlistedFromArray($result['Attribute'], true);
}
$results = array_values($results);
$i = 0;
foreach ($results as $object) {

48
app/Model/Module.php
View File

@ -50,6 +50,8 @@ class Module extends AppModel
)
);
private $httpSocket = [];
public function validateIPField($value)
{
if (!filter_var($value, FILTER_VALIDATE_IP) === false) {
@ -309,16 +311,9 @@ class Module extends AppModel
if (!$serverUrl) {
throw new Exception("Module type $moduleFamily is not enabled.");
}
App::uses('HttpSocketExtended', 'Tools');
$httpSocketSetting = ['timeout' => $timeout];
$sslSettings = array('ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_verify_peer', 'ssl_cafile');
foreach ($sslSettings as $sslSetting) {
$value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
if ($value && $value !== '') {
$httpSocketSetting[$sslSetting] = $value;
}
}
$httpSocket = new HttpSocketExtended($httpSocketSetting);
$httpSocket = $this->initHttpSocket($moduleFamily, $timeout);
$request = [];
if ($moduleFamily === 'Cortex') {
if (!empty(Configure::read('Plugin.' . $moduleFamily . '_authkey'))) {
@ -422,4 +417,37 @@ class Module extends AppModel
return false;
}
/**
* @param string $moduleFamily
* @param int $timeout
* @return HttpSocketExtended|CurlClient
*/
private function initHttpSocket($moduleFamily, $timeout)
{
$unique = "$moduleFamily:$timeout";
if (isset($this->httpSocket[$unique])) {
return $this->httpSocket[$unique];
}
$httpSocketSetting = ['timeout' => $timeout];
$sslSettings = ['ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_cafile'];
foreach ($sslSettings as $sslSetting) {
$value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
if ($value && $value !== '') {
$httpSocketSetting[$sslSetting] = $value;
}
}
if (function_exists('curl_init')) {
App::uses('CurlClient', 'Tools');
$httpSocket = new CurlClient($httpSocketSetting);
} else {
App::uses('HttpSocketExtended', 'Tools');
$httpSocket = new HttpSocketExtended($httpSocketSetting);
}
return $this->httpSocket[$unique] = $httpSocket;
}
}

17
app/Model/Organisation.php
View File

@ -76,14 +76,28 @@ class Organisation extends AppModel
);
const ORGANISATION_ASSOCIATIONS = array(
'AccessLog' => array('table' => 'access_logs', 'fields' => array('org_id')),
'AuditLog' => array('table' => 'audit_logs', 'fields' => array('org_id')),
'Correlation' => array('table' => 'correlations', 'fields' => array('org_id')),
'Cerebrate' => array('table' => 'cerebrates', 'fields' => array('org_id')),
'Dashboard' => array('table' => 'dashboards', 'fields' => array('restrict_to_org_id')),
'Event' => array('table' => 'events', 'fields' => array('org_id', 'orgc_id')),
'EventGraph' => array('table' => 'event_graph', 'fields' => array('org_id')),
'Feed' => array('table' => 'feeds', 'fields' => array('orgc_id')),
'GalaxyCluster' => array('table' => 'galaxy_clusters', 'fields' => array('org_id', 'orgc_id')),
'ObjectTemplate' => array('table' => 'object_templates', 'fields' => array('org_id')),
'Job' => array('table' => 'jobs', 'fields' => array('org_id')),
'RestClientHistory' => array('table' => 'rest_client_histories', 'fields' => array('org_id')),
'Server' => array('table' => 'servers', 'fields' => array('org_id', 'remote_org_id')),
'ShadowAttribute' => array('table' => 'shadow_attributes', 'fields' => array('org_id', 'event_org_id')),
'SharingGroup' => array('table' => 'sharing_groups', 'fields' => array('org_id')),
'SharingGroupOrg' => array('table' => 'sharing_group_orgs', 'fields' => array('org_id')),
'SharingGroupBlueprint' => array('table' => 'sharing_group_blueprints', 'fields' => array('org_id')),
'Sighting' => array('table' => 'sightings', 'fields' => array('org_id')),
'SightingdbOrg' => array('table' => 'sightingdb_orgs', 'fields' => array('org_id')),
'Thread' => array('table' => 'threads', 'fields' => array('org_id')),
'Tag' => array('table' => 'tags', 'fields' => array('org_id')),
'TagCollection' => array('table' => 'tag_collections', 'fields' => array('org_id')),
'User' => array('table' => 'users', 'fields' => array('org_id'))
);
@ -287,6 +301,9 @@ class Organisation extends AppModel
public function orgMerge($id, $request, $user)
{
$currentOrg = $this->find('first', array('recursive' => -1, 'conditions' => array('Organisation.id' => $id)));
if (isset($currentOrg['Organisation']['restricted_to_domain'])) {
$currentOrg['Organisation']['restricted_to_domain'] = json_encode($currentOrg['Organisation']['restricted_to_domain']);
}
$currentOrgUserCount = $this->User->find('count', array(
'conditions' => array('User.org_id' => $id)
));

148
app/Model/Server.php
View File

@ -472,7 +472,21 @@ class Server extends AppModel
return false;
}
private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, Event $eventModel, $server, $user, $jobId, $force = false, $headers = false, $body = false)
/**
* @param array $event
* @param int|string $eventId
* @param array $successes
* @param array $fails
* @param Event $eventModel
* @param array $server
* @param array $user
* @param int $jobId
* @param bool $force
* @param HttpSocketResponseExtended $response
* @return false|void
* @throws Exception
*/
private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, Event $eventModel, $server, $user, $jobId, $force = false, $response)
{
// check if the event already exist (using the uuid)
$existingEvent = $eventModel->find('first', [
@ -485,7 +499,7 @@ class Server extends AppModel
if (!$existingEvent) {
// add data for newly imported events
if (isset($event['Event']['protected']) && $event['Event']['protected']) {
if (!$eventModel->CryptographicKey->validateProtectedEvent($body, $user, $headers['x-pgp-signature'], $event)) {
if (!$eventModel->CryptographicKey->validateProtectedEvent($response->body, $user, $response->getHeader('x-pgp-signature'), $event)) {
$fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.');
return false;
}
@ -505,7 +519,7 @@ class Server extends AppModel
$fails[$eventId] = __('Blocked an edit to an event that was created locally. This can happen if a synchronised event that was created on this instance was modified by an administrator on the remote side.');
} else {
if ($existingEvent['Event']['protected']) {
if (!$eventModel->CryptographicKey->validateProtectedEvent($body, $user, $headers['x-pgp-signature'], $existingEvent)) {
if (!$eventModel->CryptographicKey->validateProtectedEvent($response->body, $user, $response->getHeader('x-pgp-signature'), $existingEvent)) {
$fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.');
}
}
@ -549,12 +563,10 @@ class Server extends AppModel
$params['excludeLocalTags'] = 1;
}
try {
$event = $serverSync->fetchEvent($eventId, $params);
$headers = $event->headers;
$body = $event->body;
$event = $event->json();
$response = $serverSync->fetchEvent($eventId, $params);
$event = $response->json();
} catch (Exception $e) {
$this->logException("Failed downloading the event $eventId from remote server {$serverSync->serverId()}", $e);
$this->logException("Failed to download the event $eventId from remote server {$serverSync->serverId()} '{$serverSync->serverName()}'", $e);
$fails[$eventId] = __('failed downloading the event');
return false;
}
@ -568,7 +580,7 @@ class Server extends AppModel
}
return false;
}
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $headers, $body);
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $response);
return true;
}
@ -2359,23 +2371,21 @@ class Server extends AppModel
return $setting;
}
public function serverSettingsEditValue(array $user, array $setting, $value, $forceSave = false)
/**
* @param array|string $user
* @param array $setting
* @param mixed $value
* @param bool $forceSave
* @return mixed|string|true|null
* @throws Exception
*/
public function serverSettingsEditValue($user, array $setting, $value, $forceSave = false)
{
if (isset($setting['beforeHook'])) {
$beforeResult = call_user_func_array(array($this, $setting['beforeHook']), array($setting['name'], $value));
$beforeResult = $this->{$setting['beforeHook']}($setting['name'], $value);
if ($beforeResult !== true) {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->saveOrFailSilently(array(
'org' => $user['Organisation']['name'],
'model' => 'Server',
'model_id' => 0,
'email' => $user['email'],
'action' => 'serverSettingsEdit',
'user_id' => $user['id'],
'title' => 'Server setting issue',
'change' => 'There was an issue witch changing ' . $setting['name'] . ' to ' . $value . '. The error message returned is: ' . $beforeResult . 'No changes were made.',
));
$change = 'There was an issue witch changing ' . $setting['name'] . ' to ' . $value . '. The error message returned is: ' . $beforeResult . 'No changes were made.';
$this->loadLog()->createLogEntry($user, 'serverSettingsEdit', 'Server', 0, 'Server setting issue', $change);
return $beforeResult;
}
}
@ -2384,7 +2394,7 @@ class Server extends AppModel
if ($setting['type'] === 'boolean') {
$value = (bool)$value;
} else if ($setting['type'] === 'numeric') {
$value = (int)($value);
$value = (int)$value;
}
if (isset($setting['test'])) {
if ($setting['test'] instanceof Closure) {
@ -2425,7 +2435,7 @@ class Server extends AppModel
if ($setting['afterHook'] instanceof Closure) {
$afterResult = $setting['afterHook']($setting['name'], $value, $oldValue);
} else {
$afterResult = call_user_func_array(array($this, $setting['afterHook']), array($setting['name'], $value, $oldValue));
$afterResult = $this->{$setting['afterHook']}($setting['name'], $value, $oldValue);
}
if ($afterResult !== true) {
$change = 'There was an issue after setting a new setting. The error message returned is: ' . $afterResult;
@ -2434,9 +2444,8 @@ class Server extends AppModel
}
}
return true;
} else {
return __('Something went wrong. MISP tried to save a malformed config file. Setting change reverted.');
}
return __('Something went wrong. MISP tried to save a malformed config file or you dont have permission to write to config file. Setting change reverted.');
}
/**
@ -2539,10 +2548,10 @@ class Server extends AppModel
'name' => __('Organisation logos'),
'description' => __('The logo used by an organisation on the event index, event view, discussions, proposals, etc. Make sure that the filename is in the org.png format, where org is the case-sensitive organisation name.'),
'expected' => array(),
'valid_format' => __('48x48 pixel .png files'),
'valid_format' => __('48x48 pixel .png files or .svg file'),
'path' => APP . 'webroot' . DS . 'img' . DS . 'orgs',
'regex' => '.*\.(png|PNG)$',
'regex_error' => __('Filename must be in the following format: *.png'),
'regex' => '.*\.(png|svg)$',
'regex_error' => __('Filename must be in the following format: *.png or *.svg'),
'files' => array(),
),
'img' => array(
@ -2578,6 +2587,7 @@ class Server extends AppModel
'read' => $f->isReadable(),
'write' => $f->isWritable(),
'execute' => $f->isExecutable(),
'link' => $f->isLink(),
];
}
}
@ -4155,12 +4165,13 @@ class Server extends AppModel
private function checkRemoteVersion($HttpSocket)
{
try {
$json_decoded_tags = GitTool::getLatestTags($HttpSocket);
$tags = GitTool::getLatestTags($HttpSocket);
} catch (Exception $e) {
$this->logException('Could not retrieve latest tags from GitHub', $e, LOG_NOTICE);
return false;
}
// find the latest version tag in the v[major].[minor].[hotfix] format
foreach ($json_decoded_tags as $tag) {
foreach ($tags as $tag) {
if (preg_match('/^v[0-9]+\.[0-9]+\.[0-9]+$/', $tag['name'])) {
return $this->checkVersion($tag['name']);
}
@ -4182,7 +4193,7 @@ class Server extends AppModel
try {
$latestCommit = GitTool::getLatestCommit($HttpSocket);
} catch (Exception $e) {
$latestCommit = false;
$this->logException('Could not retrieve version from GitHub', $e, LOG_NOTICE);
}
}
@ -4202,6 +4213,7 @@ class Server extends AppModel
try {
return GitTool::currentBranch();
} catch (Exception $e) {
$this->logException('Could not retrieve current Git branch', $e, LOG_NOTICE);
return false;
}
}
@ -4252,38 +4264,38 @@ class Server extends AppModel
'app/files/scripts/misp-opendata',
'app/files/scripts/python-maec',
'app/files/scripts/python-stix',
);
return in_array($submodule, $accepted_submodules_names, true);
}
/**
* @param string $submodule_name
* @param string $superproject_submodule_commit_id
* @param string $submoduleName
* @param string $superprojectSubmoduleCommitId
* @return array
* @throws Exception
*/
private function getSubmoduleGitStatus($submodule_name, $superproject_submodule_commit_id)
private function getSubmoduleGitStatus($submoduleName, $superprojectSubmoduleCommitId)
{
$path = APP . '../' . $submodule_name;
$submodule_name = (strpos($submodule_name, '/') >= 0 ? explode('/', $submodule_name) : $submodule_name);
$submodule_name = end($submodule_name);
$path = APP . '../' . $submoduleName;
$submoduleName = (strpos($submoduleName, '/') >= 0 ? explode('/', $submoduleName) : $submoduleName);
$submoduleName = end($submoduleName);
$submoduleCurrentCommitId = GitTool::submoduleCurrentCommit($path);
$submoduleCurrentCommitId = GitTool::currentCommit($path);
$currentTimestamp = GitTool::commitTimestamp($submoduleCurrentCommitId, $path);
if ($submoduleCurrentCommitId !== $superproject_submodule_commit_id) {
$remoteTimestamp = GitTool::commitTimestamp($superproject_submodule_commit_id, $path);
if ($submoduleCurrentCommitId !== $superprojectSubmoduleCommitId) {
$remoteTimestamp = GitTool::commitTimestamp($superprojectSubmoduleCommitId, $path);
} else {
$remoteTimestamp = $currentTimestamp;
}
$status = array(
'moduleName' => $submodule_name,
'moduleName' => $submoduleName,
'current' => $submoduleCurrentCommitId,
'currentTimestamp' => $currentTimestamp,
'remote' => $superproject_submodule_commit_id,
'remote' => $superprojectSubmoduleCommitId,
'remoteTimestamp' => $remoteTimestamp,
'upToDate' => '',
'upToDate' => 'error',
'isReadable' => is_readable($path) && is_readable($path . '/.git'),
);
@ -4295,15 +4307,11 @@ class Server extends AppModel
} else {
$status['upToDate'] = 'younger';
}
} else {
$status['upToDate'] = 'error';
}
if ($status['isReadable'] && !empty($status['remoteTimestamp']) && !empty($status['currentTimestamp'])) {
$date1 = new DateTime();
$date1->setTimestamp($status['remoteTimestamp']);
$date2 = new DateTime();
$date2->setTimestamp($status['currentTimestamp']);
$date1 = new DateTime("@{$status['remoteTimestamp']}");
$date2 = new DateTime("@{$status['currentTimestamp']}");
$status['timeDiff'] = $date1->diff($date2);
} else {
$status['upToDate'] = 'error';
@ -4793,11 +4801,11 @@ class Server extends AppModel
$results = [
__('User') => $user['User']['email'],
__('Role name') => isset($user['Role']['name']) ? $user['Role']['name'] : __('Unknown, outdated instance'),
__('Role name') => $user['Role']['name'] ?? __('Unknown, outdated instance'),
__('Sync flag') => isset($user['Role']['perm_sync']) ? ($user['Role']['perm_sync'] ? __('Yes') : __('No')) : __('Unknown, outdated instance'),
];
if (isset($response->headers['X-Auth-Key-Expiration'])) {
$date = new DateTime($response->headers['X-Auth-Key-Expiration']);
if ($response->getHeader('X-Auth-Key-Expiration')) {
$date = new DateTime($response->getHeader('X-Auth-Key-Expiration'));
$results[__('Auth key expiration')] = $date->format('Y-m-d H:i:s');
}
return $results;
@ -4935,6 +4943,28 @@ class Server extends AppModel
return $this->saveMany($toSave, ['validate' => false, 'fields' => ['authkey']]);
}
/**
* @param string $encryptionKey
* @return bool
* @throws Exception
*/
public function isEncryptionKeyValid($encryptionKey)
{
$servers = $this->find('list', [
'fields' => ['Server.id', 'Server.authkey'],
]);
foreach ($servers as $id => $authkey) {
if (EncryptedValue::isEncrypted($authkey)) {
try {
BetterSecurity::decrypt(substr($authkey, 2), $encryptionKey);
} catch (Exception $e) {
throw new Exception("Could not decrypt auth key for server #$id", 0, $e);
}
}
}
return true;
}
/**
* Return all Attribute and Object types
*/
@ -6143,6 +6173,14 @@ class Server extends AppModel
'type' => 'boolean',
'null' => true,
],
'block_publishing_for_same_creator' => [
'level' => self::SETTING_OPTIONAL,
'description' => __('Enabling this setting will make MISP block event publishing in the case of the publisher being the same user as the event creator.'),
'value' => false,
'test' => 'testBool',
'type' => 'boolean',
'null' => true,
],
'self_update' => [
'level' => self::SETTING_CRITICAL,
'description' => __('Enable the GUI button for MISP self-update on the Diagnostics page.'),

8
app/Model/Sighting.php
View File

@ -1017,16 +1017,16 @@ class Sighting extends AppModel
* @return TmpFileTool
* @throws Exception
*/
public function restSearch(array $user, $returnFormat, $filters)
public function restSearch(array $user, $returnFormat, array $filters)
{
$allowedContext = array('event', 'attribute');
// validate context
if (isset($filters['context']) && !in_array($filters['context'], $allowedContext, true)) {
throw new MethodNotAllowedException(__('Invalid context %s.', $filters['context']));
throw new BadRequestException(__('Invalid context %s.', $filters['context']));
}
// ensure that an id or uuid is provided if context is set
if (!empty($filters['context']) && !(isset($filters['id']) || isset($filters['uuid'])) ) {
throw new MethodNotAllowedException(__('An ID or UUID must be provided if the context is set.'));
throw new BadRequestException(__('An ID or UUID must be provided if the context is set.'));
}
if (!isset($this->validFormats[$returnFormat][1])) {
@ -1396,7 +1396,7 @@ class Sighting extends AppModel
try {
$sightings = $serverSync->fetchSightingsForEvents($chunk);
} catch (Exception $e) {
$this->logException("Failed downloading the sightings from {$serverSync->server()['Server']['name']}.", $e);
$this->logException("Failed to download sightings from {$serverSync->server()['Server']['name']}.", $e);
continue;
}

30
app/Model/SystemSetting.php
View File

@ -46,7 +46,7 @@ class SystemSetting extends AppModel
{
/** @var self $systemSetting */
$systemSetting = ClassRegistry::init('SystemSetting');
if (!$systemSetting->databaseExists()) {
if (!$systemSetting->tableExists()) {
return;
}
$settings = $systemSetting->getSettings();
@ -58,7 +58,7 @@ class SystemSetting extends AppModel
}
}
public function databaseExists()
private function tableExists()
{
$tables = ConnectionManager::getDataSource($this->useDbConfig)->listSources();
return in_array('system_settings', $tables, true);
@ -154,6 +154,32 @@ class SystemSetting extends AppModel
return $this->saveMany($toSave);
}
/**
* Check if provided encryption key is valid for all encrypted settings
* @param string $encryptionKey
* @return bool
* @throws Exception
*/
public function isEncryptionKeyValid($encryptionKey)
{
$settings = $this->find('list', [
'fields' => ['SystemSetting.setting', 'SystemSetting.value'],
]);
foreach ($settings as $setting => $value) {
if (!self::isSensitive($setting)) {
continue;
}
if (EncryptedValue::isEncrypted($value)) {
try {
BetterSecurity::decrypt(substr($value, 2), $encryptionKey);
} catch (Exception $e) {
throw new Exception("Could not decrypt `$setting` setting.", 0, $e);
}
}
}
return true;
}
/**
* Sensitive setting are passwords or api keys.
* @param string $setting Setting name

126
app/Model/User.php
View File

@ -659,21 +659,18 @@ class User extends AppModel
public function getUserById($id)
{
if (empty($id)) {
throw new NotFoundException('Invalid user ID.');
throw new InvalidArgumentException('Invalid user ID.');
}
return $this->find(
'first',
array(
'conditions' => array('User.id' => $id),
'recursive' => -1,
'contain' => array(
'Organisation',
'Role',
'Server',
'UserSetting',
)
)
);
return $this->find('first', [
'conditions' => ['User.id' => $id],
'recursive' => -1,
'contain' => [
'Organisation',
'Role',
'Server',
'UserSetting',
]
]);
}
/**
@ -740,7 +737,7 @@ class User extends AppModel
],
]);
if (empty($user)) {
return $user;
return null;
}
return $this->rearrangeToAuthForm($user);
}
@ -861,6 +858,10 @@ class User extends AppModel
return true;
}
if (!isset($user['User'])) {
throw new InvalidArgumentException("Invalid user model provided.");
}
if ($user['User']['disabled'] || !$this->checkIfUserIsValid($user['User'])) {
return true;
}
@ -937,6 +938,11 @@ class User extends AppModel
*/
public function describeAuthFields()
{
static $fields; // generate array just once
if ($fields) {
return $fields;
}
$fields = $this->schema();
// Do not include keys, because they are big and usually not necessary
unset($fields['gpgkey']);
@ -1105,13 +1111,18 @@ class User extends AppModel
return $hashed;
}
public function createInitialUser($org_id)
/**
* @param int $orgId
* @return string User auth key
* @throws Exception
*/
public function createInitialUser($orgId)
{
$authKey = $this->generateAuthKey();
$admin = array('User' => array(
'id' => 1,
'email' => 'admin@admin.test',
'org_id' => $org_id,
'org_id' => $orgId,
'password' => 'admin',
'confirm_password' => 'admin',
'authkey' => $authKey,
@ -1123,7 +1134,6 @@ class User extends AppModel
$this->validator()->remove('password'); // password is too simple, remove validation
$this->save($admin);
if (!empty(Configure::read("Security.advanced_authkeys"))) {
$this->AuthKey = ClassRegistry::init('AuthKey');
$newKey = [
'authkey' => $authKey,
'user_id' => 1,
@ -2068,12 +2078,10 @@ class User extends AppModel
return false;
}
$cutoff = $redis->get('misp:session_destroy:' . $id);
$allcutoff = $redis->get('misp:session_destroy:all');
list($cutoff, $allcutoff) = $redis->mGet(['misp:session_destroy:' . $id, 'misp:session_destroy:all']);
if (
empty($cutoff) ||
(
!empty($cutoff) &&
!empty($allcutoff) &&
$allcutoff < $cutoff
)
@ -2156,7 +2164,7 @@ class User extends AppModel
if (!ctype_alnum($token)) {
return false;
}
$redis = $this->setupRedis();
$redis = RedisTool::init();
$userId = $redis->get('misp:forgot:' . $token);
if (empty($userId)) {
return false;
@ -2167,8 +2175,78 @@ class User extends AppModel
public function purgeForgetToken($token)
{
$redis = $this->setupRedis();
$userId = $redis->del('misp:forgot:' . $token);
$redis = RedisTool::init();
$redis->del('misp:forgot:' . $token);
return true;
}
/**
* Create default Role, Organisation and User
* @return string|null Created user auth key
* @throws Exception
*/
public function init()
{
if (!$this->Role->hasAny()) {
$siteAdmin = ['Role' => [
'id' => 1,
'name' => 'Site Admin',
'permission' => 3,
'perm_add' => 1,
'perm_modify' => 1,
'perm_modify_org' => 1,
'perm_publish' => 1,
'perm_sync' => 1,
'perm_admin' => 1,
'perm_audit' => 1,
'perm_auth' => 1,
'perm_site_admin' => 1,
'perm_regexp_access' => 1,
'perm_sharing_group' => 1,
'perm_template' => 1,
'perm_tagger' => 1,
]];
$this->Role->save($siteAdmin);
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
if (!$this->isMysql()) {
$sql = "SELECT setval('roles_id_seq', (SELECT MAX(id) FROM roles));";
$this->Role->query($sql);
}
}
if (!$this->Organisation->hasAny(['Organisation.local' => true])) {
$this->runUpdates();
$org = ['Organisation' => [
'id' => 1,
'name' => !empty(Configure::read('MISP.org')) ? Configure::read('MISP.org') : 'ADMIN',
'description' => 'Automatically generated admin organisation',
'type' => 'ADMIN',
'date_created' => date('Y-m-d H:i:s'),
'local' => 1,
]];
$this->Organisation->save($org);
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
if (!$this->isMysql()) {
$sql = "SELECT setval('organisations_id_seq', (SELECT MAX(id) FROM organisations));";
$this->Organisation->query($sql);
}
$orgId = $this->Organisation->id;
}
if (!$this->hasAny()) {
if (!isset($orgId)) {
$hostOrg = $this->Organisation->find('first', array('conditions' => array('Organisation.name' => Configure::read('MISP.org'), 'Organisation.local' => true), 'recursive' => -1));
if (!empty($hostOrg)) {
$orgId = $hostOrg['Organisation']['id'];
} else {
$firstOrg = $this->Organisation->find('first', array('conditions' => array('Organisation.local' => true), 'order' => 'Organisation.id ASC'));
$orgId = $firstOrg['Organisation']['id'];
}
}
$this->runUpdates();
return $this->createInitialUser($orgId);
}
return null;
}
}

79
app/Model/UserLoginProfile.php
View File

@ -36,22 +36,55 @@ class UserLoginProfile extends AppModel
];
const BROWSER_CACHE_DIR = APP . DS . 'tmp' . DS . 'browscap';
const BROWSER_INI_FILE = APP . DS . 'files' . DS . 'browscap'. DS . 'browscap.ini'; // Browscap file managed by MISP - https://browscap.org/stream?q=Lite_PHP_BrowsCapINI
const BROWSER_INI_FILE = APP . DS . 'files' . DS . 'browscap'. DS . 'browscap.ini.gz'; // Browscap file managed by MISP - https://browscap.org/stream?q=Lite_PHP_BrowsCapINI
const GEOIP_DB_FILE = APP . DS . 'files' . DS . 'geo-open' . DS . 'GeoOpen-Country.mmdb'; // GeoIP file managed by MISP - https://data.public.lu/en/datasets/geo-open-ip-address-geolocation-per-country-in-mmdb-format/
private $userProfile;
private $knownUserProfiles = [];
private function _buildBrowscapCache()
private function browscapGetBrowser()
{
$this->log("Browscap - building new cache from browscap.ini file.", LOG_INFO);
$fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR);
$cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache);
$logger = new \Monolog\Logger('name');
$bc = new \BrowscapPHP\BrowscapUpdater($cache, $logger);
$bc->convertFile(UserLoginProfile::BROWSER_INI_FILE);
if (function_exists('apcu_fetch')) {
App::uses('ApcuCacheTool', 'Tools');
$cache = new ApcuCacheTool('misp:browscap');
} else {
$fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR);
$cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache);
}
try {
$bc = new \BrowscapPHP\Browscap($cache, $logger);
return $bc->getBrowser();
} catch (\BrowscapPHP\Exception $e) {
$this->log("Browscap - building new cache from browscap.ini file.", LOG_INFO);
$bcUpdater = new \BrowscapPHP\BrowscapUpdater($cache, $logger);
$bcUpdater->convertString(FileAccessTool::readCompressedFile(UserLoginProfile::BROWSER_INI_FILE));
}
$bc = new \BrowscapPHP\Browscap($cache, $logger);
return $bc->getBrowser();
}
/**
* @param string $ip
* @return string|null
*/
public function countryByIp($ip)
{
if (class_exists('GeoIp2\Database\Reader')) {
$geoDbReader = new GeoIp2\Database\Reader(UserLoginProfile::GEOIP_DB_FILE);
try {
$record = $geoDbReader->country($ip);
return $record->country->isoCode;
} catch (InvalidArgumentException $e) {
$this->logException("Could not get country code for IP address", $e, LOG_NOTICE);
return null;
}
}
return null;
}
public function beforeSave($options = [])
@ -76,16 +109,7 @@ class UserLoginProfile extends AppModel
if (!$this->userProfile) {
// below uses https://github.com/browscap/browscap-php
if (class_exists('\BrowscapPHP\Browscap')) {
try {
$fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR);
$cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache);
$logger = new \Monolog\Logger('name');
$bc = new \BrowscapPHP\Browscap($cache, $logger);
$browser = $bc->getBrowser();
} catch (\BrowscapPHP\Exception $e) {
$this->_buildBrowscapCache();
return $this->_getUserProfile();
}
$browser = $this->browscapGetBrowser();
} else {
// a primitive OS & browser extraction capability
$ua = $_SERVER['HTTP_USER_AGENT'] ?? null;
@ -100,18 +124,7 @@ class UserLoginProfile extends AppModel
$browser->browser = "browser";
}
$ip = $this->_remoteIp();
if (class_exists('GeoIp2\Database\Reader')) {
try {
$geoDbReader = new GeoIp2\Database\Reader(UserLoginProfile::GEOIP_DB_FILE);
$record = $geoDbReader->country($ip);
$country = $record->country->isoCode;
} catch (InvalidArgumentException $e) {
$this->logException("Could not get country code for IP address", $e);
$country = 'None';
}
} else {
$country = 'None';
}
$country = $this->countryByIp($ip) ?? 'None';
$this->userProfile = [
'user_agent' => $_SERVER['HTTP_USER_AGENT'] ?? null,
'ip' => $ip,
@ -247,13 +260,13 @@ class UserLoginProfile extends AppModel
public function emailNewLogin(array $user)
{
if (!Configure::read('MISP.disable_emailing')) {
$date_time = date('c');
$user = $this->User->getUserById($user['id']); // fetch in database format
$datetime = date('c'); // ISO 8601 date
$body = new SendEmailTemplate('userloginprofile_newlogin');
$body->set('userLoginProfile', $this->User->UserLoginProfile->_getUserProfile());
$body->set('baseurl', Configure::read('MISP.baseurl'));
$body->set('misp_org', Configure::read('MISP.org'));
$body->set('date_time', $date_time);
$body->set('date_time', $datetime);
// Fetch user that contains also PGP or S/MIME keys for e-mail encryption
$this->User->sendEmail($user, $body, false, "[" . Configure::read('MISP.org') . " MISP] New sign in.");
}

3
app/Model/Warninglist.php
View File

@ -390,8 +390,7 @@ class Warninglist extends AppModel
$warninglistId = (int)$this->id;
$result = true;
$keys = array_keys($list['list']);
if ($keys === array_keys($keys)) {
if (JsonTool::arrayIsList($list['list'])) {
foreach (array_chunk($list['list'], 1000) as $chunk) {
$valuesToInsert = [];
foreach ($chunk as $value) {

276
app/Plugin/BinaryFileCache/Engine/BinaryFileEngine.php Normal file
View File

@ -0,0 +1,276 @@
<?php
require_once CAKE_CORE_INCLUDE_PATH . '/Cake/Cache/CacheEngine.php';
require_once CAKE_CORE_INCLUDE_PATH . '/Cake/Cache/Engine/FileEngine.php';
/**
* This is faster version of FileEngine cache engine
* - stores file in binary format, so no need to change line endings
* - uses igbinary if supported for faster serialization/deserialization and smaller cache files
* - default file mask is set to 0660, so cache files are not readable by other users
* - optimised file saving and fetching
*/
class BinaryFileEngine extends FileEngine
{
const BINARY_CACHE_TIME_LENGTH = 8;
private $useIgbinary = false;
public function init($settings = [])
{
$settings += [
'engine' => 'BinaryFile',
'path' => CACHE,
'prefix' => 'cake_',
'serialize' => true,
'mask' => 0660,
];
CacheEngine::init($settings);
$this->useIgbinary = function_exists('igbinary_serialize');
if (substr($this->settings['path'], -1) !== DS) {
$this->settings['path'] .= DS;
}
if (!empty($this->_groupPrefix)) {
$this->_groupPrefix = str_replace('_', DS, $this->_groupPrefix);
}
return $this->_active();
}
/**
* @param string $key
* @param mixed $data
* @param int $duration
* @return bool
*/
public function write($key, $data, $duration)
{
if (!$this->_init) {
return false;
}
$fileInfo = $this->cacheFilePath($key);
$resource = $this->createFile($fileInfo);
if (!$resource) {
return false;
}
if (!empty($this->settings['serialize'])) {
if ($this->useIgbinary) {
$data = igbinary_serialize($data);
if ($data === null) {
return false;
}
} else {
$data = serialize($data);
}
}
$expires = pack("q", time() + $duration);
flock($resource, LOCK_EX);
ftruncate($resource, 0);
$result = fwrite($resource, $expires);
if ($result !== self::BINARY_CACHE_TIME_LENGTH) {
$this->handleWriteError($fileInfo);
fclose($resource);
return false;
}
$result = fwrite($resource, $data);
if ($result !== strlen($data)) {
$this->handleWriteError($fileInfo);
fclose($resource);
return false;
}
fclose($resource);
return true;
}
/**
* @param string $key
* @return false|mixed|string
*/
public function read($key)
{
if (!$this->_init) {
return false;
}
$fileInfo = $this->cacheFilePath($key);
$exists = file_exists($fileInfo->getPathname());
if (!$exists) {
return false;
}
$resource = $this->openFile($fileInfo);
if (!$resource) {
return false;
}
$time = time();
flock($resource, LOCK_SH);
$cacheTimeBinary = fread($resource, self::BINARY_CACHE_TIME_LENGTH);
if (!$cacheTimeBinary) {
fclose($resource);
return false;
}
$cacheTime = $this->unpackCacheTime($cacheTimeBinary);
if ($cacheTime < $time || ($time + $this->settings['duration']) < $cacheTime) {
fclose($resource);
return false; // already expired
}
$data = stream_get_contents($resource, null, self::BINARY_CACHE_TIME_LENGTH);
fclose($resource);
if (!empty($this->settings['serialize'])) {
if ($this->useIgbinary) {
$data = igbinary_unserialize($data);
} else {
$data = unserialize($data);
}
}
return $data;
}
/**
* @param string $path
* @param int $now
* @param int $threshold
* @return void
*/
protected function _clearDirectory($path, $now, $threshold)
{
$prefixLength = strlen($this->settings['prefix']);
if (!is_dir($path)) {
return;
}
$dir = dir($path);
if ($dir === false) {
return;
}
while (($entry = $dir->read()) !== false) {
if (substr($entry, 0, $prefixLength) !== $this->settings['prefix']) {
continue;
}
try {
$file = new SplFileObject($path . $entry, 'rb');
} catch (Exception $e) {
continue;
}
if ($threshold) {
$mtime = $file->getMTime();
if ($mtime > $threshold) {
continue;
}
$expires = $this->unpackCacheTime($file->fread(self::BINARY_CACHE_TIME_LENGTH));
if ($expires > $now) {
continue;
}
}
if ($file->isFile()) {
$filePath = $file->getRealPath();
$file = null;
@unlink($filePath);
}
}
}
/**
* @param SplFileInfo $fileInfo
* @return false|resource
*/
private function createFile(SplFileInfo $fileInfo)
{
$exists = file_exists($fileInfo->getPathname());
if (!$exists) {
$resource = $this->openFile($fileInfo, 'cb');
if ($resource && !chmod($fileInfo->getPathname(), (int)$this->settings['mask'])) {
trigger_error(__d(
'cake_dev', 'Could not apply permission mask "%s" on cache file "%s"',
[$fileInfo->getPathname(), $this->settings['mask']]), E_USER_WARNING);
}
return $resource;
}
return $this->openFile($fileInfo, 'cb');
}
/**
* @param SplFileInfo $fileInfo
* @param string $mode
* @return false|resource
*/
private function openFile(SplFileInfo $fileInfo, $mode = 'rb')
{
$resource = fopen($fileInfo->getPathname(), $mode);
if (!$resource) {
trigger_error(__d(
'cake_dev', 'Could not open file %s',
array($fileInfo->getPathname())), E_USER_WARNING);
}
return $resource;
}
/**
* @param string $key
* @return SplFileInfo
*/
private function cacheFilePath(string $key): SplFileInfo
{
$groups = null;
if (!empty($this->_groupPrefix)) {
$groups = vsprintf($this->_groupPrefix, $this->groups());
}
$dir = $this->settings['path'] . $groups;
if (!is_dir($dir)) {
mkdir($dir, 0775, true);
}
$suffix = '.bin';
if ($this->settings['serialize'] && $this->useIgbinary) {
$suffix = '.igbin';
}
return new SplFileInfo($dir . $key . $suffix);
}
/**
* @param SplFileInfo $fileInfo
* @return void
*/
private function handleWriteError(SplFileInfo $fileInfo)
{
unlink($fileInfo->getPathname()); // delete file in case file was just partially written
trigger_error(__d(
'cake_dev', 'Could not write to file %s',
array($fileInfo->getPathname())), E_USER_WARNING);
}
/**
* @param string $cacheTimeBinary
* @return int
*/
private function unpackCacheTime($cacheTimeBinary)
{
if ($cacheTimeBinary === false || strlen($cacheTimeBinary) !== self::BINARY_CACHE_TIME_LENGTH) {
throw new InvalidArgumentException("Invalid cache time in binary format provided '$cacheTimeBinary'");
}
return unpack("q", $cacheTimeBinary)[1];
}
}

2
app/Plugin/EcsLog/Lib/Log/Engine/EcsLog.php
View File

@ -57,7 +57,7 @@ class EcsLog implements CakeLogInterface
'log' => [
'level' => $type,
],
'message' => $message,
'message' => JsonTool::escapeNonUnicode($message),
];
static::writeMessage($message);

5
app/Plugin/OidcAuth/Lib/Oidc.php
View File

@ -28,7 +28,6 @@ class Oidc
$claims = $oidc->getVerifiedClaims();
$mispUsername = $claims->email ?? $oidc->requestUserInfo('email');
if (empty($mispUsername)) {
$sub = $claims->sub ?? 'UNKNOWN';
throw new Exception("OIDC user $sub doesn't have email address, that is required by MISP.");
@ -66,13 +65,13 @@ class Oidc
$roleProperty = $this->getConfig('roles_property', 'roles');
$roles = $claims->{$roleProperty} ?? $oidc->requestUserInfo($roleProperty);
if ($roles === null) {
$this->log($mispUsername, "Role property `$roleProperty` is missing in claims.", LOG_WARNING);
$this->log($mispUsername, "Role property `$roleProperty` is missing in claims, access prohibited.", LOG_WARNING);
return false;
}
$roleId = $this->getUserRole($roles, $mispUsername);
if ($roleId === null) {
$this->log($mispUsername, 'No role was assigned.');
$this->log($mispUsername, 'No role was assigned, access prohibited.', LOG_WARNING);
if ($user) {
$this->block($user);
}

10
app/Test/AttributeValidationToolTest.php
View File

@ -124,6 +124,16 @@ class AttributeValidationToolTest extends TestCase
]);
}
public function testRemoveCidrFromIp(): void
{
$this->assertEquals('127.0.0.1', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1/32'));
$this->assertEquals('127.0.0.1/31', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1/31'));
$this->assertEquals('example.com|1234:fd2:5621:1:89::4500', AttributeValidationTool::modifyBeforeValidation('domain|ip', 'example.com|1234:0fd2:5621:0001:0089:0000:0000:4500/128'));
$this->assertEquals('1234:fd2:5621:1:89::4500|80', AttributeValidationTool::modifyBeforeValidation('ip-src|port', '1234:0fd2:5621:0001:0089:0000:0000:4500/128|80'));
$this->assertEquals('1234:fd2:5621:1:89::4500/127|80', AttributeValidationTool::modifyBeforeValidation('ip-src|port', '1234:0fd2:5621:0001:0089:0000:0000:4500/127|80'));
$this->assertEquals('127.0.0.1', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1'));
}
public function testCompressIpv6(): void
{
$this->assertEquals('1234:fd2:5621:1:89::4500', AttributeValidationTool::modifyBeforeValidation('ip-src', '1234:0fd2:5621:0001:0089:0000:0000:4500'));

21
app/Test/ComplexTypeToolTest.php
View File

@ -527,10 +527,31 @@ EOT;
public function testCheckFreeTextNonBreakableSpace(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText("127.0.0.1\xc2\xa0127.0.0.2");
$this->assertCount(2, $results);
$this->assertEquals('127.0.0.1', $results[0]['value']);
$this->assertEquals('ip-dst', $results[0]['default_type']);
$results = $complexTypeTool->checkFreeText("127.0.0.1\xc2\xa0\xc2\xa0127.0.0.2");
$this->assertCount(2, $results);
$this->assertEquals('127.0.0.1', $results[0]['value']);
$this->assertEquals('ip-dst', $results[0]['default_type']);
}
public function testCheckFreeTextControlCharToSpace(): void
{
$complexTypeTool = new ComplexTypeTool();
$results = $complexTypeTool->checkFreeText("127.0.0.1\x1d127.0.0.2");
$this->assertCount(2, $results);
$this->assertEquals('127.0.0.1', $results[0]['value']);
$this->assertEquals('ip-dst', $results[0]['default_type']);
$results = $complexTypeTool->checkFreeText("127.0.0.1\x1d\x1d127.0.0.2");
$this->assertCount(2, $results);
$this->assertEquals('127.0.0.1', $results[0]['value']);
$this->assertEquals('ip-dst', $results[0]['default_type']);
}
public function testCheckFreeTextQuoted(): void

8
app/Test/JSONConverterToolTest.php
View File

@ -1,4 +1,5 @@
<?php
require_once __DIR__ . '/../Lib/Tools/JsonTool.php';
require_once __DIR__ . '/../Lib/Tools/JSONConverterTool.php';
use PHPUnit\Framework\TestCase;
@ -57,11 +58,6 @@ class JSONConverterToolTest extends TestCase
$jsonNormalWithoutSpaces = preg_replace("/\s+/", "", JSONConverterTool::convert($event));
$this->assertEquals($jsonNormalWithoutSpaces, $jsonStreamWithoutSpaces);
if (defined('JSON_THROW_ON_ERROR')) {
json_decode($json, true, 512, JSON_THROW_ON_ERROR);
$this->assertTrue(true);
} else {
$this->assertNotNull(json_decode($json));
}
$this->assertTrue(JsonTool::isValid($json));
}
}

4
app/View/AuthKeys/view.ctp
View File

@ -3,12 +3,12 @@ $keyUsageCsv = null;
if (isset($keyUsage)) {
$todayString = date('Y-m-d');
$today = strtotime($todayString);
$startDate = key($keyUsage); // oldest date for sparkline
$startDate = array_key_first($keyUsage); // oldest date for sparkline
$startDate = strtotime($startDate) - (3600 * 24 * 3);
$keyUsageCsv = 'Date,Close\n';
for ($date = $startDate; $date <= $today; $date += (3600 * 24)) {
$dateAsString = date('Y-m-d', $date);
$keyUsageCsv .= $dateAsString . ',' . (isset($keyUsage[$dateAsString]) ? $keyUsage[$dateAsString] : 0) . '\n';
$keyUsageCsv .= $dateAsString . ',' . ($keyUsage[$dateAsString] ?? '0') . '\n';
}
} else {
$lastUsed = null;

12
app/View/Elements/global_menu.ctp
View File

@ -277,11 +277,6 @@
'url' => $baseurl . '/servers/createSync',
'requirement' => $isAclSync && !$isSiteAdmin
),
array(
'text' => __('Import Server Settings'),
'url' => $baseurl . '/servers/import',
'requirement' => $this->Acl->canAccess('servers', 'import'),
),
array(
'text' => __('Remote Servers'),
'url' => $baseurl . '/servers/index',
@ -292,11 +287,6 @@
'url' => $baseurl . '/feeds/index',
'requirement' => $this->Acl->canAccess('feeds', 'index'),
),
array(
'text' => __('Search Feed Caches'),
'url' => $baseurl . '/feeds/searchCaches',
'requirement' => $this->Acl->canAccess('feeds', 'searchCaches'),
),
array(
'text' => __('SightingDB'),
'url' => $baseurl . '/sightingdb/index',
@ -313,7 +303,7 @@
'requirement' => $this->Acl->canAccess('cerebrates', 'index'),
),
array(
'text' => __('List Taxii Servers'),
'text' => __('TAXII Servers'),
'url' => $baseurl . '/TaxiiServers/index',
'requirement' => $this->Acl->canAccess('taxiiServers', 'index'),
),

33
app/View/Elements/healthElements/files.ctp
View File

@ -1,3 +1,10 @@
<?php
$humanReadableFilesize = function ($bytes, $dec = 2) {
$size = ['B', 'kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
$factor = floor((strlen($bytes) - 1) / 3);
return sprintf("%.{$dec}f&nbsp;%s", ($bytes / (1024 ** $factor)), $size[$factor]);
};
?>
<div style="border:1px solid #dddddd; margin-top:1px; width:100%; padding:10px">
<p><?php echo __('Below you will find a list of the uploaded files based on type.');?></p>
<?php
@ -5,13 +12,13 @@
?>
<h3><?php echo h($file['name']); ?></h3>
<div>
<b><?php echo __('Description');?></b>: <?php echo $file['description']; ?><br />
<b><?php echo __('Expected Format');?></b>: <?php echo h($file['valid_format']);?><br />
<b><?php echo __('Path');?></b>: <?php echo h($file['path']);?><br />
<b><?php echo __('Description');?></b>: <?php echo $file['description']; ?><br>
<b><?php echo __('Expected Format');?></b>: <?php echo h($file['valid_format']);?><br>
<b><?php echo __('Path');?></b>: <?php echo h($file['path']);?><br>
<?php
if (!empty($file['expected'])):
?>
<b><?php echo __('Files set for each relevant setting');?>:</b><br />
<b><?php echo __('Files set for each relevant setting');?>:</b><br>
<ul>
<?php foreach ($file['expected'] as $expectedKey => $expectedValue):
$colour = 'red';
@ -24,7 +31,7 @@
endif;
?>
</div>
<table class="table table-striped table-hover table-condensed" style="width:600px;">
<table class="table table-striped table-hover table-condensed" style="width:700px">
<tr>
<th><?php echo __('Filename');?></th>
<th><?php echo __('Used by');?></th>
@ -35,19 +42,10 @@
<?php
foreach ($file['files'] as $f):
$permission = "";
if ($f['link']) $permission .= "l";
if ($f['read']) $permission .= "r";
if ($f['write']) $permission .= "w";
if ($f['execute']) $permission .= "x";
$sizeUnit = "B";
if (($f['filesize'] / 1024) > 1) {
$f['filesize'] = $f['filesize'] / 1024;
$sizeUnit = "KB";
if (($f['filesize'] / 1024) > 1) {
$f['filesize'] = $f['filesize'] / 1024;
$sizeUnit = "MB";
}
$f['filesize'] = round($f['filesize'], 1);
}
?>
<tr>
<td><?php echo h($f['filename']);?></td>
@ -55,7 +53,7 @@
<?php
if ($k != 'orgs'):
foreach ($file['expected'] as $ek => $ev):
if ($f['filename'] == $ev) echo h($ek) . "<br />";
if ($f['filename'] == $ev) echo h($ek) . "<br>";
endforeach;
else:
echo __('N/A');
@ -63,7 +61,7 @@
?>
</td>
<td width="75px;">
<?php echo h($f['filesize']) . ' ' . $sizeUnit;?>
<?= $humanReadableFilesize($f['filesize'], 1) ?>
</td>
<td class="short">
<?php echo $permission;?>
@ -93,5 +91,4 @@
echo $this->Form->end();
endforeach;
?>
</div>

8
app/View/Pages/administration.ctp
View File

@ -1,11 +1,3 @@
<?php
if (!$isSiteAdmin) exit();
?>
<div class="actions">
<ol class="nav nav-list">
</ol>
</div>
<div class="index">
<h2><?php echo __('Administrative actions');?></h2>
<ul>

4
app/View/Users/totp_new.ctp
View File

@ -1,7 +1,7 @@
<?php echo $this->Flash->render(); ?>
<?php
$detailsHtml = __("To enable TOTP for your account, scan the following QR code with your TOTP application and validate the token.");;
$secretHtml = __("Alternatively you can enter the following secret in your TOTP application: ") . "<pre>" . $secret . "</pre>";
$detailsHtml = __("To enable TOTP for your account, scan the following QR code with your TOTP application (for example Google authenticator or KeepassXC) and validate the token.");;
$secretHtml = __("Alternatively you can enter the following secret in your TOTP application. This can be particularly handy in case you don't have a supported application in your working environment. Once the verification is done you'll also get 50 \"paper-based\" login tokens so you don't have to use a TOTP application each time: ") . "<pre>" . $secret . "</pre>";
echo $this->element('/genericElements/Form/genericForm', array(
"form" => $this->Form,

1
app/composer.json
View File

@ -36,6 +36,7 @@
"ext-rdkafka": "Required for publishing events to Kafka broker",
"ext-apcu": "To cache data in memory instead of file system",
"ext-simdjson": "To decode JSON structures faster",
"ext-curl": "For faster remote requests",
"elasticsearch/elasticsearch": "For logging to elasticsearch",
"aws/aws-sdk-php": "To upload samples to S3",
"jakub-onderka/openid-connect-php": "For OIDC authentication",

35702
app/files/browscap/browscap.ini
View File

File diff suppressed because it is too large Load Diff

BIN
app/files/browscap/browscap.ini.gz Normal file
View File

Binary file not shown.

79
app/files/scripts/mispzmq/mispzmq.py
View File

@ -4,15 +4,18 @@ from zmq.auth.thread import ThreadAuthenticator
from zmq.utils.monitor import recv_monitor_message
import sys
import redis
import json
import os
import time
import threading
import logging
import typing
import argparse
from pathlib import Path
logging.basicConfig(level=logging.INFO, format="%(asctime)s:%(levelname)s:%(name)s:%(message)s")
try:
import orjson as json
except ImportError:
import json
def check_pid(pid):
@ -55,10 +58,11 @@ class MispZmq:
socket = None
pidfile = None
r: redis.StrictRedis
redis: redis.StrictRedis
namespace: str
def __init__(self):
def __init__(self, debug=False):
logging.basicConfig(level=logging.DEBUG if debug else logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
self._logger = logging.getLogger()
self.tmp_location = Path(__file__).parent.parent / "tmp"
@ -67,7 +71,7 @@ class MispZmq:
with open(self.pidfile.as_posix()) as f:
pid = f.read()
if check_pid(pid):
raise Exception("mispzmq already running on PID {}".format(pid))
raise Exception(f"mispzmq already running on PID {pid}")
else:
# Cleanup
self.pidfile.unlink()
@ -77,17 +81,18 @@ class MispZmq:
raise Exception("The settings file is missing.")
def _setup(self):
with open((self.tmp_location / "mispzmq_settings.json").as_posix()) as settings_file:
self.settings = json.load(settings_file)
with open((self.tmp_location / "mispzmq_settings.json").as_posix(), 'rb') as settings_file:
self.settings = json.loads(settings_file.read())
self.namespace = self.settings["redis_namespace"]
# Check if TLS is being used with Redis host
redis_host = self.settings["redis_host"]
redis_ssl = redis_host.startswith("tls://")
if redis_host.startswith("tls://"):
redis_host = redis_host[6:]
self.r = redis.StrictRedis(host=redis_host, db=self.settings["redis_database"],
self.redis = redis.StrictRedis(host=redis_host, db=self.settings["redis_database"],
password=self.settings["redis_password"], port=self.settings["redis_port"],
decode_responses=True, ssl=redis_ssl)
ssl=redis_ssl)
self.timestamp_settings = time.time()
self._logger.debug("Connected to Redis {}:{}/{}".format(self.settings["redis_host"], self.settings["redis_port"],
self.settings["redis_database"]))
@ -122,34 +127,38 @@ class MispZmq:
self.socket.disable_monitor()
self.monitor_thread = None
def _handle_command(self, command):
if command == "kill":
def _handle_command(self, command: bytes):
if command == b"kill":
self._logger.info("Kill command received, shutting down.")
self.clean()
sys.exit()
elif command == "reload":
elif command == b"reload":
self._logger.info("Reload command received, reloading settings from file.")
self._setup()
self._setup_zmq()
elif command == "status":
elif command == b"status":
self._logger.info("Status command received, responding with latest stats.")
self.r.delete("{}:status".format(self.namespace))
self.r.lpush("{}:status".format(self.namespace),
self.redis.delete(f"{self.namespace}:status")
self.redis.lpush(f"{self.namespace}:status",
json.dumps({"timestamp": time.time(),
"timestampSettings": self.timestamp_settings,
"publishCount": self.publish_count,
"messageCount": self.message_count}))
else:
self._logger.warning("Received invalid command '{}'.".format(command))
self._logger.warning(f"Received invalid command '{command}'.")
def _create_pid_file(self):
with open(self.pidfile.as_posix(), "w") as f:
f.write(str(os.getpid()))
def _pub_message(self, topic, data):
self.socket.send_string("{} {}".format(topic, data))
def _pub_message(self, topic: bytes, data: typing.Union[str, bytes]):
data_to_send = bytearray()
data_to_send.extend(topic)
data_to_send.extend(b" ")
data_to_send.extend(data.encode("utf-8") if isinstance(data, str) else data)
self.socket.send(bytes(data_to_send))
def clean(self):
if self.monitor_thread:
@ -179,12 +188,14 @@ class MispZmq:
"misp_json_tag", "misp_json_warninglist", "misp_json_workflow"
]
lists = ["{}:command".format(self.namespace)]
lists = [f"{self.namespace}:command"]
for topic in topics:
lists.append("{}:data:{}".format(self.namespace, topic))
lists.append(f"{self.namespace}:data:{topic}")
key_prefix = f"{self.namespace}:".encode("utf-8")
while True:
data = self.r.blpop(lists, timeout=10)
data = self.redis.blpop(lists, timeout=10)
if data is None:
# redis timeout expired
@ -195,26 +206,30 @@ class MispZmq:
"status": status_array[status_entry],
"uptime": current_time - int(self.timestamp_settings)
}
self._pub_message("misp_json_self", json.dumps(status_message))
self._logger.debug("No message received for 10 seconds, sending ZMQ status message.")
self._pub_message(b"misp_json_self", json.dumps(status_message))
self._logger.debug("No message received from Redis for 10 seconds, sending ZMQ status message.")
else:
key, value = data
key = key.replace("{}:".format(self.namespace), "")
if key == "command":
key = key.replace(key_prefix, b"")
if key == b"command":
self._handle_command(value)
elif key.startswith("data:"):
topic = key.split(":")[1]
self._logger.debug("Received data for topic '{}', sending to ZMQ.".format(topic))
elif key.startswith(b"data:"):
topic = key.split(b":", 1)[1]
self._logger.debug("Received data for topic %s, sending to ZMQ.", topic)
self._pub_message(topic, value)
self.message_count += 1
if topic == "misp_json":
if topic == b"misp_json":
self.publish_count += 1
else:
self._logger.warning("Received invalid message '{}'.".format(key))
self._logger.warning("Received invalid message type %s.", key)
if __name__ == "__main__":
mzq = MispZmq()
arg_parser = argparse.ArgumentParser(description="MISP ZeroMQ PUB server")
arg_parser.add_argument("--debug", action="store_true", help="Enable debugging messages")
parsed = arg_parser.parse_args()
mzq = MispZmq(parsed.debug)
try:
mzq.main()
except KeyboardInterrupt:

29
app/files/scripts/mispzmq/mispzmqclient.py Normal file
View File

@ -0,0 +1,29 @@
#!/usr/bin/env python3
import sys
import zmq
import argparse
def main(port: int):
context = zmq.Context()
print("Connecting to MISP ZeroMQ server…", file=sys.stderr)
socket = context.socket(zmq.SUB)
socket.connect(f"tcp://localhost:{port}")
socket.setsockopt(zmq.SUBSCRIBE, b"misp_")
print(f"Connected to tcp://localhost:{port}", file=sys.stderr)
while True:
message = socket.recv()
print(message)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Example Python client for MISP ZMQ")
parser.add_argument("--port", default=50000, type=int)
parsed = parser.parse_args()
try:
main(parsed.port)
except KeyboardInterrupt:
pass

2
travis/database.php → build/database.php
View File

@ -60,7 +60,7 @@
class DATABASE_CONFIG {
public $default = array(
'datasource' => 'Database/Mysql',
'datasource' => 'Database/MysqlExtended',
'persistent' => false,
'host' => '127.0.0.1',
'login' => 'misp',

6
travis/email.php → build/email.php
View File

@ -1,7 +1,7 @@
<?php
class EmailConfig {
public $default = array(
public $default = [
'transport' => 'Debug',
'log' => true
);
'log' => true,
];
}

0
travis/gpg → build/gpg
View File

15
build/travis-ci-apache
View File

@ -1,15 +0,0 @@
<VirtualHost misp.local>
ServerAdmin me@me.local
ServerName misp.local
DocumentRoot %TRAVIS_BUILD_DIR%/app/webroot
<Directory %TRAVIS_BUILD_DIR%/app/webroot>
Options -Indexes
AllowOverride all
Require all granted
</Directory>
LogLevel warn
ErrorLog /var/log/apache2/misp.local_error.log
CustomLog /var/log/apache2/misp.local_access.log combined
ServerSignature Off
</VirtualHost>

5
tests/build-test.sh
View File

@ -9,16 +9,15 @@ python ./../app/files/scripts/mispzmq/mispzmqtest.py
python ./../app/files/scripts/generate_file_objects.py -c | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if len([i for i in data.values() if i is not False]) == 0 else 1)'
# Try to extract data from file
python ./../app/files/scripts/generate_file_objects.py -p /bin/ls
python ./../app/files/scripts/generate_file_objects.py -p /bin/ls | python3 -c 'import sys, json; data = json.load(sys.stdin); sys.exit(0 if "objects" in data else 1)'
# Test converting stix1 to MISP format
curl https://stixproject.github.io/documentation/idioms/c2-indicator/indicator-for-c2-ip-address.xml > ./../app/files/scripts/tmp/test-stix1.xml
curl -sS --compressed https://stixproject.github.io/documentation/idioms/c2-indicator/indicator-for-c2-ip-address.xml > ./../app/files/scripts/tmp/test-stix1.xml
python ./../app/files/scripts/stix2misp.py test-stix1.xml 1 1 ./../app/files/scripts/synonymsToTagNames.json | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if data["success"] == 1 else 1)'
rm -f ./../app/files/scripts/tmp/{test-stix1.xml,test-stix1.xml.json}
# Test converting stix2 to MISP format
curl https://raw.githubusercontent.com/oasis-open/cti-stix2-json-schemas/master/examples/indicator-for-c2-ip-address.json > ./../app/files/scripts/tmp/test-stix2.json
curl -sS --compressed https://raw.githubusercontent.com/oasis-open/cti-stix2-json-schemas/master/examples/indicator-for-c2-ip-address.json > ./../app/files/scripts/tmp/test-stix2.json
python ./../app/files/scripts/stix2/stix2misp.py -i ./../app/files/scripts/tmp/test-stix2.json --distribution 1 | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if data["success"] == 1 else 1)'
rm -f ./../app/files/scripts/tmp/{test-stix2.json,test-stix2.json.stix2}

15
tests/curl_tests.sh
View File

@ -1,15 +0,0 @@
#!/usr/bin/env bash
set -e
set -x
AUTH="$1"
# Check if user is logged
curl -i -H "Authorization: $AUTH" -H "Accept: application/json" -X GET http://misp.local/servers/getVersion
curl -i -H "Accept: application/json" -H "content-type: application/json" -H "Authorization: $AUTH" --data "@event.json" -X POST http://misp.local/events
curl -H "Authorization: $AUTH" -X GET http://misp.local/events/csv/download/1/ignore:1 | sed -e 's/^M//g' | cut -d, -f2 --complement | sort > 1.csv
cat 1.csv
cut -d, -f2 --complement event.csv | sort > compare.csv
diff compare.csv 1.csv
curl -i -H "Accept: application/json" -H "content-type: application/json" -H "Authorization: $AUTH" -X POST http://misp.local/events/delete/1

8
tests/curl_tests_GH.sh
View File

@ -7,13 +7,13 @@ AUTH="$1"
HOST="$2"
# Check if user is logged
curl -i -H "Authorization: $AUTH" -H "Accept: application/json" -X GET http://${HOST}/servers/getVersion
curl -i -H "Accept: application/json" -H "content-type: application/json" -H "Authorization: $AUTH" --data "@event.json" -X POST http://${HOST}/events > /dev/null
curl -H "Authorization: $AUTH" -X GET http://${HOST}/events/csv/download/1/ignore:1 | sed -e 's/^M//g' | cut -d, -f2 --complement | sort > 1.csv
curl -sS -i -H "Authorization: $AUTH" -H "Accept: application/json" -X GET http://${HOST}/servers/getVersion
curl -sS -i -H "Accept: application/json" -H "content-type: application/json" -H "Authorization: $AUTH" --data "@event.json" -X POST http://${HOST}/events > /dev/null
curl -sS -H "Authorization: $AUTH" -X GET http://${HOST}/events/csv/download/1/ignore:1 | sed -e 's/^M//g' | cut -d, -f2 --complement | sort > 1.csv
cat 1.csv
cut -d, -f2 --complement event.csv | sort > compare.csv
diff compare.csv 1.csv
# Test alert email generating
sudo -u www-data ../app/Console/cake Event testEventNotificationEmail 1 1 > /dev/null
# Delete created event
curl -i -H "Accept: application/json" -H "content-type: application/json" -H "Authorization: $AUTH" -X POST http://${HOST}/events/delete/1
curl -sS -i -H "Accept: application/json" -H "content-type: application/json" -H "Authorization: $AUTH" -X POST http://${HOST}/events/delete/1

9
tests/modify_config.php
View File

@ -10,9 +10,10 @@ if (!isset($argv[2])) {
if (!in_array($argv[1], ['modify', 'replace'], true)) {
fail(1, "Invalid argument '{$argv[1]}', it must be 'modify' or 'replace'.");
}
$newConfig = json_decode($argv[2], true);
if ($newConfig === null) {
fail(2, "Could not decode new config, it is not JSON: " . json_last_error_msg());
try {
$newConfig = json_decode($argv[2], true, JSON_THROW_ON_ERROR);
} catch (Exception $e) {
fail(2, "Could not decode new config, it is not JSON: " . $e->getMessage());
}
if (!is_array($newConfig)) {
fail(2, "Provided new config is not array, `" . gettype($newConfig) . "` given.");
@ -41,4 +42,4 @@ if ($argv[1] === 'modify') {
file_put_contents($configFile, "<?php\n\$config = " . var_export($merged, true) . ';', LOCK_EX);
// Returns config file before modification
echo json_encode($config);
echo json_encode($config, JSON_THROW_ON_ERROR | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);

99
tests/testlive_comprehensive_local.py
View File

@ -2,6 +2,7 @@
import os
import json
import uuid
import logging
import inspect
import subprocess
import unittest
@ -11,21 +12,19 @@ from xml.etree import ElementTree as ET
from io import BytesIO
import urllib3 # type: ignore
from datetime import datetime, timedelta
import logging
logging.disable(logging.CRITICAL)
logger = logging.getLogger('pymisp')
from typing import Union
from pymisp import PyMISP, MISPOrganisation, MISPUser, MISPRole, MISPSharingGroup, MISPEvent, MISPLog, MISPSighting, Distribution, ThreatLevel, Analysis, MISPEventReport, MISPServerError
from pymisp.tools import DomainIPObject
from pymisp.api import get_uuid_or_id_from_abstract_misp
logging.disable(logging.CRITICAL)
logger = logging.getLogger('pymisp')
urllib3.disable_warnings()
# Load access information for env variables
url = "http://" + os.environ["HOST"]
key = os.environ["AUTH"]
urllib3.disable_warnings()
def create_simple_event() -> MISPEvent:
caller_name = inspect.stack()[1].function
@ -52,13 +51,19 @@ def request(pymisp: PyMISP, request_type: str, url: str, data: dict = {}) -> dic
return pymisp._check_json_response(response)
def publish_immediately(pymisp: PyMISP, event: Union[MISPEvent, int, str, uuid.UUID], with_email: bool = False):
event_id = get_uuid_or_id_from_abstract_misp(event)
action = "alert" if with_email else "publish"
return check_response(request(pymisp, 'POST', f'events/{action}/{event_id}/disable_background_processing:1'))
class MISPSetting:
def __init__(self, admin_connector: PyMISP, new_setting: dict):
self.admin_connector = admin_connector
self.new_setting = new_setting
def __enter__(self):
self.original = self.__run("modify", json.dumps(self.new_setting))
self.original = self.__run("modify", json.dumps(self.new_setting).encode("utf-8"))
# Try to reset config cache
self.admin_connector.get_server_setting("MISP.live")
@ -68,12 +73,12 @@ class MISPSetting:
self.admin_connector.get_server_setting("MISP.live")
@staticmethod
def __run(command: str, data: str) -> str:
def __run(command: str, data: bytes) -> bytes:
dir_path = os.path.dirname(os.path.realpath(__file__))
r = subprocess.run(["php", dir_path + "/modify_config.php", command, data], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if r.returncode != 0:
raise Exception([r.returncode, r.stdout, r.stderr])
return r.stdout.decode("utf-8")
return r.stdout
class TestComprehensive(unittest.TestCase):
@ -465,8 +470,6 @@ class TestComprehensive(unittest.TestCase):
check_response(self.admin_misp_connector.delete_event(event))
def test_publish_alert_filter(self):
check_response(self.admin_misp_connector.set_server_setting('MISP.background_jobs', 0, force=True))
first = create_simple_event()
first.add_tag('test_publish_filter')
first.threat_level_id = ThreatLevel.medium
@ -499,7 +502,7 @@ class TestComprehensive(unittest.TestCase):
# Publish events
for event in (first, second, third, four):
check_response(self.admin_misp_connector.publish(event, alert=True))
publish_immediately(self.admin_misp_connector, event, with_email=True)
# Email notification should be send just to first event
mail_logs = self.admin_misp_connector.search_logs(model='User', action='email')
@ -516,8 +519,6 @@ class TestComprehensive(unittest.TestCase):
check_response(self.admin_misp_connector.update_user(self.admin_misp_connector._current_user))
# Delete filter
self.admin_misp_connector.delete_user_setting('publish_alert_filter')
# Reenable background jobs
check_response(self.admin_misp_connector.set_server_setting('MISP.background_jobs', 1, force=True))
# Delete events
for event in (first, second, third, four):
check_response(self.admin_misp_connector.delete_event(event))
@ -740,20 +741,27 @@ class TestComprehensive(unittest.TestCase):
event.add_attribute("ip-src", "1.2.4.5", to_ids=True)
event = check_response(self.admin_misp_connector.add_event(event))
result = self._search({'returnFormat': "openioc", 'eventid': event.id, "published": [0, 1]})
result = self._search_event({'returnFormat': "openioc", 'eventid': event.id, "published": [0, 1]})
ET.fromstring(result) # check if result is valid XML
self.assertTrue("1.2.4.5" in result, result)
result = self._search({'returnFormat': "yara", 'eventid': event.id, "published": [0, 1]})
result = self._search_event({'returnFormat': "yara", 'eventid': event.id, "published": [0, 1]})
self.assertTrue("1.2.4.5" in result, result)
self.assertTrue("GENERATED" in result, result)
self.assertTrue("AS-IS" in result, result)
result = self._search({'returnFormat': "yara-json", 'eventid': event.id, "published": [0, 1]})
result = self._search_event({'returnFormat': "yara-json", 'eventid': event.id, "published": [0, 1]})
self.assertIn("generated", result)
self.assertEqual(len(result["generated"]), 1, result)
self.assertIn("as-is", result)
# RPZ
result = self._search_event({'returnFormat': "rpz", 'eventid': event.id, "published": [0, 1]})
self.assertTrue("32.5.4.2.1" in result, result)
result = self._search_attribute({'returnFormat': "rpz", 'eventid': event.id, "published": [0, 1]})
self.assertTrue("32.5.4.2.1" in result, result)
check_response(self.admin_misp_connector.delete_event(event))
def test_event_report_empty_name(self):
@ -916,16 +924,18 @@ class TestComprehensive(unittest.TestCase):
def test_search_snort_suricata(self):
event = create_simple_event()
event.add_attribute('ip-src', '8.8.8.8', to_ids=True)
event = self.user_misp_connector.add_event(event)
check_response(event)
event.add_attribute('snort', 'alert tcp 192.168.1.0/24 any -> 131.171.127.1 25 (content: "hacking"; msg: "malicious packet"; sid:2000001;)', to_ids=True)
# Snort rule without msg, test for #9515
event.add_attribute('snort', 'alert tcp 192.168.1.0/24 any -> 131.171.127.1 25 (content: "hacking"; sid:2000001;)', to_ids=True)
event = check_response(self.user_misp_connector.add_event(event))
self.admin_misp_connector.publish(event, alert=False)
time.sleep(6)
snort = self._search({'returnFormat': 'snort', 'eventid': event.id})
publish_immediately(self.admin_misp_connector, event)
snort = self._search_event({'returnFormat': 'snort', 'eventid': event.id})
self.assertIsInstance(snort, str)
self.assertIn('8.8.8.8', snort)
suricata = self._search({'returnFormat': 'suricata', 'eventid': event.id})
suricata = self._search_event({'returnFormat': 'suricata', 'eventid': event.id})
self.assertIsInstance(suricata, str)
self.assertIn('8.8.8.8', suricata)
@ -962,7 +972,38 @@ class TestComprehensive(unittest.TestCase):
self.admin_misp_connector.delete_event(event)
def _search(self, query: dict):
def test_restsearch_sightings(self):
# Create test event
event = create_simple_event()
event = self.admin_misp_connector.add_event(event)
check_response(event)
# Add sighting
sighting = MISPSighting()
sighting.value = 'test'
sighting.source = 'Testcases'
sighting.type = '1'
response = self.admin_misp_connector.add_sighting(sighting, event.attributes[0])
check_response(response)
self.assertEqual(response.source, 'Testcases')
# Try to find sighting by event UUID, this is the same type of request when doing sync
search_result = self._search_sighting('event', {
'returnFormat': 'json',
'last': 0,
'includeUuid': True,
'uuid': [event.uuid],
})
self.assertEqual(len(search_result), 1, search_result)
sighting = search_result[0]["Sighting"]
self.assertIn("attribute_uuid", sighting)
self.assertIn("event_uuid", sighting)
self.assertEqual(sighting["event_uuid"], event.uuid, search_result)
self.admin_misp_connector.delete_event(event)
def _search_event(self, query: dict):
response = self.admin_misp_connector._prepare_request('POST', 'events/restSearch', data=query)
response = self.admin_misp_connector._check_response(response)
check_response(response)
@ -974,6 +1015,12 @@ class TestComprehensive(unittest.TestCase):
check_response(response)
return response
def _search_sighting(self, context: str, query: dict):
response = self.admin_misp_connector._prepare_request('POST', f'sightings/restSearch/{context}', data=query)
response = self.admin_misp_connector._check_response(response)
check_response(response)
return response
class TestLastPwChange(unittest.TestCase):
@classmethod

24
tools/misp-delegation/src/misp-delegation.py
View File

@ -33,8 +33,14 @@ def main():
global unpublish_event_on_remote
parser = argparse.ArgumentParser(Path(__file__).name)
parser.add_argument('-c', '--config', default='config.json', help='The JSON config file to use')
parser.add_argument('-l', '--log', default='INFO', help='The loglevel to use.', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR',])
options = parser.parse_args()
numeric_log_level = getattr(logging, options.log.upper(), None)
if not isinstance(numeric_log_level, int):
raise ValueError('Invalid log level: %s' % numeric_log_level)
logger.setLevel(numeric_log_level)
config = {}
config_file_path = Path(__file__).parent / options.config
with open(config_file_path, 'r') as f:
@ -74,7 +80,7 @@ def main():
return 1
# Collect events from source
logger.debug('Collecting events from source...')
logger.debug('Collecting *published* events from source...')
try:
events_on_source = collect_events_from_source(source_instance, filters, incremental_sync)
except Exception as err:
@ -185,9 +191,10 @@ def update_event_for_push(event: dict) -> dict:
elif attribute['distribution'] == 2:
event['Attribute'][i]['distribution'] = 1
for t, tag in enumerate(attribute['Tag']):
if tag['local']:
event['Attribute'][i]['Tag'].pop(t)
if 'Tag' in attribute:
for t, tag in enumerate(attribute['Tag']):
if tag['local']:
event['Attribute'][i]['Tag'].pop(t)
# Downgrade distribution for Objects and their Attributes
for i, object in enumerate(event['Object'][:]):
@ -201,9 +208,10 @@ def update_event_for_push(event: dict) -> dict:
elif attribute['distribution'] == 2:
event['Object'][i]['Attribute'][j]['distribution'] = 1
for t, tag in enumerate(attribute['Tag']):
if tag['local']:
event['Object'][i]['Attribute'][j]['Tag'].pop(t)
if 'Tag' in attribute:
for t, tag in enumerate(attribute['Tag']):
if tag['local']:
event['Object'][i]['Attribute'][j]['Tag'].pop(t)
# Downgrade distribution for EventReport
for i, report in enumerate(event['EventReport'][:]):
@ -222,7 +230,7 @@ def collect_events_from_source(source_instance: MISPInstance, filters: dict, inc
}
last_timestamp = get_last_sync_timestamp()
if incremental_sync and last_timestamp is not None:
logger.debug('Using timestamp from last synchronisation %s (%s)', last_timestamp, datetime.fromtimestamp(last_timestamp))
logger.info('Using timestamp from last synchronisation %s (%s)', last_timestamp, datetime.fromtimestamp(last_timestamp))
sync_filters['timestamp'] = last_timestamp # type: ignore
sync_filters.update(filters)
events = source_instance.POST('/events/index', payload=sync_filters)

154
travis/bootstrap.php
View File

@ -1,154 +0,0 @@
<?php
/**
* This file is loaded automatically by the app/webroot/index.php file after core.php
*
* This file should load/create any application wide configuration settings, such as
* Caching, Logging, loading additional configuration files.
*
* You should also use this file to include any files that provide global functions/constants
* that your application uses.
*/
/**
* Cache Engine Configuration
* Default settings provided below
*
* File storage engine.
*
* Cache::config('default', array(
* 'engine' => 'File', //[required]
* 'duration'=> 3600, //[optional]
* 'probability'=> 100, //[optional]
* 'path' => CACHE, //[optional] use system tmp directory - remember to use absolute path
* 'prefix' => 'cake_', //[optional] prefix every cache file with this string
* 'lock' => false, //[optional] use file locking
* 'serialize' => true, // [optional]
* 'mask' => 0666, // [optional] permission mask to use when creating cache files
* ));
*
* APC (http://pecl.php.net/package/APC)
*
* Cache::config('default', array(
* 'engine' => 'Apc', //[required]
* 'duration'=> 3600, //[optional]
* 'probability'=> 100, //[optional]
* 'prefix' => Inflector::slug(APP_DIR) . '_', //[optional] prefix every cache file with this string
* ));
*
* Xcache (http://xcache.lighttpd.net/)
*
* Cache::config('default', array(
* 'engine' => 'Xcache', //[required]
* 'duration'=> 3600, //[optional]
* 'probability'=> 100, //[optional]
* 'prefix' => Inflector::slug(APP_DIR) . '_', //[optional] prefix every cache file with this string
* 'user' => 'user', //user from xcache.admin.user settings
* 'password' => 'password', //plaintext password (xcache.admin.pass)
* ));
*
* Memcache (http://memcached.org/)
*
* Cache::config('default', array(
* 'engine' => 'Memcache', //[required]
* 'duration'=> 3600, //[optional]
* 'probability'=> 100, //[optional]
* 'prefix' => Inflector::slug(APP_DIR) . '_', //[optional] prefix every cache file with this string
* 'servers' => array(
* '127.0.0.1:11211' // localhost, default port 11211
* ), //[optional]
* 'persistent' => true, // [optional] set this to false for non-persistent connections
* 'compress' => false, // [optional] compress data in Memcache (slower, but uses less memory)
* ));
*
* Wincache (http://php.net/wincache)
*
* Cache::config('default', array(
* 'engine' => 'Wincache', //[required]
* 'duration'=> 3600, //[optional]
* 'probability'=> 100, //[optional]
* 'prefix' => Inflector::slug(APP_DIR) . '_', //[optional] prefix every cache file with this string
* ));
*
* Redis (http://http://redis.io/)
*
* Cache::config('default', array(
* 'engine' => 'Redis', //[required]
* 'duration'=> 3600, //[optional]
* 'probability'=> 100, //[optional]
* 'prefix' => Inflector::slug(APP_DIR) . '_', //[optional] prefix every cache file with this string
* 'server' => '127.0.0.1' // localhost
* 'port' => 6379 // default port 6379
* 'timeout' => 0 // timeout in seconds, 0 = unlimited
* 'persistent' => true, // [optional] set this to false for non-persistent connections
* ));
*/
Cache::config('default', array('engine' => 'File'));
Configure::load('config');
if (!Configure::read('MISP.baseurl')) {
if (!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off' || $_SERVER['SERVER_PORT'] == 443) {
Configure::write('MISP.baseurl', sprintf('https://%s:%d', $_SERVER['SERVER_ADDR'], $_SERVER['SERVER_PORT']));
} else {
Configure::write('MISP.baseurl', sprintf('http://%s:%d', $_SERVER['SERVER_ADDR'], $_SERVER['SERVER_PORT']));
}
}
/**
* Plugins need to be loaded manually, you can either load them one by one or all of them in a single call
* Uncomment one of the lines below, as you need. make sure you read the documentation on CakePlugin to use more
* advanced ways of loading plugins
*
* CakePlugin::loadAll(); // Loads all plugins at once
* CakePlugin::load('DebugKit'); //Loads a single plugin named DebugKit
*
*/
CakePlugin::load('SysLog');
CakePlugin::load('Assets'); // having Logable
CakePlugin::load('SysLogLogable');
/**
* Uncomment the following line to enable client SSL certificate authentication.
* It's also necessary to configure the plugin — for more information, please read app/Plugin/CertAuth/reame.md
*/
// CakePlugin::load('CertAuth');
/**
* You can attach event listeners to the request lifecyle as Dispatcher Filter . By Default CakePHP bundles two filters:
*
* - AssetDispatcher filter will serve your asset files (css, images, js, etc) from your themes and plugins
* - CacheDispatcher filter will read the Cache.check configure variable and try to serve cached content generated from controllers
*
* Feel free to remove or add filters as you see fit for your application. A few examples:
*
* Configure::write('Dispatcher.filters', array(
* 'MyCacheFilter', // will use MyCacheFilter class from the Routing/Filter package in your app.
* 'MyPlugin.MyFilter', // will use MyFilter class from the Routing/Filter package in MyPlugin plugin.
* array('callable' => $aFunction, 'on' => 'before', 'priority' => 9), // A valid PHP callback type to be called on beforeDispatch
* array('callable' => $anotherMethod, 'on' => 'after'), // A valid PHP callback type to be called on afterDispatch
*
* ));
*/
Configure::write('Dispatcher.filters', array(
'AssetDispatcher',
'CacheDispatcher'
));
/**
* Configures default file logging options
*/
App::uses('CakeLog', 'Log');
CakeLog::config('debug', array(
'engine' => 'FileLog',
'types' => array('notice', 'info', 'debug'),
'file' => 'debug',
));
CakeLog::config('error', array(
'engine' => 'FileLog',
'types' => array('warning', 'error', 'critical', 'alert', 'emergency'),
'file' => 'error',
));
CakePlugin::loadAll(array(
'CakeResque' => array('bootstrap' => true)
));

81
travis/config.php
View File

@ -1,81 +0,0 @@
<?php
$config = array (
'debug' => 1,
'Security' =>
array (
'level' => 'medium',
'salt' => 'Rooraenietu8Eeyo<Qu2eeNfterd-dd+',
'cipherSeed' => '',
//'auth'=>array('CertAuth.Certificate'), // additional authentication methods
),
'MISP' =>
array (
'baseurl' => 'http://misp.local',
'footerpart1' => 'Powered by MISP',
'footerpart2' => '&copy; Belgian Defense CERT & NCIRC',
'org' => 'ORGNAME',
'showorg' => true,
'background_jobs' => false,
'email' => 'email@address.com',
'contact' => 'email@address.com',
'cveurl' => 'http://web.nvd.nist.gov/view/vuln/detail?vulnId=',
'disablerestalert' => false,
'default_event_distribution' => '0',
'default_attribute_distribution' => 'event',
'tagging' => true,
'full_tags_on_event_index' => true,
'footer_logo' => '',
'take_ownership_xml_import' => false,
'unpublishedprivate' => false,
),
'GnuPG' =>
array (
'onlyencrypted' => false,
'email' => '',
'homedir' => '',
'password' => '',
'bodyonlyencrypted' => false,
),
'Proxy' =>
array (
'host' => '',
'port' => '',
'method' => '',
'user' => '',
'password' => '',
),
'SecureAuth' =>
array (
'amount' => 5,
'expire' => 300,
),
// Uncomment the following to enable client SSL certificate authentication
/*
'CertAuth' =>
array(
'ca' => array( 'FIRST.Org' ), // allowed CAs
'caId' => 'O', // which attribute will be used to verify the CA
'userModel' => 'User', // name of the User class to check if user exists
'userModelKey' => 'nids_sid', // User field that will be used for querying
'map' => array( // maps client certificate attributes to User properties
'O' => 'org',
'emailAddress'=>'email',
),
'syncUser' => true, // should the User be synchronized with an external REST API
'userDefaults'=> array( // default user attributes, only used when creating new users
'role_id' => 4,
),
'restApi' => array( // API parameters
'url' => 'https://example.com/data/users', // URL to query
'headers' => array(), // additional headers, used for authentication
'param' => array( 'email' => 'email'), // query parameters to add to the URL, mapped to USer properties
'map' => array( // maps REST result to the User properties
'uid' => 'nids_sid',
'team' => 'org',
'email' => 'email',
'pgp_public'=> 'gpgkey',
),
),
),
*/
);

285
travis/core.php
View File

@ -1,285 +0,0 @@
<?php
/**
* This is core configuration file.
*
* Use it to configure core behavior of Cake.
*
* PHP 5
*
* CakePHP(tm) : Rapid Development Framework (http://cakephp.org)
* Copyright 2005-2012, Cake Software Foundation, Inc. (http://cakefoundation.org)
*
* Licensed under The MIT License
* Redistributions of files must retain the above copyright notice.
*
* @copyright Copyright 2005-2012, Cake Software Foundation, Inc. (http://cakefoundation.org)
* @link http://cakephp.org CakePHP(tm) Project
* @package app.Config
* @since CakePHP(tm) v 0.2.9
* @license MIT License (http://www.opensource.org/licenses/mit-license.php)
*/
/**
* CakePHP Debug Level:
*
* Production Mode:
* 0: No error messages, errors, or warnings shown. Flash messages redirect.
*
* Development Mode:
* 1: Errors and warnings shown, model caches refreshed, flash messages halted.
* 2: As in 1, but also with full debug messages and SQL output.
*
* In production mode, flash messages redirect after a time interval.
* In development mode, you need to click the flash message to continue.
*/
Configure::write('debug', 2); // 0 = for production, 2 = full debug mode
/**
* Configure the Error handler used to handle errors for your application. By default
* ErrorHandler::handleError() is used. It will display errors using Debugger, when debug > 0
* and log errors with CakeLog when debug = 0.
*
* Options:
*
* - `handler` - callback - The callback to handle errors. You can set this to any callable type,
* including anonymous functions.
* - `level` - int - The level of errors you are interested in capturing.
* - `trace` - boolean - Include stack traces for errors in log files.
*
* @see ErrorHandler for more information on error handling and configuration.
*/
Configure::write('Error', array(
'handler' => 'ErrorHandler::handleError',
'level' => E_ALL & ~E_DEPRECATED,
'trace' => true
));
/**
* Configure the Exception handler used for uncaught exceptions. By default,
* ErrorHandler::handleException() is used. It will display a HTML page for the exception, and
* while debug > 0, framework errors like Missing Controller will be displayed. When debug = 0,
* framework errors will be coerced into generic HTTP errors.
*
* Options:
*
* - `handler` - callback - The callback to handle exceptions. You can set this to any callback type,
* including anonymous functions.
* - `renderer` - string - The class responsible for rendering uncaught exceptions. If you choose a custom class you
* should place the file for that class in app/Lib/Error. This class needs to implement a render method.
* - `log` - boolean - Should Exceptions be logged?
*
* @see ErrorHandler for more information on exception handling and configuration.
*/
Configure::write('Exception', array(
'handler' => 'ErrorHandler::handleException',
'renderer' => 'ExceptionRenderer',
'log' => true,
'skipLog' => array(
'NotFoundException',
)
));
/**
* Application wide charset encoding
*/
Configure::write('App.encoding', 'UTF-8');
/**
* To configure CakePHP *not* to use mod_rewrite and to
* use CakePHP pretty URLs, remove these .htaccess
* files:
*
* /.htaccess
* /app/.htaccess
* /app/webroot/.htaccess
*
* And uncomment the App.baseUrl below:
*/
//Configure::write('App.baseUrl', env('SCRIPT_NAME'));
/**
* Uncomment the define below to use CakePHP prefix routes.
*
* The value of the define determines the names of the routes
* and their associated controller actions:
*
* Set to an array of prefixes you want to use in your application. Use for
* admin or other prefixed routes.
*
* Routing.prefixes = array('admin', 'manager');
*
* Enables:
* `admin_index()` and `/admin/controller/index`
* `manager_index()` and `/manager/controller/index`
*
*/
Configure::write('Routing.prefixes', array('admin'));
/**
* Turn off all caching application-wide.
*
*/
Configure::write('Cache.disable', false);
/**
* Enable cache checking.
*
* If set to true, for view caching you must still use the controller
* public $cacheAction inside your controllers to define caching settings.
* You can either set it controller-wide by setting public $cacheAction = true,
* or in each action using $this->cacheAction = true.
*
*/
//Configure::write('Cache.check', true);
/**
* Defines the default error type when using the log() function. Used for
* differentiating error logging and debugging. Currently PHP supports LOG_DEBUG.
*/
define('LOG_ERROR', LOG_ERR);
/**
* Session configuration.
*
* Contains an array of settings to use for session configuration. The defaults key is
* used to define a default preset to use for sessions, any settings declared here will override
* the settings of the default config.
*
* ## Options
*
* - `Session.cookie` - The name of the cookie to use. Defaults to 'CAKEPHP'
* - `Session.timeout` - The number of minutes you want sessions to live for. This timeout is handled by CakePHP
* - `Session.cookieTimeout` - The number of minutes you want session cookies to live for.
* - `Session.checkAgent` - Do you want the user agent to be checked when starting sessions? You might want to set the
* value to false, when dealing with older versions of IE, Chrome Frame or certain web-browsing devices and AJAX
* - `Session.defaults` - The default configuration set to use as a basis for your session.
* There are four builtins: php, cake, cache, database.
* - `Session.handler` - Can be used to enable a custom session handler. Expects an array of callables,
* that can be used with `session_save_handler`. Using this option will automatically add `session.save_handler`
* to the ini array.
* - `Session.autoRegenerate` - Enabling this setting, turns on automatic renewal of sessions, and
* sessionids that change frequently. See CakeSession::$requestCountdown.
* - `Session.ini` - An associative array of additional ini values to set.
*
* The built in defaults are:
*
* - 'php' - Uses settings defined in your php.ini.
* - 'cake' - Saves session files in CakePHP's /tmp directory.
* - 'database' - Uses CakePHP's database sessions.
* - 'cache' - Use the Cache class to save sessions.
*
* To define a custom session handler, save it at /app/Model/Datasource/Session/<name>.php.
* Make sure the class implements `CakeSessionHandlerInterface` and set Session.handler to <name>
*
* To use database sessions, run the app/Config/Schema/sessions.php schema using
* the cake shell command: cake schema create Sessions
*
*/
Configure::write('Session', array(
'timeout' => 60, // Session timeout, default is 1 hour
'defaults' => 'php'
));
/**
* The level of CakePHP security.
*/
Configure::write('Security.level', 'medium');
/**
* A random string used in security hashing methods.
*/
Configure::write('Security.salt', 'Rooraenietu8Eeyo<Qu2eeNfterd-dd+');
/**
* A random numeric string (digits only) used to encrypt/decrypt strings.
*/
Configure::write('Security.cipherSeed', '395786739573056621429506834955');
/**
* Apply timestamps with the last modified time to static assets (js, css, images).
* Will append a querystring parameter containing the time the file was modified. This is
* useful for invalidating browser caches.
*
* Set to `true` to apply timestamps when debug > 0. Set to 'force' to always enable
* timestamping regardless of debug value.
*/
//Configure::write('Asset.timestamp', true);
/**
* Compress CSS output by removing comments, whitespace, repeating tags, etc.
* This requires a/var/cache directory to be writable by the web server for caching.
* and /vendors/csspp/csspp.php
*
* To use, prefix the CSS link URL with '/ccss/' instead of '/css/' or use HtmlHelper::css().
*/
//Configure::write('Asset.filter.css', 'css.php');
/**
* Plug in your own custom JavaScript compressor by dropping a script in your webroot to handle the
* output, and setting the config below to the name of the script.
*
* To use, prefix your JavaScript link URLs with '/cjs/' instead of '/js/' or use JavaScriptHelper::link().
*/
//Configure::write('Asset.filter.js', 'custom_javascript_output_filter.php');
/**
* The classname and database used in CakePHP's
* access control lists.
*/
Configure::write('Acl.classname', 'DbAcl');
Configure::write('Acl.database', 'default');
/**
* Uncomment this line and correct your server timezone to fix
* any date & time related errors.
*/
//date_default_timezone_set('UTC');
/**
* Pick the caching engine to use. If APC is enabled use it.
* If running via cli - apc is disabled by default. ensure it's available and enabled in this case
*
* Note: 'default' and other application caches should be configured in app/Config/bootstrap.php.
* Please check the comments in boostrap.php for more info on the cache engines available
* and their setttings.
*/
$engine = 'File';
if (function_exists('apcu_dec') && (PHP_SAPI !== 'cli' || ini_get('apc.enable_cli'))) {
require_once APP . 'Plugin/ApcuCache/Engine/ApcuEngine.php'; // it is not possible to use plugin
$engine = 'Apcu';
}
// In development mode, caches should expire quickly.
$duration = '+999 days';
if (Configure::read('debug') >= 1) {
$duration = '+10 seconds';
}
// Prefix each application on the same server with a different string, to avoid Memcache and APC conflicts.
$prefix = 'myapp_';
/**
* Configure the cache used for general framework caching. Path information,
* object listings, and translation cache files are stored with this configuration.
*/
Cache::config('_cake_core_', array(
'engine' => $engine,
'prefix' => $prefix . 'cake_core_',
'path' => CACHE . 'persistent' . DS,
'serialize' => ($engine === 'File'),
'duration' => $duration
));
/**
* Configure the cache for model and datasource caches. This cache configuration
* is used to store schema descriptions, and table listings in connections.
*/
Cache::config('_cake_model_', array(
'engine' => $engine,
'prefix' => $prefix . 'cake_model_',
'path' => CACHE . 'models' . DS,
'serialize' => ($engine === 'File'),
'duration' => $duration
));
//require_once dirname(__DIR__) . '/Vendor/autoload.php';