MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'develop' of github.com:MISP/MISP into develop

pull/9570/head
Christian Studer 2024-02-16 16:28:30 +01:00
parent 15ed02432c 5ac9c995aa
commit 1a9f2836c8
No known key found for this signature in database
GPG Key ID: 6BBED1B63A6D639F
154 changed files with 3157 additions and 37847 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

363
.github/workflows/main.yml vendored
View File

@ -6,9 +6,9 @@ name: misp
# events but only for the 2.4 and develop branches
on:
push:
branches: [ 2.4, develop, misp-stix, taxii ]
branches: [ '2.4', develop, misp-stix, taxii ]
pull_request:
branches: [ 2.4, develop, misp-stix ]
branches: [ '2.4', develop, misp-stix ]
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
@ -62,186 +62,178 @@ jobs:
php_version: ${{ matrix.php }}
run: |
sudo apt-get -y update
# Repo is missing for unknown reason
LC_ALL=C.UTF-8 sudo apt-add-repository ppa:ondrej/php -y
if [[ $php_version == "7.2" ]]; then
# hotfix due to: https://bugs.php.net/bug.php?id=81640 TODO: remove after libpcre2-8-0:10.36 gets to stable channel
sudo apt-get --fix-broken install
fi
sudo apt-get -y install curl python3 python3-pip python3-virtualenv apache2 libapache2-mod-php$php_version
# Runs a set of commands using the runners shell
- name: Install deps
run: |
sudo chown $USER:www-data $HOME/.composer
pushd app
sudo -H -u $USER composer config --no-plugins allow-plugins.composer/installers true
sudo -H -u $USER composer install --no-progress
popd
cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php
# Set perms
sudo chown -R $USER:www-data `pwd`
sudo chmod -R 775 `pwd`
sudo chmod -R g+ws `pwd`/app/tmp
sudo chmod -R g+ws `pwd`/app/tmp/cache
sudo chmod -R g+ws `pwd`/app/tmp/cache/persistent
sudo chmod -R g+ws `pwd`/app/tmp/cache/models
sudo chmod -R g+ws `pwd`/app/tmp/logs
sudo chmod -R g+ws `pwd`/app/files
sudo chmod -R g+ws `pwd`/app/files/scripts/tmp
sudo chown -R $USER:www-data `pwd`
# Resque perms
sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp
sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp
# install MySQL
sudo chmod -R 777 `pwd`/INSTALL
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';"
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant usage on *.* to misp@'%' identified by 'blah';"
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant all privileges on misp.* to misp@'%';"
mysql -h 127.0.0.1 --port 3306 -u misp -pblah misp < INSTALL/MYSQL.sql
# configure apache virtual hosts
sudo chmod -R 777 `pwd`/build
sudo mkdir -p /etc/apache2/sites-available
sudo cp -f build/github-action-ci-apache /etc/apache2/sites-available/misp.conf
sudo sed -e "s?%GITHUB_WORKSPACE%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.conf
sudo sed -e "s?%HOST%?${HOST}?g" --in-place /etc/apache2/sites-available/misp.conf
sudo a2dissite 000-default
sudo a2ensite misp.conf
cat /etc/apache2/sites-enabled/misp.conf
sudo a2enmod rewrite
sudo systemctl restart apache2
# MISP configuration
sudo chmod -R 777 `pwd`/travis
sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php
sudo cp travis/database.php app/Config/database.php
sudo cp app/Config/core.default.php app/Config/core.php
sudo cp app/Config/config.default.php app/Config/config.php
sudo cp travis/email.php app/Config/email.php
# Ensure the perms
sudo chown -R $USER:www-data `pwd`/app/Config
sudo chmod -R 777 `pwd`/app/Config
# GPG setup
sudo mkdir `pwd`/.gnupg
# /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS
sudo cp -a /dev/urandom /dev/random
sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/travis/gpg
sudo gpg --list-secret-keys --homedir `pwd`/.gnupg
# change perms
sudo chown -R $USER:www-data `pwd`
sudo chown -R www-data:www-data `pwd`/.gnupg
sudo chmod -R 700 `pwd`/.gnupg
sudo usermod -a -G www-data $USER
sudo chmod -R 777 `pwd`/app/Plugin/CakeResque/tmp/
# Ensure the perms of config files
sudo chown -R $USER:www-data `pwd`/app/Config
sudo chmod -R 777 `pwd`/app/Config
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.server_settings_skip_backup_rotate" 1'
sudo chown -R $USER:www-data `pwd`/app/Config
sudo chmod -R 777 `pwd`/app/Config
sudo chown $USER:www-data $HOME/.composer
pushd app
composer config --no-plugins allow-plugins.composer/installers true
composer install --no-progress
popd
cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php
# fix perms (?)
namei -m /home/runner/work
sudo chmod +x /home/runner/work
sudo chmod +x /home/runner
sudo chmod +x /home
sudo chmod +x /
# Set perms
sudo chown -R $USER:www-data `pwd`
sudo chmod -R 775 `pwd`
sudo chmod -R g+ws `pwd`/app/tmp
sudo chmod -R g+ws `pwd`/app/tmp/cache
sudo chmod -R g+ws `pwd`/app/tmp/cache/persistent
sudo chmod -R g+ws `pwd`/app/tmp/cache/models
sudo chmod -R g+ws `pwd`/app/tmp/logs
sudo chmod -R g+ws `pwd`/app/files
sudo chmod -R g+ws `pwd`/app/files/scripts/tmp
sudo chown -R $USER:www-data `pwd`
- name: DB Update
run: |
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.osuser" $USER'
sudo -E su $USER -c 'app/Console/cake Admin runUpdates'
sudo -E su $USER -c 'app/Console/cake Admin schemaDiagnostics'
# Resque perms
sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp
sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp
- name: Configure MISP
run: |
sudo -u $USER app/Console/cake userInit -q | sudo tee ./key.txt
echo "AUTH=`cat key.txt`" >> $GITHUB_ENV
sudo -u $USER app/Console/cake Admin setSetting "Session.autoRegenerate" 0
sudo -u $USER app/Console/cake Admin setSetting "Session.timeout" 600
sudo -u $USER app/Console/cake Admin setSetting "Session.cookieTimeout" 3600
sudo -u $USER app/Console/cake Admin setSetting "MISP.host_org_id" 1
sudo -u $USER app/Console/cake Admin setSetting "MISP.email" "info@admin.test"
sudo -u $USER app/Console/cake Admin setSetting "MISP.disable_emailing" false
sudo -u $USER app/Console/cake Admin setSetting --force "debug" true
sudo -u $USER app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false
sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1"
sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_port" 6379
sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_database" 13
sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_password" ""
sudo -u $USER app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test"
sudo -u $USER app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg"
sudo -u $USER app/Console/cake Admin setSetting "GnuPG.password" "travistest"
sudo -u $USER app/Console/cake Admin setSetting "MISP.download_gpg_from_homedir" 1
# Fill database with basic MISP schema
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';"
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant usage on *.* to misp@'%' identified by 'blah';"
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant all privileges on misp.* to misp@'%';"
mysql -h 127.0.0.1 --port 3306 -u misp -pblah misp < INSTALL/MYSQL.sql
- name: Configure ZMQ
run: |
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_host" "127.0.0.1"
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_port" 6379
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_database" 1
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_password" ""
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" 1
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_audit_notifications_enable" 1
# configure apache virtual hosts
sudo mkdir -p /etc/apache2/sites-available
sudo cp -f build/github-action-ci-apache /etc/apache2/sites-available/misp.conf
sudo sed -e "s?%GITHUB_WORKSPACE%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.conf
sudo sed -e "s?%HOST%?${HOST}?g" --in-place /etc/apache2/sites-available/misp.conf
sudo a2dissite 000-default
sudo a2ensite misp.conf
cat /etc/apache2/sites-enabled/misp.conf
sudo a2enmod rewrite
sudo systemctl start --no-block apache2
- name: Update Galaxies
run: sudo -E su $USER -c 'app/Console/cake Admin updateGalaxies'
# MISP configuration
sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php
sudo cp build/database.php app/Config/database.php
sudo cp app/Config/core.default.php app/Config/core.php
sudo cp app/Config/config.default.php app/Config/config.php
sudo cp build/email.php app/Config/email.php
- name: Update Taxonomies
run: sudo -E su $USER -c 'app/Console/cake Admin updateTaxonomies'
# GPG setup
sudo mkdir `pwd`/.gnupg
# /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS
sudo cp -a /dev/urandom /dev/random
sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/build/gpg
sudo gpg --list-secret-keys --homedir `pwd`/.gnupg
- name: Update Warninglists
run: sudo -E su $USER -c 'app/Console/cake Admin updateWarningLists --verbose'
# change perms
sudo chown -R $USER:www-data `pwd`
sudo chown -R www-data:www-data `pwd`/.gnupg
sudo chmod -R 700 `pwd`/.gnupg
sudo usermod -a -G www-data $USER
sudo chmod -R 777 `pwd`/app/Plugin/CakeResque/tmp/
# Ensure the perms of config files
sudo chown -R $USER:www-data `pwd`/app/Config
sudo chmod -R 777 `pwd`/app/Config
app/Console/cake Admin setSetting "MISP.server_settings_skip_backup_rotate" 1
sudo chown -R $USER:www-data `pwd`/app/Config
sudo chmod -R 777 `pwd`/app/Config
- name: Update Noticelists
run: sudo -E su $USER -c 'app/Console/cake Admin updateNoticeLists'
- name: Update Object Templates
run: sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates 1'
- name: Turn MISP live
run: sudo -E su $USER -c 'app/Console/cake Live 1'
- name: Check if Redis is ready
run: sudo -E su $USER -c 'app/Console/cake Admin redisReady'
- name: Start workers
run: |
sudo chmod +x app/Console/worker/start.sh
sudo -u www-data 'app/Console/worker/start.sh'
# fix perms (?)
namei -m /home/runner/work
sudo chmod +x /home/runner/work
sudo chmod +x /home/runner
sudo chmod +x /home
sudo chmod +x /
- name: Python setup
run: |
sudo chmod 777 ./key.txt
sudo chmod -R 777 ./tests
# Start workers
# Dirty install python stuff
python3 -m virtualenv -p python3 ./venv
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python"'
. ./venv/bin/activate
export PYTHONPATH=$PYTHONPATH:./app/files/scripts
pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara pytest
deactivate
# Dirty install python stuff
python3 -m virtualenv -p python3 ./venv
app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python"
. ./venv/bin/activate
export PYTHONPATH=$PYTHONPATH:./app/files/scripts
pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara pytest
deactivate
- name: DB Update
run: |
app/Console/cake Admin setSetting "MISP.osuser" $USER
app/Console/cake Admin runUpdates
app/Console/cake Admin schemaDiagnostics
- name: Configure MISP
run: |
app/Console/cake User init | sudo tee ./key.txt
echo "AUTH=`cat key.txt`" >> $GITHUB_ENV
app/Console/cake Admin setSetting "Session.autoRegenerate" 0
app/Console/cake Admin setSetting "Session.timeout" 600
app/Console/cake Admin setSetting "Session.cookieTimeout" 3600
app/Console/cake Admin setSetting "MISP.host_org_id" 1
app/Console/cake Admin setSetting "MISP.email" "info@admin.test"
app/Console/cake Admin setSetting "MISP.disable_emailing" false
app/Console/cake Admin setSetting --force "debug" true
app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false
app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1"
app/Console/cake Admin setSetting "MISP.redis_port" 6379
app/Console/cake Admin setSetting "MISP.redis_database" 13
app/Console/cake Admin setSetting "MISP.redis_password" ""
app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test"
app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg"
app/Console/cake Admin setSetting "GnuPG.password" "travistest"
app/Console/cake Admin setSetting "MISP.download_gpg_from_homedir" 1
app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_host" "127.0.0.1"
app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_port" 6379
app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_database" 1
app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_password" ""
app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" 1
app/Console/cake Admin setSetting "Plugin.ZeroMQ_audit_notifications_enable" 1
- name: Update Galaxies
run: app/Console/cake Admin updateGalaxies
- name: Update Taxonomies
run: app/Console/cake Admin updateTaxonomies
- name: Update Warninglists
run: app/Console/cake Admin updateWarningLists --verbose
- name: Update Noticelists
run: app/Console/cake Admin updateNoticeLists
- name: Update Object Templates
run: app/Console/cake Admin updateObjectTemplates 1
- name: Turn MISP live
run: app/Console/cake Admin live 1
- name: Check if Redis is ready
run: app/Console/cake Admin redisReady
- name: Start workers
run: |
sudo chmod +x app/Console/worker/start.sh
sudo -u www-data 'app/Console/worker/start.sh'
- name: Test if apache is working
run: |
sudo systemctl status apache2 --no-pager -l
sudo apache2ctl -S
curl http://${HOST}
sudo chmod -R 777 PyMISP
pushd PyMISP
echo 'url = "http://'${HOST}'"' >> tests/keys.py
echo 'key = "'${AUTH}'"' >> tests/keys.py
cat tests/keys.py
popd
. ./venv/bin/activate
pushd tests
bash ./build-test.sh
popd
deactivate
sudo systemctl status apache2 --no-pager -l
sudo apache2ctl -S
curl -sS http://${HOST}
- name: Check if dependencies working as expected
run: |
sudo chmod -R 777 PyMISP
pushd PyMISP
echo 'url = "http://'${HOST}'"' >> tests/keys.py
echo 'key = "'${AUTH}'"' >> tests/keys.py
cat tests/keys.py
popd
. ./venv/bin/activate
pushd tests
bash ./build-test.sh
popd
deactivate
- name: Run PHP tests
run: |
./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
sudo -u www-data ./app/Vendor/bin/phpunit app/Test/
./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ -e php,ctp app/
sudo -u www-data ./app/Vendor/bin/phpunit app/Test/
- name: Clone test files
uses: actions/checkout@v4
@ -249,31 +241,30 @@ jobs:
repository: viper-framework/viper-test-files
path: PyMISP/tests/viper-test-files
- name: Run tests
run: |
pushd tests
./curl_tests_GH.sh $AUTH $HOST
popd
pushd tests
./curl_tests_GH.sh $AUTH $HOST
popd
sudo chmod -R g+ws `pwd`/app/tmp/logs
sudo chmod -R g+ws `pwd`/app/tmp/logs
. ./venv/bin/activate
pushd PyMISP
cp tests/keys.py .
python -m pytest -v --durations=0 tests/test_mispevent.py
python -m pytest -v --durations=0 tests/testlive_comprehensive.py
popd
python tests/testlive_security.py -v
python tests/testlive_sync.py
python tests/testlive_comprehensive_local.py -v
cp PyMISP/tests/keys.py PyMISP/examples/events/
pushd PyMISP/examples/events/
python ./create_massive_dummy_events.py -l 5 -a 30
popd
pip install jsonschema
python tools/misp-feed/validate.py
deactivate
. ./venv/bin/activate
pushd PyMISP
cp tests/keys.py .
python -m pytest -v --durations=0 tests/test_mispevent.py
python -m pytest -v --durations=0 tests/testlive_comprehensive.py
popd
python tests/testlive_comprehensive_local.py -v
python tests/testlive_sync.py -v
python tests/testlive_security.py -v
cp PyMISP/tests/keys.py PyMISP/examples/events/
pushd PyMISP/examples/events/
python ./create_massive_dummy_events.py -l 5 -a 30
popd
pip install jsonschema
python tools/misp-feed/validate.py
deactivate
- name: Check requirements.txt
run: python tests/check_requirements.py
@ -282,13 +273,13 @@ jobs:
if: ${{ always() }}
# update logs_test.sh when adding more logsources here
run: |
tail -n +1 `pwd`/app/tmp/logs/*
tail -n +1 /var/log/apache2/*.log
tail -n +1 `pwd`/app/tmp/logs/*
tail -n +1 /var/log/apache2/*.log
sudo -u $USER app/Console/cake Log export /tmp/logs.json.gz --without-changes
zcat /tmp/logs.json.gz
app/Console/cake Log export /tmp/logs.json.gz --without-changes
zcat /tmp/logs.json.gz
- name: Errors in Logs
if: ${{ always() }}
run: |
./tests/logs_tests.sh
./tests/logs_tests.sh

9
.gitignore vendored
View File

@ -82,6 +82,15 @@ app/Lib/EventWarning/Custom/*
!/app/webroot/img/orgs/MISP.png
!/app/webroot/img/orgs/NATO.png
!/app/webroot/img/orgs/NCIRC.png
/app/files/img/custom/*
!/app/files/img/custom/empty
!/app/files/img/orgs
/app/files/img/orgs/*
!/app/files/img/orgs/ADMIN.png
!/app/files/img/orgs/MIL.be.png
!/app/files/img/orgs/MISP.png
!/app/files/img/orgs/NATO.png
!/app/files/img/orgs/NCIRC.png
/app/Config/bootstrap.php
/app/Config/database.php
/app/Config/core.php

195
.travis.yml
View File

@ -1,195 +0,0 @@
language: php
php:
- 7.2
- 7.3
- 7.4
- nightly
services:
- redis
sudo: required
dist: bionic
addons:
mariadb: '10.2'
hosts:
- misp.local
- localhost
before_install:
- git config --global user.name "TravisCI"
- export PATH="$HOME/.local/bin:$PATH"
install:
- date
- sudo apt-get -y update
# Install haveged, because Travis lacks entropy.
- sudo apt-get -y install haveged python3 python3-venv python3-pip python3-dev python3-nose python3-redis python3-lxml python3-dateutil python3-msgpack libxml2-dev libzmq3-dev zlib1g-dev apache2 curl php-mysql php-dev php-cli libapache2-mod-php libfuzzy-dev php-mbstring libonig4 php-json php-xml php-opcache php-readline php-redis php-gnupg php-gd
- sudo pip3 install --upgrade pip setuptools requests
- sudo pip3 install --upgrade -r requirements.txt
- sudo pip3 install --upgrade -r requirements-dev.txt
- pip3 install --user poetry
- phpenv rehash
- sudo mkdir $HOME/.composer ; sudo chown $USER:www-data $HOME/.composer
- pushd app
- sudo -H -u $USER php composer.phar install --no-progress
- sudo phpenmod redis
- sudo phpenmod gnupg
- popd
- cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php
# Set perms
- sudo chown -R $USER:www-data `pwd`
- sudo chmod -R 775 `pwd`
- sudo chmod -R g+ws `pwd`/app/tmp
- sudo chmod -R g+ws `pwd`/app/tmp/cache
- sudo chmod -R g+ws `pwd`/app/tmp/cache/persistent
- sudo chmod -R g+ws `pwd`/app/tmp/cache/models
- sudo chmod -R g+ws `pwd`/app/tmp/logs
- sudo chmod -R g+ws `pwd`/app/files
- sudo chmod -R g+ws `pwd`/app/files/scripts/tmp
- sudo chown -R $USER:www-data `pwd`
# Resque perms
- sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp
- sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp
# install MySQL
- sudo chmod -R 777 `pwd`/INSTALL
- mysql -u root -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';"
- mysql -u root -e 'create database misp;'
- mysql -u root -e "grant usage on *.* to misp@localhost identified by 'blah'";
- mysql -u root -e "grant all privileges on misp.* to misp@localhost;"
- mysql -u misp -pblah misp < INSTALL/MYSQL.sql
# configure apache virtual hosts
- sudo chmod -R 777 `pwd`/build
- sudo mkdir -p /etc/apache2/sites-available
- sudo cp -f build/travis-ci-apache /etc/apache2/sites-available/misp.local.conf
- sudo sed -e "s?%TRAVIS_BUILD_DIR%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.local.conf
- sudo a2dissite 000-default
- sudo a2ensite misp.local.conf
- sudo a2enmod rewrite
- sudo service apache2 restart
# MISP configuration
- sudo chmod -R 777 `pwd`/travis
- sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php
- sudo cp travis/database.php app/Config/database.php
- sudo cp app/Config/core.default.php app/Config/core.php
- sudo cp app/Config/config.default.php app/Config/config.php
- sudo cp travis/email.php app/Config/email.php
# Ensure the perms
- sudo chown -R $USER:www-data `pwd`/app/Config
- sudo chmod -R 770 `pwd`/app/Config
# GPG setup
- sudo mkdir `pwd`/.gnupg
# /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS
- sudo cp -a /dev/urandom /dev/random
- sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/travis/gpg
- sudo gpg --list-secret-keys --homedir `pwd`/.gnupg
# change perms
- sudo chown -R $USER:www-data `pwd`
- sudo chmod +x /home/travis/build
- sudo chmod +x /home/travis
- sudo chmod +x /home
- sudo chmod -R 770 `pwd`/.gnupg
# Get authkey
- sudo usermod -a -G www-data $USER
- sudo -E su $USER -c 'app/Console/cake Admin runUpdates'
- sudo -E su $USER -c 'app/Console/cake userInit -q | sudo tee ./key.txt'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.autoRegenerate" 0'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.timeout" 600'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.cookieTimeout" 3600'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.host_org_id" 1'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.email" "info@admin.test"'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.disable_emailing" false'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "debug" true'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1"'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_port" 6379'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_database" 13'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_password" ""'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test"'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg"'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.password" "travistest"'
- sudo -E su $USER -c 'app/Console/cake Admin updateGalaxies'
- sudo -E su $USER -c 'app/Console/cake Admin updateTaxonomies'
- sudo -E su $USER -c 'app/Console/cake Admin updateWarningLists'
- sudo -E su $USER -c 'app/Console/cake Admin updateNoticeLists'
- sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates 1'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" true'
- sudo -E su $USER -c 'app/Console/cake Live 1'
- sudo chmod 777 ./key.txt
- sudo chmod -R 777 ./tests
# Start workers
- sudo chmod +x app/Console/worker/start.sh
- sudo -E su $USER -c 'app/Console/worker/start.sh &'
- sleep 10
# Dirty install python stuff
- virtualenv -p python3.6 ./venv
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$TRAVIS_BUILD_DIR/venv/bin/python"'
- . ./venv/bin/activate
- pushd cti-python-stix2
- pip install .
- popd
- pushd PyMISP
- pip install .[fileobjects]
- popd
- pip install stix zmq redis plyara
- deactivate
before_script:
- curl http://misp.local
- AUTH=`cat key.txt`
- sudo chmod -R 777 PyMISP
- pushd PyMISP
- echo 'url = "http://misp.local"' >> tests/keys.py
- echo 'key = "'${AUTH}'"' >> tests/keys.py
- cat tests/keys.py
- popd
script:
- ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
- ./app/Vendor/bin/phpunit app/Test/ComplexTypeToolTest.php
- ./app/Vendor/bin/phpunit app/Test/JSONConverterToolTest.php
# Ensure the perms
- sudo chown -R $USER:www-data `pwd`/app/Config
- sudo chmod -R 770 `pwd`/app/Config
- pushd tests
- ./curl_tests.sh $AUTH
- popd
- pushd PyMISP
- git submodule init
- git submodule update
- travis_retry poetry install -E fileobjects -E openioc -E virustotal -E docs -E pdfexport
- poetry run python tests/testlive_comprehensive.py
- poetry run python tests/test_mispevent.py
- popd
- cp PyMISP/tests/keys.py PyMISP/examples/events/
- pushd PyMISP/examples/events/
- poetry run python ./create_massive_dummy_events.py -l 5 -a 30
- popd
- python3 tools/misp-feed/validate.py
after_failure:
- curl http://misp.local
- cat /etc/apache2/sites-available/misp.local.conf
- sudo tail -n +1 `pwd`/app/tmp/logs/*
- sudo ls -l /var/log/apache2
- sudo cat /var/log/apache2/error.log
- sudo cat /var/log/apache2/misp.local_error.log
- sudo cat /var/log/apache2/misp.local_access.log
- pwd
notifications:
webhooks:
urls:
- https://webhooks.gitter.im/e/05e30284086a8e948d31
on_success: change # options: [always|never|change] default: always
on_failure: always # options: [always|never|change] default: always
on_start: never # options: [always|never|change] default: always
after_success:
- sudo tail -n +1 `pwd`/app/tmp/logs/*
- coveralls
- coverage report
- coverage xml
- codecov

2
PyMISP

@ -1 +1 @@
Subproject commit 81f5d596a7dd5cb1ca7213ac4fbdf07b402420b7
Subproject commit 492cfba2d2ad015d3fcda6e16c221fdefd93eca2

2
VERSION.json
View File

@ -1 +1 @@
{"major":2, "minor":4, "hotfix":183}
{"major":2, "minor":4, "hotfix":185}

116
app/Console/Command/AdminShell.php
View File

@ -46,11 +46,11 @@ class AdminShell extends AppShell
'help' => __('Update the JSON definition of taxonomies.'),
));
$parser->addSubcommand('setSetting', [
'help' => __('Set setting in PHP config file.'),
'help' => __('Set setting in MISP config'),
'parser' => [
'arguments' => [
'name' => ['help' => __('Setting name'), 'required' => true],
'value' => ['help' => __('Setting value'), 'required' => true],
'value' => ['help' => __('Setting value')],
],
'options' => [
'force' => [
@ -72,7 +72,7 @@ class AdminShell extends AppShell
'help' => __('Set if MISP instance is live and accessible for users.'),
'parser' => [
'arguments' => [
'state' => ['help' => __('Set Live state')],
'state' => ['help' => __('Set Live state (boolean). If not provided, current state will be printed.')],
],
],
]);
@ -85,6 +85,14 @@ class AdminShell extends AppShell
],
],
]);
$parser->addSubcommand('isEncryptionKeyValid', [
'help' => __('Check if current encryption key is valid.'),
'parser' => [
'options' => [
'encryptionKey' => ['help' => __('Encryption key to test. If not provided, current key will be used.')],
],
],
]);
$parser->addSubcommand('dumpCurrentDatabaseSchema', [
'help' => __('Dump current database schema to JSON file.'),
]);
@ -109,6 +117,20 @@ class AdminShell extends AppShell
$parser->addSubcommand('configLint', [
'help' => __('Check if settings has correct value.'),
]);
$parser->addSubcommand('createZmqConfig', [
'help' => __('Create config file for ZeroMQ server.'),
]);
$parser->addSubcommand('scanAttachment', [
'help' => __('Scan attachments with AV.'),
'parser' => [
'arguments' => [
'type' => ['help' => __('all, Attribute or ShadowAttribute'), 'required' => true],
'attributeId' => ['help' => __('ID to scan.')],
'jobId' => ['help' => __('Job ID')],
],
],
]);
return $parser;
}
@ -485,32 +507,47 @@ class AdminShell extends AppShell
}
}
}
echo json_encode($result, JSON_PRETTY_PRINT) . PHP_EOL;
$this->out($this->json($result));
}
public function setSetting()
{
list($setting_name, $value) = $this->args;
if ($value === 'false') {
$value = 0;
} elseif ($value === 'true') {
$value = 1;
}
if ($this->params['null']) {
list($settingName) = $this->args;
if ($this->params['null'] && isset($this->args[1])) {
$this->error(__('Trying to set setting to null value, but value was provided.'));
} else if ($this->params['null']) {
$value = null;
} elseif (isset($this->args[1])) {
$value = $this->args[1];
} else {
$this->error(__('No setting value provided.'));
}
$cli_user = array('id' => 0, 'email' => 'SYSTEM', 'Organisation' => array('name' => 'SYSTEM'));
if (empty($setting_name) || ($value === null && !$this->params['null'])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Set setting'] . PHP_EOL);
}
$setting = $this->Server->getSettingData($setting_name);
$setting = $this->Server->getSettingData($settingName);
if (empty($setting)) {
$message = 'Invalid setting "' . $setting_name . '". Please make sure that the setting that you are attempting to change exists and if a module parameter, the modules are running.' . PHP_EOL;
$message = 'Invalid setting "' . $settingName . '". Please make sure that the setting that you are attempting to change exists and if a module parameter, the modules are running.' . PHP_EOL;
$this->error(__('Setting change rejected.'), $message);
}
$result = $this->Server->serverSettingsEditValue($cli_user, $setting, $value, $this->params['force']);
// Convert value to boolean or to int
if ($value !== null) {
if ($setting['type'] === 'boolean') {
$value = $this->toBoolean($value);
} else if ($setting['type'] === 'numeric') {
if (is_numeric($value)) {
$value = (int)$value;
} elseif ($value === 'true' || $value === 'false') {
$value = $value === 'true' ? 1 : 0; // special case for `debug` setting
} else {
$this->error(__('Setting "%s" change rejected.', $settingName), __('Provided value %s is not a number.', $value));
}
}
}
$result = $this->Server->serverSettingsEditValue('SYSTEM', $setting, $value, $this->params['force']);
if ($result === true) {
$this->out(__('Setting "%s" changed to %s', $setting_name, is_string($value) ? '"' . $value . '"' : (string)$value));
$this->out(__('Setting "%s" changed to %s', $settingName, is_string($value) ? '"' . $value . '"' : json_encode($value)));
} else {
$message = __("The setting change was rejected. MISP considers the requested setting value as invalid and would lead to the following error:\n\n\"%s\"\n\nIf you still want to force this change, please supply the --force argument.\n", $result);
$this->error(__('Setting change rejected.'), $message);
@ -648,6 +685,8 @@ class AdminShell extends AppShell
*/
public function change_authkey()
{
$this->deprecated('cake user change_authkey [user_id]');
if (empty($this->args[0])) {
echo 'MISP apikey command line tool' . PHP_EOL . 'To assign a new random API key for a user: ' . APP . 'Console/cake Admin change_authkey [user_email]' . PHP_EOL . 'To assign a fixed API key: ' . APP . 'Console/cake Admin change_authkey [user_email] [authkey]' . PHP_EOL;
die();
@ -787,6 +826,8 @@ class AdminShell extends AppShell
*/
public function UserIP()
{
$this->deprecated('cake user user_ips [user_id]');
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get IPs for user ID'] . PHP_EOL);
}
@ -814,6 +855,8 @@ class AdminShell extends AppShell
*/
public function IPUser()
{
$this->deprecated('cake user ip_user [ip]');
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get user ID for user IP'] . PHP_EOL);
}
@ -839,8 +882,8 @@ class AdminShell extends AppShell
public function scanAttachment()
{
$input = $this->args[0];
$attributeId = isset($this->args[1]) ? $this->args[1] : null;
$jobId = isset($this->args[2]) ? $this->args[2] : null;
$attributeId = $this->args[1] ?? null;
$jobId = $this->args[2] ?? null;
$this->loadModel('AttachmentScan');
$result = $this->AttachmentScan->scan($input, $attributeId, $jobId);
@ -951,7 +994,7 @@ class AdminShell extends AppShell
$newStatus = $this->toBoolean($this->args[0]);
$overallSuccess = false;
try {
$redis = $this->Server->setupRedisWithException();
$redis = RedisTool::init();
if ($newStatus) {
$redis->del('misp:live');
$this->out('Set live status to True in Redis.');
@ -980,7 +1023,7 @@ class AdminShell extends AppShell
} else {
$this->out('Current status:');
$this->out('PHP Config file: ' . (Configure::read('MISP.live') ? 'True' : 'False'));
$newStatus = $this->Server->setupRedisWithException()->get('misp:live');
$newStatus = RedisTool::init()->get('misp:live');
$this->out('Redis: ' . ($newStatus !== '0' ? 'True' : 'False'));
}
}
@ -1031,6 +1074,27 @@ class AdminShell extends AppShell
$this->out(__('New encryption key "%s" saved into config file.', $new));
}
public function isEncryptionKeyValid()
{
$encryptionKey = $this->params['encryptionKey'] ?? null;
if ($encryptionKey === null) {
$encryptionKey = Configure::read('Security.encryption_key');
}
if (!$encryptionKey) {
$this->error('No encryption key provided');
}
/** @var SystemSetting $systemSetting */
$systemSetting = ClassRegistry::init('SystemSetting');
try {
$systemSetting->isEncryptionKeyValid($encryptionKey);
$this->Server->isEncryptionKeyValid($encryptionKey);
} catch (Exception $e) {
$this->error($e->getMessage(), __('Probably provided encryption key is invalid'));
}
}
public function redisMemoryUsage()
{
$redis = RedisTool::init();
@ -1240,4 +1304,10 @@ class AdminShell extends AppShell
$this->Job->saveField('message', __('Database truncated: ' . $table));
}
}
public function createZmqConfig()
{
$this->Server->getPubSubTool()->createConfigFile();
$this->err("Config file created in " . PubSubTool::SCRIPTS_TMP);
}
}

15
app/Console/Command/AppShell.php
View File

@ -31,15 +31,13 @@ require_once dirname(__DIR__) . '/../Model/Attribute.php'; // FIXME workaround
*/
abstract class AppShell extends Shell
{
public $tasks = array('ConfigLoad');
/** @var BackgroundJobsTool */
private $BackgroundJobsTool;
public function initialize()
{
$this->ConfigLoad = $this->Tasks->load('ConfigLoad');
$this->ConfigLoad->execute();
$configLoad = $this->Tasks->load('ConfigLoad');
$configLoad->execute();
parent::initialize();
}
@ -84,6 +82,15 @@ abstract class AppShell extends Shell
}
}
/**
* @param string $newCommand
* @return void
*/
protected function deprecated($newCommand)
{
$this->err("<warning>Warning: This method is deprecated. Next time please use `$newCommand`.</warning>");
}
/**
* @return BackgroundJobsTool
* @throws Exception

2
app/Console/Command/AuthkeyShell.php
View File

@ -12,7 +12,7 @@ class AuthkeyShell extends AppShell {
public function main()
{
$this->err('This method is deprecated. Next time please use `cake user change_authkey [user] [authkey]` command.');
$this->deprecated('cake user change_authkey [user] [authkey]');
if (!isset($this->args[0]) || empty($this->args[0])) echo 'MISP authkey reset command line tool.' . PHP_EOL . 'To assign a new authkey for a user:' . PHP_EOL . APP . 'Console/cake Authkey [email] [auth_key | optional]' . PHP_EOL;
else {

2
app/Console/Command/BaseurlShell.php
View File

@ -11,7 +11,7 @@ class BaseurlShell extends AppShell {
public function main()
{
$this->err('This method is deprecated. Next time please use `cake admin setSetting MISP.baseurl [baseurl]` command.');
$this->deprecated('cake admin setSetting MISP.baseurl [baseurl]');
$baseurl = $this->args[0];
$result = $this->Server->testBaseURL($baseurl);

59
app/Console/Command/EventShell.php
View File

@ -53,10 +53,21 @@ class EventShell extends AppShell
$parser->addSubcommand('mergeTags', [
'help' => __('Merge tags'),
'parser' => [
'arguments' => array(
'arguments' => [
'source' => ['help' => __('Source tag ID or name. Source tag will be deleted.'), 'required' => true],
'destination' => ['help' => __('Destination tag ID or name.'), 'required' => true],
)
],
],
]);
$parser->addSubcommand('reportValidationIssuesAttributes', [
'help' => __('Report validation issues on attributes'),
]);
$parser->addSubcommand('normalizeIpAddress', [
'help' => __('Normalize IP address format in old events'),
'parser' => [
'options' => [
'dry-run' => ['help' => __('Just show what changes will be made.'), 'boolean' => true],
],
],
]);
return $parser;
@ -636,18 +647,28 @@ class EventShell extends AppShell
}
}
/**
* @param int $userId
* @return array
*/
private function getUser($userId)
public function reportValidationIssuesAttributes()
{
$user = $this->User->getAuthUser($userId, true);
if (empty($user)) {
$this->error("User with ID $userId does not exist.");
foreach ($this->Event->Attribute->reportValidationIssuesAttributes() as $validationIssue) {
echo $this->json($validationIssue) . "\n";
}
}
public function normalizeIpAddress()
{
$dryRun = $this->param('dry-run');
$count = 0;
foreach ($this->Event->Attribute->normalizeIpAddress($dryRun) as $attribute) {
$count++;
echo JsonTool::encode($attribute) . "\n";
}
if ($dryRun) {
$this->err(__n("%s attribute to fix", "%s attributes to fix", $count, $count));
} else {
$this->err(__n("%s attribute fixed", "%s attributes fixed", $count, $count));
}
Configure::write('CurrentUserId', $user['id']); // for audit logging purposes
return $user;
}
public function generateTopCorrelations()
@ -668,4 +689,18 @@ class EventShell extends AppShell
$this->Job->save($job);
}
}
/**
* @param int $userId
* @return array
*/
private function getUser($userId)
{
$user = $this->User->getAuthUser($userId, true);
if (empty($user)) {
$this->error("User with ID $userId does not exist.");
}
Configure::write('CurrentUserId', $user['id']); // for audit logging purposes
return $user;
}
}

5
app/Console/Command/LiveShell.php
View File

@ -1,8 +1,9 @@
<?php
/*
/**
* Enable/disable misp
*
* arg0 = [0|1]
* @deprecated Use AdminShell::live instead
*/
class LiveShell extends AppShell {
@ -10,6 +11,8 @@ class LiveShell extends AppShell {
public function main()
{
$this->deprecated('cake admin live [0|1]');
$live = $this->args[0];
if ($live != 0 && $live != 1) {
echo 'Invalid parameters. Usage: /var/www/MISP/app/Console/cake Live [0|1]';

2
app/Console/Command/PasswordShell.php
View File

@ -12,7 +12,7 @@ class PasswordShell extends AppShell {
public function main()
{
$this->err('This method is deprecated. Next time please use `cake user change_pw [user] [password]` command.');
$this->deprecated('cake user change_pw [user] [password]');
if (!isset($this->args[0]) || empty($this->args[0]) || !isset($this->args[1]) || empty($this->args[1])) echo 'MISP password reset command line tool.' . PHP_EOL . 'To assign a new password for a user:' . PHP_EOL . APP . 'Console/cake Password [email] [password]' . PHP_EOL;
else {

26
app/Console/Command/StartWorkerShell.php
View File

@ -37,24 +37,32 @@ class StartWorkerShell extends AppShell
public function main()
{
$pid = getmypid();
if ($pid === false) {
throw new RuntimeException("Could not get current process ID");
}
$this->worker = new Worker(
[
'pid' => getmypid(),
'pid' => $pid,
'queue' => $this->args[0],
'user' => ProcessTool::whoami(),
]
);
$this->maxExecutionTime = (int)$this->params['maxExecutionTime'];
$queue = $this->worker->queue();
$backgroundJobTool = $this->getBackgroundJobsTool();
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - starting to process background jobs...");
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$queue}] - starting to process background jobs...");
while (true) {
$this->checkMaxExecutionTime();
$job = $this->getBackgroundJobsTool()->dequeue($this->worker->queue());
$job = $backgroundJobTool->dequeue($queue);
if ($job) {
$this->runJob($job);
$backgroundJobTool->removeFromRunning($this->worker, $job);
}
}
}
@ -64,7 +72,7 @@ class StartWorkerShell extends AppShell
*/
private function runJob(BackgroundJob $job)
{
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}...");
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}");
try {
$job->setStatus(BackgroundJob::STATUS_RUNNING);
@ -73,12 +81,16 @@ class StartWorkerShell extends AppShell
CakeLog::info("[JOB ID: {$job->id()}] - started command `$command`.");
$this->getBackgroundJobsTool()->update($job);
$job->run();
$start = microtime(true);
$job->run(function (array $status) use ($job) {
$this->getBackgroundJobsTool()->markAsRunning($this->worker, $job, $status['pid']);
});
$duration = number_format(microtime(true) - $start, 3, '.', '');
if ($job->status() === BackgroundJob::STATUS_COMPLETED) {
CakeLog::info("[JOB ID: {$job->id()}] - completed.");
CakeLog::info("[JOB ID: {$job->id()}] - successfully completed in $duration seconds.");
} else {
CakeLog::error("[JOB ID: {$job->id()}] - failed with error code {$job->returnCode()}. STDERR: {$job->error()}. STDOUT: {$job->output()}.");
CakeLog::error("[JOB ID: {$job->id()}] - failed with error code {$job->returnCode()} after $duration seconds. STDERR: {$job->error()}. STDOUT: {$job->output()}.");
}
} catch (Exception $exception) {
CakeLog::error("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - job ID: {$job->id()} failed with exception: {$exception->getMessage()}");

2
app/Console/Command/Task/ConfigLoadTask.php
View File

@ -3,8 +3,6 @@ class ConfigLoadTask extends Shell
{
public function execute()
{
Configure::load('config');
if (Configure::read('MISP.system_setting_db')) {
App::uses('SystemSetting', 'Model');
SystemSetting::setGlobalSetting();

6
app/Console/Command/UserInitShell.php
View File

@ -1,7 +1,13 @@
<?php
/**
* @deprecated
*/
class UserInitShell extends AppShell {
public $uses = array('User', 'Role', 'Organisation', 'Server', 'ConnectionManager');
public function main() {
$this->deprecated('cake user init');
if (!Configure::read('Security.salt')) {
$this->loadModel('Server');
$this->Server->serverSettingsSaveValue('Security.salt', $this->User->generateRandomPassword(32));

135
app/Console/Command/UserShell.php
View File

@ -3,10 +3,11 @@
/**
* @property User $User
* @property Log $Log
* @property UserLoginProfile $UserLoginProfile
*/
class UserShell extends AppShell
{
public $uses = ['User', 'Log'];
public $uses = ['User', 'Log', 'UserLoginProfile'];
public function getOptionParser()
{
@ -22,16 +23,24 @@ class UserShell extends AppShell
],
]
]);
$parser->addSubcommand('init', [
'help' => __('Create default role, organisation and user when not exists.'),
]);
$parser->addSubcommand('authkey', [
'help' => __('Get information about given authkey.'),
'parser' => [
'arguments' => [
'authkey' => ['help' => __('Authentication key. If not provide, it will be read from STDIN.')],
'authkey' => ['help' => __('Authentication key. If not provided, it will be read from STDIN.')],
],
]
]);
$parser->addSubcommand('authkey_valid', [
'help' => __('Check if given authkey by STDIN is valid.'),
'parser' => [
'options' => [
'disableStdLog' => ['help' => __('Do not show logs in STDOUT or STDERR.'), 'boolean' => true],
],
],
]);
$parser->addSubcommand('block', [
'help' => __('Immediately block user.'),
@ -104,6 +113,14 @@ class UserShell extends AppShell
],
],
]);
$parser->addSubcommand('ip_country', [
'help' => __('Get country for given IP address'),
'parser' => [
'arguments' => [
'ip' => ['help' => __('IPv4 or IPv6 address.'), 'required' => true],
]
],
]);
$parser->addSubcommand('require_password_change_for_old_passwords', [
'help' => __('Trigger forced password change on next login for users with an old (older than x days) password.'),
'parser' => [
@ -121,7 +138,7 @@ class UserShell extends AppShell
public function list()
{
$userId = isset($this->args[0]) ? $this->args[0] : null;
$userId = $this->args[0] ?? null;
if ($userId) {
$conditions = ['OR' => [
'User.id' => $userId,
@ -163,13 +180,24 @@ class UserShell extends AppShell
}
}
public function init()
{
if (!Configure::read('Security.salt')) {
$this->loadModel('Server');
$this->Server->serverSettingsSaveValue('Security.salt', $this->User->generateRandomPassword(32));
}
$authKey = $this->User->init();
if ($authKey === null) {
$this->err('Script aborted: MISP instance already initialised.');
} else {
$this->out($authKey);
}
}
public function authkey()
{
if (isset($this->args[0])) {
$authkey = $this->args[0];
} else {
$authkey = fgets(STDIN); // read line from STDIN
}
$authkey = $this->args[0] ?? fgets(STDIN);
$authkey = trim($authkey);
if (strlen($authkey) !== 40) {
$this->error('Authkey has not valid format.');
@ -212,28 +240,37 @@ class UserShell extends AppShell
*/
public function authkey_valid()
{
if ($this->params['disableStdLog']) {
$this->_useLogger(false);
}
$cache = [];
$randomKey = random_bytes(16);
do {
$advancedAuthKeysEnabled = (bool)Configure::read('Security.advanced_authkeys');
while (true) {
$authkey = fgets(STDIN); // read line from STDIN
$authkey = trim($authkey);
if (strlen($authkey) !== 40) {
fwrite(STDOUT, "0\n"); // authkey is not in valid format
$this->log("Authkey in incorrect format provided.", LOG_WARNING);
echo "0\n"; // authkey is not in valid format
$this->log("Authkey in incorrect format provided, expected 40 chars long string, $authkey provided.", LOG_WARNING);
continue;
}
$time = time();
// Generate hash from authkey to not store raw authkey in memory
$keyHash = sha1($authkey . $randomKey, true);
// If authkey is in cache and is fresh, use info from cache
$time = time();
if (isset($cache[$keyHash]) && $cache[$keyHash][1] > $time) {
fwrite(STDOUT, $cache[$keyHash][0] ? "1\n" : "0\n");
echo $cache[$keyHash][0] ? "1\n" : "0\n";
continue;
}
$user = false;
for ($i = 0; $i < 5; $i++) {
try {
if (Configure::read('Security.advanced_authkeys')) {
if ($advancedAuthKeysEnabled) {
$user = $this->User->AuthKey->getAuthUserByAuthKey($authkey);
} else {
$user = $this->User->getAuthUserByAuthkey($authkey);
@ -251,18 +288,34 @@ class UserShell extends AppShell
}
}
$user = (bool)$user;
if (!$user) {
$start = substr($authkey, 0, 4);
$end = substr($authkey, -4);
$authKeyToStore = $start . str_repeat('*', 32) . $end;
$this->log("Not valid authkey $authKeyToStore provided.", LOG_WARNING);
$valid = null;
} else if ($user['disabled']) {
$valid = false;
} else {
$valid = true;
}
// Cache results for 5 seconds
$cache[$keyHash] = [$user, $time + 5];
fwrite(STDOUT, $user ? "1\n" : "0\n");
} while (true);
echo $valid ? "1\n" : "0\n";
if ($valid) {
// Cache results for 60 seconds if key is valid
$cache[$keyHash] = [true, $time + 60];
} else {
// Cache results for 5 seconds if key is invalid
$cache[$keyHash] = [false, $time + 5];
$start = substr($authkey, 0, 4);
$end = substr($authkey, -4);
$authKeyForLog = $start . str_repeat('*', 32) . $end;
if ($valid === false) {
$this->log("Authkey $authKeyForLog belongs to user {$user['id']} that is disabled.", LOG_WARNING);
} else {
$this->log("Authkey $authKeyForLog is invalid or expired.", LOG_WARNING);
}
}
}
}
public function block()
@ -305,7 +358,7 @@ class UserShell extends AppShell
$conditions = ['User.disabled' => false]; // fetch just not disabled users
$userId = isset($this->args[0]) ? $this->args[0] : null;
$userId = $this->args[0] ?? null;
if ($userId) {
$conditions['OR'] = [
'User.id' => $userId,
@ -364,7 +417,7 @@ class UserShell extends AppShell
}
$user = $this->getUser($userId);
# validate new authentication key if provided
// validate new authentication key if provided
if (!empty($newkey) && (strlen($newkey) != 40 || !ctype_alnum($newkey))) {
$this->error('The new auth key needs to be 40 characters long and only alphanumeric.');
}
@ -399,7 +452,7 @@ class UserShell extends AppShell
$this->out('<warning>Storing user IP addresses is disabled.</warning>');
}
$ips = $this->User->setupRedisWithException()->smembers('misp:user_ip:' . $user['id']);
$ips = RedisTool::init()->smembers('misp:user_ip:' . $user['id']);
if ($this->params['json']) {
$this->out($this->json($ips));
@ -422,36 +475,50 @@ class UserShell extends AppShell
$this->out('<warning>Storing user IP addresses is disabled.</warning>');
}
$userId = $this->User->setupRedisWithException()->get('misp:ip_user:' . $ip);
$userId = RedisTool::init()->get('misp:ip_user:' . $ip);
if (empty($userId)) {
$this->out('No hits.');
$this->_stop();
}
$user = $this->User->find('first', array(
$user = $this->User->find('first', [
'recursive' => -1,
'conditions' => array('User.id' => $userId),
'conditions' => ['User.id' => $userId],
'fields' => ['id', 'email'],
));
]);
if (empty($user)) {
$this->error("User with ID $userId doesn't exists anymore.");
}
$ipCountry = $this->UserLoginProfile->countryByIp($ip);
if ($this->params['json']) {
$this->out($this->json([
'ip' => $ip,
'id' => $user['User']['id'],
'email' => $user['User']['email'],
'country' => $ipCountry,
]));
} else {
$this->out(sprintf(
'%s==============================%sIP: %s%s==============================%sUser #%s: %s%s==============================%s',
PHP_EOL, PHP_EOL, $ip, PHP_EOL, PHP_EOL, $user['User']['id'], $user['User']['email'], PHP_EOL, PHP_EOL
));
$this->hr();
$this->out("IP: $ip (country $ipCountry)");
$this->hr();
$this->out("User #{$user['User']['id']}: {$user['User']['email']}");
$this->hr();
}
}
public function ip_country()
{
list($ip) = $this->args;
if (!filter_var($ip, FILTER_VALIDATE_IP)) {
$this->error("IP `$ip` is not valid IPv4 or IPv6 address");
}
$this->out($this->UserLoginProfile->countryByIp($ip));
}
public function require_password_change_for_old_passwords()
{
list($days) = $this->args;

119
app/Console/Command/WorkerShell.php Normal file
View File

@ -0,0 +1,119 @@
<?php
declare(strict_types=1);
/**
* @property Job $Job
*/
class WorkerShell extends AppShell
{
public $uses = ['Job'];
public function getOptionParser(): ConsoleOptionParser
{
$parser = parent::getOptionParser();
$parser->addSubcommand('showQueues', [
'help' => __('Show jobs in worker queues'),
]);
$parser->addSubcommand('flushQueue', [
'help' => __('Flush jobs in given queue'),
'parser' => [
'arguments' => [
'queue' => ['help' => __('Queue name'), 'required' => true],
],
],
]);
$parser->addSubcommand('showJobStatus', [
'help' => __('Show job status'),
'parser' => [
'arguments' => [
'job_id' => ['help' => __('Job ID (ID or UUID)'), 'required' => true],
],
],
]);
return $parser;
}
/**
* @throws RedisException
* @throws JsonException
*/
public function showQueues()
{
$tool = $this->getBackgroundJobsTool();
$runningJobs = $tool->runningJobs();
foreach (BackgroundJobsTool::VALID_QUEUES as $queue) {
$this->out("{$queue}:\t{$tool->getQueueSize($queue)}");
$queueJobs = $runningJobs[$queue] ?? [];
foreach ($queueJobs as $jobId => $data) {
$this->out(" - $jobId (" . JsonTool::encode($data) .")");
}
}
}
public function flushQueue()
{
$queue = $this->args[0];
try {
$this->getBackgroundJobsTool()->clearQueue($queue);
} catch (InvalidArgumentException $e) {
$this->error($e->getMessage());
}
}
public function showJobStatus()
{
$processId = $this->args[0];
if (is_numeric($processId)) {
$job = $this->Job->find('first', [
'conditions' => ['Job.id' => $processId],
'recursive' => -1,
]);
if (!$job) {
$this->error('Job not found', "Job with ID {$processId} not found");
}
$this->out($this->json($job['Job']));
$processId = $job['Job']['process_id'];
}
if (!Validation::uuid($processId)) {
$this->error('Job not found', "Job ID must be number or UUID, '$processId' given");
}
$jobStatus = $this->getBackgroundJobsTool()->getJob($processId);
if (!$jobStatus) {
$this->error('Job not found', "Job with UUID {$processId} not found");
}
$jobStatus = $jobStatus->jsonSerialize();
foreach (['createdAt', 'updatedAt'] as $timeField) {
if (isset($jobStatus[$timeField])) {
$jobStatus[$timeField] = date('c', $jobStatus[$timeField]);
}
}
if (isset($jobStatus['status'])) {
$jobStatus['status'] = $this->jobStatusToString($jobStatus['status']);
}
$this->out($this->json($jobStatus));
}
private function jobStatusToString(int $jobStatus)
{
switch ($jobStatus) {
case Job::STATUS_WAITING:
return 'waiting';
case Job::STATUS_RUNNING:
return 'running';
case Job::STATUS_FAILED:
return 'failed';
case Job::STATUS_COMPLETED:
return 'completed';
}
throw new InvalidArgumentException("Invalid job status $jobStatus");
}
}

71
app/Controller/AppController.php
View File

@ -33,13 +33,11 @@ class AppController extends Controller
public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
private $__queryVersion = '157';
public $pyMispVersion = '2.4.183';
private $__queryVersion = '158';
public $pyMispVersion = '2.4.185';
public $phpmin = '7.2';
public $phprec = '7.4';
public $phptoonew = '8.0';
public $pythonmin = '3.6';
public $pythonrec = '3.7';
private $isApiAuthed = false;
public $baseurl = '';
@ -232,6 +230,10 @@ class AppController extends Controller
$this->Security->csrfCheck = false;
$loginByAuthKeyResult = $this->__loginByAuthKey();
if ($loginByAuthKeyResult === false || $this->Auth->user() === null) {
if ($this->IndexFilter->isXhr()) {
throw new ForbiddenException('Authentication failed.');
}
if ($loginByAuthKeyResult === null) {
$this->loadModel('Log');
$this->Log->createLogEntry('SYSTEM', 'auth_fail', 'User', 0, "Failed API authentication. No authkey was provided.");
@ -601,7 +603,7 @@ class AppController extends Controller
if (!empty($user['allowed_ips'])) {
App::uses('CidrTool', 'Tools');
$cidrTool = new CidrTool($user['allowed_ips']);
$remoteIp = $this->_remoteIp();
$remoteIp = $this->User->_remoteIp();
if ($remoteIp === null) {
$this->Auth->logout();
throw new ForbiddenException('Auth key is limited to IP address, but IP address not found');
@ -694,7 +696,7 @@ class AppController extends Controller
return;
}
$remoteAddress = $this->_remoteIp();
$remoteAddress = $this->User->_remoteIp();
$pipe = $redis->pipeline();
// keep for 30 days
@ -737,7 +739,7 @@ class AppController extends Controller
$includeRequestBody = !empty(Configure::read('MISP.log_paranoid_include_post_body')) || $userMonitoringEnabled;
/** @var AccessLog $accessLog */
$accessLog = ClassRegistry::init('AccessLog');
$accessLog->logRequest($user, $this->_remoteIp(), $this->request, $includeRequestBody);
$accessLog->logRequest($user, $this->User->_remoteIp(), $this->request, $includeRequestBody);
}
if (
@ -828,29 +830,34 @@ class AppController extends Controller
private function __rateLimitCheck(array $user)
{
$info = array();
$rateLimitCheck = $this->RateLimit->check(
$user,
$this->request->params['controller'],
$this->request->action,
$info,
$this->response->type()
$this->request->params['action'],
);
if (!empty($info)) {
$this->RestResponse->setHeader('X-Rate-Limit-Limit', $info['limit']);
$this->RestResponse->setHeader('X-Rate-Limit-Remaining', $info['remaining']);
$this->RestResponse->setHeader('X-Rate-Limit-Reset', $info['reset']);
if ($rateLimitCheck) {
$headers = [
'X-Rate-Limit-Limit' => $rateLimitCheck['limit'],
'X-Rate-Limit-Remaining' => $rateLimitCheck['remaining'],
'X-Rate-Limit-Reset' => $rateLimitCheck['reset'],
];
if ($rateLimitCheck['exceeded']) {
$response = $this->RestResponse->throwException(
429,
__('Rate limit exceeded.'),
'/' . $this->request->params['controller'] . '/' . $this->request->params['action'],
false,
false,
$headers
);
$response->send();
$this->_stop();
} else {
$this->RestResponse->headers = array_merge($this->RestResponse->headers, $headers);
}
}
if ($rateLimitCheck !== true) {
$this->response->header('X-Rate-Limit-Limit', $info['limit']);
$this->response->header('X-Rate-Limit-Remaining', $info['remaining']);
$this->response->header('X-Rate-Limit-Reset', $info['reset']);
$this->response->body($rateLimitCheck);
$this->response->statusCode(429);
$this->response->send();
$this->_stop();
}
return true;
}
public function afterFilter()
@ -995,6 +1002,14 @@ class AppController extends Controller
*/
protected function _harvestParameters($options, &$exception = null, $data = [])
{
if (!empty($options['paramArray'])) {
if (!in_array('page', $options['paramArray'])) {
$options['paramArray'][] = 'page';
}
if (!in_array('limit', $options['paramArray'])) {
$options['paramArray'][] = 'limit';
}
}
$request = $options['request'] ?? $this->request;
if ($request->is('post')) {
if (empty($request->data)) {
@ -1135,14 +1150,14 @@ class AppController extends Controller
$headerNamespace = '';
}
if (isset($server[$headerNamespace . $header]) && !empty($server[$headerNamespace . $header])) {
if (Configure::read('Plugin.CustomAuth_only_allow_source') && Configure::read('Plugin.CustomAuth_only_allow_source') !== $this->_remoteIp()) {
if (Configure::read('Plugin.CustomAuth_only_allow_source') && Configure::read('Plugin.CustomAuth_only_allow_source') !== $this->User->_remoteIp()) {
$this->Log = ClassRegistry::init('Log');
$this->Log->createLogEntry(
'SYSTEM',
'auth_fail',
'User',
0,
'Failed authentication using external key (' . trim($server[$headerNamespace . $header]) . ') - the user has not arrived from the expected address. Instead the request came from: ' . $this->_remoteIp(),
'Failed authentication using external key (' . trim($server[$headerNamespace . $header]) . ') - the user has not arrived from the expected address. Instead the request came from: ' . $this->User->_remoteIp(),
null);
$this->__preAuthException($authName . ' authentication failed. Contact your MISP support for additional information at: ' . Configure::read('MISP.contact'));
}
@ -1302,7 +1317,7 @@ class AppController extends Controller
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception, $this->_legacyParams);
if (empty($filters) && $this->request->is('get')) {
throw new InvalidArgumentException(__('Restsearch queries using GET and no parameters are not allowed. If you have passed parameters via a JSON body, make sure you use POST requests.'));
throw new BadRequestException(__('Restsearch queries using GET and no parameters are not allowed. If you have passed parameters via a JSON body, make sure you use POST requests.'));
}
if (empty($filters['returnFormat'])) {
$filters['returnFormat'] = 'json';

2
app/Controller/AttributesController.php
View File

@ -1917,7 +1917,7 @@ class AttributesController extends AppController
public function reportValidationIssuesAttributes($eventId = false)
{
// search for validation problems in the attributes
$this->set('result', $this->Attribute->reportValidationIssuesAttributes($eventId));
$this->set('result', iterator_to_array($this->Attribute->reportValidationIssuesAttributes($eventId)));
}
public function generateCorrelation()

66
app/Controller/AuditLogsController.php
View File

@ -91,21 +91,6 @@ class AuditLogsController extends AppController
];
}
private function __applyAuditACL(array $user)
{
$acl = [];
if (empty($user['Role']['perm_site_admin'])) {
if (!empty($user['Role']['perm_admin'])) {
// ORG admins can see their own org info
$acl = ['AuditLog.org_id' => $user['org_id']];
} else {
// users can see their own info
$acl = ['AuditLog.user_id' => $user['id']];
}
}
return $acl;
}
public function admin_index()
{
$this->paginate['fields'][] = 'ip';
@ -134,7 +119,8 @@ class AuditLogsController extends AppController
]);
$this->paginate['conditions'] = $this->__searchConditions($params);
$acl = $this->__applyAuditACL($this->Auth->user());
$user = $this->Auth->user();
$acl = $this->__applyAuditAcl($user);
if ($acl) {
$this->paginate['conditions']['AND'][] = $acl;
}
@ -144,7 +130,7 @@ class AuditLogsController extends AppController
return $this->RestResponse->viewData($list, 'json');
}
$list = $this->__appendModelLinks($list);
$list = $this->__appendModelLinks($user, $list);
foreach ($list as $k => $item) {
$list[$k]['AuditLog']['action_human'] = $this->actions[$item['AuditLog']['action']];
}
@ -222,13 +208,19 @@ class AuditLogsController extends AppController
public function fullChange($id)
{
$acl = $this->__applyAuditAcl($this->Auth->user());
$log = $this->AuditLog->find('first', [
'conditions' => ['id' => $id],
'conditions' => [
'AND' => [
$acl,
'id' => $id
]
],
'recursive' => -1,
'fields' => ['change', 'action'],
]);
if (empty($log)) {
throw new Exception('Log not found.');
throw new NotFoundException('Log not found.');
}
$this->set('log', $log);
}
@ -246,6 +238,21 @@ class AuditLogsController extends AppController
return $this->RestResponse->viewData($data, $this->response->type());
}
private function __applyAuditAcl(array $user)
{
$acl = [];
if (empty($user['Role']['perm_site_admin'])) {
if (!empty($user['Role']['perm_admin'])) {
// ORG admins can see their own org info
$acl = ['AuditLog.org_id' => $user['org_id']];
} else {
// users can see their own info
$acl = ['AuditLog.user_id' => $user['id']];
}
}
return $acl;
}
/**
* @return array
*/
@ -435,10 +442,11 @@ class AuditLogsController extends AppController
/**
* Generate link to model view if exists and use has permission to access it.
* @param array $user
* @param array $auditLogs
* @return array
*/
private function __appendModelLinks(array $auditLogs)
private function __appendModelLinks(array $user, array $auditLogs)
{
$models = [];
foreach ($auditLogs as $auditLog) {
@ -449,7 +457,7 @@ class AuditLogsController extends AppController
}
}
$eventIds = isset($models['Event']) ? $models['Event'] : [];
$eventIds = $models['Event'] ?? [];
if (isset($models['ObjectReference'])) {
$this->loadModel('ObjectReference');
@ -461,11 +469,11 @@ class AuditLogsController extends AppController
if (isset($models['Object']) || isset($objectReferences)) {
$objectIds = array_unique(array_merge(
isset($models['Object']) ? $models['Object'] : [],
$models['Object'] ?? [],
isset($objectReferences) ? array_values($objectReferences) : []
));
$this->loadModel('MispObject');
$conditions = $this->MispObject->buildConditions($this->Auth->user());
$conditions = $this->MispObject->buildConditions($user);
$conditions['Object.id'] = $objectIds;
$objects = $this->MispObject->find('all', [
'conditions' => $conditions,
@ -473,22 +481,22 @@ class AuditLogsController extends AppController
'fields' => ['Object.id', 'Object.event_id', 'Object.uuid', 'Object.deleted'],
]);
$objects = array_column(array_column($objects, 'Object'), null, 'id');
$eventIds = array_merge($eventIds, array_column($objects, 'event_id'));
array_push($eventIds, ...array_column($objects, 'event_id'));
}
if (isset($models['Attribute'])) {
$this->loadModel('Attribute');
$attributes = $this->Attribute->fetchAttributesSimple($this->Auth->user(), [
$attributes = $this->Attribute->fetchAttributesSimple($user, [
'conditions' => ['Attribute.id' => array_unique($models['Attribute'])],
'fields' => ['Attribute.id', 'Attribute.event_id', 'Attribute.uuid', 'Attribute.deleted'],
]);
$attributes = array_column(array_column($attributes, 'Attribute'), null, 'id');
$eventIds = array_merge($eventIds, array_column($attributes, 'event_id'));
array_push($eventIds, ...array_column($attributes, 'event_id'));
}
if (isset($models['ShadowAttribute'])) {
$this->loadModel('ShadowAttribute');
$conditions = $this->ShadowAttribute->buildConditions($this->Auth->user());
$conditions = $this->ShadowAttribute->buildConditions($user);
$conditions['AND'][] = ['ShadowAttribute.id' => array_unique($models['ShadowAttribute'])];
$shadowAttributes = $this->ShadowAttribute->find('all', [
'conditions' => $conditions,
@ -496,12 +504,12 @@ class AuditLogsController extends AppController
'contain' => ['Event', 'Attribute'],
]);
$shadowAttributes = array_column(array_column($shadowAttributes, 'ShadowAttribute'), null, 'id');
$eventIds = array_merge($eventIds, array_column($shadowAttributes, 'event_id'));
array_push($eventIds, ...array_column($shadowAttributes, 'event_id'));
}
if (!empty($eventIds)) {
$this->loadModel('Event');
$conditions = $this->Event->createEventConditions($this->Auth->user());
$conditions = $this->Event->createEventConditions($user);
$conditions['Event.id'] = array_unique($eventIds);
$events = $this->Event->find('list', [
'conditions' => $conditions,

14
app/Controller/Component/IndexFilterComponent.php
View File

@ -8,7 +8,9 @@ class IndexFilterComponent extends Component
{
/** @var Controller */
public $Controller;
public $isRest = null;
/** @var bool|null */
private $isRest = null;
// Used for isApiFunction(), a check that returns true if the controller & action combo matches an action that is a non-xml and non-json automation method
// This is used to allow authentication via headers for methods not covered by _isRest() - as that only checks for JSON and XML formats
@ -93,6 +95,11 @@ class IndexFilterComponent extends Component
}
}
public function isXhr()
{
return $this->Controller->request->header('X-Requested-With') === 'XMLHttpRequest';
}
public function isJson()
{
return $this->Controller->request->header('Accept') === 'application/json' || $this->Controller->RequestHandler->prefers() === 'json';
@ -103,11 +110,6 @@ class IndexFilterComponent extends Component
return $this->Controller->request->header('Accept') === 'text/csv' || $this->Controller->RequestHandler->prefers() === 'csv';
}
public function isXml()
{
}
/**
* @param string $controller
* @param string $action

92
app/Controller/Component/RateLimitComponent.php
View File

@ -12,58 +12,60 @@ class RateLimitComponent extends Component
)
);
public $components = array('RestResponse');
/**
* @param array $user
* @param string $controller
* @param string $action
* @param array $info
* @param string $responseType
* @return bool
* @return array|null
* @throws RedisException
*/
public function check(array $user, $controller, $action, &$info = array(), $responseType)
public function check(array $user, $controller, $action)
{
if (!empty($user['Role']['enforce_rate_limit']) && isset(self::LIMITED_FUNCTIONS[$controller][$action])) {
if ($user['Role']['rate_limit_count'] == 0) {
throw new MethodNotAllowedException(__('API searches are not allowed for this user role.'));
}
try {
$redis = RedisTool::init();
} catch (Exception $e) {
return true; // redis is not available, allow access
}
$uuid = Configure::read('MISP.uuid') ?: 'no-uuid';
$keyName = 'misp:' . $uuid . ':rate_limit:' . $user['id'];
$count = $redis->get($keyName);
if ($count !== false && $count >= $user['Role']['rate_limit_count']) {
$info = array(
'limit' => $user['Role']['rate_limit_count'],
'reset' => $redis->ttl($keyName),
'remaining' => $user['Role']['rate_limit_count'] - $count,
);
return $this->RestResponse->throwException(
429,
__('Rate limit exceeded.'),
'/' . $controller . '/' . $action,
$responseType
);
} else {
if ($count === false) {
$redis->setEx($keyName, 900, 1);
} else {
$redis->setEx($keyName, $redis->ttl($keyName), intval($count) + 1);
}
}
$count += 1;
$info = array(
'limit' => $user['Role']['rate_limit_count'],
'reset' => $redis->ttl($keyName),
'remaining' => $user['Role']['rate_limit_count'] - $count
);
if (!isset(self::LIMITED_FUNCTIONS[$controller][$action])) {
return null; // no limit enforced for this controller action
}
return true;
if (empty($user['Role']['enforce_rate_limit'])) {
return null; // no limit enforced for this role
}
$rateLimit = (int)$user['Role']['rate_limit_count'];
if ($rateLimit === 0) {
throw new MethodNotAllowedException(__('API searches are not allowed for this user role.'));
}
try {
$redis = RedisTool::init();
} catch (Exception $e) {
return null; // redis is not available, allow access
}
$uuid = Configure::read('MISP.uuid') ?: 'no-uuid';
$keyName = 'misp:' . $uuid . ':rate_limit:' . $user['id'];
$count = $redis->get($keyName);
if ($count !== false && $count >= $rateLimit) {
return [
'exceeded' => true,
'limit' => $rateLimit,
'reset' => $redis->ttl($keyName),
'remaining' => $rateLimit - $count,
];
}
$newCount = $redis->incr($keyName);
if ($newCount === 1) {
$redis->expire($keyName, 900);
$reset = 900;
} else {
$reset = $redis->ttl($keyName);
}
return [
'exceeded' => false,
'limit' => $rateLimit,
'reset' => $reset,
'remaining' => $rateLimit - $newCount,
];
}
}

16
app/Controller/Component/RestResponseComponent.php
View File

@ -517,7 +517,7 @@ class RestResponseComponent extends Component
if ($id) {
$response['id'] = $id;
}
return $this->__sendResponse($response, 403, $format);
return $this->prepareResponse($response, 403, $format);
}
/**
@ -562,7 +562,7 @@ class RestResponseComponent extends Component
if ($id) {
$response['id'] = $id;
}
return $this->__sendResponse($response, 200, $format);
return $this->prepareResponse($response, 200, $format);
}
/**
@ -587,7 +587,7 @@ class RestResponseComponent extends Component
* @return CakeResponse
* @throws Exception
*/
private function __sendResponse($response, $code, $format = false, $raw = false, $download = false, $headers = array())
private function prepareResponse($response, $code, $format = false, $raw = false, $download = false, $headers = array())
{
App::uses('TmpFileTool', 'Tools');
$format = !empty($format) ? strtolower($format) : 'json';
@ -633,7 +633,7 @@ class RestResponseComponent extends Component
}
// If response is big array, encode items separately to save memory
if (is_array($response) && count($response) > 10000) {
if (is_array($response) && count($response) > 10000 && JsonTool::arrayIsList($response)) {
$output = new TmpFileTool();
$output->write('[');
@ -775,7 +775,7 @@ class RestResponseComponent extends Component
if (!empty($errors)) {
$data['errors'] = $errors;
}
return $this->__sendResponse($data, 200, $format, $raw, $download, $headers);
return $this->prepareResponse($data, 200, $format, $raw, $download, $headers);
}
/**
@ -807,7 +807,7 @@ class RestResponseComponent extends Component
'message' => $message,
'url' => $url
);
return $this->__sendResponse($message, $code, $format, $raw, false, $headers);
return $this->prepareResponse($message, $code, $format, $raw, false, $headers);
}
public function setHeader($header, $value)
@ -834,7 +834,7 @@ class RestResponseComponent extends Component
}
}
$response['url'] = $this->__generateURL($actionArray, $controller, $params);
return $this->__sendResponse($response, 200, $format);
return $this->prepareResponse($response, 200, $format);
}
private function __setup()
@ -1052,7 +1052,7 @@ class RestResponseComponent extends Component
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' ),
'help' => __('Include deleted elements')
'help' => __('Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned')
),
'delta_merge' => array(
'input' => 'radio',

13
app/Controller/Component/RestSearchComponent.php
View File

@ -152,6 +152,8 @@ class RestSearchComponent extends Component
'category',
'org',
'tags',
'first_seen',
'last_seen',
'from',
'to',
'last',
@ -178,12 +180,16 @@ class RestSearchComponent extends Component
'attackGalaxy',
'object_relation',
'metadata',
'includeAllTags'
'includeAllTags',
'object_name',
'object_template_uuid',
'object_template_version'
],
'Sighting' => [
'context',
'returnFormat',
'id',
'uuid',
'type',
'from',
'to',
@ -191,7 +197,8 @@ class RestSearchComponent extends Component
'org_id',
'source',
'includeAttribute',
'includeEvent'
'includeEvent',
'includeUuid',
],
'GalaxyCluster' => [
'page',
@ -204,7 +211,7 @@ class RestSearchComponent extends Component
'distribution',
'org',
'orgc',
'tag',
'tag_name',
'custom',
'sgReferenceOnly',
'minimal',

36
app/Controller/EventsController.php
View File

@ -834,33 +834,29 @@ class EventsController extends AppController
}
if (empty($rules['limit'])) {
$events = array();
$events = [];
$i = 1;
$rules['limit'] = 20000;
while (true) {
$rules['page'] = $i;
$rules['page'] = $i++;
$temp = $this->Event->find('all', $rules);
$resultCount = count($temp);
if ($resultCount !== 0) {
// this is faster and memory efficient than array_merge
foreach ($temp as $tempEvent) {
$events[] = $tempEvent;
}
array_push($events, ...$temp);
}
if ($resultCount < $rules['limit']) {
break;
}
$i++;
}
unset($temp);
$absolute_total = count($events);
$absoluteTotal = count($events);
} else {
$counting_rules = $rules;
unset($counting_rules['limit']);
unset($counting_rules['page']);
$absolute_total = $this->Event->find('count', $counting_rules);
$absoluteTotal = $this->Event->find('count', $counting_rules);
$events = $absolute_total === 0 ? [] : $this->Event->find('all', $rules);
$events = $absoluteTotal === 0 ? [] : $this->Event->find('all', $rules);
}
$isCsvResponse = $this->response->type() === 'text/csv';
@ -979,7 +975,7 @@ class EventsController extends AppController
$events = $export->eventIndex($events);
}
return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, ['X-Result-Count' => $absolute_total]);
return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, ['X-Result-Count' => $absoluteTotal]);
}
private function __indexColumns()
@ -2383,7 +2379,7 @@ class EventsController extends AppController
$results = $this->Event->addMISPExportFile($this->Auth->user(), $data, $isXml, $takeOwnership, $publish);
} catch (Exception $e) {
$this->log("Exception during processing MISP file import: {$e->getMessage()}");
$this->Flash->error(__('Could not process MISP export file. %s.', $e->getMessage()));
$this->Flash->error(__('Could not process MISP export file. %s', $e->getMessage()));
$this->redirect(['controller' => 'events', 'action' => 'add_misp_export']);
}
}
@ -3203,7 +3199,7 @@ class EventsController extends AppController
$event = $this->Event->find('first', [
'conditions' => Validation::uuid($id) ? ['Event.uuid' => $id] : ['Event.id' => $id],
'recursive' => -1,
'fields' => ['id', 'info', 'publish_timestamp', 'orgc_id'],
'fields' => ['id', 'info', 'publish_timestamp', 'orgc_id', 'user_id'],
]);
if (empty($event)) {
throw new NotFoundException(__('Invalid event.'));
@ -3222,6 +3218,16 @@ class EventsController extends AppController
}
}
}
if (
Configure::read('MISP.block_publishing_for_same_creator', false) &&
$this->Auth->user()['id'] == $event['Event']['user_id']
) {
$message = __('Could not publish the event, the publishing user cannot be the same as the event creator as per this instance\'s configuration.');
if (!$this->_isRest()) {
$this->Flash->error($message);
}
throw new MethodNotAllowedException($message);
}
return $event;
}
@ -3325,7 +3331,7 @@ class EventsController extends AppController
$this->Flash->info(__('Warning, you are logged in as a site admin, any export that you generate will contain the FULL UNRESTRICTED data-set. If you would like to generate an export for your own organisation, please log in with a different user.'));
}
// Check if the background jobs are enabled - if not, fall back to old export page.
if (Configure::read('MISP.background_jobs') && !Configure::read('MISP.disable_cached_exports')) {
if (Configure::read('MISP.background_jobs') && !Configure::read('MISP.disable_cached_exports', true)) {
$now = time();
// as a site admin we'll use the ADMIN identifier, not to overwrite the cached files of our own org with a file that includes too much data.
@ -3412,7 +3418,7 @@ class EventsController extends AppController
public function downloadExport($type, $extra = null)
{
if (Configure::read('MISP.disable_cached_exports')) {
if (Configure::read('MISP.disable_cached_exports', true)) {
throw new MethodNotAllowedException(__('This feature is currently disabled'));
}
if ($this->_isSiteAdmin()) {

5
app/Controller/JobsController.php
View File

@ -151,9 +151,12 @@ class JobsController extends AppController
public function cache($type)
{
if (Configure::read('MISP.disable_cached_exports')) {
if (Configure::read('MISP.disable_cached_exports', true)) {
throw new MethodNotAllowedException('This feature is currently disabled');
}
if (!$this->request->is('post')) {
throw new MethodNotAllowedException('This endpoint only accept POST.');
}
if ($this->_isSiteAdmin()) {
$target = 'All events.';
} else {

21
app/Controller/OrganisationsController.php
View File

@ -481,13 +481,32 @@ class OrganisationsController extends AppController
$extension = pathinfo($logo['name'], PATHINFO_EXTENSION);
$filename = $orgId . '.' . ($extension === 'svg' ? 'svg' : 'png');
if ($logo['size'] > 250*1024) {
$this->Flash->error(__('This organisation logo is too large, maximum file size allowed is 250kB.'));
return false;
}
if ($extension !== 'svg' && $extension !== 'png') {
$this->Flash->error(__('Invalid file extension, Only PNG and SVG images are allowed.'));
return false;
}
$imgMime = mime_content_type($logo['tmp_name']);
if ($extension === 'png' && !exif_imagetype($logo['tmp_name'])) {
$this->Flash->error(__('This is not a valid PNG image.'));
return false;
} else if ($extension === 'svg' && !($imgMime === 'image/svg+xml' || $imgMime === 'image/svg')) {
$this->Flash->error(__('This is not a valid SVG image.'));
return false;
}
if ($extension === 'svg' && !Configure::read('Security.enable_svg_logos')) {
$this->Flash->error(__('Invalid file extension, SVG images are not allowed.'));
return false;
}
if (!empty($logo['tmp_name']) && is_uploaded_file($logo['tmp_name'])) {
return move_uploaded_file($logo['tmp_name'], APP . 'webroot/img/orgs/' . $filename);
return move_uploaded_file($logo['tmp_name'], APP . 'files/img/orgs/' . $filename);
}
}

18
app/Controller/ServersController.php
View File

@ -1073,7 +1073,7 @@ class ServersController extends AppController
);
$dumpResults = array();
$tempArray = array();
foreach ($finalSettings as $k => $result) {
foreach ($finalSettings as $result) {
if ($result['level'] == 3) {
$issues['deprecated']++;
}
@ -1105,18 +1105,19 @@ class ServersController extends AppController
$diagnostic_errors = 0;
App::uses('File', 'Utility');
App::uses('Folder', 'Utility');
if ($tab === 'correlations') {
$this->loadModel('Correlation');
$correlation_metrics = $this->Correlation->collectMetrics();
$this->set('correlation_metrics', $correlation_metrics);
}
if ($tab === 'files') {
} else if ($tab === 'files') {
if (!empty(Configure::read('Security.disable_instance_file_uploads'))) {
throw new MethodNotAllowedException(__('This functionality is disabled.'));
}
$files = $this->Server->grabFiles();
$this->set('files', $files);
}
// Only run this check on the diagnostics tab
if ($tab === 'diagnostics' || $tab === 'download' || $this->_isRest()) {
$php_ini = php_ini_loaded_file();
@ -1279,12 +1280,10 @@ class ServersController extends AppController
$this->set('workerIssueCount', $workerIssueCount);
$priorityErrorColours = array(0 => 'red', 1 => 'yellow', 2 => 'green');
$this->set('priorityErrorColours', $priorityErrorColours);
$this->set('phpversion', phpversion());
$this->set('phpversion', PHP_VERSION);
$this->set('phpmin', $this->phpmin);
$this->set('phprec', $this->phprec);
$this->set('phptoonew', $this->phptoonew);
$this->set('pythonmin', $this->pythonmin);
$this->set('pythonrec', $this->pythonrec);
$this->set('title_for_layout', __('Diagnostics'));
}
@ -1770,6 +1769,7 @@ class ServersController extends AppController
$perm_sighting = isset($result['info']['perm_sighting']) ? $result['info']['perm_sighting'] : false;
$local_version = $this->Server->checkMISPVersion();
$version = explode('.', $result['info']['version']);
$uuid = isset($result['info']['uuid']) ? $result['info']['uuid'] : '?';
$mismatch = false;
$newer = false;
$parts = array('major', 'minor', 'hotfix');
@ -1805,6 +1805,7 @@ class ServersController extends AppController
'response_encoding' => isset($result['post']['content-encoding']) ? $result['post']['content-encoding'] : null,
'request_encoding' => isset($result['info']['request_encoding']) ? $result['info']['request_encoding'] : null,
'client_certificate' => $result['client_certificate'],
'uuid' => $uuid,
], 'json');
} else {
$result['status'] = 3;
@ -1863,7 +1864,7 @@ class ServersController extends AppController
}
if (Configure::read('SimpleBackgroundJobs.enabled')) {
$this->Server->getBackgroundJobsTool()->purgeQueue($worker);
$this->Server->getBackgroundJobsTool()->clearQueue($worker);
} else {
// CakeResque
$worker_array = array('cache', 'default', 'email', 'prio');
@ -1888,6 +1889,7 @@ class ServersController extends AppController
'perm_sync' => (bool) $user['Role']['perm_sync'],
'perm_sighting' => (bool) $user['Role']['perm_sighting'],
'perm_galaxy_editor' => (bool) $user['Role']['perm_galaxy_editor'],
'uuid' => $user['Role']['perm_sync'] ? Configure::read('MISP.uuid') : '-',
'request_encoding' => $this->CompressedRequestHandler->supportedEncodings(),
'filter_sightings' => true, // check if Sightings::filterSightingUuidsForPush method is supported
];
@ -2183,7 +2185,7 @@ class ServersController extends AppController
if ($this->_isRest()) {
return $this->RestResponse->saveFailResponse('Servers', 'addFromJson', false, $this->Server->validationErrors, $this->response->type());
} else {
$this->Flash->error(__('Could not save the server. Error: %s', json_encode($this->Server->validationErrors, true)));
$this->Flash->error(__('Could not save the server. Error: %s', json_encode($this->Server->validationErrors)));
$this->redirect(array('action' => 'index'));
}
}

70
app/Controller/UsersController.php
View File

@ -1244,8 +1244,6 @@ class UsersController extends AppController
// login was successful, do everything that is needed such as logging and more:
$this->_postlogin();
} else {
$dataSourceConfig = ConnectionManager::getDataSource('default')->config;
$dataSource = $dataSourceConfig['datasource'];
// don't display authError before first login attempt
if (str_replace("//", "/", $this->webroot . $this->Session->read('Auth.redirect')) == $this->webroot && $this->Session->read('Message.auth.message') == $this->Auth->authError) {
$this->Session->delete('Message.auth');
@ -1260,73 +1258,7 @@ class UsersController extends AppController
}
}
//
// Actions needed for the first access, when the database is not populated yet.
//
// populate the DB with the first role (site admin) if it's empty
if (!$this->User->Role->hasAny()) {
$siteAdmin = array('Role' => array(
'id' => 1,
'name' => 'Site Admin',
'permission' => 3,
'perm_add' => 1,
'perm_modify' => 1,
'perm_modify_org' => 1,
'perm_publish' => 1,
'perm_sync' => 1,
'perm_admin' => 1,
'perm_audit' => 1,
'perm_auth' => 1,
'perm_site_admin' => 1,
'perm_regexp_access' => 1,
'perm_sharing_group' => 1,
'perm_template' => 1,
'perm_tagger' => 1,
));
$this->User->Role->save($siteAdmin);
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
if ($dataSource === 'Database/Postgres') {
$sql = "SELECT setval('roles_id_seq', (SELECT MAX(id) FROM roles));";
$this->User->Role->query($sql);
}
}
if (!$this->User->Organisation->hasAny(array('Organisation.local' => true))) {
$this->User->runUpdates();
$date = date('Y-m-d H:i:s');
$org = array('Organisation' => array(
'id' => 1,
'name' => !empty(Configure::read('MISP.org')) ? Configure::read('MISP.org') : 'ADMIN',
'description' => 'Automatically generated admin organisation',
'type' => 'ADMIN',
'uuid' => CakeText::uuid(),
'local' => 1,
'date_created' => $date,
'sector' => '',
'nationality' => ''
));
$this->User->Organisation->save($org);
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
if ($dataSource === 'Database/Postgres') {
$sql = "SELECT setval('organisations_id_seq', (SELECT MAX(id) FROM organisations));";
$this->User->Organisation->query($sql);
}
$org_id = $this->User->Organisation->id;
}
// populate the DB with the first user if it's empty
if (!$this->User->hasAny()) {
if (!isset($org_id)) {
$hostOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.name' => Configure::read('MISP.org'), 'Organisation.local' => true), 'recursive' => -1));
if (!empty($hostOrg)) {
$org_id = $hostOrg['Organisation']['id'];
} else {
$firstOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.local' => true), 'order' => 'Organisation.id ASC'));
$org_id = $firstOrg['Organisation']['id'];
}
}
$this->User->runUpdates();
$this->User->createInitialUser($org_id);
}
$this->User->init();
}
}

191
app/Lib/Export/NidsExport.php
View File

@ -6,10 +6,7 @@ abstract class NidsExport
public $classtype = 'trojan-activity';
public $format = ""; // suricata (default), snort
public $checkWhitelist = true;
protected $format; // suricata (default), snort
public $additional_params = array(
'contain' => array(
@ -17,36 +14,31 @@ abstract class NidsExport
'fields' => array('threat_level_id')
)
),
);
public function handler($data, $options = array())
{
$continue = empty($format);
$this->checkWhitelist = false;
if ($options['scope'] === 'Attribute') {
$this->export(
array($data),
$options['user']['nids_sid'],
$options['returnFormat'],
$continue
$options['user']['nids_sid']
);
} else if ($options['scope'] === 'Event') {
if (!empty($data['EventTag'])) {
$data['Event']['EventTag'] = $data['EventTag'];
}
if (!empty($data['Attribute'])) {
$this->__convertFromEventFormat($data['Attribute'], $data, $options, $continue);
$this->convertFromEventFormat($data['Attribute'], $data, $options);
}
if (!empty($data['Object'])) {
$this->__convertFromEventFormatObject($data['Object'], $data, $options, $continue);
$this->convertFromEventFormatObject($data['Object'], $data, $options);
}
}
return '';
}
private function __convertFromEventFormat($attributes, $event, $options = array(), $continue = false) {
private function convertFromEventFormat($attributes, $event, $options = array())
{
$rearranged = array();
foreach ($attributes as $attribute) {
$attributeTag = array();
@ -62,15 +54,12 @@ abstract class NidsExport
}
$this->export(
$rearranged,
$options['user']['nids_sid'],
$options['returnFormat'],
$continue
$options['user']['nids_sid']
);
return true;
}
private function __convertFromEventFormatObject($objects, $event, $options = array(), $continue = false)
private function convertFromEventFormatObject($objects, $event, $options = array())
{
$rearranged = array();
foreach ($objects as $object) {
@ -93,20 +82,18 @@ abstract class NidsExport
'Event' => $event['Event']
);
} else { // In case no custom export exists for the object, the approach falls back to the attribute case
$this->__convertFromEventFormat($object['Attribute'], $event, $options, $continue);
$this->convertFromEventFormat($object['Attribute'], $event, $options);
}
}
$this->export(
$rearranged,
$options['user']['nids_sid'],
$options['returnFormat'],
$continue
$options['user']['nids_sid']
);
return true;
}
public function header($options = array())
public function header()
{
$this->explain();
return '';
@ -122,7 +109,7 @@ abstract class NidsExport
return '';
}
public function explain()
protected function explain()
{
$this->rules[] = '# MISP export of IDS rules - optimized for '.$this->format;
$this->rules[] = '#';
@ -136,21 +123,8 @@ abstract class NidsExport
$this->rules[] = '# ';
}
private $whitelist = null;
public function export($items, $startSid, $format="suricata", $continue = false)
protected function export($items, $startSid)
{
$this->format = $format;
if ($this->checkWhitelist && !isset($this->Whitelist)) {
$this->Whitelist = ClassRegistry::init('Whitelist');
$this->whitelist = $this->Whitelist->getBlockedValues();
}
// output a short explanation
if (!$continue) {
$this->explain();
}
// generate the rules
foreach ($items as $item) {
// retrieve all tags for this item to add them to the msg
@ -180,7 +154,6 @@ abstract class NidsExport
$sid++;
if (!empty($item['Attribute']['type'])) { // item is an 'Attribute'
switch ($item['Attribute']['type']) {
// LATER nids - test all the snort attributes
// LATER nids - add the tag keyword in the rules to capture network traffic
@ -195,7 +168,7 @@ abstract class NidsExport
break;
case 'email':
$this->emailSrcRule($ruleFormat, $item['Attribute'], $sid);
$sid++;
$sid++;
$this->emailDstRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'email-src':
@ -228,17 +201,17 @@ abstract class NidsExport
case 'ja3-fingerprint-md5':
$this->ja3Rule($ruleFormat, $item['Attribute'], $sid);
break;
case 'ja3s-fingerprint-md5': // Atribute type doesn't exists yet (2020-12-10) but ready when created.
case 'ja3s-fingerprint-md5': // Attribute type doesn't exists yet (2020-12-10) but ready when created.
$this->ja3sRule($ruleFormat, $item['Attribute'], $sid);
break;
case 'snort':
$this->snortRule($ruleFormat, $item['Attribute'], $sid, $ruleFormatMsg, $ruleFormatReference);
$this->snortRule($item['Attribute'], $sid, $ruleFormatMsg, $ruleFormatReference);
// no break
default:
break;
}
} else if(!empty($item['Attribute']['name'])) { // Item is an 'Object'
} else if (!empty($item['Attribute']['name'])) { // Item is an 'Object'
switch ($item['Attribute']['name']) {
case 'network-connection':
@ -252,34 +225,30 @@ abstract class NidsExport
}
}
}
return $this->rules;
}
public function networkConnectionRule($ruleFormat, $object, &$sid)
protected function networkConnectionRule($ruleFormat, $object, &$sid)
{
$attributes = NidsExport::getObjectAttributes($object);
if(!array_key_exists('layer4-protocol', $attributes)){
if (!array_key_exists('layer4-protocol', $attributes)) {
$attributes['layer4-protocol'] = 'ip'; // If layer-4 protocol is unknown, we roll-back to layer-3 ('ip')
}
if(!array_key_exists('ip-src', $attributes)){
if (!array_key_exists('ip-src', $attributes)) {
$attributes['ip-src'] = '$HOME_NET'; // If ip-src is unknown, we roll-back to $HOME_NET
}
if(!array_key_exists('ip-dst', $attributes)){
if (!array_key_exists('ip-dst', $attributes)) {
$attributes['ip-dst'] = '$HOME_NET'; // If ip-dst is unknown, we roll-back to $HOME_NET
}
if(!array_key_exists('src-port', $attributes)){
if (!array_key_exists('src-port', $attributes)) {
$attributes['src-port'] = 'any'; // If src-port is unknown, we roll-back to 'any'
}
if(!array_key_exists('dst-port', $attributes)){
if (!array_key_exists('dst-port', $attributes)) {
$attributes['dst-port'] = 'any'; // If dst-port is unknown, we roll-back to 'any'
}
$this->rules[] = sprintf(
$this->rules[] = sprintf(
$ruleFormat,
false,
$attributes['layer4-protocol'], // proto
@ -294,12 +263,10 @@ abstract class NidsExport
$sid, // sid
1 // rev
);
}
public function ddosRule($ruleFormat, $object, &$sid)
protected function ddosRule($ruleFormat, $object, &$sid)
{
$attributes = NidsExport::getObjectAttributes($object);
if(!array_key_exists('protocol', $attributes)){
@ -318,7 +285,7 @@ abstract class NidsExport
$attributes['dst-port'] = 'any'; // If dst-port is unknown, we roll-back to 'any'
}
$this->rules[] = sprintf(
$this->rules[] = sprintf(
$ruleFormat,
false,
$attributes['protocol'], // proto
@ -333,12 +300,10 @@ abstract class NidsExport
$sid, // sid
1 // rev
);
}
public static function getObjectAttributes($object)
protected static function getObjectAttributes($object)
{
$attributes = array();
foreach ($object['Attribute'] as $attribute) {
@ -348,7 +313,7 @@ abstract class NidsExport
return $attributes;
}
public function domainIpRule($ruleFormat, $attribute, &$sid)
protected function domainIpRule($ruleFormat, $attribute, &$sid)
{
$values = explode('|', $attribute['value']);
$attributeCopy = $attribute;
@ -361,7 +326,7 @@ abstract class NidsExport
$this->ipSrcRule($ruleFormat, $attributeCopy, $sid);
}
public function ipDstRule($ruleFormat, $attribute, &$sid)
protected function ipDstRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$ipport = NidsExport::getIpPort($attribute);
@ -382,7 +347,7 @@ abstract class NidsExport
);
}
public function ipSrcRule($ruleFormat, $attribute, &$sid)
protected function ipSrcRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$ipport = NidsExport::getIpPort($attribute);
@ -403,7 +368,7 @@ abstract class NidsExport
);
}
public function emailSrcRule($ruleFormat, $attribute, &$sid)
protected function emailSrcRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -425,7 +390,7 @@ abstract class NidsExport
);
}
public function emailDstRule($ruleFormat, $attribute, &$sid)
protected function emailDstRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -447,7 +412,7 @@ abstract class NidsExport
);
}
public function emailSubjectRule($ruleFormat, $attribute, &$sid)
protected function emailSubjectRule($ruleFormat, $attribute, &$sid)
{
// LATER nids - email-subject rule might not match because of line-wrapping
$overruled = $this->checkWhitelist($attribute['value']);
@ -470,7 +435,7 @@ abstract class NidsExport
);
}
public function emailAttachmentRule($ruleFormat, $attribute, &$sid)
protected function emailAttachmentRule($ruleFormat, $attribute, &$sid)
{
// LATER nids - email-attachment rule might not match because of line-wrapping
$overruled = $this->checkWhitelist($attribute['value']);
@ -493,7 +458,7 @@ abstract class NidsExport
);
}
public function hostnameRule($ruleFormat, $attribute, &$sid)
protected function hostnameRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -549,7 +514,7 @@ abstract class NidsExport
);
}
public function domainRule($ruleFormat, $attribute, &$sid)
protected function domainRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -605,7 +570,7 @@ abstract class NidsExport
);
}
public function urlRule($ruleFormat, $attribute, &$sid)
protected function urlRule($ruleFormat, $attribute, &$sid)
{
// TODO in hindsight, an url should not be excluded given a host or domain name.
//$hostpart = parse_url($attribute['value'], PHP_URL_HOST);
@ -630,7 +595,7 @@ abstract class NidsExport
);
}
public function userAgentRule($ruleFormat, $attribute, &$sid)
protected function userAgentRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -652,17 +617,17 @@ abstract class NidsExport
);
}
public function ja3Rule($ruleFormat, $attribute, &$sid)
protected function ja3Rule($ruleFormat, $attribute, &$sid)
{
//Empty because Snort doesn't support JA3 Rules
}
public function ja3sRule($ruleFormat, $attribute, &$sid)
protected function ja3sRule($ruleFormat, $attribute, &$sid)
{
//Empty because Snort doesn't support JA3S Rules
}
public function snortRule($ruleFormat, $attribute, &$sid, $ruleFormatMsg, $ruleFormatReference)
protected function snortRule($attribute, &$sid, $ruleFormatMsg, $ruleFormatReference)
{
// LATER nids - test using lots of snort rules, some rules don't contain all the necessary to be a valid rule.
@ -678,46 +643,46 @@ abstract class NidsExport
// tag - '/tag\s*:\s*.+?;/'
$replaceCount = array();
$tmpRule = preg_replace('/sid\s*:\s*[0-9]+\s*;/', 'sid:' . $sid . ';', $tmpRule, -1, $replaceCount['sid']);
if (null == $tmpRule) {
if (null === $tmpRule) {
return false;
} // don't output the rule on error with the regex
$tmpRule = preg_replace('/rev\s*:\s*[0-9]+\s*;/', 'rev:1;', $tmpRule, -1, $replaceCount['rev']);
if (null == $tmpRule) {
if (null === $tmpRule) {
return false;
} // don't output the rule on error with the regex
$tmpRule = preg_replace('/classtype:[a-zA-Z_-]+;/', 'classtype:' . $this->classtype . ';', $tmpRule, -1, $replaceCount['classtype']);
if (null == $tmpRule) {
if (null === $tmpRule) {
return false;
} // don't output the rule on error with the regex
$tmpRule = preg_replace('/msg\s*:\s*"(.*?)"\s*;/', sprintf($ruleFormatMsg, 'snort-rule | $1') . ';', $tmpRule, -1, $replaceCount['msg']);
if (null == $tmpRule) {
if (null === $tmpRule) {
return false;
} // don't output the rule on error with the regex
$tmpRule = preg_replace('/reference\s*:\s*.+?;/', $ruleFormatReference . ';', $tmpRule, -1, $replaceCount['reference']);
if (null == $tmpRule) {
if (null === $tmpRule) {
return false;
} // don't output the rule on error with the regex
$tmpRule = preg_replace('/reference\s*:\s*.+?;/', $ruleFormatReference . ';', $tmpRule, -1, $replaceCount['reference']);
if (null == $tmpRule) {
if (null === $tmpRule) {
return false;
} // don't output the rule on error with the regex
// FIXME nids - implement priority overwriting
// some values were not replaced, so we need to add them ourselves, and insert them in the rule
$extraForRule = "";
if (0 == $replaceCount['sid']) {
if (0 === $replaceCount['sid']) {
$extraForRule .= 'sid:' . $sid . ';';
}
if (0 == $replaceCount['rev']) {
if (0 === $replaceCount['rev']) {
$extraForRule .= 'rev:1;';
}
if (0 == $replaceCount['classtype']) {
if (0 === $replaceCount['classtype']) {
$extraForRule .= 'classtype:' . $this->classtype . ';';
}
if (0 == $replaceCount['msg']) {
$extraForRule .= $tmpMessage . ';';
if (0 === $replaceCount['msg']) {
$extraForRule .= $ruleFormatMsg . ';';
}
if (0 == $replaceCount['reference']) {
if (0 === $replaceCount['reference']) {
$extraForRule .= $ruleFormatReference . ';';
}
$tmpRule = preg_replace('/;\s*\)/', '; ' . $extraForRule . ')', $tmpRule);
@ -734,7 +699,7 @@ abstract class NidsExport
* @param string $type the type of dns name - domain (default) or hostname
* @return string raw snort compatible format of the dns name
*/
public static function dnsNameToRawFormat($name, $type='domain')
protected static function dnsNameToRawFormat($name, $type='domain')
{
$rawName = "";
if ('hostname' == $type) {
@ -747,7 +712,7 @@ abstract class NidsExport
// count the length of the part, and add |length| before
$length = strlen($explodedName);
if ($length > 255) {
log('WARNING: DNS name is too long for RFC: '.$name);
CakeLog::notice('WARNING: DNS name is too long for RFC: '.$name);
}
$hexLength = dechex($length);
if (1 == strlen($hexLength)) {
@ -768,7 +733,7 @@ abstract class NidsExport
* @param string $name dns name to be converted
* @return string raw snort compatible format of the dns name
*/
public static function dnsNameToMSDNSLogFormat($name)
protected static function dnsNameToMSDNSLogFormat($name)
{
$rawName = "";
// in MS DNS log format we can't use (0) to distinguish between hostname and domain (including subdomains)
@ -779,7 +744,7 @@ abstract class NidsExport
// count the length of the part, and add |length| before
$length = strlen($explodedName);
if ($length > 255) {
log('WARNING: DNS name is too long for RFC: '.$name);
CakeLog::notice('WARNING: DNS name is too long for RFC: '.$name);
}
$hexLength = dechex($length);
$rawName .= '(' . $hexLength . ')' . $explodedName;
@ -793,34 +758,32 @@ abstract class NidsExport
/**
* Replaces characters that are not allowed in a signature.
* example: " is converted to |22|
* @param unknown_type $value
* @param string $value
*/
public static function replaceIllegalChars($value)
protected static function replaceIllegalChars($value)
{
$replace_pairs = array(
'|' => '|7c|', // Needs to stay on top !
'"' => '|22|',
';' => '|3b|',
':' => '|3a|',
'\\' => '|5c|',
'0x' => '|30 78|'
);
'|' => '|7c|', // Needs to stay on top !
'"' => '|22|',
';' => '|3b|',
':' => '|3a|',
'\\' => '|5c|',
'0x' => '|30 78|'
);
return strtr($value, $replace_pairs);
}
public function checkWhitelist($value)
/**
* @deprecated
* @param $value
* @return false
*/
protected function checkWhitelist($value)
{
if ($this->checkWhitelist && is_array($this->whitelist)) {
foreach ($this->whitelist as $wlitem) {
if (preg_match($wlitem, $value)) {
return true;
}
}
}
return false;
}
public static function getProtocolPort($protocol, $customPort)
protected static function getProtocolPort($protocol, $customPort)
{
if ($customPort == null) {
switch ($protocol) {
@ -840,7 +803,7 @@ abstract class NidsExport
}
}
public static function getCustomIP($customIP)
protected static function getCustomIP($customIP)
{
if (filter_var($customIP, FILTER_VALIDATE_IP)) {
return $customIP;
@ -853,7 +816,7 @@ abstract class NidsExport
* @param array $attribute
* @return array|string[]
*/
public static function getIpPort($attribute)
protected static function getIpPort($attribute)
{
if (strpos($attribute['type'], 'port') !== false) {
return explode('|', $attribute['value']);

8
app/Lib/Export/NidsSnortExport.php
View File

@ -4,11 +4,5 @@ App::uses('NidsExport', 'Export');
class NidsSnortExport extends NidsExport
{
public function export($items, $startSid, $format = "suricata", $continue = false)
{
// set the specific format
$this->format = 'snort';
// call the generic function
return parent::export($items, $startSid, $format, $continue);
}
protected $format = 'snort';
}

20
app/Lib/Export/NidsSuricataExport.php
View File

@ -3,16 +3,10 @@ App::uses('NidsExport', 'Export');
class NidsSuricataExport extends NidsExport
{
public function export($items, $startSid, $format = "suricata", $continue = false)
{
// set the specific format
$this->format = "suricata";
// call the generic function
return parent::export($items, $startSid, $format, $continue);
}
protected $format = "suricata";
// below overwrite functions from NidsExport
public function hostnameRule($ruleFormat, $attribute, &$sid)
protected function hostnameRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -53,7 +47,7 @@ class NidsSuricataExport extends NidsExport
);
}
public function domainRule($ruleFormat, $attribute, &$sid)
protected function domainRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -94,7 +88,7 @@ class NidsSuricataExport extends NidsExport
);
}
public function urlRule($ruleFormat, $attribute, &$sid)
protected function urlRule($ruleFormat, $attribute, &$sid)
{
$createRule = true;
$overruled = $this->checkWhitelist($attribute['value']);
@ -207,7 +201,7 @@ class NidsSuricataExport extends NidsExport
}
}
public function userAgentRule($ruleFormat, $attribute, &$sid)
protected function userAgentRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -230,7 +224,7 @@ class NidsSuricataExport extends NidsExport
);
}
public function ja3Rule($ruleFormat, $attribute, &$sid)
protected function ja3Rule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
@ -253,7 +247,7 @@ class NidsSuricataExport extends NidsExport
}
// For Future use once JA3S Hash Attribute type is created
public function ja3sRule($ruleFormat, $attribute, &$sid)
protected function ja3sRule($ruleFormat, $attribute, &$sid)
{
$overruled = $this->checkWhitelist($attribute['value']);
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule

176
app/Lib/Export/RPZExport.php
View File

@ -2,107 +2,104 @@
class RPZExport
{
private $__policies = array(
'Local-Data' => array(
'explanation' => 'returns the defined alternate location.',
'action' => '$walled_garden',
'setting_id' => 3,
),
'NXDOMAIN' => array(
'explanation' => 'return NXDOMAIN (name does not exist) irrespective of actual result received.',
'action' => '.',
'setting_id' => 1,
),
'NODATA' => array(
'explanation' => 'returns NODATA (name exists but no answers returned) irrespective of actual result received.',
'action' => '*.',
'setting_id' => 2,
),
'DROP' => array(
'explanation' => 'timeout.',
'action' => 'rpz-drop.',
'setting_id' => 0,
),
'PASSTHRU' => array(
'explanation' => 'lets queries through, but allows for logging the hits (useful for testing).',
'action' => 'rpz-passthru.',
'setting_id' => 4,
),
'TCP-only' => array(
'explanation' => 'force the client to use TCP.',
'action' => 'rpz-tcp-only.',
'setting_id' => 5,
),
const POLICIES = array(
'Local-Data' => array(
'explanation' => 'returns the defined alternate location.',
'action' => '$walled_garden',
'setting_id' => 3,
),
'NXDOMAIN' => array(
'explanation' => 'return NXDOMAIN (name does not exist) irrespective of actual result received.',
'action' => '.',
'setting_id' => 1,
),
'NODATA' => array(
'explanation' => 'returns NODATA (name exists but no answers returned) irrespective of actual result received.',
'action' => '*.',
'setting_id' => 2,
),
'DROP' => array(
'explanation' => 'timeout.',
'action' => 'rpz-drop.',
'setting_id' => 0,
),
'PASSTHRU' => array(
'explanation' => 'lets queries through, but allows for logging the hits (useful for testing).',
'action' => 'rpz-passthru.',
'setting_id' => 4,
),
'TCP-only' => array(
'explanation' => 'force the client to use TCP.',
'action' => 'rpz-tcp-only.',
'setting_id' => 5,
),
);
private $__items = array();
private $items = array();
public $additional_params = array(
'flatten' => 1
);
private $__rpzSettings = array();
private $__valid_policies = array('NXDOMAIN', 'NODATA', 'DROP', 'Local-Data', 'PASSTHRU', 'TCP-only');
private $rpzSettings = array();
private $__server = null;
public $validTypes = array(
const VALID_TYPES = array(
'ip-src' => array(
'value' => 'ip'
'value' => 'ip'
),
'ip-dst' => array(
'value' => 'ip'
'value' => 'ip'
),
'domain' => array(
'value' => 'domain'
'value' => 'domain'
),
'domain|ip' => array(
'value1' => 'domain',
'value2' => 'ip'
'value1' => 'domain',
'value2' => 'ip'
),
'hostname' => array(
'value' => 'hostname'
'value' => 'hostname'
)
);
public function handler($data, $options = array())
{
if ($options['scope'] === 'Attribute') {
return $this->__attributeHandler($data, $options);
$this->attributeHandler($data);
} else {
return $this->__eventHandler($data, $options);
$this->eventHandler($data);
}
return '';
}
private function __eventHandler($event, $options = array()) {
private function eventHandler($event)
{
foreach ($event['Attribute'] as $attribute) {
if (isset($this->validTypes[$attribute['type']])) {
if ($attribute['type'] == 'domain|ip') {
if (isset(self::VALID_TYPES[$attribute['type']])) {
if ($attribute['type'] === 'domain|ip') {
$temp = explode('|', $attribute['value']);
$attribute['value1'] = $temp[0];
$attribute['value2'] = $temp[1];
}
$this->__attributeHandler(array('Attribute' => $attribute, $options));
$this->attributeHandler(array('Attribute' => $attribute));
}
}
return '';
}
private function __attributeHandler($attribute, $options = array())
private function attributeHandler($attribute)
{
if (isset($attribute['Attribute'])) {
$attribute = $attribute['Attribute'];
}
if (isset($this->validTypes[$attribute['type']])) {
foreach ($this->validTypes[$attribute['type']] as $field => $mapping) {
// get rid of the in_array check
if (empty($this->__items[$mapping]) || !isset($this->__items[$mapping][$attribute[$field]])) {
$this->__items[$mapping][$attribute[$field]] = true;
if (isset(self::VALID_TYPES[$attribute['type']])) {
foreach (self::VALID_TYPES[$attribute['type']] as $field => $mapping) {
if (!isset($this->items[$mapping][$attribute[$field]])) {
$this->items[$mapping][$attribute[$field]] = true;
}
}
}
return '';
}
public function header($options = array())
@ -117,16 +114,16 @@ class RPZExport
}
}
if (isset($options['filters'][$v])) {
$this->__rpzSettings[$v] = $options['filters'][$v];
$this->rpzSettings[$v] = $options['filters'][$v];
} else {
$tempSetting = Configure::read('Plugin.RPZ_' . $v);
if (isset($tempSetting)) {
$this->__rpzSettings[$v] = Configure::read('Plugin.RPZ_' . $v);
$this->rpzSettings[$v] = $tempSetting;
} else {
if (empty($this->__server)) {
$this->__server = ClassRegistry::init('Server');
}
$this->__rpzSettings[$v] = $this->__server->serverSettings['Plugin']['RPZ_' . $v]['value'];
$this->rpzSettings[$v] = $this->__server->serverSettings['Plugin']['RPZ_' . $v]['value'];
}
}
}
@ -135,10 +132,7 @@ class RPZExport
public function footer($options = array())
{
foreach ($this->__items as $k => $v) {
$this->__items[$k] = array_keys($this->__items[$k]);
}
return $this->export($this->__items, $this->__rpzSettings);
return $this->export($this->items, $this->rpzSettings);
}
public function separator()
@ -146,39 +140,32 @@ class RPZExport
return '';
}
public function getPolicyById($id)
private function getPolicyById($id)
{
foreach ($this->__policies as $k => $v) {
if ($id == $v['setting_id']) {
foreach (self::POLICIES as $k => $v) {
if ($id === $v['setting_id']) {
return $k;
}
}
return null;
}
public function getIdByPolicy($policy)
private function getIdByPolicy($policy)
{
return $this->__policies[$policy]['setting_id'];
return self::POLICIES[$policy]['setting_id'];
}
public function explain($type, $policy)
private function explain($type, $policy)
{
$explanations = array(
'ip' => '; The following list of IP addresses will ',
'domain' => '; The following domain names and all of their sub-domains will ',
'hostname' => '; The following hostnames will '
);
$policy_explanations = array(
'Local-Data' => 'returns the defined alternate location.',
'NXDOMAIN' => 'return NXDOMAIN (name does not exist) irrespective of actual result received.',
'NODATA' => 'returns NODATA (name exists but no answers returned) irrespective of actual result received.',
'DROP' => 'timeout.',
'PASSTHRU' => 'lets queries through, but allows for logging the hits (useful for testing).',
'TCP-only' => 'force the client to use TCP.',
);
return $explanations[$type] . $this->__policies[$policy]['explanation'] . PHP_EOL;
return $explanations[$type] . self::POLICIES[$policy]['explanation'] . PHP_EOL;
}
public function buildHeader($rpzSettings)
private function buildHeader(array $rpzSettings)
{
$rpzSettings['serial'] = str_replace('$date', date('Ymd'), $rpzSettings['serial']);
$rpzSettings['serial'] = str_replace('$time', time(), $rpzSettings['serial']);
@ -196,55 +183,55 @@ class RPZExport
return $header;
}
public function export($items, $rpzSettings)
private function export(array $items, array $rpzSettings)
{
$result = $this->buildHeader($rpzSettings);
$policy = $this->getPolicyById($rpzSettings['policy']);
$action = $this->__policies[$policy]['action'];
if ($policy == 'Local-Data') {
$action = self::POLICIES[$policy]['action'];
if ($policy === 'Local-Data') {
$action = str_replace('$walled_garden', $rpzSettings['walled_garden'], $action);
}
if (isset($items['ip'])) {
$result .= $this->explain('ip', $policy);
foreach ($items['ip'] as $item) {
$result .= $this->__convertIP($item, $action);
foreach ($items['ip'] as $item => $foo) {
$result .= $this->convertIp($item, $action);
}
$result .= PHP_EOL;
}
if (isset($items['domain'])) {
$result .= $this->explain('domain', $policy);
foreach ($items['domain'] as $item) {
$result .= $this->__convertdomain($item, $action);
foreach ($items['domain'] as $item => $foo) {
$result .= $this->convertDomain($item, $action);
}
$result .= PHP_EOL;
}
if (isset($items['hostname'])) {
$result .= $this->explain('hostname', $policy);
foreach ($items['hostname'] as $item) {
$result .= $this->__converthostname($item, $action);
foreach ($items['hostname'] as $item => $foo) {
$result .= $this->convertHostname($item, $action);
}
$result .= PHP_EOL;
}
return $result;
}
private function __convertdomain($input, $action)
private function convertDomain($input, $action)
{
return $input . ' CNAME ' . $action . PHP_EOL . '*.' . $input . ' CNAME ' . $action . PHP_EOL;
}
private function __converthostname($input, $action)
private function convertHostname($input, $action)
{
return $input . ' CNAME ' . $action . PHP_EOL;
}
private function __convertIP($input, $action)
private function convertIp($input, $action)
{
$type = filter_var($input, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) ? 'ipv6' : 'ipv4';
if ($type == 'ipv6') {
$isIpv6 = filter_var($input, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6);
if ($isIpv6) {
$prefix = '128';
} else {
$prefix = '32';
@ -252,7 +239,8 @@ class RPZExport
if (strpos($input, '/')) {
list($input, $prefix) = explode('/', $input);
}
return $prefix . '.' . $this->{'__' . $type}($input) . '.rpz-ip CNAME ' . $action . PHP_EOL;
$converted = $isIpv6 ? $this->__ipv6($input) : $this->__ipv4($input);
return $prefix . '.' . $converted . '.rpz-ip CNAME ' . $action . PHP_EOL;
}
private function __ipv6($input)

198
app/Lib/Tools/ApcuCacheTool.php Normal file
View File

@ -0,0 +1,198 @@
<?php
declare(strict_types=1);
class ApcuCacheTool implements \Psr\SimpleCache\CacheInterface
{
/** @var string */
private $prefix;
/**
* @param string $prefix
*/
public function __construct(string $prefix)
{
$this->prefix = $prefix;
}
/**
* Fetches a value from the cache.
*
* @param string $key The unique key of this item in the cache.
* @param mixed $default Default value to return if the key does not exist.
*
* @return mixed The value of the item from the cache, or $default in case of cache miss.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if the $key string is not a legal value.
*/
public function get($key, $default = null)
{
$value = \apcu_fetch("$this->prefix:$key", $success);
if ($success) {
return $value;
}
return $default;
}
/**
* Persists data in the cache, uniquely referenced by a key with an optional expiration TTL time.
*
* @param string $key The key of the item to store.
* @param mixed $value The value of the item to store, must be serializable.
* @param null|int|\DateInterval $ttl Optional. The TTL value of this item. If no value is sent and
* the driver supports TTL then the library may set a default value
* for it or let the driver take care of that.
*
* @return bool True on success and false on failure.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if the $key string is not a legal value.
*/
public function set($key, $value, $ttl = null)
{
return \apcu_store("$this->prefix:$key", $value, $this->tllToInt($ttl));
}
/**
* Delete an item from the cache by its unique key.
*
* @param string $key The unique cache key of the item to delete.
*
* @return bool True if the item was successfully removed. False if there was an error.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if the $key string is not a legal value.
*/
public function delete($key)
{
return \apcu_delete("$this->prefix:$key");
}
/**
* Wipes clean the entire cache's keys.
*
* @return bool True on success and false on failure.
*/
public function clear()
{
$iterator = new APCUIterator(
'/^' . preg_quote($this->prefix . ':', '/') . '/',
APC_ITER_NONE
);
return \apcu_delete($iterator);
}
/**
* Obtains multiple cache items by their unique keys.
*
* @param iterable $keys A list of keys that can obtained in a single operation.
* @param mixed $default Default value to return for keys that do not exist.
*
* @return iterable A list of key => value pairs. Cache keys that do not exist or are stale will have $default as value.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if $keys is neither an array nor a Traversable,
* or if any of the $keys are not a legal value.
*/
public function getMultiple($keys, $default = null)
{
$keysToFetch = $this->keysToFetch($keys);
$values = \apcu_fetch($keysToFetch);
foreach ($keysToFetch as $keyToFetch) {
if (!isset($values[$keyToFetch])) {
$values[$keyToFetch] = $default;
}
}
return $values;
}
/**
* Persists a set of key => value pairs in the cache, with an optional TTL.
*
* @param iterable $values A list of key => value pairs for a multiple-set operation.
* @param null|int|\DateInterval $ttl Optional. The TTL value of this item. If no value is sent and
* the driver supports TTL then the library may set a default value
* for it or let the driver take care of that.
*
* @return bool True on success and false on failure.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if $values is neither an array nor a Traversable,
* or if any of the $values are not a legal value.
*/
public function setMultiple($values, $ttl = null)
{
$dataToSave = [];
foreach ($values as $key => $value) {
$dataToSave["$this->prefix:$key"] = $value;
}
return \apcu_store($dataToSave, null, $this->tllToInt($ttl));
}
/**
* Deletes multiple cache items in a single operation.
*
* @param iterable $keys A list of string-based keys to be deleted.
*
* @return bool True if the items were successfully removed. False if there was an error.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if $keys is neither an array nor a Traversable,
* or if any of the $keys are not a legal value.
*/
public function deleteMultiple($keys)
{
$keysToDelete = $this->keysToFetch($keys);
return \apcu_delete($keysToDelete);
}
/**
* Determines whether an item is present in the cache.
*
* NOTE: It is recommended that has() is only to be used for cache warming type purposes
* and not to be used within your live applications operations for get/set, as this method
* is subject to a race condition where your has() will return true and immediately after,
* another script can remove it making the state of your app out of date.
*
* @param string $key The cache item key.
*
* @return bool
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if the $key string is not a legal value.
*/
public function has($key)
{
return \apcu_exists("$this->prefix:$key");
}
/**
* @param iterable $keys
* @return array
*/
private function keysToFetch(iterable $keys): array
{
$keysToFetch = [];
foreach ($keys as $key) {
$keysToFetch[] = "$this->prefix:$key";
}
return $keysToFetch;
}
/**
* @param null|int|\DateInterval $ttl
* @return int
*/
private function tllToInt($ttl = null): int
{
if ($ttl === null) {
return 0;
} elseif (is_int($ttl)) {
return $ttl;
} elseif ($ttl instanceof \DateInterval) {
return $ttl->days * 86400 + $ttl->h * 3600 + $ttl->i * 60 + $ttl->s;
} else {
throw new \Psr\SimpleCache\InvalidArgumentException("Invalid ttl value '$ttl' provided.");
}
}
}

27
app/Lib/Tools/AttributeValidationTool.php
View File

@ -41,7 +41,7 @@ class AttributeValidationTool
switch ($type) {
case 'ip-src':
case 'ip-dst':
return self::compressIpv6($value);
return self::normalizeIp($value);
case 'md5':
case 'sha1':
case 'sha224':
@ -98,7 +98,7 @@ class AttributeValidationTool
$parts[0] = $punyCode;
}
}
$parts[1] = self::compressIpv6($parts[1]);
$parts[1] = self::normalizeIp($parts[1]);
return "$parts[0]|$parts[1]";
case 'filename|md5':
case 'filename|sha1':
@ -175,7 +175,7 @@ class AttributeValidationTool
} else {
return $value;
}
return self::compressIpv6($parts[0]) . '|' . $parts[1];
return self::normalizeIp($parts[0]) . '|' . $parts[1];
case 'mac-address':
case 'mac-eui-64':
$value = str_replace(array('.', ':', '-', ' '), '', strtolower($value));
@ -700,11 +700,30 @@ class AttributeValidationTool
* @param string $value
* @return string
*/
private static function compressIpv6($value)
private static function normalizeIp($value)
{
// If IP is a CIDR
if (strpos($value, '/')) {
list($ip, $range) = explode('/', $value, 2);
// Compress IPv6
if (strpos($ip, ':') && $converted = inet_pton($ip)) {
$ip = inet_ntop($converted);
}
// If IP is in CIDR format, but the network is 32 for IPv4 or 128 for IPv6, normalize to non CIDR type
if (($range === '32' && strpos($value, '.')) || ($range === '128' && strpos($value, ':'))) {
return $ip;
}
return "$ip/$range";
}
// Compress IPv6
if (strpos($value, ':') && $converted = inet_pton($value)) {
return inet_ntop($converted);
}
return $value;
}

30
app/Lib/Tools/BackgroundJobs/BackgroundJob.php
View File

@ -66,8 +66,9 @@ class BackgroundJob implements JsonSerializable
/**
* Run the job command
* @param callable|null $runningCallback
*/
public function run(): void
public function run(callable $runningCallback = null): void
{
$descriptorSpec = [
1 => ["pipe", "w"], // stdout
@ -88,7 +89,7 @@ class BackgroundJob implements JsonSerializable
['BACKGROUND_JOB_ID' => $this->id]
);
$this->pool($process, $pipes);
$this->pool($process, $pipes, $runningCallback);
if ($this->returnCode === 0 && empty($stderr)) {
$this->setStatus(BackgroundJob::STATUS_COMPLETED);
@ -98,7 +99,13 @@ class BackgroundJob implements JsonSerializable
}
}
private function pool($process, array $pipes)
/**
* @param resource $process
* @param array $pipes
* @param callable|null $runningCallback
* @return void
*/
private function pool($process, array $pipes, callable $runningCallback = null)
{
stream_set_blocking($pipes[1], false);
stream_set_blocking($pipes[2], false);
@ -106,6 +113,14 @@ class BackgroundJob implements JsonSerializable
$this->output = '';
$this->error = '';
if ($runningCallback) {
$status = proc_get_status($process);
if ($status === false) {
throw new RuntimeException("Could not get process status");
}
$runningCallback($status);
}
while (true) {
$read = [$pipes[1], $pipes[2]];
$write = null;
@ -118,6 +133,12 @@ class BackgroundJob implements JsonSerializable
$this->error .= stream_get_contents($pipes[2]);
}
$status = proc_get_status($process);
if ($status === false) {
throw new RuntimeException("Could not get process status");
}
if ($runningCallback) {
$runningCallback($status);
}
if (!$status['running']) {
// Just in case read rest data from stream
$this->output .= stream_get_contents($pipes[1]);
@ -153,6 +174,9 @@ class BackgroundJob implements JsonSerializable
return ['id', 'command', 'args', 'createdAt', 'updatedAt', 'status', 'output', 'error', 'metadata'];
}
/**
* @return string Background job ID in UUID format
*/
public function id(): string
{
return $this->id;

2
app/Lib/Tools/BackgroundJobs/Worker.php
View File

@ -65,7 +65,7 @@ class Worker implements JsonSerializable
];
}
public function pid(): ?int
public function pid(): int
{
return $this->pid;
}

67
app/Lib/Tools/BackgroundJobsTool.php
View File

@ -91,7 +91,8 @@ class BackgroundJobsTool
];
const JOB_STATUS_PREFIX = 'job_status',
DATA_CONTENT_PREFIX = 'data_content';
DATA_CONTENT_PREFIX = 'data_content',
RUNNING_JOB_PREFIX = 'running';
/** @var array */
private $settings;
@ -277,6 +278,54 @@ class BackgroundJobsTool
return null;
}
/**
* @param Worker $worker
* @param BackgroundJob $job
* @param int|null $pid
* @return void
* @throws RedisException
*/
public function markAsRunning(Worker $worker, BackgroundJob $job, $pid = null)
{
$key = self::RUNNING_JOB_PREFIX . ':' . $worker->queue() . ':' . $job->id();
$this->RedisConnection->setex($key, 60, [
'worker_pid' => $worker->pid(),
'process_pid' => $pid,
]);
}
/**
* @param Worker $worker
* @param BackgroundJob $job
* @return void
* @throws RedisException
*/
public function removeFromRunning(Worker $worker, BackgroundJob $job)
{
$key = self::RUNNING_JOB_PREFIX . ':' . $worker->queue() . ':' . $job->id();
$this->RedisConnection->del($key);
}
/**
* Return current running jobs
* @return array
* @throws RedisException
*/
public function runningJobs(): array
{
$pattern = $this->RedisConnection->_prefix(self::RUNNING_JOB_PREFIX . ':*');
$keys = RedisTool::keysByPattern($this->RedisConnection, $pattern);
$jobIds = [];
foreach ($keys as $key) {
$parts = explode(':', $key);
$queue = $parts[2];
$jobId = $parts[3];
$jobIds[$queue][$jobId] = $this->RedisConnection->get(self::RUNNING_JOB_PREFIX . ":$queue:$jobId");
}
return $jobIds;
}
/**
* Get the job status.
*
@ -500,19 +549,6 @@ class BackgroundJobsTool
$this->getSupervisor()->startProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart);
}
/**
* Purge queue
*
* @param string $queue
* @return void
*/
public function purgeQueue(string $queue)
{
$this->validateQueue($queue);
$this->RedisConnection->del($queue);
}
/**
* Return Background Jobs status
*
@ -728,8 +764,7 @@ class BackgroundJobsTool
*
* @param integer $pid
* @return \Supervisor\Process
*
* @throws NotFoundException
* @throws NotFoundException|Exception
*/
private function getProcessByPid(int $pid): \Supervisor\Process
{

4
app/Lib/Tools/BetterCakeEventManager.php
View File

@ -52,10 +52,10 @@ class BetterCakeEventManager extends CakeEventManager
$result = [];
foreach ($priorities as $priority) {
if (isset($globalListeners[$priority])) {
$result = array_merge($result, $globalListeners[$priority]);
array_push($result, ...$globalListeners[$priority]);
}
if (isset($localListeners[$priority])) {
$result = array_merge($result, $localListeners[$priority]);
array_push($result, ...$localListeners[$priority]);
}
}
return $result;

28
app/Lib/Tools/BetterSecurity.php
View File

@ -7,8 +7,8 @@ class BetterSecurity
/**
* @param string $plain
* @param string $key
* @return string
* @param string $key Encryption key
* @return string Cipher text with IV and tag
* @throws Exception
*/
public static function encrypt($plain, $key)
@ -33,17 +33,17 @@ class BetterSecurity
}
/**
* @param string $cipher
* @param string $key
* @param string $cipherText Cipher text with IV and tag
* @param string $key Decryption key
* @return string
* @throws Exception
*/
public static function decrypt($cipher, $key)
public static function decrypt($cipherText, $key)
{
if (strlen($key) < 32) {
throw new Exception('Invalid key, key must be at least 256 bits (32 bytes) long.');
}
if (empty($cipher)) {
if (empty($cipherText)) {
throw new Exception('The data to decrypt cannot be empty.');
}
@ -52,12 +52,18 @@ class BetterSecurity
$ivSize = openssl_cipher_iv_length(self::METHOD);
// Split out hmac for comparison
$iv = substr($cipher, 0, $ivSize);
$tag = substr($cipher, $ivSize, self::TAG_SIZE);
$cipher = substr($cipher, $ivSize + self::TAG_SIZE);
if (strlen($cipherText) < $ivSize + self::TAG_SIZE) {
$length = strlen($cipherText);
$minLength = $ivSize + self::TAG_SIZE;
throw new Exception("Provided cipher text is too short, $length bytes provided, expected at least $minLength bytes.");
}
$decrypted = openssl_decrypt($cipher, self::METHOD, $key, true, $iv, $tag);
// Split out hmac for comparison
$iv = substr($cipherText, 0, $ivSize);
$tag = substr($cipherText, $ivSize, self::TAG_SIZE);
$cipherText = substr($cipherText, $ivSize + self::TAG_SIZE);
$decrypted = openssl_decrypt($cipherText, self::METHOD, $key, OPENSSL_RAW_DATA, $iv, $tag);
if ($decrypted === false) {
throw new Exception('Could not decrypt. Maybe invalid encryption key?');
}

11
app/Lib/Tools/ComplexTypeTool.php
View File

@ -308,14 +308,13 @@ class ComplexTypeTool
*/
private function parseFreetext($input)
{
$input = str_replace("\xc2\xa0", ' ', $input); // non breaking space to normal space
$input = preg_replace('/\p{C}+/u', ' ', $input);
$iocArray = preg_split("/\r\n|\n|\r|\s|\s+|,|\<|\>|;/", $input);
// convert non breaking space to normal space and all unicode chars from "other" category
$input = preg_replace("/\p{C}+|\xc2\xa0/u", ' ', $input);
$iocArray = preg_split("/\r\n|\n|\r|\s|\s+|,|<|>|;/", $input);
preg_match_all('/\"([^\"]+)\"/', $input, $matches);
foreach ($matches[1] as $match) {
$iocArray[] = $match;
}
array_push($iocArray, ...$matches[1]);
return $iocArray;
}

373
app/Lib/Tools/CurlClient.php Normal file
View File

@ -0,0 +1,373 @@
<?php
App::uses('HttpSocketExtended', 'Tools');
class CurlClient extends HttpSocketExtended
{
/** @var resource */
private $ch;
/** @var int */
private $timeout = 30;
/** @var string|null */
private $caFile;
/** @var string|null */
private $localCert;
/** @var int */
private $cryptoMethod;
/** @var bool */
private $allowSelfSigned;
/** @var bool */
private $verifyPeer;
/** @var bool */
private $compress = true;
/** @var array */
private $proxy = [];
/**
* @param array $params
* @noinspection PhpMissingParentConstructorInspection
*/
public function __construct(array $params)
{
if (isset($params['timeout'])) {
$this->timeout = $params['timeout'];
}
if (isset($params['ssl_cafile'])) {
$this->caFile = $params['ssl_cafile'];
}
if (isset($params['ssl_local_cert'])) {
$this->localCert = $params['ssl_local_cert'];
}
if (isset($params['compress'])) {
$this->compress = $params['compress'];
}
if (isset($params['ssl_crypto_method'])) {
$this->cryptoMethod = $this->convertCryptoMethod($params['ssl_crypto_method']);
}
if (isset($params['ssl_allow_self_signed'])) {
$this->allowSelfSigned = $params['ssl_allow_self_signed'];
}
if (isset($params['ssl_verify_peer'])) {
$this->verifyPeer = $params['ssl_verify_peer'];
}
}
/**
* @param string $uri
* @param array $query
* @param array $request
* @return HttpSocketResponseExtended
*/
public function head($uri = null, $query = [], $request = [])
{
return $this->internalRequest('HEAD', $uri, $query, $request);
}
/**
* @param string $uri
* @param array $query
* @param array $request
* @return HttpSocketResponseExtended
*/
public function get($uri = null, $query = [], $request = [])
{
return $this->internalRequest('GET', $uri, $query, $request);
}
/**
* @param string $uri
* @param array $data
* @param array $request
* @return HttpSocketResponseExtended
*/
public function post($uri = null, $data = [], $request = [])
{
return $this->internalRequest('POST', $uri, $data, $request);
}
/**
* @param string $uri
* @param array$data
* @param $request
* @return HttpSocketResponseExtended
*/
public function put($uri = null, $data = [], $request = [])
{
return $this->internalRequest('PUT', $uri, $data, $request);
}
/**
* @param string $uri
* @param array $data
* @param array $request
* @return HttpSocketResponseExtended
*/
public function patch($uri = null, $data = [], $request = [])
{
return $this->internalRequest('PATCH', $uri, $data, $request);
}
/**
* @param string $uri
* @param array $data
* @param array $request
* @return HttpSocketResponseExtended
*/
public function delete($uri = null, $data = array(), $request = array())
{
return $this->internalRequest('DELETE', $uri, $data, $request);
}
public function url($url = null, $uriTemplate = null)
{
throw new Exception('Not implemented');
}
public function request($request = array())
{
throw new Exception('Not implemented');
}
public function setContentResource($resource)
{
throw new Exception('Not implemented');
}
public function getMetaData()
{
return null; // not supported by curl extension
}
/**
* @param string $host
* @param int $port
* @param string $method
* @param string $user
* @param string $pass
* @return void
*/
public function configProxy($host, $port = 3128, $method = null, $user = null, $pass = null)
{
if (empty($host)) {
$this->proxy = [];
return;
}
if (is_array($host)) {
$this->proxy = $host + ['host' => null];
return;
}
$this->proxy = compact('host', 'port', 'method', 'user', 'pass');
}
/**
* @param string $method
* @param string $url
* @param array|string $query
* @param array $request
* @return HttpSocketResponseExtended
*/
private function internalRequest($method, $url, $query, $request)
{
if (empty($url)) {
throw new InvalidArgumentException("No URL provided.");
}
if (!$this->ch) {
// Share handle between requests to allow keep connection alive between requests
$this->ch = curl_init();
if (!$this->ch) {
throw new \RuntimeException("Could not initialize curl");
}
} else {
// Reset options, so we can do another request
curl_reset($this->ch);
}
if (($method === 'GET' || $method === 'HEAD') && !empty($query)) {
$url .= '?' . http_build_query($query, '', '&', PHP_QUERY_RFC3986);
}
$options = $this->generateOptions();
$options[CURLOPT_URL] = $url;
$options[CURLOPT_CUSTOMREQUEST] = $method;
if (($method === 'POST' || $method === 'DELETE' || $method === 'PUT' || $method === 'PATCH') && !empty($query)) {
$options[CURLOPT_POSTFIELDS] = $query;
}
if (!empty($request['header'])) {
$headers = [];
foreach ($request['header'] as $key => $value) {
if (is_array($value)) {
$value = implode(', ', $value);
}
$headers[] = "$key: $value";
}
$options[CURLOPT_HTTPHEADER] = $headers;
}
// Parse response headers
$responseHeaders = [];
$options[CURLOPT_HEADERFUNCTION] = function ($curl, $header) use (&$responseHeaders){
$len = strlen($header);
$header = explode(':', $header, 2);
if (count($header) < 2) { // ignore invalid headers
return $len;
}
$key = strtolower(trim($header[0]));
$value = trim($header[1]);
if (isset($responseHeaders[$key])) {
$responseHeaders[$key] = array_merge((array)$responseHeaders[$key], [$value]);
} else {
$responseHeaders[$key] = $value;
}
return $len;
};
if (!curl_setopt_array($this->ch, $options)) {
throw new \RuntimeException('curl error: Could not set options');
}
// Download the given URL, and return output
$output = curl_exec($this->ch);
if ($output === false) {
$errorCode = curl_errno($this->ch);
$errorMessage = curl_error($this->ch);
if (!empty($errorMessage)) {
$errorMessage = ": $errorMessage";
}
throw new SocketException("curl error $errorCode '" . curl_strerror($errorCode) . "'" . $errorMessage);
}
$code = curl_getinfo($this->ch, CURLINFO_HTTP_CODE);
return $this->constructResponse($output, $responseHeaders, $code);
}
public function disconnect()
{
if ($this->ch) {
curl_close($this->ch);
$this->ch = null;
}
}
/**
* @param string $body
* @param array $headers
* @param int $code
* @return HttpSocketResponseExtended
*/
private function constructResponse($body, array $headers, $code)
{
if (isset($responseHeaders['content-encoding']) && $responseHeaders['content-encoding'] === 'zstd') {
if (!function_exists('zstd_uncompress')) {
throw new SocketException('Response is zstd encoded, but PHP do not support zstd decoding.');
}
$body = zstd_uncompress($body);
if ($body === false) {
throw new SocketException('Could not decode zstd encoded response.');
}
}
$response = new HttpSocketResponseExtended();
$response->code = $code;
$response->body = $body;
$response->headers = $headers;
return $response;
}
/**
* @param int $cryptoMethod
* @return int
*/
private function convertCryptoMethod($cryptoMethod)
{
switch ($cryptoMethod) {
case STREAM_CRYPTO_METHOD_TLSv1_0_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_1_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
return CURL_SSLVERSION_TLSv1;
case STREAM_CRYPTO_METHOD_TLSv1_1_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
return CURL_SSLVERSION_TLSv1_1;
case STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
return CURL_SSLVERSION_TLSv1_2;
case STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
return CURL_SSLVERSION_TLSv1_3;
default:
throw new InvalidArgumentException("Unsupported crypto method value $cryptoMethod");
}
}
/**
* @return array
*/
private function generateOptions()
{
$options = [
CURLOPT_FOLLOWLOCATION => true, // Allows to follow redirect
CURLOPT_MAXREDIRS => 10,
CURLOPT_RETURNTRANSFER => true, // Should cURL return or print out the data? (true = return, false = print)
CURLOPT_HEADER => false, // Include header in result?
CURLOPT_TIMEOUT => $this->timeout, // Timeout in seconds
CURLOPT_PROTOCOLS => CURLPROTO_HTTPS | CURLPROTO_HTTP, // be sure that only HTTP and HTTPS protocols are enabled,
];
if ($this->caFile) {
$options[CURLOPT_CAINFO] = $this->caFile;
}
if ($this->localCert) {
$options[CURLOPT_SSLCERT] = $this->localCert;
}
if ($this->cryptoMethod) {
$options[CURLOPT_SSLVERSION] = $this->cryptoMethod;
}
if ($this->compress) {
$options[CURLOPT_ACCEPT_ENCODING] = $this->supportedEncodings();
}
if ($this->allowSelfSigned) {
$options[CURLOPT_SSL_VERIFYPEER] = $this->verifyPeer;
$options[CURLOPT_SSL_VERIFYHOST] = 0;
}
if (!empty($this->proxy)) {
$options[CURLOPT_PROXY] = "{$this->proxy['host']}:{$this->proxy['port']}";
if (!empty($this->proxy['method']) && isset($this->proxy['user'], $this->proxy['pass'])) {
$options[CURLOPT_PROXYUSERPWD] = "{$this->proxy['user']}:{$this->proxy['pass']}";
}
}
return $options;
}
/**
* @return string
*/
private function supportedEncodings()
{
$encodings = [];
// zstd is not supported by curl itself, but add support if PHP zstd extension is installed
if (function_exists('zstd_uncompress')) {
$encodings[] = 'zstd';
}
// brotli and gzip is supported by curl itself if it is compiled with these features
$info = curl_version();
if (defined('CURL_VERSION_BROTLI') && $info['features'] & CURL_VERSION_BROTLI) {
$encodings[] = 'br';
}
if ($info['features'] & CURL_VERSION_LIBZ) {
$encodings[] = 'gzip, deflate';
}
return implode(', ', $encodings);
}
}

2
app/Lib/Tools/EventTimelineTool.php
View File

@ -146,7 +146,7 @@
'group' => 'object_attribute',
'timestamp' => $obj_attr['timestamp'],
'attribute_type' => $obj_attr['type'],
'date_sighting' => $sightingsAttributeMap[$attr['id']] ?? [],
'date_sighting' => $sightingsAttributeMap[$obj_attr['id']] ?? [],
'is_image' => $this->__eventModel->Attribute->isImage($obj_attr),
);
$toPush_obj['Attribute'][] = $toPush_attr;

65
app/Lib/Tools/GitTool.php
View File

@ -12,11 +12,7 @@ class GitTool
public static function getLatestTags(HttpSocketExtended $HttpSocket)
{
$url = 'https://api.github.com/repos/MISP/MISP/tags?per_page=10';
$response = $HttpSocket->get($url);
if (!$response->isOk()) {
throw new HttpSocketHttpException($response, $url);
}
return $response->json();
return self::gitHubRequest($HttpSocket, $url);
}
/**
@ -28,11 +24,7 @@ class GitTool
public static function getLatestCommit(HttpSocketExtended $HttpSocket)
{
$url = 'https://api.github.com/repos/MISP/MISP/commits?per_page=1';
$response = $HttpSocket->get($url);
if (!$response->isOk()) {
throw new HttpSocketHttpException($response, $url);
}
$data = $response->json();
$data = self::gitHubRequest($HttpSocket, $url);
if (!isset($data[0]['sha'])) {
throw new Exception("Response do not contains requested data.");
}
@ -40,20 +32,49 @@ class GitTool
}
/**
* @param HttpSocketExtended $HttpSocket
* @param string $url
* @return array
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
*/
private static function gitHubRequest(HttpSocketExtended $HttpSocket, $url)
{
$response = $HttpSocket->get($url, [], ['header' => ['User-Agent' => 'MISP']]);
if (!$response->isOk()) {
throw new HttpSocketHttpException($response, $url);
}
return $response->json();
}
/**
* Returns current SHA1 hash of current commit
* `git rev-parse HEAD`
* @param string $repoPath
* @return string
* @throws Exception
*/
public static function currentCommit()
public static function currentCommit($repoPath)
{
$head = rtrim(FileAccessTool::readFromFile(ROOT . '/.git/HEAD'));
if (is_file($repoPath . '/.git')) {
$fileContent = FileAccessTool::readFromFile($repoPath . '/.git');
if (substr($fileContent, 0, 8) === 'gitdir: ') {
$gitDir = $repoPath . '/' . trim(substr($fileContent, 8)) . '/';
} else {
throw new Exception("$repoPath/.git is file, but contains non expected content $fileContent");
}
} else {
$gitDir = $repoPath . '/.git/';
}
$head = rtrim(FileAccessTool::readFromFile($gitDir . 'HEAD'));
if (substr($head, 0, 5) === 'ref: ') {
$path = substr($head, 5);
return rtrim(FileAccessTool::readFromFile(ROOT . '/.git/' . $path));
return rtrim(FileAccessTool::readFromFile($gitDir . $path));
} else if (strlen($head) === 40) {
return $head;
} else {
throw new Exception("Invalid head $head");
throw new Exception("Invalid head '$head' in $gitDir/HEAD");
}
}
@ -94,30 +115,18 @@ class GitTool
return $output;
}
/**
* @param string $submodule Path to Git repo
* @return string|null
*/
public static function submoduleCurrentCommit($submodule)
{
try {
$commit = ProcessTool::execute(['git', 'rev-parse', 'HEAD'], $submodule);
} catch (ProcessException $e) {
return null;
}
return rtrim($commit);
}
/**
* @param string $commit
* @param string|null $submodule Path to Git repo
* @return int|null
* @throws Exception
*/
public static function commitTimestamp($commit, $submodule = null)
{
try {
$timestamp = ProcessTool::execute(['git', 'show', '-s', '--pretty=format:%ct', $commit], $submodule);
} catch (ProcessException $e) {
CakeLog::notice("Could not get Git commit timestamp for $submodule: {$e->getMessage()}");
return null;
}
return (int)rtrim($timestamp);

5
app/Lib/Tools/HttpSocketExtended.php
View File

@ -18,10 +18,15 @@ class HttpSocketHttpException extends Exception
{
$this->response = $response;
$this->url = $url;
$message = "Remote server returns HTTP error code $response->code";
if ($url) {
$message .= " for URL $url";
}
if ($response->body) {
$message .= ': ' . substr($response->body, 0, 100);
}
parent::__construct($message, (int)$response->code);
}

2
app/Lib/Tools/JSONConverterTool.php
View File

@ -153,7 +153,7 @@ class JSONConverterTool
return;
}
yield '{"Event":{';
$firstKey = key($event['Event']);
$firstKey = array_key_first($event['Event']);
foreach ($event['Event'] as $key => $value) {
if ($key === 'Attribute' || $key === 'Object') { // Encode every object or attribute separately
yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":[";

50
app/Lib/Tools/JsonTool.php
View File

@ -9,10 +9,7 @@ class JsonTool
*/
public static function encode($value, $prettyPrint = false)
{
$flags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE;
if (defined('JSON_THROW_ON_ERROR')) {
$flags |= JSON_THROW_ON_ERROR; // Throw exception on error if supported
}
$flags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR;
if ($prettyPrint) {
$flags |= JSON_PRETTY_PRINT;
}
@ -34,16 +31,8 @@ class JsonTool
} catch (SimdJsonException $e) {
throw new JsonException($e->getMessage(), $e->getCode(), $e);
}
} elseif (defined('JSON_THROW_ON_ERROR')) {
// JSON_THROW_ON_ERROR is supported since PHP 7.3
return json_decode($value, true, 512, JSON_THROW_ON_ERROR);
} else {
$decoded = json_decode($value, true);
if ($decoded === null) {
throw new UnexpectedValueException('Could not parse JSON: ' . json_last_error_msg(), json_last_error());
}
return $decoded;
}
return json_decode($value, true, 512, JSON_THROW_ON_ERROR);
}
/**
@ -78,4 +67,39 @@ class JsonTool
return false;
}
}
/**
* @see https://www.php.net/manual/en/function.array-is-list.php
* @param array $array
* @return bool
*/
public static function arrayIsList(array $array)
{
if (function_exists('array_is_list')) {
return array_is_list($array);
}
$i = -1;
foreach ($array as $k => $v) {
++$i;
if ($k !== $i) {
return false;
}
}
return true;
}
/**
* JSON supports just unicode strings. This helper method converts non unicode chars to Unicode Replacement Character U+FFFD (UTF-8)
* @param string $string
* @return string
*/
public static function escapeNonUnicode($string)
{
if (mb_check_encoding($string, 'UTF-8')) {
return $string; // string is valid unicode
}
return htmlspecialchars_decode(htmlspecialchars($string, ENT_SUBSTITUTE, 'UTF-8'));
}
}

15
app/Lib/Tools/ProcessTool.php
View File

@ -8,14 +8,14 @@ class ProcessException extends Exception
private $stdout;
/**
* @param string|array $command
* @param array $command
* @param int $returnCode
* @param string $stderr
* @param string $stdout
*/
public function __construct($command, $returnCode, $stderr, $stdout)
public function __construct(array $command, $returnCode, $stderr, $stdout)
{
$commandForException = is_array($command) ? implode(' ', $command) : $command;
$commandForException = implode(' ', $command);
$message = "Command '$commandForException' finished with error code $returnCode.\nSTDERR: '$stderr'\nSTDOUT: '$stdout'";
$this->stderr = $stderr;
$this->stdout = $stdout;
@ -56,11 +56,6 @@ class ProcessTool
self::logMessage('Running command ' . implode(' ', $command));
}
// PHP older than 7.4 do not support proc_open with array, so we need to convert values to string manually
if (PHP_VERSION_ID < 70400) {
$command = array_map('escapeshellarg', $command);
$command = implode(' ', $command);
}
$process = proc_open($command, $descriptorSpec, $pipes, $cwd);
if (!$process) {
$commandForException = self::commandFormat($command);
@ -136,8 +131,8 @@ class ProcessTool
* @param array|string $command
* @return string
*/
private static function commandFormat($command)
private static function commandFormat(array $command)
{
return is_array($command) ? implode(' ', $command) : $command;
return implode(' ', $command);
}
}

31
app/Lib/Tools/PubSubTool.php
View File

@ -178,8 +178,12 @@ class PubSubTool
public function killService()
{
$settings = $this->getSetSettings();
if ($settings['supervisor_managed']) {
throw new RuntimeException('ZeroMQ server is managed by supervisor, it is not possible to restart it.');
}
if ($this->checkIfRunning()) {
$settings = $this->getSetSettings();
$redis = $this->createRedisConnection($settings);
$redis->rPush('command', 'kill');
sleep(1);
@ -213,12 +217,16 @@ class PubSubTool
public function restartServer()
{
$settings = $this->getSetSettings();
if ($settings['supervisor_managed']) {
throw new RuntimeException('ZeroMQ server is managed by supervisor, it is not possible to restart it.');
}
if (!$this->checkIfRunning()) {
if (!$this->killService()) {
return 'Could not kill the previous instance of the ZeroMQ script.';
}
}
$settings = $this->getSetSettings();
$this->setupPubServer($settings);
if ($this->checkIfRunning() === false) {
return 'Failed starting the ZeroMQ script.';
@ -226,12 +234,22 @@ class PubSubTool
return true;
}
public function createConfigFile()
{
$settings = $this->getSetSettings();
$this->saveSettingToFile($settings);
}
/**
* @param array $settings
* @throws Exception
*/
private function setupPubServer(array $settings)
{
if ($settings['supervisor_managed']) {
return; // server is managed by supervisor, we don't need to check if is running or start it when not
}
if ($this->checkIfRunning() === false) {
if ($this->checkIfRunning(self::OLD_PID_LOCATION)) {
// Old version is running, kill it and start again new one.
@ -250,6 +268,7 @@ class PubSubTool
* @param string|array $data
* @return bool
* @throws JsonException
* @throws RedisException
*/
private function pushToRedis($ns, $data)
{
@ -295,9 +314,12 @@ class PubSubTool
FileAccessTool::writeToFile($settingFilePath, JsonTool::encode($settings));
}
/**
* @return array
*/
private function getSetSettings()
{
$settings = array(
$settings = [
'redis_host' => 'localhost',
'redis_port' => 6379,
'redis_password' => '',
@ -307,7 +329,8 @@ class PubSubTool
'port' => '50000',
'username' => null,
'password' => null,
);
'supervisor_managed' => false,
];
$pluginConfig = Configure::read('Plugin');
foreach ($settings as $key => $setting) {

23
app/Lib/Tools/RedisTool.php
View File

@ -57,25 +57,34 @@ class RedisTool
/**
* @param Redis $redis
* @param string|array $pattern
* @return int|Redis Number of deleted keys or instance of Redis if used in MULTI mode
* @return Generator<string>
* @throws RedisException
*/
public static function deleteKeysByPattern(Redis $redis, $pattern)
public static function keysByPattern(Redis $redis, $pattern)
{
if (is_string($pattern)) {
$pattern = [$pattern];
}
$allKeys = [];
foreach ($pattern as $p) {
$iterator = null;
while (false !== ($keys = $redis->scan($iterator, $p, 1000))) {
foreach ($keys as $key) {
$allKeys[] = $key;
yield $key;
}
}
}
}
/**
* @param Redis $redis
* @param string|array $pattern
* @return int|Redis Number of deleted keys or instance of Redis if used in MULTI mode
* @throws RedisException
*/
public static function deleteKeysByPattern(Redis $redis, $pattern)
{
$allKeys = iterator_to_array(self::keysByPattern($redis, $pattern));
if (empty($allKeys)) {
return 0;
}
@ -153,11 +162,7 @@ class RedisTool
return false;
}
if (self::$serializer === null) {
self::$serializer = Configure::read('MISP.redis_serializer') ?: false;
}
if (self::$serializer === 'igbinary') {
if ($string[0] === "\x00") {
return igbinary_unserialize($string);
} else {
return JsonTool::decode($string);

8
app/Lib/Tools/ServerSyncTool.php
View File

@ -355,6 +355,14 @@ class ServerSyncTool
return $this->server['Server']['id'];
}
/**
* @return string
*/
public function serverName()
{
return $this->server['Server']['name'];
}
/**
* @return array
*/

11
app/Lib/Tools/SyncTool.php
View File

@ -1,5 +1,4 @@
<?php
class SyncTool
{
@ -84,8 +83,14 @@ class SyncTool
$params['ssl_crypto_method'] = $version;
}
App::uses('HttpSocketExtended', 'Tools');
$HttpSocket = new HttpSocketExtended($params);
if (function_exists('curl_init')) {
App::uses('CurlClient', 'Tools');
$HttpSocket = new CurlClient($params);
} else {
App::uses('HttpSocketExtended', 'Tools');
$HttpSocket = new HttpSocketExtended($params);
}
$proxy = Configure::read('Proxy');
if (empty($params['skip_proxy']) && isset($proxy['host']) && !empty($proxy['host'])) {
$HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']);

2
app/Locale/ara/LC_MESSAGES/default.po
View File

@ -14679,7 +14679,7 @@ msgstr ""
#: View/Events/automation.ctp:67
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr ""
#: View/Events/automation.ctp:68

2
app/Locale/cze/LC_MESSAGES/default.po
View File

@ -14651,7 +14651,7 @@ msgstr ""
#: View/Events/automation.ctp:67
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr ""
#: View/Events/automation.ctp:68

2
app/Locale/dan/LC_MESSAGES/default.po
View File

@ -14624,7 +14624,7 @@ msgstr ""
#: View/Events/automation.ctp:67
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr ""
#: View/Events/automation.ctp:68

2
app/Locale/default.pot
View File

@ -15934,7 +15934,7 @@ msgstr ""
#: View/Events/automation.ctp:67
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr ""
#: View/Events/automation.ctp:68

2
app/Locale/deu/LC_MESSAGES/default.po
View File

@ -14622,7 +14622,7 @@ msgstr ""
#: View/Events/automation.ctp:67
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr ""
#: View/Events/automation.ctp:68

2
app/Locale/fra/LC_MESSAGES/default.po
View File

@ -14626,7 +14626,7 @@ msgstr "Par défaut (0), tout les attributs qui correspondent aux autres paramè
#: View/Events/automation.ctp:67
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr "Si le paramètre est défini à 1, cela va retourner les attributs mis à la corbeille ainsi que les attributs actifs. En utilisant \"only\" en tant que paramètre, cela va seulement retourner les données mises à la corbeille."
#: View/Events/automation.ctp:68

2
app/Locale/hun/LC_MESSAGES/default.po
View File

@ -14622,7 +14622,7 @@ msgstr ""
#: View/Events/automation.ctp:67
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr ""
#: View/Events/automation.ctp:68

2
app/Locale/ita/LC_MESSAGES/default.po
View File

@ -14625,7 +14625,7 @@ msgstr "Per impostazione predefinita (0) tutti gli attributi restituiti rispondo
#: View/Events/automation.ctp:67
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr "Se questo parametro è impostato a 1, verranno restituiti attributi \"soft-deleted\" insieme a quelli attivi. Utilizzando \"only\" come parametro verranno restituiti solo gli attributi \"soft-deleted\"."
#: View/Events/automation.ctp:68

2
app/Locale/jpn/LC_MESSAGES/default.po
View File

@ -14608,7 +14608,7 @@ msgstr "デフォルトの (0) では、to_ids の設定に関係なく、他の
#: View/Events/automation.ctp:67
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr "このパラメーターを 1 に設定すると、ソフト削除されたアトリビュートがアクティなアトリビュートと共に返されます。\"only\"をパラメーターとして使用すると、返されるデータはソフト削除されたデータのみに制限されます。"
#: View/Events/automation.ctp:68

2
app/Locale/kor/LC_MESSAGES/default.po
View File

@ -14609,7 +14609,7 @@ msgstr ""
#: View/Events/automation.ctp:67
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr ""
#: View/Events/automation.ctp:68

2
app/Locale/no/LC_MESSAGES/default.po
View File

@ -14625,7 +14625,7 @@ msgstr "Som standard (0) returneres alle attributter som samsvarer med de andre
#: View/Events/automation.ctp:67
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr "Hvis denne parameteren er satt til 1, vil den returnere myke slettede attributter sammen med aktive. Ved å bruke \"only\" som en parameter, vil det begrense det returnerte datasettet til bare slettede data."
#: View/Events/automation.ctp:68

2
app/Locale/pol/LC_MESSAGES/default.po
View File

@ -14651,7 +14651,7 @@ msgstr ""
#: View/Events/automation.ctp:67
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr ""
#: View/Events/automation.ctp:68

2
app/Locale/pt/LC_MESSAGES/default.po
View File

@ -7964,7 +7964,7 @@ msgstr ""
#: View/Events/automation.ctp:42
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr ""
#: View/Events/automation.ctp:43

2
app/Locale/pt_BR/LC_MESSAGES/default.po
View File

@ -14623,7 +14623,7 @@ msgstr ""
#: View/Events/automation.ctp:67
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr ""
#: View/Events/automation.ctp:68

2
app/Locale/ro/LC_MESSAGES/default.po
View File

@ -14636,7 +14636,7 @@ msgstr ""
#: View/Events/automation.ctp:67
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr ""
#: View/Events/automation.ctp:68

2
app/Locale/rus/LC_MESSAGES/default.po
View File

@ -14650,7 +14650,7 @@ msgstr "По-умолчанию (а также при значении 0) в п
#: View/Events/automation.ctp:67
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr "По-умолчанию (а также при значении 0) в поиск попадают только активные атрибуты. Если параметр равен 1, то в поиск попадут дополнительно удаленные атрибуты. Если используется ключевое слово \"only\", то в результаты поиска попадут только удаленные атрибуты. "
#: View/Events/automation.ctp:68

2
app/Locale/si-LK/LC_MESSAGES/default.po
View File

@ -14638,7 +14638,7 @@ msgstr "පෙරනිමියෙන් (0) to_ids සිටුවම් න
#: View/Events/automation.ctp:67
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr "මෙම පරාමිතිය 1 ලෙස සකසා ඇත්නම්, එය සක්‍රිය ඒවා සමඟ මෘදු-මකා දැමූ ගුණාංග ලබා දෙනු ඇත. පරාමිතියක් ලෙස \"පමණක්\" භාවිතා කිරීමෙන් එය ආපසු ලබා දෙන දත්ත කට්ටලය මෘදු-මකා දැමූ දත්ත වලට පමණක් සීමා කරයි."
#: View/Events/automation.ctp:68

2
app/Locale/spa/LC_MESSAGES/default.po
View File

@ -14624,7 +14624,7 @@ msgstr ""
#: View/Events/automation.ctp:67
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr ""
#: View/Events/automation.ctp:68

2
app/Locale/swe/LC_MESSAGES/default.po
View File

@ -10898,7 +10898,7 @@ msgstr ""
#: View/Events/automation.ctp:52
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr ""
#: View/Events/automation.ctp:53

2
app/Locale/th_TH/LC_MESSAGES/default.po
View File

@ -14620,7 +14620,7 @@ msgstr ""
#: View/Events/automation.ctp:67
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr ""
#: View/Events/automation.ctp:68

2
app/Locale/ukr/LC_MESSAGES/default.po
View File

@ -6030,7 +6030,7 @@ msgid "By default (0) all attributes are returned that match the other filter pa
msgstr ""
#: View/Events/automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr ""
#: View/Events/automation.ctp:316

2
app/Locale/zh-s/LC_MESSAGES/default.po
View File

@ -14619,7 +14619,7 @@ msgstr "默认情况下(0), 返回所有与其他过滤器参数匹配的属性,
#: View/Events/automation.ctp:67
#: View/Events/legacy_automation.ctp:315
msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
msgstr "如果这个参数被设置为1, 它将返回软删除的属性和活动属性. 如果使用\"only\"作为参数, 则返回的数据集将只限于软删除的数据."
#: View/Events/automation.ctp:68

48
app/Model/AccessLog.php
View File

@ -71,12 +71,6 @@ class AccessLog extends AppModel
{
$accessLog = &$this->data['AccessLog'];
$this->externalLog($accessLog);
if (Configure::read('MISP.log_paranoid_skip_db')) {
return;
}
// Truncate
foreach (['request_id', 'user_agent', 'url'] as $field) {
if (isset($accessLog[$field]) && strlen($accessLog[$field]) > 255) {
@ -202,7 +196,7 @@ class AccessLog extends AppModel
if ($includeSqlQueries && !empty($sqlLog['log'])) {
foreach ($sqlLog['log'] as &$log) {
$log['query'] = $this->escapeNonUnicode($log['query']);
$log['query'] = JsonTool::escapeNonUnicode($log['query']);
unset($log['affected']); // affected is the same as numRows
unset($log['params']); // no need to save for your use case
}
@ -214,6 +208,12 @@ class AccessLog extends AppModel
$data['query_count'] = $queryCount;
$data['duration'] = (int)((microtime(true) - $requestTime->format('U.u')) * 1000); // in milliseconds
$this->externalLog($data);
if (Configure::read('MISP.log_paranoid_skip_db')) {
return true; // do not save access log to database
}
try {
return $this->save($data, ['atomic' => false]);
} catch (Exception $e) {
@ -226,7 +226,7 @@ class AccessLog extends AppModel
* @param array $data
* @return void
*/
public function externalLog(array $data)
private function externalLog(array $data)
{
if ($this->pubToZmq('audit')) {
$this->getPubSubTool()->publish($data, 'audit', 'log');
@ -310,36 +310,4 @@ class AccessLog extends AppModel
}
return $data;
}
/**
* @param $string
* @return string
*/
private function escapeNonUnicode($string)
{
if (json_encode($string, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_LINE_TERMINATORS) !== false) {
return $string; // string is valid unicode
}
if (function_exists('mb_str_split')) {
$result = mb_str_split($string);
} else {
$result = [];
$length = mb_strlen($string);
for ($i = 0; $i < $length; $i++) {
$result[] = mb_substr($string, $i, 1);
}
}
$string = '';
foreach ($result as $char) {
if (strlen($char) === 1 && !preg_match('/[[:print:]]/', $char)) {
$string .= '\x' . bin2hex($char);
} else {
$string .= $char;
}
}
return $string;
}
}

28
app/Model/AdminSetting.php
View File

@ -90,6 +90,7 @@ class AdminSetting extends AppModel
$time = time();
$this->__deleteScriptTmpFiles($time);
$this->__deleteTaxiiTmpFiles($time);
$this->__deleteCachedExportFiles($time);
}
private function __deleteScriptTmpFiles($time) {
@ -107,6 +108,29 @@ class AdminSetting extends AppModel
}
}
private function __deleteCachedExportFiles($time) {
$cache_path = APP . 'tmp/cached_exports';
$cache_dir = new Folder($cache_path);
$cache_data = $cache_dir->read(false, false);
if (!empty($cache_data[0])) {
foreach ($cache_data[0] as $cache_export_dir) {
$tmp_dir = new Folder($cache_path . '/' . $cache_export_dir);
$cache_export_dir_contents = $tmp_dir->read(false, false);
if (!empty(count($cache_export_dir_contents[1]))) {
$files_count = count($cache_export_dir_contents[1]);
$files_removed = 0;
foreach ($cache_export_dir_contents[1] as $tmp_file) {
$tmp_file = new File($cache_path . '/' . $cache_export_dir . '/' . $tmp_file);
if ($time > $tmp_file->lastChange() + 3600) {
$tmp_file->delete();
$files_removed += 1;
}
}
}
}
}
}
private function __deleteTaxiiTmpFiles($time) {
$taxii_path = APP . 'files/scripts/tmp/Taxii';
$taxii_dir = new Folder($taxii_path);
@ -114,13 +138,13 @@ class AdminSetting extends AppModel
if (!empty($taxii_contents[0])) {
foreach ($taxii_contents[0] as $taxii_temp_dir) {
if (preg_match('/^[a-zA-Z0-9]{12}$/', $taxii_temp_dir)) {
$tmp_dir = new Folder($taxii_path . $taxii_temp_dir);
$tmp_dir = new Folder($taxii_path . '/' .$taxii_temp_dir);
$taxii_temp_dir_contents = $tmp_dir->read(false, false);
if (!empty(count($taxii_temp_dir_contents[1]))) {
$files_count = count($taxii_temp_dir_contents[1]);
$files_removed = 0;
foreach ($taxii_temp_dir_contents[1] as $tmp_file) {
$tmp_file = new File($taxii_path . $taxii_temp_dir . '/' . $tmp_file);
$tmp_file = new File($taxii_path . '/' . $taxii_temp_dir . '/' . $tmp_file);
if ($time > $tmp_file->lastChange() + 3600) {
$tmp_file->delete();
$files_removed += 1;

3
app/Model/Allowedlist.php
View File

@ -89,6 +89,9 @@ class Allowedlist extends AppModel
if ($isAttributeArray) {
// loop through each attribute and unset the ones that are allowedlisted
foreach ($data as $k => $attribute) {
if (empty($attribute['Attribute'])) {
$attribute = ['Attribute' => $attribute];
}
// loop through each allowedlist item and run a preg match against the attribute value. If it matches, unset the attribute
foreach ($allowedlists as $wlitem) {
if (preg_match($wlitem, $attribute['Attribute']['value'])) {

94
app/Model/AppModel.php
View File

@ -27,6 +27,7 @@ App::uses('FileAccessTool', 'Tools');
App::uses('JsonTool', 'Tools');
App::uses('RedisTool', 'Tools');
App::uses('BetterCakeEventManager', 'Tools');
App::uses('Folder', 'Utility');
class AppModel extends Model
{
@ -86,7 +87,7 @@ class AppModel extends Model
99 => false, 100 => false, 101 => false, 102 => false, 103 => false, 104 => false,
105 => false, 106 => false, 107 => false, 108 => false, 109 => false, 110 => false,
111 => false, 112 => false, 113 => true, 114 => false, 115 => false, 116 => false,
117 => false, 118 => false
117 => false, 118 => false, 119 => false, 120 => false
);
const ADVANCED_UPDATES_DESCRIPTION = array(
@ -272,6 +273,9 @@ class AppModel extends Model
$this->removeDuplicatedUUIDs();
$dbUpdateSuccess = $this->updateDatabase('createUUIDsConstraints');
break;
case 120:
$dbUpdateSuccess = $this->moveImages();
break;
default:
$dbUpdateSuccess = $this->updateDatabase($command);
break;
@ -2006,6 +2010,9 @@ class AppModel extends Model
case 118:
$sqlArray[] = "ALTER TABLE `event_reports` MODIFY `content` mediumtext;";
break;
case 119:
$sqlArray[] = "ALTER TABLE `access_logs` MODIFY `action` varchar(191) NOT NULL";
break;
case 'fixNonEmptySharingGroupID':
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@ -2374,9 +2381,9 @@ class AppModel extends Model
}
// alternative to the build in notempty/notblank validation functions, compatible with cakephp <= 2.6 and cakephp and cakephp >= 2.7
public function valueNotEmpty($value)
public function valueNotEmpty(array $value)
{
$field = array_keys($value)[0];
$field = array_key_first($value);
$value = trim($value[$field]);
if (!empty($value)) {
return true;
@ -2384,27 +2391,27 @@ class AppModel extends Model
return ucfirst($field) . ' cannot be empty.';
}
public function valueIsJson($value)
public function valueIsJson(array $value)
{
$value = array_values($value)[0];
$value = current($value);
if (!JsonTool::isValid($value)) {
return __('Invalid JSON.');
}
return true;
}
public function valueIsID($value)
public function valueIsID(array $value)
{
$field = array_keys($value)[0];
$field = array_key_first($value);
if (!is_numeric($value[$field]) || $value[$field] < 0) {
return 'Invalid ' . ucfirst($field) . ' ID';
}
return true;
}
public function stringNotEmpty($value)
public function stringNotEmpty(array $value)
{
$field = array_keys($value)[0];
$field = array_key_first($value);
$value = trim($value[$field]);
if (!isset($value) || ($value == false && $value !== "0")) {
return ucfirst($field) . ' cannot be empty.';
@ -3267,14 +3274,13 @@ class AppModel extends Model
* Returns MISP version from VERSION.json file as array with major, minor and hotfix keys.
*
* @return array
* @throws JsonException
* @throws Exception
*/
public function checkMISPVersion()
{
static $versionArray;
if ($versionArray === null) {
$content = FileAccessTool::readFromFile(ROOT . DS . 'VERSION.json');
$versionArray = JsonTool::decode($content);
$versionArray = FileAccessTool::readJsonFromFile(ROOT . DS . 'VERSION.json', true);
}
return $versionArray;
}
@ -3290,7 +3296,7 @@ class AppModel extends Model
if ($commit === null) {
App::uses('GitTool', 'Tools');
try {
$commit = GitTool::currentCommit();
$commit = GitTool::currentCommit(ROOT);
} catch (Exception $e) {
$this->logException('Could not get current git commit', $e, LOG_NOTICE);
$commit = false;
@ -3714,7 +3720,7 @@ class AppModel extends Model
if (!$isRule) {
$args = func_get_args();
$fields = $args[1];
$or = isset($args[2]) ? $args[2] : true;
$or = $args[2] ?? true;
}
}
if (!is_array($fields)) {
@ -3859,8 +3865,7 @@ class AppModel extends Model
protected function isMysql()
{
$dataSource = ConnectionManager::getDataSource('default');
$dataSourceName = $dataSource->config['datasource'];
return $dataSourceName === 'Database/Mysql' || $dataSourceName === 'Database/MysqlObserver' || $dataSourceName === 'Database/MysqlExtended' || $dataSource instanceof Mysql;
return $dataSource instanceof Mysql;
}
/**
@ -3996,21 +4001,21 @@ class AppModel extends Model
");
}
public function findOrder($order, $order_model, $valid_order_fields)
public function findOrder($order, $orderModel, $validOrderFields)
{
if (!is_array($order)) {
$order_rules = explode(' ', strtolower($order));
$order_field = explode('.', $order_rules[0]);
$order_field = end($order_field);
if (in_array($order_field, $valid_order_fields)) {
$orderRules = explode(' ', strtolower($order));
$orderField = explode('.', $orderRules[0]);
$orderField = end($orderField);
if (in_array($orderField, $validOrderFields, true)) {
$direction = 'asc';
if (!empty($order_rules[1]) && trim($order_rules[1]) === 'desc') {
if (!empty($orderRules[1]) && trim($orderRules[1]) === 'desc') {
$direction = 'desc';
}
} else {
return null;
}
return $order_model . '.' . $order_field . ' ' . $direction;
return $orderModel . '.' . $orderField . ' ' . $direction;
}
return null;
}
@ -4020,6 +4025,12 @@ class AppModel extends Model
*/
public function _remoteIp()
{
static $remoteIp;
if ($remoteIp) {
return $remoteIp;
}
$clientIpHeader = Configure::read('MISP.log_client_ip_header');
if ($clientIpHeader && isset($_SERVER[$clientIpHeader])) {
$headerValue = $_SERVER[$clientIpHeader];
@ -4027,9 +4038,12 @@ class AppModel extends Model
if (($commaPos = strpos($headerValue, ',')) !== false) {
$headerValue = substr($headerValue, 0, $commaPos);
}
return trim($headerValue);
$remoteIp = trim($headerValue);
} else {
$remoteIp = $_SERVER['REMOTE_ADDR'] ?? null;
}
return $_SERVER['REMOTE_ADDR'] ?? null;
return $remoteIp;
}
public function find($type = 'first', $query = array())
@ -4062,8 +4076,36 @@ class AppModel extends Model
return false;
}
public function checkParam($param)
private function checkParam($param)
{
return preg_match('/^[\w\_\-\. ]+$/', $param);
}
public function moveImages()
{
$oldImageDir = APP . 'webroot/img';
$newImageDir = APP . 'files/img';
$oldOrgDir = new Folder($oldImageDir . '/orgs');
$oldCustomDir = new Folder($oldImageDir . '/custom');
$result = false;
$result = $oldOrgDir->copy([
'from' => $oldImageDir . '/orgs',
'to' => $newImageDir . '/orgs',
'scheme' => Folder::OVERWRITE,
'recursive' => true
]);
if ($result) {
$oldOrgDir->delete();
}
$result = $oldCustomDir->copy([
'from' => $oldImageDir . '/custom',
'to' => $newImageDir . '/custom',
'scheme' => Folder::OVERWRITE,
'recursive' => true
]);
if ($result) {
$oldCustomDir->delete();
}
return true;
}
}

26
app/Model/AttachmentScan.php
View File

@ -189,6 +189,7 @@ class AttachmentScan extends AppModel
/** @var Job $job */
$job = ClassRegistry::init('Job');
if ($jobId && !$job->exists($jobId)) {
$this->log("Job with ID $jobId not found in database", LOG_NOTICE);
$jobId = null;
}
@ -252,12 +253,12 @@ class AttachmentScan extends AppModel
$infected = $this->scanAttachment($type, $attribute[$type], $moduleInfo);
if ($infected === true) {
$virusFound++;
$scanned++;
} else if ($infected === false) {
$scanned++;
}
$scanned++;
} catch (NotFoundException $e) {
// skip if file doesn't exists
} catch (Exception $e) {
$this->logException("Could not scan attachment for $type {$attribute['Attribute']['id']}", $e);
$this->logException("Could not scan attachment for $type {$attribute['Attribute']['id']}", $e, LOG_WARNING);
$fails++;
}
@ -297,14 +298,14 @@ class AttachmentScan extends AppModel
$job = ClassRegistry::init('Job');
$jobId = $job->createJob(
'SYSTEM',
Job::WORKER_DEFAULT,
Job::WORKER_PRIO,
'virus_scan',
($type === self::TYPE_ATTRIBUTE ? 'Attribute: ' : 'Shadow attribute: ') . $attribute['id'],
'Scanning...'
);
$this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::PRIO_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[
'scanAttachment',
@ -319,10 +320,12 @@ class AttachmentScan extends AppModel
}
/**
* Return true if attachment is infected, null if attachment was not scanned and false if attachment is OK
*
* @param string $type
* @param array $attribute
* @param array $moduleInfo
* @return bool|null Return true if attachment is infected.
* @return bool|null
* @throws Exception
*/
private function scanAttachment($type, array $attribute, array $moduleInfo)
@ -351,11 +354,10 @@ class AttachmentScan extends AppModel
return false; // empty file is automatically considered as not infected
}
/* if ($file->size() > 50 * 1024 * 1024) {
$this->log("File '$file->path' is bigger than 50 MB, will be not scanned.", LOG_NOTICE);
return false;
}*/
if ($fileSize > 25 * 1024 * 1024) {
$this->log("File '$file->path' is bigger than 25 MB, will be not scanned.", LOG_NOTICE);
return null;
}
$fileContent = $file->read();
if ($fileContent === false) {

145
app/Model/Attribute.php
View File

@ -434,7 +434,7 @@ class Attribute extends AppModel
public function afterSave($created, $options = array())
{
// Passing event in `parentEvent` field will speed up correlation
$passedEvent = isset($options['parentEvent']) ? $options['parentEvent'] : false;
$passedEvent = $options['parentEvent'] ?? false;
$attribute = $this->data['Attribute'];
@ -545,6 +545,28 @@ class Attribute extends AppModel
return $result;
}
/**
* This method is called after all data are successfully saved into database
* @return void
* @throws Exception
*/
private function afterDatabaseSave(array $data)
{
$attribute = $data['Attribute'];
if (isset($attribute['type']) && $this->typeIsAttachment($attribute['type'])) {
$this->loadAttachmentScan()->backgroundScan(AttachmentScan::TYPE_ATTRIBUTE, $attribute);
}
}
public function save($data = null, $validate = true, $fieldList = array())
{
$result = parent::save($data, $validate, $fieldList);
if ($result) {
$this->afterDatabaseSave($result);
}
return $result;
}
public function beforeDelete($cascade = true)
{
// delete attachments from the disk
@ -786,7 +808,7 @@ class Attribute extends AppModel
// check whether the variable is null or datetime
public function datetimeOrNull($fields)
{
$seen = array_values($fields)[0];
$seen = current($fields);
if ($seen === null) {
return true;
}
@ -881,7 +903,6 @@ class Attribute extends AppModel
}
$result = $this->loadAttachmentTool()->save($attribute['event_id'], $attribute['id'], $attribute['data']);
if ($result) {
$this->loadAttachmentScan()->backgroundScan(AttachmentScan::TYPE_ATTRIBUTE, $attribute);
// Clean thumbnail cache
if ($this->isImage($attribute) && Configure::read('MISP.thumbnail_in_redis')) {
$redis = RedisTool::init();
@ -1224,38 +1245,96 @@ class Attribute extends AppModel
$this->Correlation->purgeCorrelations($eventId);
}
public function reportValidationIssuesAttributes($eventId)
/**
* This method is useful if you want to iterate all attributes sorted by ID
* @param array $conditions
* @param array $fields
* @param bool|string $callbacks
* @return Generator<array>|void
*/
public function fetchAttributesInChunks(array $conditions = [], array $fields = [], $callbacks = true)
{
$query = [
'recursive' => -1,
'conditions' => $conditions,
'limit' => 500,
'order' => ['Attribute.id'],
'fields' => $fields,
'callbacks' => $callbacks,
];
while (true) {
$attributes = $this->find('all', $query);
foreach ($attributes as $attribute) {
yield $attribute;
}
$count = count($attributes);
if ($count < 500) {
return;
}
$lastAttribute = $attributes[$count - 1];
$query['conditions']['Attribute.id >'] = $lastAttribute['Attribute']['id'];
}
}
/**
* @param int|null $eventId
* @return Generator
*/
public function reportValidationIssuesAttributes($eventId = null)
{
$conditions = array();
if ($eventId && is_numeric($eventId)) {
$conditions = array('event_id' => $eventId);
}
$attributeIds = $this->find('column', array(
'fields' => array('id'),
'conditions' => $conditions
));
$chunks = array_chunk($attributeIds, 500);
$attributes = $this->fetchAttributesInChunks($conditions);
$result = array();
foreach ($chunks as $chunk) {
$attributes = $this->find('all', array('recursive' => -1, 'conditions' => array('id' => $chunk)));
foreach ($attributes as $attribute) {
$this->set($attribute);
if (!$this->validates()) {
$resultErrors = array();
foreach ($this->validationErrors as $field => $error) {
$resultErrors[$field] = array('value' => $attribute['Attribute'][$field], 'error' => $error[0]);
}
$result[] = [
'id' => $attribute['Attribute']['id'],
'error' => $resultErrors,
'details' => 'Event ID: [' . $attribute['Attribute']['event_id'] . "] - Category: [" . $attribute['Attribute']['category'] . "] - Type: [" . $attribute['Attribute']['type'] . "] - Value: [" . $attribute['Attribute']['value'] . ']',
];
foreach ($attributes as $attribute) {
$this->set($attribute);
if (!$this->validates()) {
$resultErrors = [];
foreach ($this->validationErrors as $field => $error) {
$resultErrors[$field] = ['value' => $attribute['Attribute'][$field], 'error' => $error[0]];
}
yield [
'id' => $attribute['Attribute']['id'],
'error' => $resultErrors,
'details' => 'Event ID: [' . $attribute['Attribute']['event_id'] . "] - Category: [" . $attribute['Attribute']['category'] . "] - Type: [" . $attribute['Attribute']['type'] . "] - Value: [" . $attribute['Attribute']['value'] . ']',
];
}
}
}
/**
* @param bool $dryRun If true, no changes will be made to
* @return Generator
* @throws Exception
*/
public function normalizeIpAddress($dryRun = false)
{
$attributes = $this->fetchAttributesInChunks([
'Attribute.type' => ['ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'domain|ip'],
]);
foreach ($attributes as $attribute) {
$value = $attribute['Attribute']['value'];
$normalizedValue = AttributeValidationTool::modifyBeforeValidation($attribute['Attribute']['type'], $value);
if ($value !== $normalizedValue) {
if (!$dryRun) {
$attribute['Attribute']['value'] = $normalizedValue;
$this->save($attribute, true, ['value1', 'value2']);
}
yield [
'id' => (int) $attribute['Attribute']['id'],
'event_id' => (int) $attribute['Attribute']['event_id'],
'type' => $attribute['Attribute']['type'],
'value' => $value,
'normalized_value' => $normalizedValue,
];
}
}
return $result;
}
/**
@ -1610,6 +1689,7 @@ class Attribute extends AppModel
* @param array $user
* @param array $options
* @param int|false $result_count If false, count is not fetched
* @param bool $real_count
* @return array
* @throws Exception
*/
@ -2322,11 +2402,15 @@ class Attribute extends AppModel
$timestamp[0] = $timestamp[1];
$timestamp[1] = $temp;
}
$conditions['AND'][] = array($scope . ' >=' => $timestamp[0]);
if ($timestamp[0] != 0) {
$conditions['AND'][] = array($scope . ' >=' => $timestamp[0]);
}
$conditions['AND'][] = array($scope . ' <=' => $timestamp[1]);
} else {
$timestamp = $this->resolveTimeDelta($timestamp);
$conditions['AND'][] = array($scope . ' >=' => $timestamp);
if ($timestamp !== 0) {
$conditions['AND'][] = array($scope . ' >=' => $timestamp);
}
}
if ($returnRaw) {
return $timestamp;
@ -2348,7 +2432,7 @@ class Attribute extends AppModel
$conditions['AND'][] = array($scope . ' <=' => $timestamp[1]);
} else {
$timestamp = intval($this->resolveTimeDelta($timestamp)) * 1000000; // seen in stored in micro-seconds in the DB
if ($scope == 'Attribute.first_seen') {
if ($scope == 'Attribute.first_seen' || $scope == 'Object.first_seen') {
$conditions['AND'][] = array($scope . ' >=' => $timestamp);
} else {
$conditions['AND'][] = array($scope . ' <=' => $timestamp);
@ -3096,8 +3180,7 @@ class Attribute extends AppModel
$exportTool->additional_params
);
}
ClassRegistry::init('ConnectionManager');
$db = ConnectionManager::getDataSource('default');
$tmpfile = new TmpFileTool();
$tmpfile->write($exportTool->header($exportToolParams));
$loop = false;
@ -3673,7 +3756,7 @@ class Attribute extends AppModel
);
}
private function findAttributeByValue($attribute)
private function findAttributeByValue(array $attribute)
{
$type = $attribute['type'];
$conditions = [

5
app/Model/AuditLog.php
View File

@ -10,6 +10,7 @@ class AuditLog extends AppModel
{
const BROTLI_HEADER = "\xce\xb2\xcf\x81";
const COMPRESS_MIN_LENGTH = 256;
const CHANGE_MAX_SIZE = 64 * 1024; // MySQL type blob
const ACTION_ADD = 'add',
ACTION_EDIT = 'edit',
@ -235,6 +236,10 @@ class AuditLog extends AppModel
if (isset($auditLog['change'])) {
$auditLog['change'] = $this->encodeChange($auditLog['change']);
if (strlen($auditLog['change']) > self::CHANGE_MAX_SIZE) {
// Change is too big to save in database, skipping
$auditLog['change'] = null;
}
}
}

2
app/Model/AuthKey.php
View File

@ -207,7 +207,7 @@ class AuthKey extends AppModel
*/
private function updateUniqueIp(array $authkey)
{
if (Configure::read("MISP.disable_seen_ips_authkeys")) {
if (PHP_SAPI === 'cli' || Configure::read("MISP.disable_seen_ips_authkeys")) {
return;
}

34
app/Model/Correlation.php
View File

@ -153,7 +153,7 @@ class Correlation extends AppModel
if (!empty($eventIds)) {
$eventCount = count($eventIds);
foreach ($eventIds as $j => $currentEventId) {
$attributeCount += $this->__iteratedCorrelation(
$attributeCount += $this->iteratedCorrelation(
$jobId,
$full,
$attributeId,
@ -179,7 +179,7 @@ class Correlation extends AppModel
* @return int
* @throws Exception
*/
private function __iteratedCorrelation(
private function iteratedCorrelation(
$jobId = false,
$full = false,
$attributeId = null,
@ -215,30 +215,14 @@ class Correlation extends AppModel
if ($attributeId) {
$attributeConditions['Attribute.id'] = $attributeId;
}
$query = [
'recursive' => -1,
'conditions' => $attributeConditions,
// fetch just necessary fields to save memory
'fields' => $this->getFieldRules(),
'order' => 'Attribute.id',
'limit' => 5000,
'callbacks' => false, // memory leak fix
];
$attributes = $this->Attribute->fetchAttributesInChunks($attributeConditions, $this->getFieldRules(), false);
$attributeCount = 0;
do {
$attributes = $this->Attribute->find('all', $query);
foreach ($attributes as $attribute) {
$this->afterSaveCorrelation($attribute['Attribute'], $full, $event);
}
$fetchedAttributes = count($attributes);
unset($attributes);
$attributeCount += $fetchedAttributes;
if ($fetchedAttributes === 5000) { // maximum number of attributes fetched, continue in next loop
$query['conditions']['Attribute.id >'] = $attribute['Attribute']['id'];
} else {
break;
}
} while (true);
foreach ($attributes as $attribute) {
$this->afterSaveCorrelation($attribute['Attribute'], $full, $event);
++$attributeCount;
}
// Generating correlations can take long time, so clear caches after each event to refresh them
$this->cidrListCache = null;

26
app/Model/Datasource/Database/MysqlExtended.php
View File

@ -7,6 +7,14 @@ App::uses('Mysql', 'Model/Datasource/Database');
*/
class MysqlExtended extends Mysql
{
const PDO_MAP = [
'integer' => PDO::PARAM_INT,
'float' => PDO::PARAM_STR,
'boolean' => PDO::PARAM_BOOL,
'string' => PDO::PARAM_STR,
'text' => PDO::PARAM_STR
];
/**
* Output MD5 as binary, that is faster and uses less memory
* @param string $value
@ -157,15 +165,9 @@ class MysqlExtended extends Mysql
public function insertMulti($table, $fields, $values)
{
$table = $this->fullTableName($table);
$holder = implode(',', array_fill(0, count($fields), '?'));
$holder = substr(str_repeat('?,', count($fields)), 0, -1);
$fields = implode(',', array_map([$this, 'name'], $fields));
$pdoMap = [
'integer' => PDO::PARAM_INT,
'float' => PDO::PARAM_STR,
'boolean' => PDO::PARAM_BOOL,
'string' => PDO::PARAM_STR,
'text' => PDO::PARAM_STR
];
$columnMap = [];
foreach ($values[key($values)] as $key => $val) {
if (is_int($val)) {
@ -174,21 +176,21 @@ class MysqlExtended extends Mysql
$columnMap[$key] = PDO::PARAM_BOOL;
} else {
$type = $this->introspectType($val);
$columnMap[$key] = $pdoMap[$type];
$columnMap[$key] = self::PDO_MAP[$type];
}
}
$sql = "INSERT INTO $table ($fields) VALUES ";
$sql .= implode(',', array_fill(0, count($values), "($holder)"));
$sql .= substr(str_repeat("($holder),", count($values)), 0, -1);
$statement = $this->_connection->prepare($sql);
$valuesList = array();
$i = 1;
$i = 0;
foreach ($values as $value) {
foreach ($value as $col => $val) {
if ($this->fullDebug) {
$valuesList[] = $val;
}
$statement->bindValue($i++, $val, $columnMap[$col]);
$statement->bindValue(++$i, $val, $columnMap[$col]);
}
}
$result = $statement->execute();

16
app/Model/Event.php
View File

@ -3743,7 +3743,10 @@ class Event extends AppModel
unset($this->Attribute->validate['value']['uniqueValue']); // unset this - we are saving a new event, there are no values to compare against and event_id is not set in the attributes
}
unset($data['Event']['id']);
if (isset($data['Event']['published']) && $data['Event']['published'] && $user['Role']['perm_publish'] == 0) {
if (
(Configure::read('MISP.block_publishing_for_same_creator', false) && !$user['Role']['perm_sync']) ||
(isset($data['Event']['published']) && $data['Event']['published'] && $user['Role']['perm_publish'] == 0)
) {
$data['Event']['published'] = 0;
}
if (isset($data['Event']['uuid'])) {
@ -4059,7 +4062,10 @@ class Event extends AppModel
} else {
return array('error' => 'Event could not be saved: Could not find the local event.');
}
if (!empty($data['Event']['published']) && !$user['Role']['perm_publish']) {
if (
(Configure::read('MISP.block_publishing_for_same_creator', false) && !$user['Role']['perm_sync'] && $user['id'] == $existingEvent['Event']['user_id']) ||
(!empty($data['Event']['published']) && !$user['Role']['perm_publish'])
) {
$data['Event']['published'] = 0;
}
if (!isset($data['Event']['published'])) {
@ -4190,7 +4196,7 @@ class Event extends AppModel
if ((true != Configure::read('MISP.disablerestalert')) && (empty($server) || empty($server['Server']['publish_without_email']))) {
$this->sendAlertEmailRouter($id, $user, $existingEvent['Event']['publish_timestamp']);
}
$this->publish($existingEvent['Event']['id']);
$this->publish($existingEvent['Event']['id'], $passAlong);
}
if ($jobId) {
$eventLock->deleteBackgroundJobLock($data['Event']['id'], $jobId);
@ -5952,7 +5958,9 @@ class Event extends AppModel
$this->add_original_file($decoded['original'], $originalFile, $created_id, $stixVersion);
}
if ($publish && $user['Role']['perm_publish']) {
$this->publish($created_id);
if (!Configure::read('MISP.block_publishing_for_same_creator', false) || $user['Role']['perm_sync']) {
$this->publish($created_id);
}
}
return $created_id;
} else if (is_numeric($result)) {

3
app/Model/Feed.php
View File

@ -2062,6 +2062,7 @@ class Feed extends AppModel
$contentType = $response->getHeader('content-type');
if ($contentType === 'application/zip') {
$zipFilePath = FileAccessTool::writeToTempFile($response->body);
unset($response->body); // cleanup variable to reduce memory usage
try {
$response->body = $this->unzipFirstFile($zipFilePath);
@ -2198,7 +2199,7 @@ class Feed extends AppModel
ZipArchive::ER_READ => 'read error',
ZipArchive::ER_SEEK => 'seek error',
];
$message = isset($errorCodes[$result]) ? $errorCodes[$result] : 'error ' . $result;
$message = $errorCodes[$result] ?? 'error ' . $result;
throw new Exception("Remote server returns ZIP file, that cannot be open ($message)");
}

54
app/Model/FuzzyCorrelateSsdeep.php
View File

@ -9,38 +9,32 @@ class FuzzyCorrelateSsdeep extends AppModel
public function ssdeep_prepare($hash)
{
list($block_size, $hash) = explode(':', $hash, 2);
$chars = array();
for ($i = 0; $i < strlen($hash); $i++) {
if (!in_array($hash[$i], $chars, true)) {
$chars[] = $hash[$i];
}
}
list($blockSize, $hash) = explode(':', $hash, 2);
$uniqueChars = array_unique(str_split($hash), SORT_REGULAR);
$search = true;
while ($search) {
$search = false;
foreach ($chars as $c) {
foreach ($uniqueChars as $c) {
if (strpos($hash, $c . $c . $c . $c)) {
$hash = str_replace($c . $c . $c . $c, $c . $c . $c, $hash);
$search = true;
}
}
}
$hash = explode(':', $hash);
$block_data = $hash[0];
$double_block_data = $hash[1];
// (struct.unpack("<Q", base64.b64decode(h[i:i + 7] + "=") + "\x00\x00\x00")[0] for i in range(len(h) - 6)))
$result = array(
$block_size,
$this->get_all_7_char_chunks($block_data),
$this->get_all_7_char_chunks($double_block_data)
);
return $result;
$hash = explode(':', $hash);
list($block_data, $double_block_data) = $hash;
return [
$blockSize,
$this->getAll7CharChunks($block_data),
$this->getAll7CharChunks($double_block_data)
];
}
public function get_all_7_char_chunks($hash)
private function getAll7CharChunks($hash)
{
$results = array();
for ($i = 0; $i < strlen($hash) - 6; $i++) {
@ -56,16 +50,22 @@ class FuzzyCorrelateSsdeep extends AppModel
return $results;
}
/**
* @param string $hash
* @param int $attributeId
* @return array
*/
public function query_ssdeep_chunks($hash, $attributeId)
{
$chunks = $this->ssdeep_prepare($hash);
$bothPartChunks = array_merge($chunks[1], $chunks[2]);
// Original algo from article https://www.virusbulletin.com/virusbulletin/2015/11/optimizing-ssdeep-use-scale
// also propose to insert chunk size to database, but current database schema doesn't contain that column.
// This optimisation can be add in future versions.
$result = $this->find('column', array(
'conditions' => array(
'FuzzyCorrelateSsdeep.chunk' => array_merge($chunks[1], $chunks[2]),
'FuzzyCorrelateSsdeep.chunk' => $bothPartChunks,
),
'fields' => array('FuzzyCorrelateSsdeep.attribute_id'),
'unique' => true,
@ -73,15 +73,11 @@ class FuzzyCorrelateSsdeep extends AppModel
$toSave = [];
$attributeId = (int) $attributeId;
foreach (array(1, 2) as $type) {
foreach ($chunks[$type] as $chunk) {
$toSave[] = [$attributeId, $chunk];
}
}
if (!empty($toSave)) {
$db = $this->getDataSource();
$db->insertMulti($this->table, ['attribute_id', 'chunk'], $toSave);
foreach ($bothPartChunks as $chunk) {
$toSave[] = [$attributeId, $chunk];
}
$db = $this->getDataSource();
$db->insertMulti($this->table, ['attribute_id', 'chunk'], $toSave);
return $result;
}

50
app/Model/Galaxy.php
View File

@ -264,7 +264,7 @@ class Galaxy extends AppModel
$fields = array('galaxy_cluster_id', 'key', 'value');
$db->insertMulti('galaxy_elements', $fields, $elements);
}
$allRelations = array_merge($allRelations, $relations);
array_push($allRelations, ...$relations);
}
// Save relation as last part when all clusters are created
if (!empty($allRelations)) {
@ -287,24 +287,42 @@ class Galaxy extends AppModel
if (empty($galaxy['uuid'])) {
return false;
}
$existingGalaxy = $this->find('first', array(
$existingGalaxy = $this->find('first', [
'recursive' => -1,
'conditions' => array('Galaxy.uuid' => $galaxy['uuid'])
));
if (empty($existingGalaxy)) {
if ($user['Role']['perm_site_admin'] || $user['Role']['perm_galaxy_editor']) {
$this->create();
unset($galaxy['id']);
$this->save($galaxy);
$existingGalaxy = $this->find('first', array(
'recursive' => -1,
'conditions' => array('Galaxy.id' => $this->id)
));
} else {
return false;
'conditions' => ['Galaxy.uuid' => $galaxy['uuid']],
]);
unset($galaxy['id']);
if (!empty($existingGalaxy)) {
// check if provided galaxy has the same fields as galaxy that are saved in database
$fieldsToSave = [];
foreach (array_keys(array_intersect_key($existingGalaxy, $galaxy)) as $key) {
if ($existingGalaxy['Galaxy'][$key] != $galaxy[$key]) {
$fieldsToSave[$key] = $galaxy[$key];
}
}
} else {
$fieldsToSave = $galaxy;
}
return $existingGalaxy;
if (empty($fieldsToSave) && !empty($existingGalaxy)) {
return $existingGalaxy; // galaxy already exists and galaxy fields are the same
}
if (!$user['Role']['perm_site_admin'] && !$user['Role']['perm_galaxy_editor']) {
return false; // user has no permission to modify galaxy
}
if (empty($existingGalaxy)) {
$this->create();
}
$this->save($fieldsToSave);
return $this->find('first', [
'recursive' => -1,
'conditions' => ['Galaxy.id' => $this->id],
]);
}
/**

31
app/Model/GalaxyElement.php
View File

@ -48,37 +48,6 @@ class GalaxyElement extends AppModel
$this->saveMany($tempElements);
}
public function update($galaxy_id, $oldClusters, $newClusters)
{
$elementsToSave = array();
// Since we are dealing with flat files as the end all be all content, we are safe to just drop all of the old clusters and recreate them.
foreach ($oldClusters as $oldCluster) {
$this->deleteAll(array('GalaxyElement.galaxy_cluster_id' => $oldCluster['GalaxyCluster']['id']));
}
foreach ($newClusters as $newCluster) {
$tempCluster = array();
foreach ($newCluster as $key => $value) {
// Don't store the reserved fields as elements
if ($key == 'description' || $key == 'value') {
continue;
}
if (is_array($value)) {
foreach ($value as $arrayElement) {
$tempCluster[] = array('key' => $key, 'value' => $arrayElement);
}
} else {
$tempCluster[] = array('key' => $key, 'value' => $value);
}
}
foreach ($tempCluster as $key => $value) {
$tempCluster[$key]['galaxy_cluster_id'] = $oldCluster['GalaxyCluster']['id'];
}
$elementsToSave = array_merge($elementsToSave, $tempCluster);
}
$this->saveMany($elementsToSave);
}
public function captureElements($user, $elements, $clusterId)
{
$tempElements = array();

10
app/Model/MispObject.php
View File

@ -153,6 +153,8 @@ class MispObject extends AppModel
'object_name' => array('function' => 'set_filter_object_name'),
'object_template_uuid' => array('function' => 'set_filter_object_template_uuid'),
'object_template_version' => array('function' => 'set_filter_object_template_version'),
'first_seen' => array('function' => 'set_filter_seen'),
'last_seen' => array('function' => 'set_filter_seen'),
'deleted' => array('function' => 'set_filter_deleted')
),
'Event' => array(
@ -181,8 +183,8 @@ class MispObject extends AppModel
'deleted' => array('function' => 'set_filter_deleted'),
'timestamp' => array('function' => 'set_filter_timestamp'),
'attribute_timestamp' => array('function' => 'set_filter_timestamp'),
'first_seen' => array('function' => 'set_filter_seen'),
'last_seen' => array('function' => 'set_filter_seen'),
//'first_seen' => array('function' => 'set_filter_seen'),
//'last_seen' => array('function' => 'set_filter_seen'),
'to_ids' => array('function' => 'set_filter_to_ids'),
'comment' => array('function' => 'set_filter_comment')
)
@ -1678,7 +1680,9 @@ class MispObject extends AppModel
$results = $this->Sightingdb->attachToObjects($results, $user);
}
$params['page'] += 1;
$results = $this->Allowedlist->removeAllowedlistedFromArray($results, true);
foreach ($results as $k => $result) {
$results[$k]['Attribute'] = $this->Allowedlist->removeAllowedlistedFromArray($result['Attribute'], true);
}
$results = array_values($results);
$i = 0;
foreach ($results as $object) {

48
app/Model/Module.php
View File

@ -50,6 +50,8 @@ class Module extends AppModel
)
);
private $httpSocket = [];
public function validateIPField($value)
{
if (!filter_var($value, FILTER_VALIDATE_IP) === false) {
@ -309,16 +311,9 @@ class Module extends AppModel
if (!$serverUrl) {
throw new Exception("Module type $moduleFamily is not enabled.");
}
App::uses('HttpSocketExtended', 'Tools');
$httpSocketSetting = ['timeout' => $timeout];
$sslSettings = array('ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_verify_peer', 'ssl_cafile');
foreach ($sslSettings as $sslSetting) {
$value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
if ($value && $value !== '') {
$httpSocketSetting[$sslSetting] = $value;
}
}
$httpSocket = new HttpSocketExtended($httpSocketSetting);
$httpSocket = $this->initHttpSocket($moduleFamily, $timeout);
$request = [];
if ($moduleFamily === 'Cortex') {
if (!empty(Configure::read('Plugin.' . $moduleFamily . '_authkey'))) {
@ -422,4 +417,37 @@ class Module extends AppModel
return false;
}
/**
* @param string $moduleFamily
* @param int $timeout
* @return HttpSocketExtended|CurlClient
*/
private function initHttpSocket($moduleFamily, $timeout)
{
$unique = "$moduleFamily:$timeout";
if (isset($this->httpSocket[$unique])) {
return $this->httpSocket[$unique];
}
$httpSocketSetting = ['timeout' => $timeout];
$sslSettings = ['ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_cafile'];
foreach ($sslSettings as $sslSetting) {
$value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
if ($value && $value !== '') {
$httpSocketSetting[$sslSetting] = $value;
}
}
if (function_exists('curl_init')) {
App::uses('CurlClient', 'Tools');
$httpSocket = new CurlClient($httpSocketSetting);
} else {
App::uses('HttpSocketExtended', 'Tools');
$httpSocket = new HttpSocketExtended($httpSocketSetting);
}
return $this->httpSocket[$unique] = $httpSocket;
}
}

17
app/Model/Organisation.php
View File

@ -76,14 +76,28 @@ class Organisation extends AppModel
);
const ORGANISATION_ASSOCIATIONS = array(
'AccessLog' => array('table' => 'access_logs', 'fields' => array('org_id')),
'AuditLog' => array('table' => 'audit_logs', 'fields' => array('org_id')),
'Correlation' => array('table' => 'correlations', 'fields' => array('org_id')),
'Cerebrate' => array('table' => 'cerebrates', 'fields' => array('org_id')),
'Dashboard' => array('table' => 'dashboards', 'fields' => array('restrict_to_org_id')),
'Event' => array('table' => 'events', 'fields' => array('org_id', 'orgc_id')),
'EventGraph' => array('table' => 'event_graph', 'fields' => array('org_id')),
'Feed' => array('table' => 'feeds', 'fields' => array('orgc_id')),
'GalaxyCluster' => array('table' => 'galaxy_clusters', 'fields' => array('org_id', 'orgc_id')),
'ObjectTemplate' => array('table' => 'object_templates', 'fields' => array('org_id')),
'Job' => array('table' => 'jobs', 'fields' => array('org_id')),
'RestClientHistory' => array('table' => 'rest_client_histories', 'fields' => array('org_id')),
'Server' => array('table' => 'servers', 'fields' => array('org_id', 'remote_org_id')),
'ShadowAttribute' => array('table' => 'shadow_attributes', 'fields' => array('org_id', 'event_org_id')),
'SharingGroup' => array('table' => 'sharing_groups', 'fields' => array('org_id')),
'SharingGroupOrg' => array('table' => 'sharing_group_orgs', 'fields' => array('org_id')),
'SharingGroupBlueprint' => array('table' => 'sharing_group_blueprints', 'fields' => array('org_id')),
'Sighting' => array('table' => 'sightings', 'fields' => array('org_id')),
'SightingdbOrg' => array('table' => 'sightingdb_orgs', 'fields' => array('org_id')),
'Thread' => array('table' => 'threads', 'fields' => array('org_id')),
'Tag' => array('table' => 'tags', 'fields' => array('org_id')),
'TagCollection' => array('table' => 'tag_collections', 'fields' => array('org_id')),
'User' => array('table' => 'users', 'fields' => array('org_id'))
);
@ -287,6 +301,9 @@ class Organisation extends AppModel
public function orgMerge($id, $request, $user)
{
$currentOrg = $this->find('first', array('recursive' => -1, 'conditions' => array('Organisation.id' => $id)));
if (isset($currentOrg['Organisation']['restricted_to_domain'])) {
$currentOrg['Organisation']['restricted_to_domain'] = json_encode($currentOrg['Organisation']['restricted_to_domain']);
}
$currentOrgUserCount = $this->User->find('count', array(
'conditions' => array('User.org_id' => $id)
));

154
app/Model/Server.php
View File

@ -472,7 +472,21 @@ class Server extends AppModel
return false;
}
private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, Event $eventModel, $server, $user, $jobId, $force = false, $headers = false, $body = false)
/**
* @param array $event
* @param int|string $eventId
* @param array $successes
* @param array $fails
* @param Event $eventModel
* @param array $server
* @param array $user
* @param int $jobId
* @param bool $force
* @param HttpSocketResponseExtended $response
* @return false|void
* @throws Exception
*/
private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, Event $eventModel, $server, $user, $jobId, $force = false, $response)
{
// check if the event already exist (using the uuid)
$existingEvent = $eventModel->find('first', [
@ -485,7 +499,7 @@ class Server extends AppModel
if (!$existingEvent) {
// add data for newly imported events
if (isset($event['Event']['protected']) && $event['Event']['protected']) {
if (!$eventModel->CryptographicKey->validateProtectedEvent($body, $user, $headers['x-pgp-signature'], $event)) {
if (!$eventModel->CryptographicKey->validateProtectedEvent($response->body, $user, $response->getHeader('x-pgp-signature'), $event)) {
$fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.');
return false;
}
@ -505,7 +519,7 @@ class Server extends AppModel
$fails[$eventId] = __('Blocked an edit to an event that was created locally. This can happen if a synchronised event that was created on this instance was modified by an administrator on the remote side.');
} else {
if ($existingEvent['Event']['protected']) {
if (!$eventModel->CryptographicKey->validateProtectedEvent($body, $user, $headers['x-pgp-signature'], $existingEvent)) {
if (!$eventModel->CryptographicKey->validateProtectedEvent($response->body, $user, $response->getHeader('x-pgp-signature'), $existingEvent)) {
$fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.');
}
}
@ -549,12 +563,10 @@ class Server extends AppModel
$params['excludeLocalTags'] = 1;
}
try {
$event = $serverSync->fetchEvent($eventId, $params);
$headers = $event->headers;
$body = $event->body;
$event = $event->json();
$response = $serverSync->fetchEvent($eventId, $params);
$event = $response->json();
} catch (Exception $e) {
$this->logException("Failed downloading the event $eventId from remote server {$serverSync->serverId()}", $e);
$this->logException("Failed to download the event $eventId from remote server {$serverSync->serverId()} '{$serverSync->serverName()}'", $e);
$fails[$eventId] = __('failed downloading the event');
return false;
}
@ -568,7 +580,7 @@ class Server extends AppModel
}
return false;
}
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $headers, $body);
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $response);
return true;
}
@ -2359,23 +2371,21 @@ class Server extends AppModel
return $setting;
}
public function serverSettingsEditValue(array $user, array $setting, $value, $forceSave = false)
/**
* @param array|string $user
* @param array $setting
* @param mixed $value
* @param bool $forceSave
* @return mixed|string|true|null
* @throws Exception
*/
public function serverSettingsEditValue($user, array $setting, $value, $forceSave = false)
{
if (isset($setting['beforeHook'])) {
$beforeResult = call_user_func_array(array($this, $setting['beforeHook']), array($setting['name'], $value));
$beforeResult = $this->{$setting['beforeHook']}($setting['name'], $value);
if ($beforeResult !== true) {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->saveOrFailSilently(array(
'org' => $user['Organisation']['name'],
'model' => 'Server',
'model_id' => 0,
'email' => $user['email'],
'action' => 'serverSettingsEdit',
'user_id' => $user['id'],
'title' => 'Server setting issue',
'change' => 'There was an issue witch changing ' . $setting['name'] . ' to ' . $value . '. The error message returned is: ' . $beforeResult . 'No changes were made.',
));
$change = 'There was an issue witch changing ' . $setting['name'] . ' to ' . $value . '. The error message returned is: ' . $beforeResult . 'No changes were made.';
$this->loadLog()->createLogEntry($user, 'serverSettingsEdit', 'Server', 0, 'Server setting issue', $change);
return $beforeResult;
}
}
@ -2384,7 +2394,7 @@ class Server extends AppModel
if ($setting['type'] === 'boolean') {
$value = (bool)$value;
} else if ($setting['type'] === 'numeric') {
$value = (int)($value);
$value = (int)$value;
}
if (isset($setting['test'])) {
if ($setting['test'] instanceof Closure) {
@ -2425,7 +2435,7 @@ class Server extends AppModel
if ($setting['afterHook'] instanceof Closure) {
$afterResult = $setting['afterHook']($setting['name'], $value, $oldValue);
} else {
$afterResult = call_user_func_array(array($this, $setting['afterHook']), array($setting['name'], $value, $oldValue));
$afterResult = $this->{$setting['afterHook']}($setting['name'], $value, $oldValue);
}
if ($afterResult !== true) {
$change = 'There was an issue after setting a new setting. The error message returned is: ' . $afterResult;
@ -2434,9 +2444,8 @@ class Server extends AppModel
}
}
return true;
} else {
return __('Something went wrong. MISP tried to save a malformed config file. Setting change reverted.');
}
return __('Something went wrong. MISP tried to save a malformed config file or you dont have permission to write to config file. Setting change reverted.');
}
/**
@ -2539,10 +2548,10 @@ class Server extends AppModel
'name' => __('Organisation logos'),
'description' => __('The logo used by an organisation on the event index, event view, discussions, proposals, etc. Make sure that the filename is in the org.png format, where org is the case-sensitive organisation name.'),
'expected' => array(),
'valid_format' => __('48x48 pixel .png files'),
'valid_format' => __('48x48 pixel .png files or .svg file'),
'path' => APP . 'webroot' . DS . 'img' . DS . 'orgs',
'regex' => '.*\.(png|PNG)$',
'regex_error' => __('Filename must be in the following format: *.png'),
'regex' => '.*\.(png|svg)$',
'regex_error' => __('Filename must be in the following format: *.png or *.svg'),
'files' => array(),
),
'img' => array(
@ -2578,6 +2587,7 @@ class Server extends AppModel
'read' => $f->isReadable(),
'write' => $f->isWritable(),
'execute' => $f->isExecutable(),
'link' => $f->isLink(),
];
}
}
@ -4155,12 +4165,13 @@ class Server extends AppModel
private function checkRemoteVersion($HttpSocket)
{
try {
$json_decoded_tags = GitTool::getLatestTags($HttpSocket);
$tags = GitTool::getLatestTags($HttpSocket);
} catch (Exception $e) {
$this->logException('Could not retrieve latest tags from GitHub', $e, LOG_NOTICE);
return false;
}
// find the latest version tag in the v[major].[minor].[hotfix] format
foreach ($json_decoded_tags as $tag) {
foreach ($tags as $tag) {
if (preg_match('/^v[0-9]+\.[0-9]+\.[0-9]+$/', $tag['name'])) {
return $this->checkVersion($tag['name']);
}
@ -4182,7 +4193,7 @@ class Server extends AppModel
try {
$latestCommit = GitTool::getLatestCommit($HttpSocket);
} catch (Exception $e) {
$latestCommit = false;
$this->logException('Could not retrieve version from GitHub', $e, LOG_NOTICE);
}
}
@ -4202,6 +4213,7 @@ class Server extends AppModel
try {
return GitTool::currentBranch();
} catch (Exception $e) {
$this->logException('Could not retrieve current Git branch', $e, LOG_NOTICE);
return false;
}
}
@ -4252,38 +4264,38 @@ class Server extends AppModel
'app/files/scripts/misp-opendata',
'app/files/scripts/python-maec',
'app/files/scripts/python-stix',
);
return in_array($submodule, $accepted_submodules_names, true);
}
/**
* @param string $submodule_name
* @param string $superproject_submodule_commit_id
* @param string $submoduleName
* @param string $superprojectSubmoduleCommitId
* @return array
* @throws Exception
*/
private function getSubmoduleGitStatus($submodule_name, $superproject_submodule_commit_id)
private function getSubmoduleGitStatus($submoduleName, $superprojectSubmoduleCommitId)
{
$path = APP . '../' . $submodule_name;
$submodule_name = (strpos($submodule_name, '/') >= 0 ? explode('/', $submodule_name) : $submodule_name);
$submodule_name = end($submodule_name);
$path = APP . '../' . $submoduleName;
$submoduleName = (strpos($submoduleName, '/') >= 0 ? explode('/', $submoduleName) : $submoduleName);
$submoduleName = end($submoduleName);
$submoduleCurrentCommitId = GitTool::submoduleCurrentCommit($path);
$submoduleCurrentCommitId = GitTool::currentCommit($path);
$currentTimestamp = GitTool::commitTimestamp($submoduleCurrentCommitId, $path);
if ($submoduleCurrentCommitId !== $superproject_submodule_commit_id) {
$remoteTimestamp = GitTool::commitTimestamp($superproject_submodule_commit_id, $path);
if ($submoduleCurrentCommitId !== $superprojectSubmoduleCommitId) {
$remoteTimestamp = GitTool::commitTimestamp($superprojectSubmoduleCommitId, $path);
} else {
$remoteTimestamp = $currentTimestamp;
}
$status = array(
'moduleName' => $submodule_name,
'moduleName' => $submoduleName,
'current' => $submoduleCurrentCommitId,
'currentTimestamp' => $currentTimestamp,
'remote' => $superproject_submodule_commit_id,
'remote' => $superprojectSubmoduleCommitId,
'remoteTimestamp' => $remoteTimestamp,
'upToDate' => '',
'upToDate' => 'error',
'isReadable' => is_readable($path) && is_readable($path . '/.git'),
);
@ -4295,15 +4307,11 @@ class Server extends AppModel
} else {
$status['upToDate'] = 'younger';
}
} else {
$status['upToDate'] = 'error';
}
if ($status['isReadable'] && !empty($status['remoteTimestamp']) && !empty($status['currentTimestamp'])) {
$date1 = new DateTime();
$date1->setTimestamp($status['remoteTimestamp']);
$date2 = new DateTime();
$date2->setTimestamp($status['currentTimestamp']);
$date1 = new DateTime("@{$status['remoteTimestamp']}");
$date2 = new DateTime("@{$status['currentTimestamp']}");
$status['timeDiff'] = $date1->diff($date2);
} else {
$status['upToDate'] = 'error';
@ -4793,11 +4801,11 @@ class Server extends AppModel
$results = [
__('User') => $user['User']['email'],
__('Role name') => isset($user['Role']['name']) ? $user['Role']['name'] : __('Unknown, outdated instance'),
__('Role name') => $user['Role']['name'] ?? __('Unknown, outdated instance'),
__('Sync flag') => isset($user['Role']['perm_sync']) ? ($user['Role']['perm_sync'] ? __('Yes') : __('No')) : __('Unknown, outdated instance'),
];
if (isset($response->headers['X-Auth-Key-Expiration'])) {
$date = new DateTime($response->headers['X-Auth-Key-Expiration']);
if ($response->getHeader('X-Auth-Key-Expiration')) {
$date = new DateTime($response->getHeader('X-Auth-Key-Expiration'));
$results[__('Auth key expiration')] = $date->format('Y-m-d H:i:s');
}
return $results;
@ -4935,6 +4943,28 @@ class Server extends AppModel
return $this->saveMany($toSave, ['validate' => false, 'fields' => ['authkey']]);
}
/**
* @param string $encryptionKey
* @return bool
* @throws Exception
*/
public function isEncryptionKeyValid($encryptionKey)
{
$servers = $this->find('list', [
'fields' => ['Server.id', 'Server.authkey'],
]);
foreach ($servers as $id => $authkey) {
if (EncryptedValue::isEncrypted($authkey)) {
try {
BetterSecurity::decrypt(substr($authkey, 2), $encryptionKey);
} catch (Exception $e) {
throw new Exception("Could not decrypt auth key for server #$id", 0, $e);
}
}
}
return true;
}
/**
* Return all Attribute and Object types
*/
@ -5143,9 +5173,9 @@ class Server extends AppModel
'type' => 'string',
),
'disable_cached_exports' => array(
'level' => 1,
'description' => __('Cached exports can take up a considerable amount of space and can be disabled instance wide using this setting. Disabling the cached exports is not recommended as it\'s a valuable feature, however, if your server is having free space issues it might make sense to take this step.'),
'value' => false,
'level' => 2,
'description' => __('Cached exports can take up a considerable amount of space and can be disabled instance wide using this setting. Even tough the feature is deprecated and will be removed in the future, you can still decide to enable it.'),
'value' => true,
'null' => true,
'test' => 'testDisableCache',
'type' => 'boolean',
@ -6143,6 +6173,14 @@ class Server extends AppModel
'type' => 'boolean',
'null' => true,
],
'block_publishing_for_same_creator' => [
'level' => self::SETTING_OPTIONAL,
'description' => __('Enabling this setting will make MISP block event publishing in the case of the publisher being the same user as the event creator.'),
'value' => false,
'test' => 'testBool',
'type' => 'boolean',
'null' => true,
],
'self_update' => [
'level' => self::SETTING_CRITICAL,
'description' => __('Enable the GUI button for MISP self-update on the Diagnostics page.'),

47
app/Model/Sighting.php
View File

@ -1017,16 +1017,16 @@ class Sighting extends AppModel
* @return TmpFileTool
* @throws Exception
*/
public function restSearch(array $user, $returnFormat, $filters)
public function restSearch(array $user, $returnFormat, array $filters)
{
$allowedContext = array('event', 'attribute');
// validate context
if (isset($filters['context']) && !in_array($filters['context'], $allowedContext, true)) {
throw new MethodNotAllowedException(__('Invalid context %s.', $filters['context']));
throw new BadRequestException(__('Invalid context %s.', $filters['context']));
}
// ensure that an id or uuid is provided if context is set
if (!empty($filters['context']) && !(isset($filters['id']) || isset($filters['uuid'])) ) {
throw new MethodNotAllowedException(__('An ID or UUID must be provided if the context is set.'));
throw new BadRequestException(__('An ID or UUID must be provided if the context is set.'));
}
if (!isset($this->validFormats[$returnFormat][1])) {
@ -1102,8 +1102,12 @@ class Sighting extends AppModel
$conditions['Attribute.uuid'] = $filters['uuid'];
$contain[] = 'Attribute';
} elseif ($filters['context'] === 'event') {
$conditions['Event.uuid'] = $filters['uuid'];
$contain[] = 'Event';
$temp = $this->Event->find('column', [
'recursive' => -1,
'fields' => ['Event.id'],
'conditions' => ['Event.uuid IN' => $filters['uuid']]
]);
$conditions['Sighting.event_id'] = empty($temp) ? -1 : $temp;
}
}
@ -1131,15 +1135,30 @@ class Sighting extends AppModel
$tmpfile = new TmpFileTool();
$tmpfile->write($exportTool->header($exportToolParams));
$separator = $exportTool->separator($exportToolParams);
// fetch sightings matching the query without ACL checks
$sightingIds = $this->find('column', [
'conditions' => $conditions,
'fields' => ['Sighting.id'],
'contain' => $contain,
'order' => 'Sighting.id',
]);
if (!empty($conditions['Sighting.event_id']) && is_array($conditions['Sighting.event_id'])) {
$conditions_copy = $conditions;
$sightingIds = [];
foreach ($conditions['Sighting.event_id'] as $e_id) {
$conditions_copy['Sighting.event_id'] = $e_id;
$tempIds = $this->find('column', [
'conditions' => $conditions,
'fields' => ['Sighting.id'],
'contain' => $contain
]);
if (!empty($tempIds)) {
$sightingIds = array_merge($sightingIds, $tempIds);
}
}
} else {
$sightingIds = $this->find('column', [
'conditions' => $conditions,
'fields' => ['Sighting.id'],
'contain' => $contain
]);
}
foreach (array_chunk($sightingIds, 500) as $chunk) {
// fetch sightings with ACL checks and sighting policies
$sightings = $this->getSightings($user, $chunk, $includeEvent, $includeAttribute, $includeUuid);
@ -1396,7 +1415,7 @@ class Sighting extends AppModel
try {
$sightings = $serverSync->fetchSightingsForEvents($chunk);
} catch (Exception $e) {
$this->logException("Failed downloading the sightings from {$serverSync->server()['Server']['name']}.", $e);
$this->logException("Failed to download sightings from {$serverSync->server()['Server']['name']}.", $e);
continue;
}

30
app/Model/SystemSetting.php
View File

@ -46,7 +46,7 @@ class SystemSetting extends AppModel
{
/** @var self $systemSetting */
$systemSetting = ClassRegistry::init('SystemSetting');
if (!$systemSetting->databaseExists()) {
if (!$systemSetting->tableExists()) {
return;
}
$settings = $systemSetting->getSettings();
@ -58,7 +58,7 @@ class SystemSetting extends AppModel
}
}
public function databaseExists()
private function tableExists()
{
$tables = ConnectionManager::getDataSource($this->useDbConfig)->listSources();
return in_array('system_settings', $tables, true);
@ -154,6 +154,32 @@ class SystemSetting extends AppModel
return $this->saveMany($toSave);
}
/**
* Check if provided encryption key is valid for all encrypted settings
* @param string $encryptionKey
* @return bool
* @throws Exception
*/
public function isEncryptionKeyValid($encryptionKey)
{
$settings = $this->find('list', [
'fields' => ['SystemSetting.setting', 'SystemSetting.value'],
]);
foreach ($settings as $setting => $value) {
if (!self::isSensitive($setting)) {
continue;
}
if (EncryptedValue::isEncrypted($value)) {
try {
BetterSecurity::decrypt(substr($value, 2), $encryptionKey);
} catch (Exception $e) {
throw new Exception("Could not decrypt `$setting` setting.", 0, $e);
}
}
}
return true;
}
/**
* Sensitive setting are passwords or api keys.
* @param string $setting Setting name

126
app/Model/User.php
View File

@ -659,21 +659,18 @@ class User extends AppModel
public function getUserById($id)
{
if (empty($id)) {
throw new NotFoundException('Invalid user ID.');
throw new InvalidArgumentException('Invalid user ID.');
}
return $this->find(
'first',
array(
'conditions' => array('User.id' => $id),
'recursive' => -1,
'contain' => array(
'Organisation',
'Role',
'Server',
'UserSetting',
)
)
);
return $this->find('first', [
'conditions' => ['User.id' => $id],
'recursive' => -1,
'contain' => [
'Organisation',
'Role',
'Server',
'UserSetting',
]
]);
}
/**
@ -740,7 +737,7 @@ class User extends AppModel
],
]);
if (empty($user)) {
return $user;
return null;
}
return $this->rearrangeToAuthForm($user);
}
@ -861,6 +858,10 @@ class User extends AppModel
return true;
}
if (!isset($user['User'])) {
throw new InvalidArgumentException("Invalid user model provided.");
}
if ($user['User']['disabled'] || !$this->checkIfUserIsValid($user['User'])) {
return true;
}
@ -937,6 +938,11 @@ class User extends AppModel
*/
public function describeAuthFields()
{
static $fields; // generate array just once
if ($fields) {
return $fields;
}
$fields = $this->schema();
// Do not include keys, because they are big and usually not necessary
unset($fields['gpgkey']);
@ -1105,13 +1111,18 @@ class User extends AppModel
return $hashed;
}
public function createInitialUser($org_id)
/**
* @param int $orgId
* @return string User auth key
* @throws Exception
*/
public function createInitialUser($orgId)
{
$authKey = $this->generateAuthKey();
$admin = array('User' => array(
'id' => 1,
'email' => 'admin@admin.test',
'org_id' => $org_id,
'org_id' => $orgId,
'password' => 'admin',
'confirm_password' => 'admin',
'authkey' => $authKey,
@ -1123,7 +1134,6 @@ class User extends AppModel
$this->validator()->remove('password'); // password is too simple, remove validation
$this->save($admin);
if (!empty(Configure::read("Security.advanced_authkeys"))) {
$this->AuthKey = ClassRegistry::init('AuthKey');
$newKey = [
'authkey' => $authKey,
'user_id' => 1,
@ -2068,12 +2078,10 @@ class User extends AppModel
return false;
}
$cutoff = $redis->get('misp:session_destroy:' . $id);
$allcutoff = $redis->get('misp:session_destroy:all');
list($cutoff, $allcutoff) = $redis->mGet(['misp:session_destroy:' . $id, 'misp:session_destroy:all']);
if (
empty($cutoff) ||
(
!empty($cutoff) &&
!empty($allcutoff) &&
$allcutoff < $cutoff
)
@ -2156,7 +2164,7 @@ class User extends AppModel
if (!ctype_alnum($token)) {
return false;
}
$redis = $this->setupRedis();
$redis = RedisTool::init();
$userId = $redis->get('misp:forgot:' . $token);
if (empty($userId)) {
return false;
@ -2167,8 +2175,78 @@ class User extends AppModel
public function purgeForgetToken($token)
{
$redis = $this->setupRedis();
$userId = $redis->del('misp:forgot:' . $token);
$redis = RedisTool::init();
$redis->del('misp:forgot:' . $token);
return true;
}
/**
* Create default Role, Organisation and User
* @return string|null Created user auth key
* @throws Exception
*/
public function init()
{
if (!$this->Role->hasAny()) {
$siteAdmin = ['Role' => [
'id' => 1,
'name' => 'Site Admin',
'permission' => 3,
'perm_add' => 1,
'perm_modify' => 1,
'perm_modify_org' => 1,
'perm_publish' => 1,
'perm_sync' => 1,
'perm_admin' => 1,
'perm_audit' => 1,
'perm_auth' => 1,
'perm_site_admin' => 1,
'perm_regexp_access' => 1,
'perm_sharing_group' => 1,
'perm_template' => 1,
'perm_tagger' => 1,
]];
$this->Role->save($siteAdmin);
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
if (!$this->isMysql()) {
$sql = "SELECT setval('roles_id_seq', (SELECT MAX(id) FROM roles));";
$this->Role->query($sql);
}
}
if (!$this->Organisation->hasAny(['Organisation.local' => true])) {
$this->runUpdates();
$org = ['Organisation' => [
'id' => 1,
'name' => !empty(Configure::read('MISP.org')) ? Configure::read('MISP.org') : 'ADMIN',
'description' => 'Automatically generated admin organisation',
'type' => 'ADMIN',
'date_created' => date('Y-m-d H:i:s'),
'local' => 1,
]];
$this->Organisation->save($org);
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
if (!$this->isMysql()) {
$sql = "SELECT setval('organisations_id_seq', (SELECT MAX(id) FROM organisations));";
$this->Organisation->query($sql);
}
$orgId = $this->Organisation->id;
}
if (!$this->hasAny()) {
if (!isset($orgId)) {
$hostOrg = $this->Organisation->find('first', array('conditions' => array('Organisation.name' => Configure::read('MISP.org'), 'Organisation.local' => true), 'recursive' => -1));
if (!empty($hostOrg)) {
$orgId = $hostOrg['Organisation']['id'];
} else {
$firstOrg = $this->Organisation->find('first', array('conditions' => array('Organisation.local' => true), 'order' => 'Organisation.id ASC'));
$orgId = $firstOrg['Organisation']['id'];
}
}
$this->runUpdates();
return $this->createInitialUser($orgId);
}
return null;
}
}

79
app/Model/UserLoginProfile.php
View File

@ -36,22 +36,55 @@ class UserLoginProfile extends AppModel
];
const BROWSER_CACHE_DIR = APP . DS . 'tmp' . DS . 'browscap';
const BROWSER_INI_FILE = APP . DS . 'files' . DS . 'browscap'. DS . 'browscap.ini'; // Browscap file managed by MISP - https://browscap.org/stream?q=Lite_PHP_BrowsCapINI
const BROWSER_INI_FILE = APP . DS . 'files' . DS . 'browscap'. DS . 'browscap.ini.gz'; // Browscap file managed by MISP - https://browscap.org/stream?q=Lite_PHP_BrowsCapINI
const GEOIP_DB_FILE = APP . DS . 'files' . DS . 'geo-open' . DS . 'GeoOpen-Country.mmdb'; // GeoIP file managed by MISP - https://data.public.lu/en/datasets/geo-open-ip-address-geolocation-per-country-in-mmdb-format/
private $userProfile;
private $knownUserProfiles = [];
private function _buildBrowscapCache()
private function browscapGetBrowser()
{
$this->log("Browscap - building new cache from browscap.ini file.", LOG_INFO);
$fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR);
$cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache);
$logger = new \Monolog\Logger('name');
$bc = new \BrowscapPHP\BrowscapUpdater($cache, $logger);
$bc->convertFile(UserLoginProfile::BROWSER_INI_FILE);
if (function_exists('apcu_fetch')) {
App::uses('ApcuCacheTool', 'Tools');
$cache = new ApcuCacheTool('misp:browscap');
} else {
$fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR);
$cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache);
}
try {
$bc = new \BrowscapPHP\Browscap($cache, $logger);
return $bc->getBrowser();
} catch (\BrowscapPHP\Exception $e) {
$this->log("Browscap - building new cache from browscap.ini file.", LOG_INFO);
$bcUpdater = new \BrowscapPHP\BrowscapUpdater($cache, $logger);
$bcUpdater->convertString(FileAccessTool::readCompressedFile(UserLoginProfile::BROWSER_INI_FILE));
}
$bc = new \BrowscapPHP\Browscap($cache, $logger);
return $bc->getBrowser();
}
/**
* @param string $ip
* @return string|null
*/
public function countryByIp($ip)
{
if (class_exists('GeoIp2\Database\Reader')) {
$geoDbReader = new GeoIp2\Database\Reader(UserLoginProfile::GEOIP_DB_FILE);
try {
$record = $geoDbReader->country($ip);
return $record->country->isoCode;
} catch (InvalidArgumentException $e) {
$this->logException("Could not get country code for IP address", $e, LOG_NOTICE);
return null;
}
}
return null;
}
public function beforeSave($options = [])
@ -76,16 +109,7 @@ class UserLoginProfile extends AppModel
if (!$this->userProfile) {
// below uses https://github.com/browscap/browscap-php
if (class_exists('\BrowscapPHP\Browscap')) {
try {
$fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR);
$cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache);
$logger = new \Monolog\Logger('name');
$bc = new \BrowscapPHP\Browscap($cache, $logger);
$browser = $bc->getBrowser();
} catch (\BrowscapPHP\Exception $e) {
$this->_buildBrowscapCache();
return $this->_getUserProfile();
}
$browser = $this->browscapGetBrowser();
} else {
// a primitive OS & browser extraction capability
$ua = $_SERVER['HTTP_USER_AGENT'] ?? null;
@ -100,18 +124,7 @@ class UserLoginProfile extends AppModel
$browser->browser = "browser";
}
$ip = $this->_remoteIp();
if (class_exists('GeoIp2\Database\Reader')) {
try {
$geoDbReader = new GeoIp2\Database\Reader(UserLoginProfile::GEOIP_DB_FILE);
$record = $geoDbReader->country($ip);
$country = $record->country->isoCode;
} catch (InvalidArgumentException $e) {
$this->logException("Could not get country code for IP address", $e);
$country = 'None';
}
} else {
$country = 'None';
}
$country = $this->countryByIp($ip) ?? 'None';
$this->userProfile = [
'user_agent' => $_SERVER['HTTP_USER_AGENT'] ?? null,
'ip' => $ip,
@ -247,13 +260,13 @@ class UserLoginProfile extends AppModel
public function emailNewLogin(array $user)
{
if (!Configure::read('MISP.disable_emailing')) {
$date_time = date('c');
$user = $this->User->getUserById($user['id']); // fetch in database format
$datetime = date('c'); // ISO 8601 date
$body = new SendEmailTemplate('userloginprofile_newlogin');
$body->set('userLoginProfile', $this->User->UserLoginProfile->_getUserProfile());
$body->set('baseurl', Configure::read('MISP.baseurl'));
$body->set('misp_org', Configure::read('MISP.org'));
$body->set('date_time', $date_time);
$body->set('date_time', $datetime);
// Fetch user that contains also PGP or S/MIME keys for e-mail encryption
$this->User->sendEmail($user, $body, false, "[" . Configure::read('MISP.org') . " MISP] New sign in.");
}

3
app/Model/Warninglist.php
View File

@ -390,8 +390,7 @@ class Warninglist extends AppModel
$warninglistId = (int)$this->id;
$result = true;
$keys = array_keys($list['list']);
if ($keys === array_keys($keys)) {
if (JsonTool::arrayIsList($list['list'])) {
foreach (array_chunk($list['list'], 1000) as $chunk) {
$valuesToInsert = [];
foreach ($chunk as $value) {

276
app/Plugin/BinaryFileCache/Engine/BinaryFileEngine.php Normal file
View File

@ -0,0 +1,276 @@
<?php
require_once CAKE_CORE_INCLUDE_PATH . '/Cake/Cache/CacheEngine.php';
require_once CAKE_CORE_INCLUDE_PATH . '/Cake/Cache/Engine/FileEngine.php';
/**
* This is faster version of FileEngine cache engine
* - stores file in binary format, so no need to change line endings
* - uses igbinary if supported for faster serialization/deserialization and smaller cache files
* - default file mask is set to 0660, so cache files are not readable by other users
* - optimised file saving and fetching
*/
class BinaryFileEngine extends FileEngine
{
const BINARY_CACHE_TIME_LENGTH = 8;
private $useIgbinary = false;
public function init($settings = [])
{
$settings += [
'engine' => 'BinaryFile',
'path' => CACHE,
'prefix' => 'cake_',
'serialize' => true,
'mask' => 0660,
];
CacheEngine::init($settings);
$this->useIgbinary = function_exists('igbinary_serialize');
if (substr($this->settings['path'], -1) !== DS) {
$this->settings['path'] .= DS;
}
if (!empty($this->_groupPrefix)) {
$this->_groupPrefix = str_replace('_', DS, $this->_groupPrefix);
}
return $this->_active();
}
/**
* @param string $key
* @param mixed $data
* @param int $duration
* @return bool
*/
public function write($key, $data, $duration)
{
if (!$this->_init) {
return false;
}
$fileInfo = $this->cacheFilePath($key);
$resource = $this->createFile($fileInfo);
if (!$resource) {
return false;
}
if (!empty($this->settings['serialize'])) {
if ($this->useIgbinary) {
$data = igbinary_serialize($data);
if ($data === null) {
return false;
}
} else {
$data = serialize($data);
}
}
$expires = pack("q", time() + $duration);
flock($resource, LOCK_EX);
ftruncate($resource, 0);
$result = fwrite($resource, $expires);
if ($result !== self::BINARY_CACHE_TIME_LENGTH) {
$this->handleWriteError($fileInfo);
fclose($resource);
return false;
}
$result = fwrite($resource, $data);
if ($result !== strlen($data)) {
$this->handleWriteError($fileInfo);
fclose($resource);
return false;
}
fclose($resource);
return true;
}
/**
* @param string $key
* @return false|mixed|string
*/
public function read($key)
{
if (!$this->_init) {
return false;
}
$fileInfo = $this->cacheFilePath($key);
$exists = file_exists($fileInfo->getPathname());
if (!$exists) {
return false;
}
$resource = $this->openFile($fileInfo);
if (!$resource) {
return false;
}
$time = time();
flock($resource, LOCK_SH);
$cacheTimeBinary = fread($resource, self::BINARY_CACHE_TIME_LENGTH);
if (!$cacheTimeBinary) {
fclose($resource);
return false;
}
$cacheTime = $this->unpackCacheTime($cacheTimeBinary);
if ($cacheTime < $time || ($time + $this->settings['duration']) < $cacheTime) {
fclose($resource);
return false; // already expired
}
$data = stream_get_contents($resource, null, self::BINARY_CACHE_TIME_LENGTH);
fclose($resource);
if (!empty($this->settings['serialize'])) {
if ($this->useIgbinary) {
$data = igbinary_unserialize($data);
} else {
$data = unserialize($data);
}
}
return $data;
}
/**
* @param string $path
* @param int $now
* @param int $threshold
* @return void
*/
protected function _clearDirectory($path, $now, $threshold)
{
$prefixLength = strlen($this->settings['prefix']);
if (!is_dir($path)) {
return;
}
$dir = dir($path);
if ($dir === false) {
return;
}
while (($entry = $dir->read()) !== false) {
if (substr($entry, 0, $prefixLength) !== $this->settings['prefix']) {
continue;
}
try {
$file = new SplFileObject($path . $entry, 'rb');
} catch (Exception $e) {
continue;
}
if ($threshold) {
$mtime = $file->getMTime();
if ($mtime > $threshold) {
continue;
}
$expires = $this->unpackCacheTime($file->fread(self::BINARY_CACHE_TIME_LENGTH));
if ($expires > $now) {
continue;
}
}
if ($file->isFile()) {
$filePath = $file->getRealPath();
$file = null;
@unlink($filePath);
}
}
}
/**
* @param SplFileInfo $fileInfo
* @return false|resource
*/
private function createFile(SplFileInfo $fileInfo)
{
$exists = file_exists($fileInfo->getPathname());
if (!$exists) {
$resource = $this->openFile($fileInfo, 'cb');
if ($resource && !chmod($fileInfo->getPathname(), (int)$this->settings['mask'])) {
trigger_error(__d(
'cake_dev', 'Could not apply permission mask "%s" on cache file "%s"',
[$fileInfo->getPathname(), $this->settings['mask']]), E_USER_WARNING);
}
return $resource;
}
return $this->openFile($fileInfo, 'cb');
}
/**
* @param SplFileInfo $fileInfo
* @param string $mode
* @return false|resource
*/
private function openFile(SplFileInfo $fileInfo, $mode = 'rb')
{
$resource = fopen($fileInfo->getPathname(), $mode);
if (!$resource) {
trigger_error(__d(
'cake_dev', 'Could not open file %s',
array($fileInfo->getPathname())), E_USER_WARNING);
}
return $resource;
}
/**
* @param string $key
* @return SplFileInfo
*/
private function cacheFilePath(string $key): SplFileInfo
{
$groups = null;
if (!empty($this->_groupPrefix)) {
$groups = vsprintf($this->_groupPrefix, $this->groups());
}
$dir = $this->settings['path'] . $groups;
if (!is_dir($dir)) {
mkdir($dir, 0775, true);
}
$suffix = '.bin';
if ($this->settings['serialize'] && $this->useIgbinary) {
$suffix = '.igbin';
}
return new SplFileInfo($dir . $key . $suffix);
}
/**
* @param SplFileInfo $fileInfo
* @return void
*/
private function handleWriteError(SplFileInfo $fileInfo)
{
unlink($fileInfo->getPathname()); // delete file in case file was just partially written
trigger_error(__d(
'cake_dev', 'Could not write to file %s',
array($fileInfo->getPathname())), E_USER_WARNING);
}
/**
* @param string $cacheTimeBinary
* @return int
*/
private function unpackCacheTime($cacheTimeBinary)
{
if ($cacheTimeBinary === false || strlen($cacheTimeBinary) !== self::BINARY_CACHE_TIME_LENGTH) {
throw new InvalidArgumentException("Invalid cache time in binary format provided '$cacheTimeBinary'");
}
return unpack("q", $cacheTimeBinary)[1];
}
}

2
app/Plugin/EcsLog/Lib/Log/Engine/EcsLog.php
View File

@ -57,7 +57,7 @@ class EcsLog implements CakeLogInterface
'log' => [
'level' => $type,
],
'message' => $message,
'message' => JsonTool::escapeNonUnicode($message),
];
static::writeMessage($message);

Some files were not shown because too many files have changed in this diff Show More