mirror of https://github.com/MISP/MISP
Merge branch 'develop' of github.com:MISP/MISP into develop
commit
4be80d39a8
|
@ -6,9 +6,9 @@ name: misp
|
||||||
# events but only for the 2.4 and develop branches
|
# events but only for the 2.4 and develop branches
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches: [ 2.4, develop, misp-stix, taxii ]
|
branches: [ '2.4', develop, misp-stix, taxii ]
|
||||||
pull_request:
|
pull_request:
|
||||||
branches: [ 2.4, develop, misp-stix ]
|
branches: [ '2.4', develop, misp-stix ]
|
||||||
|
|
||||||
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
|
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
|
||||||
jobs:
|
jobs:
|
||||||
|
@ -62,12 +62,6 @@ jobs:
|
||||||
php_version: ${{ matrix.php }}
|
php_version: ${{ matrix.php }}
|
||||||
run: |
|
run: |
|
||||||
sudo apt-get -y update
|
sudo apt-get -y update
|
||||||
# Repo is missing for unknown reason
|
|
||||||
LC_ALL=C.UTF-8 sudo apt-add-repository ppa:ondrej/php -y
|
|
||||||
if [[ $php_version == "7.2" ]]; then
|
|
||||||
# hotfix due to: https://bugs.php.net/bug.php?id=81640 TODO: remove after libpcre2-8-0:10.36 gets to stable channel
|
|
||||||
sudo apt-get --fix-broken install
|
|
||||||
fi
|
|
||||||
sudo apt-get -y install curl python3 python3-pip python3-virtualenv apache2 libapache2-mod-php$php_version
|
sudo apt-get -y install curl python3 python3-pip python3-virtualenv apache2 libapache2-mod-php$php_version
|
||||||
|
|
||||||
# Runs a set of commands using the runners shell
|
# Runs a set of commands using the runners shell
|
||||||
|
@ -75,10 +69,11 @@ jobs:
|
||||||
run: |
|
run: |
|
||||||
sudo chown $USER:www-data $HOME/.composer
|
sudo chown $USER:www-data $HOME/.composer
|
||||||
pushd app
|
pushd app
|
||||||
sudo -H -u $USER composer config --no-plugins allow-plugins.composer/installers true
|
composer config --no-plugins allow-plugins.composer/installers true
|
||||||
sudo -H -u $USER composer install --no-progress
|
composer install --no-progress
|
||||||
popd
|
popd
|
||||||
cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php
|
cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php
|
||||||
|
|
||||||
# Set perms
|
# Set perms
|
||||||
sudo chown -R $USER:www-data `pwd`
|
sudo chown -R $USER:www-data `pwd`
|
||||||
sudo chmod -R 775 `pwd`
|
sudo chmod -R 775 `pwd`
|
||||||
|
@ -90,17 +85,18 @@ jobs:
|
||||||
sudo chmod -R g+ws `pwd`/app/files
|
sudo chmod -R g+ws `pwd`/app/files
|
||||||
sudo chmod -R g+ws `pwd`/app/files/scripts/tmp
|
sudo chmod -R g+ws `pwd`/app/files/scripts/tmp
|
||||||
sudo chown -R $USER:www-data `pwd`
|
sudo chown -R $USER:www-data `pwd`
|
||||||
|
|
||||||
# Resque perms
|
# Resque perms
|
||||||
sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp
|
sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp
|
||||||
sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp
|
sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp
|
||||||
# install MySQL
|
|
||||||
sudo chmod -R 777 `pwd`/INSTALL
|
# Fill database with basic MISP schema
|
||||||
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';"
|
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';"
|
||||||
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant usage on *.* to misp@'%' identified by 'blah';"
|
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant usage on *.* to misp@'%' identified by 'blah';"
|
||||||
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant all privileges on misp.* to misp@'%';"
|
mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant all privileges on misp.* to misp@'%';"
|
||||||
mysql -h 127.0.0.1 --port 3306 -u misp -pblah misp < INSTALL/MYSQL.sql
|
mysql -h 127.0.0.1 --port 3306 -u misp -pblah misp < INSTALL/MYSQL.sql
|
||||||
|
|
||||||
# configure apache virtual hosts
|
# configure apache virtual hosts
|
||||||
sudo chmod -R 777 `pwd`/build
|
|
||||||
sudo mkdir -p /etc/apache2/sites-available
|
sudo mkdir -p /etc/apache2/sites-available
|
||||||
sudo cp -f build/github-action-ci-apache /etc/apache2/sites-available/misp.conf
|
sudo cp -f build/github-action-ci-apache /etc/apache2/sites-available/misp.conf
|
||||||
sudo sed -e "s?%GITHUB_WORKSPACE%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.conf
|
sudo sed -e "s?%GITHUB_WORKSPACE%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.conf
|
||||||
|
@ -109,23 +105,22 @@ jobs:
|
||||||
sudo a2ensite misp.conf
|
sudo a2ensite misp.conf
|
||||||
cat /etc/apache2/sites-enabled/misp.conf
|
cat /etc/apache2/sites-enabled/misp.conf
|
||||||
sudo a2enmod rewrite
|
sudo a2enmod rewrite
|
||||||
sudo systemctl restart apache2
|
sudo systemctl start --no-block apache2
|
||||||
|
|
||||||
# MISP configuration
|
# MISP configuration
|
||||||
sudo chmod -R 777 `pwd`/travis
|
|
||||||
sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php
|
sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php
|
||||||
sudo cp travis/database.php app/Config/database.php
|
sudo cp build/database.php app/Config/database.php
|
||||||
sudo cp app/Config/core.default.php app/Config/core.php
|
sudo cp app/Config/core.default.php app/Config/core.php
|
||||||
sudo cp app/Config/config.default.php app/Config/config.php
|
sudo cp app/Config/config.default.php app/Config/config.php
|
||||||
sudo cp travis/email.php app/Config/email.php
|
sudo cp build/email.php app/Config/email.php
|
||||||
# Ensure the perms
|
|
||||||
sudo chown -R $USER:www-data `pwd`/app/Config
|
|
||||||
sudo chmod -R 777 `pwd`/app/Config
|
|
||||||
# GPG setup
|
# GPG setup
|
||||||
sudo mkdir `pwd`/.gnupg
|
sudo mkdir `pwd`/.gnupg
|
||||||
# /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS
|
# /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS
|
||||||
sudo cp -a /dev/urandom /dev/random
|
sudo cp -a /dev/urandom /dev/random
|
||||||
sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/travis/gpg
|
sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/build/gpg
|
||||||
sudo gpg --list-secret-keys --homedir `pwd`/.gnupg
|
sudo gpg --list-secret-keys --homedir `pwd`/.gnupg
|
||||||
|
|
||||||
# change perms
|
# change perms
|
||||||
sudo chown -R $USER:www-data `pwd`
|
sudo chown -R $USER:www-data `pwd`
|
||||||
sudo chown -R www-data:www-data `pwd`/.gnupg
|
sudo chown -R www-data:www-data `pwd`/.gnupg
|
||||||
|
@ -135,7 +130,7 @@ jobs:
|
||||||
# Ensure the perms of config files
|
# Ensure the perms of config files
|
||||||
sudo chown -R $USER:www-data `pwd`/app/Config
|
sudo chown -R $USER:www-data `pwd`/app/Config
|
||||||
sudo chmod -R 777 `pwd`/app/Config
|
sudo chmod -R 777 `pwd`/app/Config
|
||||||
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.server_settings_skip_backup_rotate" 1'
|
app/Console/cake Admin setSetting "MISP.server_settings_skip_backup_rotate" 1
|
||||||
sudo chown -R $USER:www-data `pwd`/app/Config
|
sudo chown -R $USER:www-data `pwd`/app/Config
|
||||||
sudo chmod -R 777 `pwd`/app/Config
|
sudo chmod -R 777 `pwd`/app/Config
|
||||||
|
|
||||||
|
@ -146,86 +141,83 @@ jobs:
|
||||||
sudo chmod +x /home
|
sudo chmod +x /home
|
||||||
sudo chmod +x /
|
sudo chmod +x /
|
||||||
|
|
||||||
|
- name: Python setup
|
||||||
|
run: |
|
||||||
|
# Dirty install python stuff
|
||||||
|
python3 -m virtualenv -p python3 ./venv
|
||||||
|
app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python"
|
||||||
|
. ./venv/bin/activate
|
||||||
|
export PYTHONPATH=$PYTHONPATH:./app/files/scripts
|
||||||
|
pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara pytest
|
||||||
|
deactivate
|
||||||
|
|
||||||
- name: DB Update
|
- name: DB Update
|
||||||
run: |
|
run: |
|
||||||
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.osuser" $USER'
|
app/Console/cake Admin setSetting "MISP.osuser" $USER
|
||||||
sudo -E su $USER -c 'app/Console/cake Admin runUpdates'
|
app/Console/cake Admin runUpdates
|
||||||
sudo -E su $USER -c 'app/Console/cake Admin schemaDiagnostics'
|
app/Console/cake Admin schemaDiagnostics
|
||||||
|
|
||||||
- name: Configure MISP
|
- name: Configure MISP
|
||||||
run: |
|
run: |
|
||||||
sudo -u $USER app/Console/cake userInit -q | sudo tee ./key.txt
|
app/Console/cake User init | sudo tee ./key.txt
|
||||||
echo "AUTH=`cat key.txt`" >> $GITHUB_ENV
|
echo "AUTH=`cat key.txt`" >> $GITHUB_ENV
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "Session.autoRegenerate" 0
|
app/Console/cake Admin setSetting "Session.autoRegenerate" 0
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "Session.timeout" 600
|
app/Console/cake Admin setSetting "Session.timeout" 600
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "Session.cookieTimeout" 3600
|
app/Console/cake Admin setSetting "Session.cookieTimeout" 3600
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "MISP.host_org_id" 1
|
app/Console/cake Admin setSetting "MISP.host_org_id" 1
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "MISP.email" "info@admin.test"
|
app/Console/cake Admin setSetting "MISP.email" "info@admin.test"
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "MISP.disable_emailing" false
|
app/Console/cake Admin setSetting "MISP.disable_emailing" false
|
||||||
sudo -u $USER app/Console/cake Admin setSetting --force "debug" true
|
app/Console/cake Admin setSetting --force "debug" true
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false
|
app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1"
|
app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1"
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_port" 6379
|
app/Console/cake Admin setSetting "MISP.redis_port" 6379
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_database" 13
|
app/Console/cake Admin setSetting "MISP.redis_database" 13
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_password" ""
|
app/Console/cake Admin setSetting "MISP.redis_password" ""
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test"
|
app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test"
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg"
|
app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg"
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "GnuPG.password" "travistest"
|
app/Console/cake Admin setSetting "GnuPG.password" "travistest"
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "MISP.download_gpg_from_homedir" 1
|
app/Console/cake Admin setSetting "MISP.download_gpg_from_homedir" 1
|
||||||
|
app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_host" "127.0.0.1"
|
||||||
- name: Configure ZMQ
|
app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_port" 6379
|
||||||
run: |
|
app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_database" 1
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_host" "127.0.0.1"
|
app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_password" ""
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_port" 6379
|
app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" 1
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_database" 1
|
app/Console/cake Admin setSetting "Plugin.ZeroMQ_audit_notifications_enable" 1
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_password" ""
|
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" 1
|
|
||||||
sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_audit_notifications_enable" 1
|
|
||||||
|
|
||||||
- name: Update Galaxies
|
- name: Update Galaxies
|
||||||
run: sudo -E su $USER -c 'app/Console/cake Admin updateGalaxies'
|
run: app/Console/cake Admin updateGalaxies
|
||||||
|
|
||||||
- name: Update Taxonomies
|
- name: Update Taxonomies
|
||||||
run: sudo -E su $USER -c 'app/Console/cake Admin updateTaxonomies'
|
run: app/Console/cake Admin updateTaxonomies
|
||||||
|
|
||||||
- name: Update Warninglists
|
- name: Update Warninglists
|
||||||
run: sudo -E su $USER -c 'app/Console/cake Admin updateWarningLists --verbose'
|
run: app/Console/cake Admin updateWarningLists --verbose
|
||||||
|
|
||||||
- name: Update Noticelists
|
- name: Update Noticelists
|
||||||
run: sudo -E su $USER -c 'app/Console/cake Admin updateNoticeLists'
|
run: app/Console/cake Admin updateNoticeLists
|
||||||
|
|
||||||
- name: Update Object Templates
|
- name: Update Object Templates
|
||||||
run: sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates 1'
|
run: app/Console/cake Admin updateObjectTemplates 1
|
||||||
|
|
||||||
- name: Turn MISP live
|
- name: Turn MISP live
|
||||||
run: sudo -E su $USER -c 'app/Console/cake Live 1'
|
run: app/Console/cake Admin live 1
|
||||||
|
|
||||||
- name: Check if Redis is ready
|
- name: Check if Redis is ready
|
||||||
run: sudo -E su $USER -c 'app/Console/cake Admin redisReady'
|
run: app/Console/cake Admin redisReady
|
||||||
|
|
||||||
- name: Start workers
|
- name: Start workers
|
||||||
run: |
|
run: |
|
||||||
sudo chmod +x app/Console/worker/start.sh
|
sudo chmod +x app/Console/worker/start.sh
|
||||||
sudo -u www-data 'app/Console/worker/start.sh'
|
sudo -u www-data 'app/Console/worker/start.sh'
|
||||||
|
|
||||||
- name: Python setup
|
|
||||||
run: |
|
|
||||||
sudo chmod 777 ./key.txt
|
|
||||||
sudo chmod -R 777 ./tests
|
|
||||||
# Start workers
|
|
||||||
# Dirty install python stuff
|
|
||||||
python3 -m virtualenv -p python3 ./venv
|
|
||||||
sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python"'
|
|
||||||
. ./venv/bin/activate
|
|
||||||
export PYTHONPATH=$PYTHONPATH:./app/files/scripts
|
|
||||||
pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara pytest
|
|
||||||
deactivate
|
|
||||||
|
|
||||||
- name: Test if apache is working
|
- name: Test if apache is working
|
||||||
run: |
|
run: |
|
||||||
sudo systemctl status apache2 --no-pager -l
|
sudo systemctl status apache2 --no-pager -l
|
||||||
sudo apache2ctl -S
|
sudo apache2ctl -S
|
||||||
curl http://${HOST}
|
curl -sS http://${HOST}
|
||||||
|
|
||||||
|
- name: Check if dependencies working as expected
|
||||||
|
run: |
|
||||||
sudo chmod -R 777 PyMISP
|
sudo chmod -R 777 PyMISP
|
||||||
pushd PyMISP
|
pushd PyMISP
|
||||||
echo 'url = "http://'${HOST}'"' >> tests/keys.py
|
echo 'url = "http://'${HOST}'"' >> tests/keys.py
|
||||||
|
@ -240,7 +232,7 @@ jobs:
|
||||||
|
|
||||||
- name: Run PHP tests
|
- name: Run PHP tests
|
||||||
run: |
|
run: |
|
||||||
./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
|
./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ -e php,ctp app/
|
||||||
sudo -u www-data ./app/Vendor/bin/phpunit app/Test/
|
sudo -u www-data ./app/Vendor/bin/phpunit app/Test/
|
||||||
|
|
||||||
- name: Clone test files
|
- name: Clone test files
|
||||||
|
@ -249,7 +241,6 @@ jobs:
|
||||||
repository: viper-framework/viper-test-files
|
repository: viper-framework/viper-test-files
|
||||||
path: PyMISP/tests/viper-test-files
|
path: PyMISP/tests/viper-test-files
|
||||||
|
|
||||||
|
|
||||||
- name: Run tests
|
- name: Run tests
|
||||||
run: |
|
run: |
|
||||||
pushd tests
|
pushd tests
|
||||||
|
@ -285,7 +276,7 @@ jobs:
|
||||||
tail -n +1 `pwd`/app/tmp/logs/*
|
tail -n +1 `pwd`/app/tmp/logs/*
|
||||||
tail -n +1 /var/log/apache2/*.log
|
tail -n +1 /var/log/apache2/*.log
|
||||||
|
|
||||||
sudo -u $USER app/Console/cake Log export /tmp/logs.json.gz --without-changes
|
app/Console/cake Log export /tmp/logs.json.gz --without-changes
|
||||||
zcat /tmp/logs.json.gz
|
zcat /tmp/logs.json.gz
|
||||||
|
|
||||||
- name: Errors in Logs
|
- name: Errors in Logs
|
||||||
|
|
195
.travis.yml
195
.travis.yml
|
@ -1,195 +0,0 @@
|
||||||
language: php
|
|
||||||
|
|
||||||
php:
|
|
||||||
- 7.2
|
|
||||||
- 7.3
|
|
||||||
- 7.4
|
|
||||||
- nightly
|
|
||||||
|
|
||||||
services:
|
|
||||||
- redis
|
|
||||||
|
|
||||||
sudo: required
|
|
||||||
dist: bionic
|
|
||||||
|
|
||||||
addons:
|
|
||||||
mariadb: '10.2'
|
|
||||||
hosts:
|
|
||||||
- misp.local
|
|
||||||
- localhost
|
|
||||||
|
|
||||||
before_install:
|
|
||||||
- git config --global user.name "TravisCI"
|
|
||||||
- export PATH="$HOME/.local/bin:$PATH"
|
|
||||||
|
|
||||||
install:
|
|
||||||
- date
|
|
||||||
- sudo apt-get -y update
|
|
||||||
# Install haveged, because Travis lacks entropy.
|
|
||||||
- sudo apt-get -y install haveged python3 python3-venv python3-pip python3-dev python3-nose python3-redis python3-lxml python3-dateutil python3-msgpack libxml2-dev libzmq3-dev zlib1g-dev apache2 curl php-mysql php-dev php-cli libapache2-mod-php libfuzzy-dev php-mbstring libonig4 php-json php-xml php-opcache php-readline php-redis php-gnupg php-gd
|
|
||||||
- sudo pip3 install --upgrade pip setuptools requests
|
|
||||||
- sudo pip3 install --upgrade -r requirements.txt
|
|
||||||
- sudo pip3 install --upgrade -r requirements-dev.txt
|
|
||||||
- pip3 install --user poetry
|
|
||||||
- phpenv rehash
|
|
||||||
- sudo mkdir $HOME/.composer ; sudo chown $USER:www-data $HOME/.composer
|
|
||||||
- pushd app
|
|
||||||
- sudo -H -u $USER php composer.phar install --no-progress
|
|
||||||
- sudo phpenmod redis
|
|
||||||
- sudo phpenmod gnupg
|
|
||||||
- popd
|
|
||||||
- cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php
|
|
||||||
# Set perms
|
|
||||||
- sudo chown -R $USER:www-data `pwd`
|
|
||||||
- sudo chmod -R 775 `pwd`
|
|
||||||
- sudo chmod -R g+ws `pwd`/app/tmp
|
|
||||||
- sudo chmod -R g+ws `pwd`/app/tmp/cache
|
|
||||||
- sudo chmod -R g+ws `pwd`/app/tmp/cache/persistent
|
|
||||||
- sudo chmod -R g+ws `pwd`/app/tmp/cache/models
|
|
||||||
- sudo chmod -R g+ws `pwd`/app/tmp/logs
|
|
||||||
- sudo chmod -R g+ws `pwd`/app/files
|
|
||||||
- sudo chmod -R g+ws `pwd`/app/files/scripts/tmp
|
|
||||||
- sudo chown -R $USER:www-data `pwd`
|
|
||||||
# Resque perms
|
|
||||||
- sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp
|
|
||||||
- sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp
|
|
||||||
# install MySQL
|
|
||||||
- sudo chmod -R 777 `pwd`/INSTALL
|
|
||||||
- mysql -u root -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';"
|
|
||||||
- mysql -u root -e 'create database misp;'
|
|
||||||
- mysql -u root -e "grant usage on *.* to misp@localhost identified by 'blah'";
|
|
||||||
- mysql -u root -e "grant all privileges on misp.* to misp@localhost;"
|
|
||||||
- mysql -u misp -pblah misp < INSTALL/MYSQL.sql
|
|
||||||
# configure apache virtual hosts
|
|
||||||
- sudo chmod -R 777 `pwd`/build
|
|
||||||
- sudo mkdir -p /etc/apache2/sites-available
|
|
||||||
- sudo cp -f build/travis-ci-apache /etc/apache2/sites-available/misp.local.conf
|
|
||||||
- sudo sed -e "s?%TRAVIS_BUILD_DIR%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.local.conf
|
|
||||||
- sudo a2dissite 000-default
|
|
||||||
- sudo a2ensite misp.local.conf
|
|
||||||
- sudo a2enmod rewrite
|
|
||||||
- sudo service apache2 restart
|
|
||||||
# MISP configuration
|
|
||||||
- sudo chmod -R 777 `pwd`/travis
|
|
||||||
- sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php
|
|
||||||
- sudo cp travis/database.php app/Config/database.php
|
|
||||||
- sudo cp app/Config/core.default.php app/Config/core.php
|
|
||||||
- sudo cp app/Config/config.default.php app/Config/config.php
|
|
||||||
- sudo cp travis/email.php app/Config/email.php
|
|
||||||
# Ensure the perms
|
|
||||||
- sudo chown -R $USER:www-data `pwd`/app/Config
|
|
||||||
- sudo chmod -R 770 `pwd`/app/Config
|
|
||||||
# GPG setup
|
|
||||||
- sudo mkdir `pwd`/.gnupg
|
|
||||||
# /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS
|
|
||||||
- sudo cp -a /dev/urandom /dev/random
|
|
||||||
- sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/travis/gpg
|
|
||||||
- sudo gpg --list-secret-keys --homedir `pwd`/.gnupg
|
|
||||||
# change perms
|
|
||||||
- sudo chown -R $USER:www-data `pwd`
|
|
||||||
- sudo chmod +x /home/travis/build
|
|
||||||
- sudo chmod +x /home/travis
|
|
||||||
- sudo chmod +x /home
|
|
||||||
- sudo chmod -R 770 `pwd`/.gnupg
|
|
||||||
# Get authkey
|
|
||||||
- sudo usermod -a -G www-data $USER
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin runUpdates'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake userInit -q | sudo tee ./key.txt'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.autoRegenerate" 0'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.timeout" 600'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.cookieTimeout" 3600'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.host_org_id" 1'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.email" "info@admin.test"'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.disable_emailing" false'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "debug" true'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1"'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_port" 6379'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_database" 13'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_password" ""'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test"'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg"'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.password" "travistest"'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin updateGalaxies'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin updateTaxonomies'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin updateWarningLists'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin updateNoticeLists'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates 1'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" true'
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Live 1'
|
|
||||||
- sudo chmod 777 ./key.txt
|
|
||||||
- sudo chmod -R 777 ./tests
|
|
||||||
# Start workers
|
|
||||||
- sudo chmod +x app/Console/worker/start.sh
|
|
||||||
- sudo -E su $USER -c 'app/Console/worker/start.sh &'
|
|
||||||
- sleep 10
|
|
||||||
# Dirty install python stuff
|
|
||||||
- virtualenv -p python3.6 ./venv
|
|
||||||
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$TRAVIS_BUILD_DIR/venv/bin/python"'
|
|
||||||
- . ./venv/bin/activate
|
|
||||||
- pushd cti-python-stix2
|
|
||||||
- pip install .
|
|
||||||
- popd
|
|
||||||
- pushd PyMISP
|
|
||||||
- pip install .[fileobjects]
|
|
||||||
- popd
|
|
||||||
- pip install stix zmq redis plyara
|
|
||||||
- deactivate
|
|
||||||
|
|
||||||
before_script:
|
|
||||||
- curl http://misp.local
|
|
||||||
- AUTH=`cat key.txt`
|
|
||||||
- sudo chmod -R 777 PyMISP
|
|
||||||
- pushd PyMISP
|
|
||||||
- echo 'url = "http://misp.local"' >> tests/keys.py
|
|
||||||
- echo 'key = "'${AUTH}'"' >> tests/keys.py
|
|
||||||
- cat tests/keys.py
|
|
||||||
- popd
|
|
||||||
|
|
||||||
script:
|
|
||||||
- ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
|
|
||||||
- ./app/Vendor/bin/phpunit app/Test/ComplexTypeToolTest.php
|
|
||||||
- ./app/Vendor/bin/phpunit app/Test/JSONConverterToolTest.php
|
|
||||||
# Ensure the perms
|
|
||||||
- sudo chown -R $USER:www-data `pwd`/app/Config
|
|
||||||
- sudo chmod -R 770 `pwd`/app/Config
|
|
||||||
- pushd tests
|
|
||||||
- ./curl_tests.sh $AUTH
|
|
||||||
- popd
|
|
||||||
- pushd PyMISP
|
|
||||||
- git submodule init
|
|
||||||
- git submodule update
|
|
||||||
- travis_retry poetry install -E fileobjects -E openioc -E virustotal -E docs -E pdfexport
|
|
||||||
- poetry run python tests/testlive_comprehensive.py
|
|
||||||
- poetry run python tests/test_mispevent.py
|
|
||||||
- popd
|
|
||||||
- cp PyMISP/tests/keys.py PyMISP/examples/events/
|
|
||||||
- pushd PyMISP/examples/events/
|
|
||||||
- poetry run python ./create_massive_dummy_events.py -l 5 -a 30
|
|
||||||
- popd
|
|
||||||
- python3 tools/misp-feed/validate.py
|
|
||||||
|
|
||||||
after_failure:
|
|
||||||
- curl http://misp.local
|
|
||||||
- cat /etc/apache2/sites-available/misp.local.conf
|
|
||||||
- sudo tail -n +1 `pwd`/app/tmp/logs/*
|
|
||||||
- sudo ls -l /var/log/apache2
|
|
||||||
- sudo cat /var/log/apache2/error.log
|
|
||||||
- sudo cat /var/log/apache2/misp.local_error.log
|
|
||||||
- sudo cat /var/log/apache2/misp.local_access.log
|
|
||||||
- pwd
|
|
||||||
|
|
||||||
notifications:
|
|
||||||
webhooks:
|
|
||||||
urls:
|
|
||||||
- https://webhooks.gitter.im/e/05e30284086a8e948d31
|
|
||||||
on_success: change # options: [always|never|change] default: always
|
|
||||||
on_failure: always # options: [always|never|change] default: always
|
|
||||||
on_start: never # options: [always|never|change] default: always
|
|
||||||
|
|
||||||
after_success:
|
|
||||||
- sudo tail -n +1 `pwd`/app/tmp/logs/*
|
|
||||||
- coveralls
|
|
||||||
- coverage report
|
|
||||||
- coverage xml
|
|
||||||
- codecov
|
|
|
@ -46,11 +46,11 @@ class AdminShell extends AppShell
|
||||||
'help' => __('Update the JSON definition of taxonomies.'),
|
'help' => __('Update the JSON definition of taxonomies.'),
|
||||||
));
|
));
|
||||||
$parser->addSubcommand('setSetting', [
|
$parser->addSubcommand('setSetting', [
|
||||||
'help' => __('Set setting in PHP config file.'),
|
'help' => __('Set setting in MISP config'),
|
||||||
'parser' => [
|
'parser' => [
|
||||||
'arguments' => [
|
'arguments' => [
|
||||||
'name' => ['help' => __('Setting name'), 'required' => true],
|
'name' => ['help' => __('Setting name'), 'required' => true],
|
||||||
'value' => ['help' => __('Setting value'), 'required' => true],
|
'value' => ['help' => __('Setting value')],
|
||||||
],
|
],
|
||||||
'options' => [
|
'options' => [
|
||||||
'force' => [
|
'force' => [
|
||||||
|
@ -72,7 +72,7 @@ class AdminShell extends AppShell
|
||||||
'help' => __('Set if MISP instance is live and accessible for users.'),
|
'help' => __('Set if MISP instance is live and accessible for users.'),
|
||||||
'parser' => [
|
'parser' => [
|
||||||
'arguments' => [
|
'arguments' => [
|
||||||
'state' => ['help' => __('Set Live state')],
|
'state' => ['help' => __('Set Live state (boolean). If not provided, current state will be printed.')],
|
||||||
],
|
],
|
||||||
],
|
],
|
||||||
]);
|
]);
|
||||||
|
@ -85,6 +85,14 @@ class AdminShell extends AppShell
|
||||||
],
|
],
|
||||||
],
|
],
|
||||||
]);
|
]);
|
||||||
|
$parser->addSubcommand('isEncryptionKeyValid', [
|
||||||
|
'help' => __('Check if current encryption key is valid.'),
|
||||||
|
'parser' => [
|
||||||
|
'options' => [
|
||||||
|
'encryptionKey' => ['help' => __('Encryption key to test. If not provided, current key will be used.')],
|
||||||
|
],
|
||||||
|
],
|
||||||
|
]);
|
||||||
$parser->addSubcommand('dumpCurrentDatabaseSchema', [
|
$parser->addSubcommand('dumpCurrentDatabaseSchema', [
|
||||||
'help' => __('Dump current database schema to JSON file.'),
|
'help' => __('Dump current database schema to JSON file.'),
|
||||||
]);
|
]);
|
||||||
|
@ -109,6 +117,20 @@ class AdminShell extends AppShell
|
||||||
$parser->addSubcommand('configLint', [
|
$parser->addSubcommand('configLint', [
|
||||||
'help' => __('Check if settings has correct value.'),
|
'help' => __('Check if settings has correct value.'),
|
||||||
]);
|
]);
|
||||||
|
$parser->addSubcommand('createZmqConfig', [
|
||||||
|
'help' => __('Create config file for ZeroMQ server.'),
|
||||||
|
]);
|
||||||
|
$parser->addSubcommand('scanAttachment', [
|
||||||
|
'help' => __('Scan attachments with AV.'),
|
||||||
|
'parser' => [
|
||||||
|
'arguments' => [
|
||||||
|
'type' => ['help' => __('all, Attribute or ShadowAttribute'), 'required' => true],
|
||||||
|
'attributeId' => ['help' => __('ID to scan.')],
|
||||||
|
'jobId' => ['help' => __('Job ID')],
|
||||||
|
|
||||||
|
],
|
||||||
|
],
|
||||||
|
]);
|
||||||
return $parser;
|
return $parser;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -485,32 +507,47 @@ class AdminShell extends AppShell
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
echo json_encode($result, JSON_PRETTY_PRINT) . PHP_EOL;
|
$this->out($this->json($result));
|
||||||
}
|
}
|
||||||
|
|
||||||
public function setSetting()
|
public function setSetting()
|
||||||
{
|
{
|
||||||
list($setting_name, $value) = $this->args;
|
list($settingName) = $this->args;
|
||||||
if ($value === 'false') {
|
|
||||||
$value = 0;
|
if ($this->params['null'] && isset($this->args[1])) {
|
||||||
} elseif ($value === 'true') {
|
$this->error(__('Trying to set setting to null value, but value was provided.'));
|
||||||
$value = 1;
|
} else if ($this->params['null']) {
|
||||||
}
|
|
||||||
if ($this->params['null']) {
|
|
||||||
$value = null;
|
$value = null;
|
||||||
|
} elseif (isset($this->args[1])) {
|
||||||
|
$value = $this->args[1];
|
||||||
|
} else {
|
||||||
|
$this->error(__('No setting value provided.'));
|
||||||
}
|
}
|
||||||
$cli_user = array('id' => 0, 'email' => 'SYSTEM', 'Organisation' => array('name' => 'SYSTEM'));
|
|
||||||
if (empty($setting_name) || ($value === null && !$this->params['null'])) {
|
$setting = $this->Server->getSettingData($settingName);
|
||||||
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Set setting'] . PHP_EOL);
|
|
||||||
}
|
|
||||||
$setting = $this->Server->getSettingData($setting_name);
|
|
||||||
if (empty($setting)) {
|
if (empty($setting)) {
|
||||||
$message = 'Invalid setting "' . $setting_name . '". Please make sure that the setting that you are attempting to change exists and if a module parameter, the modules are running.' . PHP_EOL;
|
$message = 'Invalid setting "' . $settingName . '". Please make sure that the setting that you are attempting to change exists and if a module parameter, the modules are running.' . PHP_EOL;
|
||||||
$this->error(__('Setting change rejected.'), $message);
|
$this->error(__('Setting change rejected.'), $message);
|
||||||
}
|
}
|
||||||
$result = $this->Server->serverSettingsEditValue($cli_user, $setting, $value, $this->params['force']);
|
|
||||||
|
// Convert value to boolean or to int
|
||||||
|
if ($value !== null) {
|
||||||
|
if ($setting['type'] === 'boolean') {
|
||||||
|
$value = $this->toBoolean($value);
|
||||||
|
} else if ($setting['type'] === 'numeric') {
|
||||||
|
if (is_numeric($value)) {
|
||||||
|
$value = (int)$value;
|
||||||
|
} elseif ($value === 'true' || $value === 'false') {
|
||||||
|
$value = $value === 'true' ? 1 : 0; // special case for `debug` setting
|
||||||
|
} else {
|
||||||
|
$this->error(__('Setting "%s" change rejected.', $settingName), __('Provided value %s is not a number.', $value));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$result = $this->Server->serverSettingsEditValue('SYSTEM', $setting, $value, $this->params['force']);
|
||||||
if ($result === true) {
|
if ($result === true) {
|
||||||
$this->out(__('Setting "%s" changed to %s', $setting_name, is_string($value) ? '"' . $value . '"' : (string)$value));
|
$this->out(__('Setting "%s" changed to %s', $settingName, is_string($value) ? '"' . $value . '"' : json_encode($value)));
|
||||||
} else {
|
} else {
|
||||||
$message = __("The setting change was rejected. MISP considers the requested setting value as invalid and would lead to the following error:\n\n\"%s\"\n\nIf you still want to force this change, please supply the --force argument.\n", $result);
|
$message = __("The setting change was rejected. MISP considers the requested setting value as invalid and would lead to the following error:\n\n\"%s\"\n\nIf you still want to force this change, please supply the --force argument.\n", $result);
|
||||||
$this->error(__('Setting change rejected.'), $message);
|
$this->error(__('Setting change rejected.'), $message);
|
||||||
|
@ -648,6 +685,8 @@ class AdminShell extends AppShell
|
||||||
*/
|
*/
|
||||||
public function change_authkey()
|
public function change_authkey()
|
||||||
{
|
{
|
||||||
|
$this->deprecated('cake user change_authkey [user_id]');
|
||||||
|
|
||||||
if (empty($this->args[0])) {
|
if (empty($this->args[0])) {
|
||||||
echo 'MISP apikey command line tool' . PHP_EOL . 'To assign a new random API key for a user: ' . APP . 'Console/cake Admin change_authkey [user_email]' . PHP_EOL . 'To assign a fixed API key: ' . APP . 'Console/cake Admin change_authkey [user_email] [authkey]' . PHP_EOL;
|
echo 'MISP apikey command line tool' . PHP_EOL . 'To assign a new random API key for a user: ' . APP . 'Console/cake Admin change_authkey [user_email]' . PHP_EOL . 'To assign a fixed API key: ' . APP . 'Console/cake Admin change_authkey [user_email] [authkey]' . PHP_EOL;
|
||||||
die();
|
die();
|
||||||
|
@ -787,6 +826,8 @@ class AdminShell extends AppShell
|
||||||
*/
|
*/
|
||||||
public function UserIP()
|
public function UserIP()
|
||||||
{
|
{
|
||||||
|
$this->deprecated('cake user user_ips [user_id]');
|
||||||
|
|
||||||
if (empty($this->args[0])) {
|
if (empty($this->args[0])) {
|
||||||
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get IPs for user ID'] . PHP_EOL);
|
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get IPs for user ID'] . PHP_EOL);
|
||||||
}
|
}
|
||||||
|
@ -814,6 +855,8 @@ class AdminShell extends AppShell
|
||||||
*/
|
*/
|
||||||
public function IPUser()
|
public function IPUser()
|
||||||
{
|
{
|
||||||
|
$this->deprecated('cake user ip_user [ip]');
|
||||||
|
|
||||||
if (empty($this->args[0])) {
|
if (empty($this->args[0])) {
|
||||||
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get user ID for user IP'] . PHP_EOL);
|
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get user ID for user IP'] . PHP_EOL);
|
||||||
}
|
}
|
||||||
|
@ -839,8 +882,8 @@ class AdminShell extends AppShell
|
||||||
public function scanAttachment()
|
public function scanAttachment()
|
||||||
{
|
{
|
||||||
$input = $this->args[0];
|
$input = $this->args[0];
|
||||||
$attributeId = isset($this->args[1]) ? $this->args[1] : null;
|
$attributeId = $this->args[1] ?? null;
|
||||||
$jobId = isset($this->args[2]) ? $this->args[2] : null;
|
$jobId = $this->args[2] ?? null;
|
||||||
|
|
||||||
$this->loadModel('AttachmentScan');
|
$this->loadModel('AttachmentScan');
|
||||||
$result = $this->AttachmentScan->scan($input, $attributeId, $jobId);
|
$result = $this->AttachmentScan->scan($input, $attributeId, $jobId);
|
||||||
|
@ -951,7 +994,7 @@ class AdminShell extends AppShell
|
||||||
$newStatus = $this->toBoolean($this->args[0]);
|
$newStatus = $this->toBoolean($this->args[0]);
|
||||||
$overallSuccess = false;
|
$overallSuccess = false;
|
||||||
try {
|
try {
|
||||||
$redis = $this->Server->setupRedisWithException();
|
$redis = RedisTool::init();
|
||||||
if ($newStatus) {
|
if ($newStatus) {
|
||||||
$redis->del('misp:live');
|
$redis->del('misp:live');
|
||||||
$this->out('Set live status to True in Redis.');
|
$this->out('Set live status to True in Redis.');
|
||||||
|
@ -980,7 +1023,7 @@ class AdminShell extends AppShell
|
||||||
} else {
|
} else {
|
||||||
$this->out('Current status:');
|
$this->out('Current status:');
|
||||||
$this->out('PHP Config file: ' . (Configure::read('MISP.live') ? 'True' : 'False'));
|
$this->out('PHP Config file: ' . (Configure::read('MISP.live') ? 'True' : 'False'));
|
||||||
$newStatus = $this->Server->setupRedisWithException()->get('misp:live');
|
$newStatus = RedisTool::init()->get('misp:live');
|
||||||
$this->out('Redis: ' . ($newStatus !== '0' ? 'True' : 'False'));
|
$this->out('Redis: ' . ($newStatus !== '0' ? 'True' : 'False'));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1031,6 +1074,27 @@ class AdminShell extends AppShell
|
||||||
$this->out(__('New encryption key "%s" saved into config file.', $new));
|
$this->out(__('New encryption key "%s" saved into config file.', $new));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function isEncryptionKeyValid()
|
||||||
|
{
|
||||||
|
$encryptionKey = $this->params['encryptionKey'] ?? null;
|
||||||
|
if ($encryptionKey === null) {
|
||||||
|
$encryptionKey = Configure::read('Security.encryption_key');
|
||||||
|
}
|
||||||
|
if (!$encryptionKey) {
|
||||||
|
$this->error('No encryption key provided');
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @var SystemSetting $systemSetting */
|
||||||
|
$systemSetting = ClassRegistry::init('SystemSetting');
|
||||||
|
|
||||||
|
try {
|
||||||
|
$systemSetting->isEncryptionKeyValid($encryptionKey);
|
||||||
|
$this->Server->isEncryptionKeyValid($encryptionKey);
|
||||||
|
} catch (Exception $e) {
|
||||||
|
$this->error($e->getMessage(), __('Probably provided encryption key is invalid'));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public function redisMemoryUsage()
|
public function redisMemoryUsage()
|
||||||
{
|
{
|
||||||
$redis = RedisTool::init();
|
$redis = RedisTool::init();
|
||||||
|
@ -1240,4 +1304,10 @@ class AdminShell extends AppShell
|
||||||
$this->Job->saveField('message', __('Database truncated: ' . $table));
|
$this->Job->saveField('message', __('Database truncated: ' . $table));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function createZmqConfig()
|
||||||
|
{
|
||||||
|
$this->Server->getPubSubTool()->createConfigFile();
|
||||||
|
$this->err("Config file created in " . PubSubTool::SCRIPTS_TMP);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -31,15 +31,13 @@ require_once dirname(__DIR__) . '/../Model/Attribute.php'; // FIXME workaround
|
||||||
*/
|
*/
|
||||||
abstract class AppShell extends Shell
|
abstract class AppShell extends Shell
|
||||||
{
|
{
|
||||||
public $tasks = array('ConfigLoad');
|
|
||||||
|
|
||||||
/** @var BackgroundJobsTool */
|
/** @var BackgroundJobsTool */
|
||||||
private $BackgroundJobsTool;
|
private $BackgroundJobsTool;
|
||||||
|
|
||||||
public function initialize()
|
public function initialize()
|
||||||
{
|
{
|
||||||
$this->ConfigLoad = $this->Tasks->load('ConfigLoad');
|
$configLoad = $this->Tasks->load('ConfigLoad');
|
||||||
$this->ConfigLoad->execute();
|
$configLoad->execute();
|
||||||
|
|
||||||
parent::initialize();
|
parent::initialize();
|
||||||
}
|
}
|
||||||
|
@ -84,6 +82,15 @@ abstract class AppShell extends Shell
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $newCommand
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
protected function deprecated($newCommand)
|
||||||
|
{
|
||||||
|
$this->err("<warning>Warning: This method is deprecated. Next time please use `$newCommand`.</warning>");
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return BackgroundJobsTool
|
* @return BackgroundJobsTool
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
|
|
|
@ -12,7 +12,7 @@ class AuthkeyShell extends AppShell {
|
||||||
|
|
||||||
public function main()
|
public function main()
|
||||||
{
|
{
|
||||||
$this->err('This method is deprecated. Next time please use `cake user change_authkey [user] [authkey]` command.');
|
$this->deprecated('cake user change_authkey [user] [authkey]');
|
||||||
|
|
||||||
if (!isset($this->args[0]) || empty($this->args[0])) echo 'MISP authkey reset command line tool.' . PHP_EOL . 'To assign a new authkey for a user:' . PHP_EOL . APP . 'Console/cake Authkey [email] [auth_key | optional]' . PHP_EOL;
|
if (!isset($this->args[0]) || empty($this->args[0])) echo 'MISP authkey reset command line tool.' . PHP_EOL . 'To assign a new authkey for a user:' . PHP_EOL . APP . 'Console/cake Authkey [email] [auth_key | optional]' . PHP_EOL;
|
||||||
else {
|
else {
|
||||||
|
|
|
@ -11,7 +11,7 @@ class BaseurlShell extends AppShell {
|
||||||
|
|
||||||
public function main()
|
public function main()
|
||||||
{
|
{
|
||||||
$this->err('This method is deprecated. Next time please use `cake admin setSetting MISP.baseurl [baseurl]` command.');
|
$this->deprecated('cake admin setSetting MISP.baseurl [baseurl]');
|
||||||
|
|
||||||
$baseurl = $this->args[0];
|
$baseurl = $this->args[0];
|
||||||
$result = $this->Server->testBaseURL($baseurl);
|
$result = $this->Server->testBaseURL($baseurl);
|
||||||
|
|
|
@ -53,10 +53,21 @@ class EventShell extends AppShell
|
||||||
$parser->addSubcommand('mergeTags', [
|
$parser->addSubcommand('mergeTags', [
|
||||||
'help' => __('Merge tags'),
|
'help' => __('Merge tags'),
|
||||||
'parser' => [
|
'parser' => [
|
||||||
'arguments' => array(
|
'arguments' => [
|
||||||
'source' => ['help' => __('Source tag ID or name. Source tag will be deleted.'), 'required' => true],
|
'source' => ['help' => __('Source tag ID or name. Source tag will be deleted.'), 'required' => true],
|
||||||
'destination' => ['help' => __('Destination tag ID or name.'), 'required' => true],
|
'destination' => ['help' => __('Destination tag ID or name.'), 'required' => true],
|
||||||
)
|
],
|
||||||
|
],
|
||||||
|
]);
|
||||||
|
$parser->addSubcommand('reportValidationIssuesAttributes', [
|
||||||
|
'help' => __('Report validation issues on attributes'),
|
||||||
|
]);
|
||||||
|
$parser->addSubcommand('normalizeIpAddress', [
|
||||||
|
'help' => __('Normalize IP address format in old events'),
|
||||||
|
'parser' => [
|
||||||
|
'options' => [
|
||||||
|
'dry-run' => ['help' => __('Just show what changes will be made.'), 'boolean' => true],
|
||||||
|
],
|
||||||
],
|
],
|
||||||
]);
|
]);
|
||||||
return $parser;
|
return $parser;
|
||||||
|
@ -636,18 +647,28 @@ class EventShell extends AppShell
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
public function reportValidationIssuesAttributes()
|
||||||
* @param int $userId
|
|
||||||
* @return array
|
|
||||||
*/
|
|
||||||
private function getUser($userId)
|
|
||||||
{
|
{
|
||||||
$user = $this->User->getAuthUser($userId, true);
|
foreach ($this->Event->Attribute->reportValidationIssuesAttributes() as $validationIssue) {
|
||||||
if (empty($user)) {
|
echo $this->json($validationIssue) . "\n";
|
||||||
$this->error("User with ID $userId does not exist.");
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public function normalizeIpAddress()
|
||||||
|
{
|
||||||
|
$dryRun = $this->param('dry-run');
|
||||||
|
|
||||||
|
$count = 0;
|
||||||
|
foreach ($this->Event->Attribute->normalizeIpAddress($dryRun) as $attribute) {
|
||||||
|
$count++;
|
||||||
|
echo JsonTool::encode($attribute) . "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($dryRun) {
|
||||||
|
$this->err(__n("%s attribute to fix", "%s attributes to fix", $count, $count));
|
||||||
|
} else {
|
||||||
|
$this->err(__n("%s attribute fixed", "%s attributes fixed", $count, $count));
|
||||||
}
|
}
|
||||||
Configure::write('CurrentUserId', $user['id']); // for audit logging purposes
|
|
||||||
return $user;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public function generateTopCorrelations()
|
public function generateTopCorrelations()
|
||||||
|
@ -668,4 +689,18 @@ class EventShell extends AppShell
|
||||||
$this->Job->save($job);
|
$this->Job->save($job);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param int $userId
|
||||||
|
* @return array
|
||||||
|
*/
|
||||||
|
private function getUser($userId)
|
||||||
|
{
|
||||||
|
$user = $this->User->getAuthUser($userId, true);
|
||||||
|
if (empty($user)) {
|
||||||
|
$this->error("User with ID $userId does not exist.");
|
||||||
|
}
|
||||||
|
Configure::write('CurrentUserId', $user['id']); // for audit logging purposes
|
||||||
|
return $user;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,8 +1,9 @@
|
||||||
<?php
|
<?php
|
||||||
/*
|
/**
|
||||||
* Enable/disable misp
|
* Enable/disable misp
|
||||||
*
|
*
|
||||||
* arg0 = [0|1]
|
* arg0 = [0|1]
|
||||||
|
* @deprecated Use AdminShell::live instead
|
||||||
*/
|
*/
|
||||||
class LiveShell extends AppShell {
|
class LiveShell extends AppShell {
|
||||||
|
|
||||||
|
@ -10,6 +11,8 @@ class LiveShell extends AppShell {
|
||||||
|
|
||||||
public function main()
|
public function main()
|
||||||
{
|
{
|
||||||
|
$this->deprecated('cake admin live [0|1]');
|
||||||
|
|
||||||
$live = $this->args[0];
|
$live = $this->args[0];
|
||||||
if ($live != 0 && $live != 1) {
|
if ($live != 0 && $live != 1) {
|
||||||
echo 'Invalid parameters. Usage: /var/www/MISP/app/Console/cake Live [0|1]';
|
echo 'Invalid parameters. Usage: /var/www/MISP/app/Console/cake Live [0|1]';
|
||||||
|
|
|
@ -12,7 +12,7 @@ class PasswordShell extends AppShell {
|
||||||
|
|
||||||
public function main()
|
public function main()
|
||||||
{
|
{
|
||||||
$this->err('This method is deprecated. Next time please use `cake user change_pw [user] [password]` command.');
|
$this->deprecated('cake user change_pw [user] [password]');
|
||||||
|
|
||||||
if (!isset($this->args[0]) || empty($this->args[0]) || !isset($this->args[1]) || empty($this->args[1])) echo 'MISP password reset command line tool.' . PHP_EOL . 'To assign a new password for a user:' . PHP_EOL . APP . 'Console/cake Password [email] [password]' . PHP_EOL;
|
if (!isset($this->args[0]) || empty($this->args[0]) || !isset($this->args[1]) || empty($this->args[1])) echo 'MISP password reset command line tool.' . PHP_EOL . 'To assign a new password for a user:' . PHP_EOL . APP . 'Console/cake Password [email] [password]' . PHP_EOL;
|
||||||
else {
|
else {
|
||||||
|
|
|
@ -37,24 +37,32 @@ class StartWorkerShell extends AppShell
|
||||||
|
|
||||||
public function main()
|
public function main()
|
||||||
{
|
{
|
||||||
|
$pid = getmypid();
|
||||||
|
if ($pid === false) {
|
||||||
|
throw new RuntimeException("Could not get current process ID");
|
||||||
|
}
|
||||||
|
|
||||||
$this->worker = new Worker(
|
$this->worker = new Worker(
|
||||||
[
|
[
|
||||||
'pid' => getmypid(),
|
'pid' => $pid,
|
||||||
'queue' => $this->args[0],
|
'queue' => $this->args[0],
|
||||||
'user' => ProcessTool::whoami(),
|
'user' => ProcessTool::whoami(),
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
|
|
||||||
$this->maxExecutionTime = (int)$this->params['maxExecutionTime'];
|
$this->maxExecutionTime = (int)$this->params['maxExecutionTime'];
|
||||||
|
$queue = $this->worker->queue();
|
||||||
|
$backgroundJobTool = $this->getBackgroundJobsTool();
|
||||||
|
|
||||||
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - starting to process background jobs...");
|
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$queue}] - starting to process background jobs...");
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
$this->checkMaxExecutionTime();
|
$this->checkMaxExecutionTime();
|
||||||
|
|
||||||
$job = $this->getBackgroundJobsTool()->dequeue($this->worker->queue());
|
$job = $backgroundJobTool->dequeue($queue);
|
||||||
if ($job) {
|
if ($job) {
|
||||||
$this->runJob($job);
|
$this->runJob($job);
|
||||||
|
$backgroundJobTool->removeFromRunning($this->worker, $job);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -64,7 +72,7 @@ class StartWorkerShell extends AppShell
|
||||||
*/
|
*/
|
||||||
private function runJob(BackgroundJob $job)
|
private function runJob(BackgroundJob $job)
|
||||||
{
|
{
|
||||||
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}...");
|
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}");
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$job->setStatus(BackgroundJob::STATUS_RUNNING);
|
$job->setStatus(BackgroundJob::STATUS_RUNNING);
|
||||||
|
@ -73,12 +81,16 @@ class StartWorkerShell extends AppShell
|
||||||
CakeLog::info("[JOB ID: {$job->id()}] - started command `$command`.");
|
CakeLog::info("[JOB ID: {$job->id()}] - started command `$command`.");
|
||||||
$this->getBackgroundJobsTool()->update($job);
|
$this->getBackgroundJobsTool()->update($job);
|
||||||
|
|
||||||
$job->run();
|
$start = microtime(true);
|
||||||
|
$job->run(function (array $status) use ($job) {
|
||||||
|
$this->getBackgroundJobsTool()->markAsRunning($this->worker, $job, $status['pid']);
|
||||||
|
});
|
||||||
|
$duration = number_format(microtime(true) - $start, 3, '.', '');
|
||||||
|
|
||||||
if ($job->status() === BackgroundJob::STATUS_COMPLETED) {
|
if ($job->status() === BackgroundJob::STATUS_COMPLETED) {
|
||||||
CakeLog::info("[JOB ID: {$job->id()}] - completed.");
|
CakeLog::info("[JOB ID: {$job->id()}] - successfully completed in $duration seconds.");
|
||||||
} else {
|
} else {
|
||||||
CakeLog::error("[JOB ID: {$job->id()}] - failed with error code {$job->returnCode()}. STDERR: {$job->error()}. STDOUT: {$job->output()}.");
|
CakeLog::error("[JOB ID: {$job->id()}] - failed with error code {$job->returnCode()} after $duration seconds. STDERR: {$job->error()}. STDOUT: {$job->output()}.");
|
||||||
}
|
}
|
||||||
} catch (Exception $exception) {
|
} catch (Exception $exception) {
|
||||||
CakeLog::error("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - job ID: {$job->id()} failed with exception: {$exception->getMessage()}");
|
CakeLog::error("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - job ID: {$job->id()} failed with exception: {$exception->getMessage()}");
|
||||||
|
|
|
@ -3,8 +3,6 @@ class ConfigLoadTask extends Shell
|
||||||
{
|
{
|
||||||
public function execute()
|
public function execute()
|
||||||
{
|
{
|
||||||
Configure::load('config');
|
|
||||||
|
|
||||||
if (Configure::read('MISP.system_setting_db')) {
|
if (Configure::read('MISP.system_setting_db')) {
|
||||||
App::uses('SystemSetting', 'Model');
|
App::uses('SystemSetting', 'Model');
|
||||||
SystemSetting::setGlobalSetting();
|
SystemSetting::setGlobalSetting();
|
||||||
|
|
|
@ -1,7 +1,13 @@
|
||||||
<?php
|
<?php
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @deprecated
|
||||||
|
*/
|
||||||
class UserInitShell extends AppShell {
|
class UserInitShell extends AppShell {
|
||||||
public $uses = array('User', 'Role', 'Organisation', 'Server', 'ConnectionManager');
|
public $uses = array('User', 'Role', 'Organisation', 'Server', 'ConnectionManager');
|
||||||
public function main() {
|
public function main() {
|
||||||
|
$this->deprecated('cake user init');
|
||||||
|
|
||||||
if (!Configure::read('Security.salt')) {
|
if (!Configure::read('Security.salt')) {
|
||||||
$this->loadModel('Server');
|
$this->loadModel('Server');
|
||||||
$this->Server->serverSettingsSaveValue('Security.salt', $this->User->generateRandomPassword(32));
|
$this->Server->serverSettingsSaveValue('Security.salt', $this->User->generateRandomPassword(32));
|
||||||
|
|
|
@ -3,10 +3,11 @@
|
||||||
/**
|
/**
|
||||||
* @property User $User
|
* @property User $User
|
||||||
* @property Log $Log
|
* @property Log $Log
|
||||||
|
* @property UserLoginProfile $UserLoginProfile
|
||||||
*/
|
*/
|
||||||
class UserShell extends AppShell
|
class UserShell extends AppShell
|
||||||
{
|
{
|
||||||
public $uses = ['User', 'Log'];
|
public $uses = ['User', 'Log', 'UserLoginProfile'];
|
||||||
|
|
||||||
public function getOptionParser()
|
public function getOptionParser()
|
||||||
{
|
{
|
||||||
|
@ -22,16 +23,24 @@ class UserShell extends AppShell
|
||||||
],
|
],
|
||||||
]
|
]
|
||||||
]);
|
]);
|
||||||
|
$parser->addSubcommand('init', [
|
||||||
|
'help' => __('Create default role, organisation and user when not exists.'),
|
||||||
|
]);
|
||||||
$parser->addSubcommand('authkey', [
|
$parser->addSubcommand('authkey', [
|
||||||
'help' => __('Get information about given authkey.'),
|
'help' => __('Get information about given authkey.'),
|
||||||
'parser' => [
|
'parser' => [
|
||||||
'arguments' => [
|
'arguments' => [
|
||||||
'authkey' => ['help' => __('Authentication key. If not provide, it will be read from STDIN.')],
|
'authkey' => ['help' => __('Authentication key. If not provided, it will be read from STDIN.')],
|
||||||
],
|
],
|
||||||
]
|
]
|
||||||
]);
|
]);
|
||||||
$parser->addSubcommand('authkey_valid', [
|
$parser->addSubcommand('authkey_valid', [
|
||||||
'help' => __('Check if given authkey by STDIN is valid.'),
|
'help' => __('Check if given authkey by STDIN is valid.'),
|
||||||
|
'parser' => [
|
||||||
|
'options' => [
|
||||||
|
'disableStdLog' => ['help' => __('Do not show logs in STDOUT or STDERR.'), 'boolean' => true],
|
||||||
|
],
|
||||||
|
],
|
||||||
]);
|
]);
|
||||||
$parser->addSubcommand('block', [
|
$parser->addSubcommand('block', [
|
||||||
'help' => __('Immediately block user.'),
|
'help' => __('Immediately block user.'),
|
||||||
|
@ -104,6 +113,14 @@ class UserShell extends AppShell
|
||||||
],
|
],
|
||||||
],
|
],
|
||||||
]);
|
]);
|
||||||
|
$parser->addSubcommand('ip_country', [
|
||||||
|
'help' => __('Get country for given IP address'),
|
||||||
|
'parser' => [
|
||||||
|
'arguments' => [
|
||||||
|
'ip' => ['help' => __('IPv4 or IPv6 address.'), 'required' => true],
|
||||||
|
]
|
||||||
|
],
|
||||||
|
]);
|
||||||
$parser->addSubcommand('require_password_change_for_old_passwords', [
|
$parser->addSubcommand('require_password_change_for_old_passwords', [
|
||||||
'help' => __('Trigger forced password change on next login for users with an old (older than x days) password.'),
|
'help' => __('Trigger forced password change on next login for users with an old (older than x days) password.'),
|
||||||
'parser' => [
|
'parser' => [
|
||||||
|
@ -121,7 +138,7 @@ class UserShell extends AppShell
|
||||||
|
|
||||||
public function list()
|
public function list()
|
||||||
{
|
{
|
||||||
$userId = isset($this->args[0]) ? $this->args[0] : null;
|
$userId = $this->args[0] ?? null;
|
||||||
if ($userId) {
|
if ($userId) {
|
||||||
$conditions = ['OR' => [
|
$conditions = ['OR' => [
|
||||||
'User.id' => $userId,
|
'User.id' => $userId,
|
||||||
|
@ -163,13 +180,24 @@ class UserShell extends AppShell
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function init()
|
||||||
|
{
|
||||||
|
if (!Configure::read('Security.salt')) {
|
||||||
|
$this->loadModel('Server');
|
||||||
|
$this->Server->serverSettingsSaveValue('Security.salt', $this->User->generateRandomPassword(32));
|
||||||
|
}
|
||||||
|
|
||||||
|
$authKey = $this->User->init();
|
||||||
|
if ($authKey === null) {
|
||||||
|
$this->err('Script aborted: MISP instance already initialised.');
|
||||||
|
} else {
|
||||||
|
$this->out($authKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public function authkey()
|
public function authkey()
|
||||||
{
|
{
|
||||||
if (isset($this->args[0])) {
|
$authkey = $this->args[0] ?? fgets(STDIN);
|
||||||
$authkey = $this->args[0];
|
|
||||||
} else {
|
|
||||||
$authkey = fgets(STDIN); // read line from STDIN
|
|
||||||
}
|
|
||||||
$authkey = trim($authkey);
|
$authkey = trim($authkey);
|
||||||
if (strlen($authkey) !== 40) {
|
if (strlen($authkey) !== 40) {
|
||||||
$this->error('Authkey has not valid format.');
|
$this->error('Authkey has not valid format.');
|
||||||
|
@ -212,28 +240,37 @@ class UserShell extends AppShell
|
||||||
*/
|
*/
|
||||||
public function authkey_valid()
|
public function authkey_valid()
|
||||||
{
|
{
|
||||||
|
if ($this->params['disableStdLog']) {
|
||||||
|
$this->_useLogger(false);
|
||||||
|
}
|
||||||
|
|
||||||
$cache = [];
|
$cache = [];
|
||||||
$randomKey = random_bytes(16);
|
$randomKey = random_bytes(16);
|
||||||
do {
|
$advancedAuthKeysEnabled = (bool)Configure::read('Security.advanced_authkeys');
|
||||||
|
|
||||||
|
while (true) {
|
||||||
$authkey = fgets(STDIN); // read line from STDIN
|
$authkey = fgets(STDIN); // read line from STDIN
|
||||||
$authkey = trim($authkey);
|
$authkey = trim($authkey);
|
||||||
if (strlen($authkey) !== 40) {
|
if (strlen($authkey) !== 40) {
|
||||||
fwrite(STDOUT, "0\n"); // authkey is not in valid format
|
echo "0\n"; // authkey is not in valid format
|
||||||
$this->log("Authkey in incorrect format provided.", LOG_WARNING);
|
$this->log("Authkey in incorrect format provided, expected 40 chars long string, $authkey provided.", LOG_WARNING);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
$time = time();
|
|
||||||
// Generate hash from authkey to not store raw authkey in memory
|
// Generate hash from authkey to not store raw authkey in memory
|
||||||
$keyHash = sha1($authkey . $randomKey, true);
|
$keyHash = sha1($authkey . $randomKey, true);
|
||||||
|
|
||||||
|
// If authkey is in cache and is fresh, use info from cache
|
||||||
|
$time = time();
|
||||||
if (isset($cache[$keyHash]) && $cache[$keyHash][1] > $time) {
|
if (isset($cache[$keyHash]) && $cache[$keyHash][1] > $time) {
|
||||||
fwrite(STDOUT, $cache[$keyHash][0] ? "1\n" : "0\n");
|
echo $cache[$keyHash][0] ? "1\n" : "0\n";
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
$user = false;
|
$user = false;
|
||||||
for ($i = 0; $i < 5; $i++) {
|
for ($i = 0; $i < 5; $i++) {
|
||||||
try {
|
try {
|
||||||
if (Configure::read('Security.advanced_authkeys')) {
|
if ($advancedAuthKeysEnabled) {
|
||||||
$user = $this->User->AuthKey->getAuthUserByAuthKey($authkey);
|
$user = $this->User->AuthKey->getAuthUserByAuthKey($authkey);
|
||||||
} else {
|
} else {
|
||||||
$user = $this->User->getAuthUserByAuthkey($authkey);
|
$user = $this->User->getAuthUserByAuthkey($authkey);
|
||||||
|
@ -251,18 +288,34 @@ class UserShell extends AppShell
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
$user = (bool)$user;
|
|
||||||
if (!$user) {
|
if (!$user) {
|
||||||
$start = substr($authkey, 0, 4);
|
$valid = null;
|
||||||
$end = substr($authkey, -4);
|
} else if ($user['disabled']) {
|
||||||
$authKeyToStore = $start . str_repeat('*', 32) . $end;
|
$valid = false;
|
||||||
$this->log("Not valid authkey $authKeyToStore provided.", LOG_WARNING);
|
} else {
|
||||||
|
$valid = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Cache results for 5 seconds
|
echo $valid ? "1\n" : "0\n";
|
||||||
$cache[$keyHash] = [$user, $time + 5];
|
|
||||||
fwrite(STDOUT, $user ? "1\n" : "0\n");
|
if ($valid) {
|
||||||
} while (true);
|
// Cache results for 60 seconds if key is valid
|
||||||
|
$cache[$keyHash] = [true, $time + 60];
|
||||||
|
} else {
|
||||||
|
// Cache results for 5 seconds if key is invalid
|
||||||
|
$cache[$keyHash] = [false, $time + 5];
|
||||||
|
|
||||||
|
$start = substr($authkey, 0, 4);
|
||||||
|
$end = substr($authkey, -4);
|
||||||
|
$authKeyForLog = $start . str_repeat('*', 32) . $end;
|
||||||
|
|
||||||
|
if ($valid === false) {
|
||||||
|
$this->log("Authkey $authKeyForLog belongs to user {$user['id']} that is disabled.", LOG_WARNING);
|
||||||
|
} else {
|
||||||
|
$this->log("Authkey $authKeyForLog is invalid or expired.", LOG_WARNING);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public function block()
|
public function block()
|
||||||
|
@ -305,7 +358,7 @@ class UserShell extends AppShell
|
||||||
|
|
||||||
$conditions = ['User.disabled' => false]; // fetch just not disabled users
|
$conditions = ['User.disabled' => false]; // fetch just not disabled users
|
||||||
|
|
||||||
$userId = isset($this->args[0]) ? $this->args[0] : null;
|
$userId = $this->args[0] ?? null;
|
||||||
if ($userId) {
|
if ($userId) {
|
||||||
$conditions['OR'] = [
|
$conditions['OR'] = [
|
||||||
'User.id' => $userId,
|
'User.id' => $userId,
|
||||||
|
@ -364,7 +417,7 @@ class UserShell extends AppShell
|
||||||
}
|
}
|
||||||
$user = $this->getUser($userId);
|
$user = $this->getUser($userId);
|
||||||
|
|
||||||
# validate new authentication key if provided
|
// validate new authentication key if provided
|
||||||
if (!empty($newkey) && (strlen($newkey) != 40 || !ctype_alnum($newkey))) {
|
if (!empty($newkey) && (strlen($newkey) != 40 || !ctype_alnum($newkey))) {
|
||||||
$this->error('The new auth key needs to be 40 characters long and only alphanumeric.');
|
$this->error('The new auth key needs to be 40 characters long and only alphanumeric.');
|
||||||
}
|
}
|
||||||
|
@ -399,7 +452,7 @@ class UserShell extends AppShell
|
||||||
$this->out('<warning>Storing user IP addresses is disabled.</warning>');
|
$this->out('<warning>Storing user IP addresses is disabled.</warning>');
|
||||||
}
|
}
|
||||||
|
|
||||||
$ips = $this->User->setupRedisWithException()->smembers('misp:user_ip:' . $user['id']);
|
$ips = RedisTool::init()->smembers('misp:user_ip:' . $user['id']);
|
||||||
|
|
||||||
if ($this->params['json']) {
|
if ($this->params['json']) {
|
||||||
$this->out($this->json($ips));
|
$this->out($this->json($ips));
|
||||||
|
@ -422,36 +475,50 @@ class UserShell extends AppShell
|
||||||
$this->out('<warning>Storing user IP addresses is disabled.</warning>');
|
$this->out('<warning>Storing user IP addresses is disabled.</warning>');
|
||||||
}
|
}
|
||||||
|
|
||||||
$userId = $this->User->setupRedisWithException()->get('misp:ip_user:' . $ip);
|
$userId = RedisTool::init()->get('misp:ip_user:' . $ip);
|
||||||
if (empty($userId)) {
|
if (empty($userId)) {
|
||||||
$this->out('No hits.');
|
$this->out('No hits.');
|
||||||
$this->_stop();
|
$this->_stop();
|
||||||
}
|
}
|
||||||
|
|
||||||
$user = $this->User->find('first', array(
|
$user = $this->User->find('first', [
|
||||||
'recursive' => -1,
|
'recursive' => -1,
|
||||||
'conditions' => array('User.id' => $userId),
|
'conditions' => ['User.id' => $userId],
|
||||||
'fields' => ['id', 'email'],
|
'fields' => ['id', 'email'],
|
||||||
));
|
]);
|
||||||
|
|
||||||
if (empty($user)) {
|
if (empty($user)) {
|
||||||
$this->error("User with ID $userId doesn't exists anymore.");
|
$this->error("User with ID $userId doesn't exists anymore.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$ipCountry = $this->UserLoginProfile->countryByIp($ip);
|
||||||
|
|
||||||
if ($this->params['json']) {
|
if ($this->params['json']) {
|
||||||
$this->out($this->json([
|
$this->out($this->json([
|
||||||
'ip' => $ip,
|
'ip' => $ip,
|
||||||
'id' => $user['User']['id'],
|
'id' => $user['User']['id'],
|
||||||
'email' => $user['User']['email'],
|
'email' => $user['User']['email'],
|
||||||
|
'country' => $ipCountry,
|
||||||
]));
|
]));
|
||||||
} else {
|
} else {
|
||||||
$this->out(sprintf(
|
$this->hr();
|
||||||
'%s==============================%sIP: %s%s==============================%sUser #%s: %s%s==============================%s',
|
$this->out("IP: $ip (country $ipCountry)");
|
||||||
PHP_EOL, PHP_EOL, $ip, PHP_EOL, PHP_EOL, $user['User']['id'], $user['User']['email'], PHP_EOL, PHP_EOL
|
$this->hr();
|
||||||
));
|
$this->out("User #{$user['User']['id']}: {$user['User']['email']}");
|
||||||
|
$this->hr();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function ip_country()
|
||||||
|
{
|
||||||
|
list($ip) = $this->args;
|
||||||
|
if (!filter_var($ip, FILTER_VALIDATE_IP)) {
|
||||||
|
$this->error("IP `$ip` is not valid IPv4 or IPv6 address");
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->out($this->UserLoginProfile->countryByIp($ip));
|
||||||
|
}
|
||||||
|
|
||||||
public function require_password_change_for_old_passwords()
|
public function require_password_change_for_old_passwords()
|
||||||
{
|
{
|
||||||
list($days) = $this->args;
|
list($days) = $this->args;
|
||||||
|
|
|
@ -0,0 +1,119 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @property Job $Job
|
||||||
|
*/
|
||||||
|
class WorkerShell extends AppShell
|
||||||
|
{
|
||||||
|
public $uses = ['Job'];
|
||||||
|
|
||||||
|
public function getOptionParser(): ConsoleOptionParser
|
||||||
|
{
|
||||||
|
$parser = parent::getOptionParser();
|
||||||
|
$parser->addSubcommand('showQueues', [
|
||||||
|
'help' => __('Show jobs in worker queues'),
|
||||||
|
]);
|
||||||
|
$parser->addSubcommand('flushQueue', [
|
||||||
|
'help' => __('Flush jobs in given queue'),
|
||||||
|
'parser' => [
|
||||||
|
'arguments' => [
|
||||||
|
'queue' => ['help' => __('Queue name'), 'required' => true],
|
||||||
|
],
|
||||||
|
],
|
||||||
|
]);
|
||||||
|
$parser->addSubcommand('showJobStatus', [
|
||||||
|
'help' => __('Show job status'),
|
||||||
|
'parser' => [
|
||||||
|
'arguments' => [
|
||||||
|
'job_id' => ['help' => __('Job ID (ID or UUID)'), 'required' => true],
|
||||||
|
],
|
||||||
|
],
|
||||||
|
]);
|
||||||
|
return $parser;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @throws RedisException
|
||||||
|
* @throws JsonException
|
||||||
|
*/
|
||||||
|
public function showQueues()
|
||||||
|
{
|
||||||
|
$tool = $this->getBackgroundJobsTool();
|
||||||
|
$runningJobs = $tool->runningJobs();
|
||||||
|
|
||||||
|
foreach (BackgroundJobsTool::VALID_QUEUES as $queue) {
|
||||||
|
$this->out("{$queue}:\t{$tool->getQueueSize($queue)}");
|
||||||
|
$queueJobs = $runningJobs[$queue] ?? [];
|
||||||
|
foreach ($queueJobs as $jobId => $data) {
|
||||||
|
$this->out(" - $jobId (" . JsonTool::encode($data) .")");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public function flushQueue()
|
||||||
|
{
|
||||||
|
$queue = $this->args[0];
|
||||||
|
try {
|
||||||
|
$this->getBackgroundJobsTool()->clearQueue($queue);
|
||||||
|
} catch (InvalidArgumentException $e) {
|
||||||
|
$this->error($e->getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public function showJobStatus()
|
||||||
|
{
|
||||||
|
$processId = $this->args[0];
|
||||||
|
if (is_numeric($processId)) {
|
||||||
|
$job = $this->Job->find('first', [
|
||||||
|
'conditions' => ['Job.id' => $processId],
|
||||||
|
'recursive' => -1,
|
||||||
|
]);
|
||||||
|
if (!$job) {
|
||||||
|
$this->error('Job not found', "Job with ID {$processId} not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->out($this->json($job['Job']));
|
||||||
|
$processId = $job['Job']['process_id'];
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!Validation::uuid($processId)) {
|
||||||
|
$this->error('Job not found', "Job ID must be number or UUID, '$processId' given");
|
||||||
|
}
|
||||||
|
|
||||||
|
$jobStatus = $this->getBackgroundJobsTool()->getJob($processId);
|
||||||
|
if (!$jobStatus) {
|
||||||
|
$this->error('Job not found', "Job with UUID {$processId} not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
$jobStatus = $jobStatus->jsonSerialize();
|
||||||
|
|
||||||
|
foreach (['createdAt', 'updatedAt'] as $timeField) {
|
||||||
|
if (isset($jobStatus[$timeField])) {
|
||||||
|
$jobStatus[$timeField] = date('c', $jobStatus[$timeField]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isset($jobStatus['status'])) {
|
||||||
|
$jobStatus['status'] = $this->jobStatusToString($jobStatus['status']);
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->out($this->json($jobStatus));
|
||||||
|
}
|
||||||
|
|
||||||
|
private function jobStatusToString(int $jobStatus)
|
||||||
|
{
|
||||||
|
switch ($jobStatus) {
|
||||||
|
case Job::STATUS_WAITING:
|
||||||
|
return 'waiting';
|
||||||
|
case Job::STATUS_RUNNING:
|
||||||
|
return 'running';
|
||||||
|
case Job::STATUS_FAILED:
|
||||||
|
return 'failed';
|
||||||
|
case Job::STATUS_COMPLETED:
|
||||||
|
return 'completed';
|
||||||
|
}
|
||||||
|
throw new InvalidArgumentException("Invalid job status $jobStatus");
|
||||||
|
}
|
||||||
|
}
|
|
@ -38,8 +38,6 @@ class AppController extends Controller
|
||||||
public $phpmin = '7.2';
|
public $phpmin = '7.2';
|
||||||
public $phprec = '7.4';
|
public $phprec = '7.4';
|
||||||
public $phptoonew = '8.0';
|
public $phptoonew = '8.0';
|
||||||
public $pythonmin = '3.6';
|
|
||||||
public $pythonrec = '3.7';
|
|
||||||
private $isApiAuthed = false;
|
private $isApiAuthed = false;
|
||||||
|
|
||||||
public $baseurl = '';
|
public $baseurl = '';
|
||||||
|
@ -232,6 +230,10 @@ class AppController extends Controller
|
||||||
$this->Security->csrfCheck = false;
|
$this->Security->csrfCheck = false;
|
||||||
$loginByAuthKeyResult = $this->__loginByAuthKey();
|
$loginByAuthKeyResult = $this->__loginByAuthKey();
|
||||||
if ($loginByAuthKeyResult === false || $this->Auth->user() === null) {
|
if ($loginByAuthKeyResult === false || $this->Auth->user() === null) {
|
||||||
|
if ($this->IndexFilter->isXhr()) {
|
||||||
|
throw new ForbiddenException('Authentication failed.');
|
||||||
|
}
|
||||||
|
|
||||||
if ($loginByAuthKeyResult === null) {
|
if ($loginByAuthKeyResult === null) {
|
||||||
$this->loadModel('Log');
|
$this->loadModel('Log');
|
||||||
$this->Log->createLogEntry('SYSTEM', 'auth_fail', 'User', 0, "Failed API authentication. No authkey was provided.");
|
$this->Log->createLogEntry('SYSTEM', 'auth_fail', 'User', 0, "Failed API authentication. No authkey was provided.");
|
||||||
|
@ -601,7 +603,7 @@ class AppController extends Controller
|
||||||
if (!empty($user['allowed_ips'])) {
|
if (!empty($user['allowed_ips'])) {
|
||||||
App::uses('CidrTool', 'Tools');
|
App::uses('CidrTool', 'Tools');
|
||||||
$cidrTool = new CidrTool($user['allowed_ips']);
|
$cidrTool = new CidrTool($user['allowed_ips']);
|
||||||
$remoteIp = $this->_remoteIp();
|
$remoteIp = $this->User->_remoteIp();
|
||||||
if ($remoteIp === null) {
|
if ($remoteIp === null) {
|
||||||
$this->Auth->logout();
|
$this->Auth->logout();
|
||||||
throw new ForbiddenException('Auth key is limited to IP address, but IP address not found');
|
throw new ForbiddenException('Auth key is limited to IP address, but IP address not found');
|
||||||
|
@ -694,7 +696,7 @@ class AppController extends Controller
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
$remoteAddress = $this->_remoteIp();
|
$remoteAddress = $this->User->_remoteIp();
|
||||||
|
|
||||||
$pipe = $redis->pipeline();
|
$pipe = $redis->pipeline();
|
||||||
// keep for 30 days
|
// keep for 30 days
|
||||||
|
@ -737,7 +739,7 @@ class AppController extends Controller
|
||||||
$includeRequestBody = !empty(Configure::read('MISP.log_paranoid_include_post_body')) || $userMonitoringEnabled;
|
$includeRequestBody = !empty(Configure::read('MISP.log_paranoid_include_post_body')) || $userMonitoringEnabled;
|
||||||
/** @var AccessLog $accessLog */
|
/** @var AccessLog $accessLog */
|
||||||
$accessLog = ClassRegistry::init('AccessLog');
|
$accessLog = ClassRegistry::init('AccessLog');
|
||||||
$accessLog->logRequest($user, $this->_remoteIp(), $this->request, $includeRequestBody);
|
$accessLog->logRequest($user, $this->User->_remoteIp(), $this->request, $includeRequestBody);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
|
@ -828,29 +830,34 @@ class AppController extends Controller
|
||||||
|
|
||||||
private function __rateLimitCheck(array $user)
|
private function __rateLimitCheck(array $user)
|
||||||
{
|
{
|
||||||
$info = array();
|
|
||||||
$rateLimitCheck = $this->RateLimit->check(
|
$rateLimitCheck = $this->RateLimit->check(
|
||||||
$user,
|
$user,
|
||||||
$this->request->params['controller'],
|
$this->request->params['controller'],
|
||||||
$this->request->action,
|
$this->request->params['action'],
|
||||||
$info,
|
|
||||||
$this->response->type()
|
|
||||||
);
|
);
|
||||||
if (!empty($info)) {
|
|
||||||
$this->RestResponse->setHeader('X-Rate-Limit-Limit', $info['limit']);
|
if ($rateLimitCheck) {
|
||||||
$this->RestResponse->setHeader('X-Rate-Limit-Remaining', $info['remaining']);
|
$headers = [
|
||||||
$this->RestResponse->setHeader('X-Rate-Limit-Reset', $info['reset']);
|
'X-Rate-Limit-Limit' => $rateLimitCheck['limit'],
|
||||||
}
|
'X-Rate-Limit-Remaining' => $rateLimitCheck['remaining'],
|
||||||
if ($rateLimitCheck !== true) {
|
'X-Rate-Limit-Reset' => $rateLimitCheck['reset'],
|
||||||
$this->response->header('X-Rate-Limit-Limit', $info['limit']);
|
];
|
||||||
$this->response->header('X-Rate-Limit-Remaining', $info['remaining']);
|
|
||||||
$this->response->header('X-Rate-Limit-Reset', $info['reset']);
|
if ($rateLimitCheck['exceeded']) {
|
||||||
$this->response->body($rateLimitCheck);
|
$response = $this->RestResponse->throwException(
|
||||||
$this->response->statusCode(429);
|
429,
|
||||||
$this->response->send();
|
__('Rate limit exceeded.'),
|
||||||
|
'/' . $this->request->params['controller'] . '/' . $this->request->params['action'],
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
$headers
|
||||||
|
);
|
||||||
|
$response->send();
|
||||||
$this->_stop();
|
$this->_stop();
|
||||||
|
} else {
|
||||||
|
$this->RestResponse->headers = array_merge($this->RestResponse->headers, $headers);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public function afterFilter()
|
public function afterFilter()
|
||||||
|
@ -1143,14 +1150,14 @@ class AppController extends Controller
|
||||||
$headerNamespace = '';
|
$headerNamespace = '';
|
||||||
}
|
}
|
||||||
if (isset($server[$headerNamespace . $header]) && !empty($server[$headerNamespace . $header])) {
|
if (isset($server[$headerNamespace . $header]) && !empty($server[$headerNamespace . $header])) {
|
||||||
if (Configure::read('Plugin.CustomAuth_only_allow_source') && Configure::read('Plugin.CustomAuth_only_allow_source') !== $this->_remoteIp()) {
|
if (Configure::read('Plugin.CustomAuth_only_allow_source') && Configure::read('Plugin.CustomAuth_only_allow_source') !== $this->User->_remoteIp()) {
|
||||||
$this->Log = ClassRegistry::init('Log');
|
$this->Log = ClassRegistry::init('Log');
|
||||||
$this->Log->createLogEntry(
|
$this->Log->createLogEntry(
|
||||||
'SYSTEM',
|
'SYSTEM',
|
||||||
'auth_fail',
|
'auth_fail',
|
||||||
'User',
|
'User',
|
||||||
0,
|
0,
|
||||||
'Failed authentication using external key (' . trim($server[$headerNamespace . $header]) . ') - the user has not arrived from the expected address. Instead the request came from: ' . $this->_remoteIp(),
|
'Failed authentication using external key (' . trim($server[$headerNamespace . $header]) . ') - the user has not arrived from the expected address. Instead the request came from: ' . $this->User->_remoteIp(),
|
||||||
null);
|
null);
|
||||||
$this->__preAuthException($authName . ' authentication failed. Contact your MISP support for additional information at: ' . Configure::read('MISP.contact'));
|
$this->__preAuthException($authName . ' authentication failed. Contact your MISP support for additional information at: ' . Configure::read('MISP.contact'));
|
||||||
}
|
}
|
||||||
|
@ -1310,7 +1317,7 @@ class AppController extends Controller
|
||||||
$exception = false;
|
$exception = false;
|
||||||
$filters = $this->_harvestParameters($filterData, $exception, $this->_legacyParams);
|
$filters = $this->_harvestParameters($filterData, $exception, $this->_legacyParams);
|
||||||
if (empty($filters) && $this->request->is('get')) {
|
if (empty($filters) && $this->request->is('get')) {
|
||||||
throw new InvalidArgumentException(__('Restsearch queries using GET and no parameters are not allowed. If you have passed parameters via a JSON body, make sure you use POST requests.'));
|
throw new BadRequestException(__('Restsearch queries using GET and no parameters are not allowed. If you have passed parameters via a JSON body, make sure you use POST requests.'));
|
||||||
}
|
}
|
||||||
if (empty($filters['returnFormat'])) {
|
if (empty($filters['returnFormat'])) {
|
||||||
$filters['returnFormat'] = 'json';
|
$filters['returnFormat'] = 'json';
|
||||||
|
|
|
@ -1917,7 +1917,7 @@ class AttributesController extends AppController
|
||||||
public function reportValidationIssuesAttributes($eventId = false)
|
public function reportValidationIssuesAttributes($eventId = false)
|
||||||
{
|
{
|
||||||
// search for validation problems in the attributes
|
// search for validation problems in the attributes
|
||||||
$this->set('result', $this->Attribute->reportValidationIssuesAttributes($eventId));
|
$this->set('result', iterator_to_array($this->Attribute->reportValidationIssuesAttributes($eventId)));
|
||||||
}
|
}
|
||||||
|
|
||||||
public function generateCorrelation()
|
public function generateCorrelation()
|
||||||
|
|
|
@ -134,7 +134,8 @@ class AuditLogsController extends AppController
|
||||||
]);
|
]);
|
||||||
|
|
||||||
$this->paginate['conditions'] = $this->__searchConditions($params);
|
$this->paginate['conditions'] = $this->__searchConditions($params);
|
||||||
$acl = $this->__applyAuditACL($this->Auth->user());
|
$user = $this->Auth->user();
|
||||||
|
$acl = $this->__applyAuditACL($user);
|
||||||
if ($acl) {
|
if ($acl) {
|
||||||
$this->paginate['conditions']['AND'][] = $acl;
|
$this->paginate['conditions']['AND'][] = $acl;
|
||||||
}
|
}
|
||||||
|
@ -144,7 +145,7 @@ class AuditLogsController extends AppController
|
||||||
return $this->RestResponse->viewData($list, 'json');
|
return $this->RestResponse->viewData($list, 'json');
|
||||||
}
|
}
|
||||||
|
|
||||||
$list = $this->__appendModelLinks($list);
|
$list = $this->__appendModelLinks($user, $list);
|
||||||
foreach ($list as $k => $item) {
|
foreach ($list as $k => $item) {
|
||||||
$list[$k]['AuditLog']['action_human'] = $this->actions[$item['AuditLog']['action']];
|
$list[$k]['AuditLog']['action_human'] = $this->actions[$item['AuditLog']['action']];
|
||||||
}
|
}
|
||||||
|
@ -435,10 +436,11 @@ class AuditLogsController extends AppController
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Generate link to model view if exists and use has permission to access it.
|
* Generate link to model view if exists and use has permission to access it.
|
||||||
|
* @param array $user
|
||||||
* @param array $auditLogs
|
* @param array $auditLogs
|
||||||
* @return array
|
* @return array
|
||||||
*/
|
*/
|
||||||
private function __appendModelLinks(array $auditLogs)
|
private function __appendModelLinks(array $user, array $auditLogs)
|
||||||
{
|
{
|
||||||
$models = [];
|
$models = [];
|
||||||
foreach ($auditLogs as $auditLog) {
|
foreach ($auditLogs as $auditLog) {
|
||||||
|
@ -449,7 +451,7 @@ class AuditLogsController extends AppController
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
$eventIds = isset($models['Event']) ? $models['Event'] : [];
|
$eventIds = $models['Event'] ?? [];
|
||||||
|
|
||||||
if (isset($models['ObjectReference'])) {
|
if (isset($models['ObjectReference'])) {
|
||||||
$this->loadModel('ObjectReference');
|
$this->loadModel('ObjectReference');
|
||||||
|
@ -461,11 +463,11 @@ class AuditLogsController extends AppController
|
||||||
|
|
||||||
if (isset($models['Object']) || isset($objectReferences)) {
|
if (isset($models['Object']) || isset($objectReferences)) {
|
||||||
$objectIds = array_unique(array_merge(
|
$objectIds = array_unique(array_merge(
|
||||||
isset($models['Object']) ? $models['Object'] : [],
|
$models['Object'] ?? [],
|
||||||
isset($objectReferences) ? array_values($objectReferences) : []
|
isset($objectReferences) ? array_values($objectReferences) : []
|
||||||
));
|
));
|
||||||
$this->loadModel('MispObject');
|
$this->loadModel('MispObject');
|
||||||
$conditions = $this->MispObject->buildConditions($this->Auth->user());
|
$conditions = $this->MispObject->buildConditions($user);
|
||||||
$conditions['Object.id'] = $objectIds;
|
$conditions['Object.id'] = $objectIds;
|
||||||
$objects = $this->MispObject->find('all', [
|
$objects = $this->MispObject->find('all', [
|
||||||
'conditions' => $conditions,
|
'conditions' => $conditions,
|
||||||
|
@ -473,22 +475,22 @@ class AuditLogsController extends AppController
|
||||||
'fields' => ['Object.id', 'Object.event_id', 'Object.uuid', 'Object.deleted'],
|
'fields' => ['Object.id', 'Object.event_id', 'Object.uuid', 'Object.deleted'],
|
||||||
]);
|
]);
|
||||||
$objects = array_column(array_column($objects, 'Object'), null, 'id');
|
$objects = array_column(array_column($objects, 'Object'), null, 'id');
|
||||||
$eventIds = array_merge($eventIds, array_column($objects, 'event_id'));
|
array_push($eventIds, ...array_column($objects, 'event_id'));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isset($models['Attribute'])) {
|
if (isset($models['Attribute'])) {
|
||||||
$this->loadModel('Attribute');
|
$this->loadModel('Attribute');
|
||||||
$attributes = $this->Attribute->fetchAttributesSimple($this->Auth->user(), [
|
$attributes = $this->Attribute->fetchAttributesSimple($user, [
|
||||||
'conditions' => ['Attribute.id' => array_unique($models['Attribute'])],
|
'conditions' => ['Attribute.id' => array_unique($models['Attribute'])],
|
||||||
'fields' => ['Attribute.id', 'Attribute.event_id', 'Attribute.uuid', 'Attribute.deleted'],
|
'fields' => ['Attribute.id', 'Attribute.event_id', 'Attribute.uuid', 'Attribute.deleted'],
|
||||||
]);
|
]);
|
||||||
$attributes = array_column(array_column($attributes, 'Attribute'), null, 'id');
|
$attributes = array_column(array_column($attributes, 'Attribute'), null, 'id');
|
||||||
$eventIds = array_merge($eventIds, array_column($attributes, 'event_id'));
|
array_push($eventIds, ...array_column($attributes, 'event_id'));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isset($models['ShadowAttribute'])) {
|
if (isset($models['ShadowAttribute'])) {
|
||||||
$this->loadModel('ShadowAttribute');
|
$this->loadModel('ShadowAttribute');
|
||||||
$conditions = $this->ShadowAttribute->buildConditions($this->Auth->user());
|
$conditions = $this->ShadowAttribute->buildConditions($user);
|
||||||
$conditions['AND'][] = ['ShadowAttribute.id' => array_unique($models['ShadowAttribute'])];
|
$conditions['AND'][] = ['ShadowAttribute.id' => array_unique($models['ShadowAttribute'])];
|
||||||
$shadowAttributes = $this->ShadowAttribute->find('all', [
|
$shadowAttributes = $this->ShadowAttribute->find('all', [
|
||||||
'conditions' => $conditions,
|
'conditions' => $conditions,
|
||||||
|
@ -496,12 +498,12 @@ class AuditLogsController extends AppController
|
||||||
'contain' => ['Event', 'Attribute'],
|
'contain' => ['Event', 'Attribute'],
|
||||||
]);
|
]);
|
||||||
$shadowAttributes = array_column(array_column($shadowAttributes, 'ShadowAttribute'), null, 'id');
|
$shadowAttributes = array_column(array_column($shadowAttributes, 'ShadowAttribute'), null, 'id');
|
||||||
$eventIds = array_merge($eventIds, array_column($shadowAttributes, 'event_id'));
|
array_push($eventIds, ...array_column($shadowAttributes, 'event_id'));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!empty($eventIds)) {
|
if (!empty($eventIds)) {
|
||||||
$this->loadModel('Event');
|
$this->loadModel('Event');
|
||||||
$conditions = $this->Event->createEventConditions($this->Auth->user());
|
$conditions = $this->Event->createEventConditions($user);
|
||||||
$conditions['Event.id'] = array_unique($eventIds);
|
$conditions['Event.id'] = array_unique($eventIds);
|
||||||
$events = $this->Event->find('list', [
|
$events = $this->Event->find('list', [
|
||||||
'conditions' => $conditions,
|
'conditions' => $conditions,
|
||||||
|
|
|
@ -8,7 +8,9 @@ class IndexFilterComponent extends Component
|
||||||
{
|
{
|
||||||
/** @var Controller */
|
/** @var Controller */
|
||||||
public $Controller;
|
public $Controller;
|
||||||
public $isRest = null;
|
|
||||||
|
/** @var bool|null */
|
||||||
|
private $isRest = null;
|
||||||
|
|
||||||
// Used for isApiFunction(), a check that returns true if the controller & action combo matches an action that is a non-xml and non-json automation method
|
// Used for isApiFunction(), a check that returns true if the controller & action combo matches an action that is a non-xml and non-json automation method
|
||||||
// This is used to allow authentication via headers for methods not covered by _isRest() - as that only checks for JSON and XML formats
|
// This is used to allow authentication via headers for methods not covered by _isRest() - as that only checks for JSON and XML formats
|
||||||
|
@ -93,6 +95,11 @@ class IndexFilterComponent extends Component
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function isXhr()
|
||||||
|
{
|
||||||
|
return $this->Controller->request->header('X-Requested-With') === 'XMLHttpRequest';
|
||||||
|
}
|
||||||
|
|
||||||
public function isJson()
|
public function isJson()
|
||||||
{
|
{
|
||||||
return $this->Controller->request->header('Accept') === 'application/json' || $this->Controller->RequestHandler->prefers() === 'json';
|
return $this->Controller->request->header('Accept') === 'application/json' || $this->Controller->RequestHandler->prefers() === 'json';
|
||||||
|
@ -103,11 +110,6 @@ class IndexFilterComponent extends Component
|
||||||
return $this->Controller->request->header('Accept') === 'text/csv' || $this->Controller->RequestHandler->prefers() === 'csv';
|
return $this->Controller->request->header('Accept') === 'text/csv' || $this->Controller->RequestHandler->prefers() === 'csv';
|
||||||
}
|
}
|
||||||
|
|
||||||
public function isXml()
|
|
||||||
{
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param string $controller
|
* @param string $controller
|
||||||
* @param string $action
|
* @param string $action
|
||||||
|
|
|
@ -12,58 +12,60 @@ class RateLimitComponent extends Component
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
|
|
||||||
public $components = array('RestResponse');
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param array $user
|
* @param array $user
|
||||||
* @param string $controller
|
* @param string $controller
|
||||||
* @param string $action
|
* @param string $action
|
||||||
* @param array $info
|
* @return array|null
|
||||||
* @param string $responseType
|
|
||||||
* @return bool
|
|
||||||
* @throws RedisException
|
* @throws RedisException
|
||||||
*/
|
*/
|
||||||
public function check(array $user, $controller, $action, &$info = array(), $responseType)
|
public function check(array $user, $controller, $action)
|
||||||
{
|
{
|
||||||
if (!empty($user['Role']['enforce_rate_limit']) && isset(self::LIMITED_FUNCTIONS[$controller][$action])) {
|
if (!isset(self::LIMITED_FUNCTIONS[$controller][$action])) {
|
||||||
if ($user['Role']['rate_limit_count'] == 0) {
|
return null; // no limit enforced for this controller action
|
||||||
|
}
|
||||||
|
|
||||||
|
if (empty($user['Role']['enforce_rate_limit'])) {
|
||||||
|
return null; // no limit enforced for this role
|
||||||
|
}
|
||||||
|
|
||||||
|
$rateLimit = (int)$user['Role']['rate_limit_count'];
|
||||||
|
if ($rateLimit === 0) {
|
||||||
throw new MethodNotAllowedException(__('API searches are not allowed for this user role.'));
|
throw new MethodNotAllowedException(__('API searches are not allowed for this user role.'));
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$redis = RedisTool::init();
|
$redis = RedisTool::init();
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
return true; // redis is not available, allow access
|
return null; // redis is not available, allow access
|
||||||
}
|
}
|
||||||
|
|
||||||
$uuid = Configure::read('MISP.uuid') ?: 'no-uuid';
|
$uuid = Configure::read('MISP.uuid') ?: 'no-uuid';
|
||||||
$keyName = 'misp:' . $uuid . ':rate_limit:' . $user['id'];
|
$keyName = 'misp:' . $uuid . ':rate_limit:' . $user['id'];
|
||||||
$count = $redis->get($keyName);
|
$count = $redis->get($keyName);
|
||||||
if ($count !== false && $count >= $user['Role']['rate_limit_count']) {
|
|
||||||
$info = array(
|
|
||||||
'limit' => $user['Role']['rate_limit_count'],
|
|
||||||
'reset' => $redis->ttl($keyName),
|
|
||||||
'remaining' => $user['Role']['rate_limit_count'] - $count,
|
|
||||||
);
|
|
||||||
return $this->RestResponse->throwException(
|
|
||||||
429,
|
|
||||||
__('Rate limit exceeded.'),
|
|
||||||
'/' . $controller . '/' . $action,
|
|
||||||
$responseType
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
if ($count === false) {
|
|
||||||
$redis->setEx($keyName, 900, 1);
|
|
||||||
} else {
|
|
||||||
$redis->setEx($keyName, $redis->ttl($keyName), intval($count) + 1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$count += 1;
|
|
||||||
$info = array(
|
|
||||||
'limit' => $user['Role']['rate_limit_count'],
|
|
||||||
'reset' => $redis->ttl($keyName),
|
|
||||||
'remaining' => $user['Role']['rate_limit_count'] - $count
|
|
||||||
);
|
|
||||||
|
|
||||||
|
if ($count !== false && $count >= $rateLimit) {
|
||||||
|
return [
|
||||||
|
'exceeded' => true,
|
||||||
|
'limit' => $rateLimit,
|
||||||
|
'reset' => $redis->ttl($keyName),
|
||||||
|
'remaining' => $rateLimit - $count,
|
||||||
|
];
|
||||||
}
|
}
|
||||||
return true;
|
|
||||||
|
$newCount = $redis->incr($keyName);
|
||||||
|
if ($newCount === 1) {
|
||||||
|
$redis->expire($keyName, 900);
|
||||||
|
$reset = 900;
|
||||||
|
} else {
|
||||||
|
$reset = $redis->ttl($keyName);
|
||||||
|
}
|
||||||
|
|
||||||
|
return [
|
||||||
|
'exceeded' => false,
|
||||||
|
'limit' => $rateLimit,
|
||||||
|
'reset' => $reset,
|
||||||
|
'remaining' => $rateLimit - $newCount,
|
||||||
|
];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -517,7 +517,7 @@ class RestResponseComponent extends Component
|
||||||
if ($id) {
|
if ($id) {
|
||||||
$response['id'] = $id;
|
$response['id'] = $id;
|
||||||
}
|
}
|
||||||
return $this->__sendResponse($response, 403, $format);
|
return $this->prepareResponse($response, 403, $format);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -562,7 +562,7 @@ class RestResponseComponent extends Component
|
||||||
if ($id) {
|
if ($id) {
|
||||||
$response['id'] = $id;
|
$response['id'] = $id;
|
||||||
}
|
}
|
||||||
return $this->__sendResponse($response, 200, $format);
|
return $this->prepareResponse($response, 200, $format);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -587,7 +587,7 @@ class RestResponseComponent extends Component
|
||||||
* @return CakeResponse
|
* @return CakeResponse
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
private function __sendResponse($response, $code, $format = false, $raw = false, $download = false, $headers = array())
|
private function prepareResponse($response, $code, $format = false, $raw = false, $download = false, $headers = array())
|
||||||
{
|
{
|
||||||
App::uses('TmpFileTool', 'Tools');
|
App::uses('TmpFileTool', 'Tools');
|
||||||
$format = !empty($format) ? strtolower($format) : 'json';
|
$format = !empty($format) ? strtolower($format) : 'json';
|
||||||
|
@ -633,7 +633,7 @@ class RestResponseComponent extends Component
|
||||||
}
|
}
|
||||||
|
|
||||||
// If response is big array, encode items separately to save memory
|
// If response is big array, encode items separately to save memory
|
||||||
if (is_array($response) && count($response) > 10000) {
|
if (is_array($response) && count($response) > 10000 && JsonTool::arrayIsList($response)) {
|
||||||
$output = new TmpFileTool();
|
$output = new TmpFileTool();
|
||||||
$output->write('[');
|
$output->write('[');
|
||||||
|
|
||||||
|
@ -775,7 +775,7 @@ class RestResponseComponent extends Component
|
||||||
if (!empty($errors)) {
|
if (!empty($errors)) {
|
||||||
$data['errors'] = $errors;
|
$data['errors'] = $errors;
|
||||||
}
|
}
|
||||||
return $this->__sendResponse($data, 200, $format, $raw, $download, $headers);
|
return $this->prepareResponse($data, 200, $format, $raw, $download, $headers);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -807,7 +807,7 @@ class RestResponseComponent extends Component
|
||||||
'message' => $message,
|
'message' => $message,
|
||||||
'url' => $url
|
'url' => $url
|
||||||
);
|
);
|
||||||
return $this->__sendResponse($message, $code, $format, $raw, false, $headers);
|
return $this->prepareResponse($message, $code, $format, $raw, false, $headers);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function setHeader($header, $value)
|
public function setHeader($header, $value)
|
||||||
|
@ -834,7 +834,7 @@ class RestResponseComponent extends Component
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
$response['url'] = $this->__generateURL($actionArray, $controller, $params);
|
$response['url'] = $this->__generateURL($actionArray, $controller, $params);
|
||||||
return $this->__sendResponse($response, 200, $format);
|
return $this->prepareResponse($response, 200, $format);
|
||||||
}
|
}
|
||||||
|
|
||||||
private function __setup()
|
private function __setup()
|
||||||
|
|
|
@ -152,6 +152,8 @@ class RestSearchComponent extends Component
|
||||||
'category',
|
'category',
|
||||||
'org',
|
'org',
|
||||||
'tags',
|
'tags',
|
||||||
|
'first_seen',
|
||||||
|
'last_seen',
|
||||||
'from',
|
'from',
|
||||||
'to',
|
'to',
|
||||||
'last',
|
'last',
|
||||||
|
@ -184,6 +186,7 @@ class RestSearchComponent extends Component
|
||||||
'context',
|
'context',
|
||||||
'returnFormat',
|
'returnFormat',
|
||||||
'id',
|
'id',
|
||||||
|
'uuid',
|
||||||
'type',
|
'type',
|
||||||
'from',
|
'from',
|
||||||
'to',
|
'to',
|
||||||
|
@ -191,7 +194,8 @@ class RestSearchComponent extends Component
|
||||||
'org_id',
|
'org_id',
|
||||||
'source',
|
'source',
|
||||||
'includeAttribute',
|
'includeAttribute',
|
||||||
'includeEvent'
|
'includeEvent',
|
||||||
|
'includeUuid',
|
||||||
],
|
],
|
||||||
'GalaxyCluster' => [
|
'GalaxyCluster' => [
|
||||||
'page',
|
'page',
|
||||||
|
@ -204,7 +208,7 @@ class RestSearchComponent extends Component
|
||||||
'distribution',
|
'distribution',
|
||||||
'org',
|
'org',
|
||||||
'orgc',
|
'orgc',
|
||||||
'tag',
|
'tag_name',
|
||||||
'custom',
|
'custom',
|
||||||
'sgReferenceOnly',
|
'sgReferenceOnly',
|
||||||
'minimal',
|
'minimal',
|
||||||
|
|
|
@ -834,33 +834,29 @@ class EventsController extends AppController
|
||||||
}
|
}
|
||||||
|
|
||||||
if (empty($rules['limit'])) {
|
if (empty($rules['limit'])) {
|
||||||
$events = array();
|
$events = [];
|
||||||
$i = 1;
|
$i = 1;
|
||||||
$rules['limit'] = 20000;
|
$rules['limit'] = 20000;
|
||||||
while (true) {
|
while (true) {
|
||||||
$rules['page'] = $i;
|
$rules['page'] = $i++;
|
||||||
$temp = $this->Event->find('all', $rules);
|
$temp = $this->Event->find('all', $rules);
|
||||||
$resultCount = count($temp);
|
$resultCount = count($temp);
|
||||||
if ($resultCount !== 0) {
|
if ($resultCount !== 0) {
|
||||||
// this is faster and memory efficient than array_merge
|
array_push($events, ...$temp);
|
||||||
foreach ($temp as $tempEvent) {
|
|
||||||
$events[] = $tempEvent;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if ($resultCount < $rules['limit']) {
|
if ($resultCount < $rules['limit']) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
$i++;
|
|
||||||
}
|
}
|
||||||
unset($temp);
|
unset($temp);
|
||||||
$absolute_total = count($events);
|
$absoluteTotal = count($events);
|
||||||
} else {
|
} else {
|
||||||
$counting_rules = $rules;
|
$counting_rules = $rules;
|
||||||
unset($counting_rules['limit']);
|
unset($counting_rules['limit']);
|
||||||
unset($counting_rules['page']);
|
unset($counting_rules['page']);
|
||||||
$absolute_total = $this->Event->find('count', $counting_rules);
|
$absoluteTotal = $this->Event->find('count', $counting_rules);
|
||||||
|
|
||||||
$events = $absolute_total === 0 ? [] : $this->Event->find('all', $rules);
|
$events = $absoluteTotal === 0 ? [] : $this->Event->find('all', $rules);
|
||||||
}
|
}
|
||||||
|
|
||||||
$isCsvResponse = $this->response->type() === 'text/csv';
|
$isCsvResponse = $this->response->type() === 'text/csv';
|
||||||
|
@ -979,7 +975,7 @@ class EventsController extends AppController
|
||||||
$events = $export->eventIndex($events);
|
$events = $export->eventIndex($events);
|
||||||
}
|
}
|
||||||
|
|
||||||
return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, ['X-Result-Count' => $absolute_total]);
|
return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, ['X-Result-Count' => $absoluteTotal]);
|
||||||
}
|
}
|
||||||
|
|
||||||
private function __indexColumns()
|
private function __indexColumns()
|
||||||
|
@ -2383,7 +2379,7 @@ class EventsController extends AppController
|
||||||
$results = $this->Event->addMISPExportFile($this->Auth->user(), $data, $isXml, $takeOwnership, $publish);
|
$results = $this->Event->addMISPExportFile($this->Auth->user(), $data, $isXml, $takeOwnership, $publish);
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
$this->log("Exception during processing MISP file import: {$e->getMessage()}");
|
$this->log("Exception during processing MISP file import: {$e->getMessage()}");
|
||||||
$this->Flash->error(__('Could not process MISP export file. %s.', $e->getMessage()));
|
$this->Flash->error(__('Could not process MISP export file. %s', $e->getMessage()));
|
||||||
$this->redirect(['controller' => 'events', 'action' => 'add_misp_export']);
|
$this->redirect(['controller' => 'events', 'action' => 'add_misp_export']);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -3203,7 +3199,7 @@ class EventsController extends AppController
|
||||||
$event = $this->Event->find('first', [
|
$event = $this->Event->find('first', [
|
||||||
'conditions' => Validation::uuid($id) ? ['Event.uuid' => $id] : ['Event.id' => $id],
|
'conditions' => Validation::uuid($id) ? ['Event.uuid' => $id] : ['Event.id' => $id],
|
||||||
'recursive' => -1,
|
'recursive' => -1,
|
||||||
'fields' => ['id', 'info', 'publish_timestamp', 'orgc_id'],
|
'fields' => ['id', 'info', 'publish_timestamp', 'orgc_id', 'user_id'],
|
||||||
]);
|
]);
|
||||||
if (empty($event)) {
|
if (empty($event)) {
|
||||||
throw new NotFoundException(__('Invalid event.'));
|
throw new NotFoundException(__('Invalid event.'));
|
||||||
|
@ -3222,6 +3218,16 @@ class EventsController extends AppController
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (
|
||||||
|
Configure::read('MISP.block_publishing_for_same_creator', false) &&
|
||||||
|
$this->Auth->user()['id'] == $event['Event']['user_id']
|
||||||
|
) {
|
||||||
|
$message = __('Could not publish the event, the publishing user cannot be the same as the event creator as per this instance\'s configuration.');
|
||||||
|
if (!$this->_isRest()) {
|
||||||
|
$this->Flash->error($message);
|
||||||
|
}
|
||||||
|
throw new MethodNotAllowedException($message);
|
||||||
|
}
|
||||||
|
|
||||||
return $event;
|
return $event;
|
||||||
}
|
}
|
||||||
|
|
|
@ -1073,7 +1073,7 @@ class ServersController extends AppController
|
||||||
);
|
);
|
||||||
$dumpResults = array();
|
$dumpResults = array();
|
||||||
$tempArray = array();
|
$tempArray = array();
|
||||||
foreach ($finalSettings as $k => $result) {
|
foreach ($finalSettings as $result) {
|
||||||
if ($result['level'] == 3) {
|
if ($result['level'] == 3) {
|
||||||
$issues['deprecated']++;
|
$issues['deprecated']++;
|
||||||
}
|
}
|
||||||
|
@ -1105,18 +1105,19 @@ class ServersController extends AppController
|
||||||
$diagnostic_errors = 0;
|
$diagnostic_errors = 0;
|
||||||
App::uses('File', 'Utility');
|
App::uses('File', 'Utility');
|
||||||
App::uses('Folder', 'Utility');
|
App::uses('Folder', 'Utility');
|
||||||
|
|
||||||
if ($tab === 'correlations') {
|
if ($tab === 'correlations') {
|
||||||
$this->loadModel('Correlation');
|
$this->loadModel('Correlation');
|
||||||
$correlation_metrics = $this->Correlation->collectMetrics();
|
$correlation_metrics = $this->Correlation->collectMetrics();
|
||||||
$this->set('correlation_metrics', $correlation_metrics);
|
$this->set('correlation_metrics', $correlation_metrics);
|
||||||
}
|
} else if ($tab === 'files') {
|
||||||
if ($tab === 'files') {
|
|
||||||
if (!empty(Configure::read('Security.disable_instance_file_uploads'))) {
|
if (!empty(Configure::read('Security.disable_instance_file_uploads'))) {
|
||||||
throw new MethodNotAllowedException(__('This functionality is disabled.'));
|
throw new MethodNotAllowedException(__('This functionality is disabled.'));
|
||||||
}
|
}
|
||||||
$files = $this->Server->grabFiles();
|
$files = $this->Server->grabFiles();
|
||||||
$this->set('files', $files);
|
$this->set('files', $files);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only run this check on the diagnostics tab
|
// Only run this check on the diagnostics tab
|
||||||
if ($tab === 'diagnostics' || $tab === 'download' || $this->_isRest()) {
|
if ($tab === 'diagnostics' || $tab === 'download' || $this->_isRest()) {
|
||||||
$php_ini = php_ini_loaded_file();
|
$php_ini = php_ini_loaded_file();
|
||||||
|
@ -1279,12 +1280,10 @@ class ServersController extends AppController
|
||||||
$this->set('workerIssueCount', $workerIssueCount);
|
$this->set('workerIssueCount', $workerIssueCount);
|
||||||
$priorityErrorColours = array(0 => 'red', 1 => 'yellow', 2 => 'green');
|
$priorityErrorColours = array(0 => 'red', 1 => 'yellow', 2 => 'green');
|
||||||
$this->set('priorityErrorColours', $priorityErrorColours);
|
$this->set('priorityErrorColours', $priorityErrorColours);
|
||||||
$this->set('phpversion', phpversion());
|
$this->set('phpversion', PHP_VERSION);
|
||||||
$this->set('phpmin', $this->phpmin);
|
$this->set('phpmin', $this->phpmin);
|
||||||
$this->set('phprec', $this->phprec);
|
$this->set('phprec', $this->phprec);
|
||||||
$this->set('phptoonew', $this->phptoonew);
|
$this->set('phptoonew', $this->phptoonew);
|
||||||
$this->set('pythonmin', $this->pythonmin);
|
|
||||||
$this->set('pythonrec', $this->pythonrec);
|
|
||||||
$this->set('title_for_layout', __('Diagnostics'));
|
$this->set('title_for_layout', __('Diagnostics'));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1863,7 +1862,7 @@ class ServersController extends AppController
|
||||||
}
|
}
|
||||||
|
|
||||||
if (Configure::read('SimpleBackgroundJobs.enabled')) {
|
if (Configure::read('SimpleBackgroundJobs.enabled')) {
|
||||||
$this->Server->getBackgroundJobsTool()->purgeQueue($worker);
|
$this->Server->getBackgroundJobsTool()->clearQueue($worker);
|
||||||
} else {
|
} else {
|
||||||
// CakeResque
|
// CakeResque
|
||||||
$worker_array = array('cache', 'default', 'email', 'prio');
|
$worker_array = array('cache', 'default', 'email', 'prio');
|
||||||
|
@ -2183,7 +2182,7 @@ class ServersController extends AppController
|
||||||
if ($this->_isRest()) {
|
if ($this->_isRest()) {
|
||||||
return $this->RestResponse->saveFailResponse('Servers', 'addFromJson', false, $this->Server->validationErrors, $this->response->type());
|
return $this->RestResponse->saveFailResponse('Servers', 'addFromJson', false, $this->Server->validationErrors, $this->response->type());
|
||||||
} else {
|
} else {
|
||||||
$this->Flash->error(__('Could not save the server. Error: %s', json_encode($this->Server->validationErrors, true)));
|
$this->Flash->error(__('Could not save the server. Error: %s', json_encode($this->Server->validationErrors)));
|
||||||
$this->redirect(array('action' => 'index'));
|
$this->redirect(array('action' => 'index'));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1244,8 +1244,6 @@ class UsersController extends AppController
|
||||||
// login was successful, do everything that is needed such as logging and more:
|
// login was successful, do everything that is needed such as logging and more:
|
||||||
$this->_postlogin();
|
$this->_postlogin();
|
||||||
} else {
|
} else {
|
||||||
$dataSourceConfig = ConnectionManager::getDataSource('default')->config;
|
|
||||||
$dataSource = $dataSourceConfig['datasource'];
|
|
||||||
// don't display authError before first login attempt
|
// don't display authError before first login attempt
|
||||||
if (str_replace("//", "/", $this->webroot . $this->Session->read('Auth.redirect')) == $this->webroot && $this->Session->read('Message.auth.message') == $this->Auth->authError) {
|
if (str_replace("//", "/", $this->webroot . $this->Session->read('Auth.redirect')) == $this->webroot && $this->Session->read('Message.auth.message') == $this->Auth->authError) {
|
||||||
$this->Session->delete('Message.auth');
|
$this->Session->delete('Message.auth');
|
||||||
|
@ -1260,73 +1258,7 @@ class UsersController extends AppController
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//
|
$this->User->init();
|
||||||
// Actions needed for the first access, when the database is not populated yet.
|
|
||||||
//
|
|
||||||
|
|
||||||
// populate the DB with the first role (site admin) if it's empty
|
|
||||||
if (!$this->User->Role->hasAny()) {
|
|
||||||
$siteAdmin = array('Role' => array(
|
|
||||||
'id' => 1,
|
|
||||||
'name' => 'Site Admin',
|
|
||||||
'permission' => 3,
|
|
||||||
'perm_add' => 1,
|
|
||||||
'perm_modify' => 1,
|
|
||||||
'perm_modify_org' => 1,
|
|
||||||
'perm_publish' => 1,
|
|
||||||
'perm_sync' => 1,
|
|
||||||
'perm_admin' => 1,
|
|
||||||
'perm_audit' => 1,
|
|
||||||
'perm_auth' => 1,
|
|
||||||
'perm_site_admin' => 1,
|
|
||||||
'perm_regexp_access' => 1,
|
|
||||||
'perm_sharing_group' => 1,
|
|
||||||
'perm_template' => 1,
|
|
||||||
'perm_tagger' => 1,
|
|
||||||
));
|
|
||||||
$this->User->Role->save($siteAdmin);
|
|
||||||
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
|
|
||||||
if ($dataSource === 'Database/Postgres') {
|
|
||||||
$sql = "SELECT setval('roles_id_seq', (SELECT MAX(id) FROM roles));";
|
|
||||||
$this->User->Role->query($sql);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!$this->User->Organisation->hasAny(array('Organisation.local' => true))) {
|
|
||||||
$this->User->runUpdates();
|
|
||||||
$date = date('Y-m-d H:i:s');
|
|
||||||
$org = array('Organisation' => array(
|
|
||||||
'id' => 1,
|
|
||||||
'name' => !empty(Configure::read('MISP.org')) ? Configure::read('MISP.org') : 'ADMIN',
|
|
||||||
'description' => 'Automatically generated admin organisation',
|
|
||||||
'type' => 'ADMIN',
|
|
||||||
'uuid' => CakeText::uuid(),
|
|
||||||
'local' => 1,
|
|
||||||
'date_created' => $date,
|
|
||||||
'sector' => '',
|
|
||||||
'nationality' => ''
|
|
||||||
));
|
|
||||||
$this->User->Organisation->save($org);
|
|
||||||
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
|
|
||||||
if ($dataSource === 'Database/Postgres') {
|
|
||||||
$sql = "SELECT setval('organisations_id_seq', (SELECT MAX(id) FROM organisations));";
|
|
||||||
$this->User->Organisation->query($sql);
|
|
||||||
}
|
|
||||||
$org_id = $this->User->Organisation->id;
|
|
||||||
}
|
|
||||||
// populate the DB with the first user if it's empty
|
|
||||||
if (!$this->User->hasAny()) {
|
|
||||||
if (!isset($org_id)) {
|
|
||||||
$hostOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.name' => Configure::read('MISP.org'), 'Organisation.local' => true), 'recursive' => -1));
|
|
||||||
if (!empty($hostOrg)) {
|
|
||||||
$org_id = $hostOrg['Organisation']['id'];
|
|
||||||
} else {
|
|
||||||
$firstOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.local' => true), 'order' => 'Organisation.id ASC'));
|
|
||||||
$org_id = $firstOrg['Organisation']['id'];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$this->User->runUpdates();
|
|
||||||
$this->User->createInitialUser($org_id);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -6,10 +6,7 @@ abstract class NidsExport
|
||||||
|
|
||||||
public $classtype = 'trojan-activity';
|
public $classtype = 'trojan-activity';
|
||||||
|
|
||||||
public $format = ""; // suricata (default), snort
|
protected $format; // suricata (default), snort
|
||||||
|
|
||||||
|
|
||||||
public $checkWhitelist = true;
|
|
||||||
|
|
||||||
public $additional_params = array(
|
public $additional_params = array(
|
||||||
'contain' => array(
|
'contain' => array(
|
||||||
|
@ -17,36 +14,31 @@ abstract class NidsExport
|
||||||
'fields' => array('threat_level_id')
|
'fields' => array('threat_level_id')
|
||||||
)
|
)
|
||||||
),
|
),
|
||||||
|
|
||||||
);
|
);
|
||||||
|
|
||||||
public function handler($data, $options = array())
|
public function handler($data, $options = array())
|
||||||
{
|
{
|
||||||
$continue = empty($format);
|
|
||||||
$this->checkWhitelist = false;
|
|
||||||
if ($options['scope'] === 'Attribute') {
|
if ($options['scope'] === 'Attribute') {
|
||||||
$this->export(
|
$this->export(
|
||||||
array($data),
|
array($data),
|
||||||
$options['user']['nids_sid'],
|
$options['user']['nids_sid']
|
||||||
$options['returnFormat'],
|
|
||||||
$continue
|
|
||||||
);
|
);
|
||||||
} else if ($options['scope'] === 'Event') {
|
} else if ($options['scope'] === 'Event') {
|
||||||
if (!empty($data['EventTag'])) {
|
if (!empty($data['EventTag'])) {
|
||||||
$data['Event']['EventTag'] = $data['EventTag'];
|
$data['Event']['EventTag'] = $data['EventTag'];
|
||||||
}
|
}
|
||||||
if (!empty($data['Attribute'])) {
|
if (!empty($data['Attribute'])) {
|
||||||
$this->__convertFromEventFormat($data['Attribute'], $data, $options, $continue);
|
$this->convertFromEventFormat($data['Attribute'], $data, $options);
|
||||||
}
|
}
|
||||||
if (!empty($data['Object'])) {
|
if (!empty($data['Object'])) {
|
||||||
$this->__convertFromEventFormatObject($data['Object'], $data, $options, $continue);
|
$this->convertFromEventFormatObject($data['Object'], $data, $options);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return '';
|
return '';
|
||||||
}
|
}
|
||||||
|
|
||||||
private function __convertFromEventFormat($attributes, $event, $options = array(), $continue = false) {
|
private function convertFromEventFormat($attributes, $event, $options = array())
|
||||||
|
{
|
||||||
$rearranged = array();
|
$rearranged = array();
|
||||||
foreach ($attributes as $attribute) {
|
foreach ($attributes as $attribute) {
|
||||||
$attributeTag = array();
|
$attributeTag = array();
|
||||||
|
@ -62,15 +54,12 @@ abstract class NidsExport
|
||||||
}
|
}
|
||||||
$this->export(
|
$this->export(
|
||||||
$rearranged,
|
$rearranged,
|
||||||
$options['user']['nids_sid'],
|
$options['user']['nids_sid']
|
||||||
$options['returnFormat'],
|
|
||||||
$continue
|
|
||||||
);
|
);
|
||||||
return true;
|
return true;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private function __convertFromEventFormatObject($objects, $event, $options = array(), $continue = false)
|
private function convertFromEventFormatObject($objects, $event, $options = array())
|
||||||
{
|
{
|
||||||
$rearranged = array();
|
$rearranged = array();
|
||||||
foreach ($objects as $object) {
|
foreach ($objects as $object) {
|
||||||
|
@ -93,20 +82,18 @@ abstract class NidsExport
|
||||||
'Event' => $event['Event']
|
'Event' => $event['Event']
|
||||||
);
|
);
|
||||||
} else { // In case no custom export exists for the object, the approach falls back to the attribute case
|
} else { // In case no custom export exists for the object, the approach falls back to the attribute case
|
||||||
$this->__convertFromEventFormat($object['Attribute'], $event, $options, $continue);
|
$this->convertFromEventFormat($object['Attribute'], $event, $options);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
$this->export(
|
$this->export(
|
||||||
$rearranged,
|
$rearranged,
|
||||||
$options['user']['nids_sid'],
|
$options['user']['nids_sid']
|
||||||
$options['returnFormat'],
|
|
||||||
$continue
|
|
||||||
);
|
);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
public function header($options = array())
|
public function header()
|
||||||
{
|
{
|
||||||
$this->explain();
|
$this->explain();
|
||||||
return '';
|
return '';
|
||||||
|
@ -122,7 +109,7 @@ abstract class NidsExport
|
||||||
return '';
|
return '';
|
||||||
}
|
}
|
||||||
|
|
||||||
public function explain()
|
protected function explain()
|
||||||
{
|
{
|
||||||
$this->rules[] = '# MISP export of IDS rules - optimized for '.$this->format;
|
$this->rules[] = '# MISP export of IDS rules - optimized for '.$this->format;
|
||||||
$this->rules[] = '#';
|
$this->rules[] = '#';
|
||||||
|
@ -136,21 +123,8 @@ abstract class NidsExport
|
||||||
$this->rules[] = '# ';
|
$this->rules[] = '# ';
|
||||||
}
|
}
|
||||||
|
|
||||||
private $whitelist = null;
|
protected function export($items, $startSid)
|
||||||
|
|
||||||
|
|
||||||
public function export($items, $startSid, $format="suricata", $continue = false)
|
|
||||||
{
|
{
|
||||||
$this->format = $format;
|
|
||||||
if ($this->checkWhitelist && !isset($this->Whitelist)) {
|
|
||||||
$this->Whitelist = ClassRegistry::init('Whitelist');
|
|
||||||
$this->whitelist = $this->Whitelist->getBlockedValues();
|
|
||||||
}
|
|
||||||
|
|
||||||
// output a short explanation
|
|
||||||
if (!$continue) {
|
|
||||||
$this->explain();
|
|
||||||
}
|
|
||||||
// generate the rules
|
// generate the rules
|
||||||
foreach ($items as $item) {
|
foreach ($items as $item) {
|
||||||
// retrieve all tags for this item to add them to the msg
|
// retrieve all tags for this item to add them to the msg
|
||||||
|
@ -180,7 +154,6 @@ abstract class NidsExport
|
||||||
$sid++;
|
$sid++;
|
||||||
|
|
||||||
if (!empty($item['Attribute']['type'])) { // item is an 'Attribute'
|
if (!empty($item['Attribute']['type'])) { // item is an 'Attribute'
|
||||||
|
|
||||||
switch ($item['Attribute']['type']) {
|
switch ($item['Attribute']['type']) {
|
||||||
// LATER nids - test all the snort attributes
|
// LATER nids - test all the snort attributes
|
||||||
// LATER nids - add the tag keyword in the rules to capture network traffic
|
// LATER nids - add the tag keyword in the rules to capture network traffic
|
||||||
|
@ -228,17 +201,17 @@ abstract class NidsExport
|
||||||
case 'ja3-fingerprint-md5':
|
case 'ja3-fingerprint-md5':
|
||||||
$this->ja3Rule($ruleFormat, $item['Attribute'], $sid);
|
$this->ja3Rule($ruleFormat, $item['Attribute'], $sid);
|
||||||
break;
|
break;
|
||||||
case 'ja3s-fingerprint-md5': // Atribute type doesn't exists yet (2020-12-10) but ready when created.
|
case 'ja3s-fingerprint-md5': // Attribute type doesn't exists yet (2020-12-10) but ready when created.
|
||||||
$this->ja3sRule($ruleFormat, $item['Attribute'], $sid);
|
$this->ja3sRule($ruleFormat, $item['Attribute'], $sid);
|
||||||
break;
|
break;
|
||||||
case 'snort':
|
case 'snort':
|
||||||
$this->snortRule($ruleFormat, $item['Attribute'], $sid, $ruleFormatMsg, $ruleFormatReference);
|
$this->snortRule($item['Attribute'], $sid, $ruleFormatMsg, $ruleFormatReference);
|
||||||
// no break
|
// no break
|
||||||
default:
|
default:
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
} else if(!empty($item['Attribute']['name'])) { // Item is an 'Object'
|
} else if (!empty($item['Attribute']['name'])) { // Item is an 'Object'
|
||||||
|
|
||||||
switch ($item['Attribute']['name']) {
|
switch ($item['Attribute']['name']) {
|
||||||
case 'network-connection':
|
case 'network-connection':
|
||||||
|
@ -252,30 +225,26 @@ abstract class NidsExport
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return $this->rules;
|
protected function networkConnectionRule($ruleFormat, $object, &$sid)
|
||||||
}
|
|
||||||
|
|
||||||
public function networkConnectionRule($ruleFormat, $object, &$sid)
|
|
||||||
{
|
{
|
||||||
|
|
||||||
$attributes = NidsExport::getObjectAttributes($object);
|
$attributes = NidsExport::getObjectAttributes($object);
|
||||||
|
|
||||||
if(!array_key_exists('layer4-protocol', $attributes)){
|
if (!array_key_exists('layer4-protocol', $attributes)) {
|
||||||
$attributes['layer4-protocol'] = 'ip'; // If layer-4 protocol is unknown, we roll-back to layer-3 ('ip')
|
$attributes['layer4-protocol'] = 'ip'; // If layer-4 protocol is unknown, we roll-back to layer-3 ('ip')
|
||||||
}
|
}
|
||||||
if(!array_key_exists('ip-src', $attributes)){
|
if (!array_key_exists('ip-src', $attributes)) {
|
||||||
$attributes['ip-src'] = '$HOME_NET'; // If ip-src is unknown, we roll-back to $HOME_NET
|
$attributes['ip-src'] = '$HOME_NET'; // If ip-src is unknown, we roll-back to $HOME_NET
|
||||||
}
|
}
|
||||||
if(!array_key_exists('ip-dst', $attributes)){
|
if (!array_key_exists('ip-dst', $attributes)) {
|
||||||
$attributes['ip-dst'] = '$HOME_NET'; // If ip-dst is unknown, we roll-back to $HOME_NET
|
$attributes['ip-dst'] = '$HOME_NET'; // If ip-dst is unknown, we roll-back to $HOME_NET
|
||||||
}
|
}
|
||||||
if(!array_key_exists('src-port', $attributes)){
|
if (!array_key_exists('src-port', $attributes)) {
|
||||||
$attributes['src-port'] = 'any'; // If src-port is unknown, we roll-back to 'any'
|
$attributes['src-port'] = 'any'; // If src-port is unknown, we roll-back to 'any'
|
||||||
}
|
}
|
||||||
if(!array_key_exists('dst-port', $attributes)){
|
if (!array_key_exists('dst-port', $attributes)) {
|
||||||
$attributes['dst-port'] = 'any'; // If dst-port is unknown, we roll-back to 'any'
|
$attributes['dst-port'] = 'any'; // If dst-port is unknown, we roll-back to 'any'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -294,12 +263,10 @@ abstract class NidsExport
|
||||||
$sid, // sid
|
$sid, // sid
|
||||||
1 // rev
|
1 // rev
|
||||||
);
|
);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public function ddosRule($ruleFormat, $object, &$sid)
|
protected function ddosRule($ruleFormat, $object, &$sid)
|
||||||
{
|
{
|
||||||
|
|
||||||
$attributes = NidsExport::getObjectAttributes($object);
|
$attributes = NidsExport::getObjectAttributes($object);
|
||||||
|
|
||||||
if(!array_key_exists('protocol', $attributes)){
|
if(!array_key_exists('protocol', $attributes)){
|
||||||
|
@ -333,12 +300,10 @@ abstract class NidsExport
|
||||||
$sid, // sid
|
$sid, // sid
|
||||||
1 // rev
|
1 // rev
|
||||||
);
|
);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static function getObjectAttributes($object)
|
protected static function getObjectAttributes($object)
|
||||||
{
|
{
|
||||||
|
|
||||||
$attributes = array();
|
$attributes = array();
|
||||||
|
|
||||||
foreach ($object['Attribute'] as $attribute) {
|
foreach ($object['Attribute'] as $attribute) {
|
||||||
|
@ -348,7 +313,7 @@ abstract class NidsExport
|
||||||
return $attributes;
|
return $attributes;
|
||||||
}
|
}
|
||||||
|
|
||||||
public function domainIpRule($ruleFormat, $attribute, &$sid)
|
protected function domainIpRule($ruleFormat, $attribute, &$sid)
|
||||||
{
|
{
|
||||||
$values = explode('|', $attribute['value']);
|
$values = explode('|', $attribute['value']);
|
||||||
$attributeCopy = $attribute;
|
$attributeCopy = $attribute;
|
||||||
|
@ -361,7 +326,7 @@ abstract class NidsExport
|
||||||
$this->ipSrcRule($ruleFormat, $attributeCopy, $sid);
|
$this->ipSrcRule($ruleFormat, $attributeCopy, $sid);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function ipDstRule($ruleFormat, $attribute, &$sid)
|
protected function ipDstRule($ruleFormat, $attribute, &$sid)
|
||||||
{
|
{
|
||||||
$overruled = $this->checkWhitelist($attribute['value']);
|
$overruled = $this->checkWhitelist($attribute['value']);
|
||||||
$ipport = NidsExport::getIpPort($attribute);
|
$ipport = NidsExport::getIpPort($attribute);
|
||||||
|
@ -382,7 +347,7 @@ abstract class NidsExport
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function ipSrcRule($ruleFormat, $attribute, &$sid)
|
protected function ipSrcRule($ruleFormat, $attribute, &$sid)
|
||||||
{
|
{
|
||||||
$overruled = $this->checkWhitelist($attribute['value']);
|
$overruled = $this->checkWhitelist($attribute['value']);
|
||||||
$ipport = NidsExport::getIpPort($attribute);
|
$ipport = NidsExport::getIpPort($attribute);
|
||||||
|
@ -403,7 +368,7 @@ abstract class NidsExport
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function emailSrcRule($ruleFormat, $attribute, &$sid)
|
protected function emailSrcRule($ruleFormat, $attribute, &$sid)
|
||||||
{
|
{
|
||||||
$overruled = $this->checkWhitelist($attribute['value']);
|
$overruled = $this->checkWhitelist($attribute['value']);
|
||||||
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
||||||
|
@ -425,7 +390,7 @@ abstract class NidsExport
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function emailDstRule($ruleFormat, $attribute, &$sid)
|
protected function emailDstRule($ruleFormat, $attribute, &$sid)
|
||||||
{
|
{
|
||||||
$overruled = $this->checkWhitelist($attribute['value']);
|
$overruled = $this->checkWhitelist($attribute['value']);
|
||||||
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
||||||
|
@ -447,7 +412,7 @@ abstract class NidsExport
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function emailSubjectRule($ruleFormat, $attribute, &$sid)
|
protected function emailSubjectRule($ruleFormat, $attribute, &$sid)
|
||||||
{
|
{
|
||||||
// LATER nids - email-subject rule might not match because of line-wrapping
|
// LATER nids - email-subject rule might not match because of line-wrapping
|
||||||
$overruled = $this->checkWhitelist($attribute['value']);
|
$overruled = $this->checkWhitelist($attribute['value']);
|
||||||
|
@ -470,7 +435,7 @@ abstract class NidsExport
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function emailAttachmentRule($ruleFormat, $attribute, &$sid)
|
protected function emailAttachmentRule($ruleFormat, $attribute, &$sid)
|
||||||
{
|
{
|
||||||
// LATER nids - email-attachment rule might not match because of line-wrapping
|
// LATER nids - email-attachment rule might not match because of line-wrapping
|
||||||
$overruled = $this->checkWhitelist($attribute['value']);
|
$overruled = $this->checkWhitelist($attribute['value']);
|
||||||
|
@ -493,7 +458,7 @@ abstract class NidsExport
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function hostnameRule($ruleFormat, $attribute, &$sid)
|
protected function hostnameRule($ruleFormat, $attribute, &$sid)
|
||||||
{
|
{
|
||||||
$overruled = $this->checkWhitelist($attribute['value']);
|
$overruled = $this->checkWhitelist($attribute['value']);
|
||||||
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
||||||
|
@ -549,7 +514,7 @@ abstract class NidsExport
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function domainRule($ruleFormat, $attribute, &$sid)
|
protected function domainRule($ruleFormat, $attribute, &$sid)
|
||||||
{
|
{
|
||||||
$overruled = $this->checkWhitelist($attribute['value']);
|
$overruled = $this->checkWhitelist($attribute['value']);
|
||||||
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
||||||
|
@ -605,7 +570,7 @@ abstract class NidsExport
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function urlRule($ruleFormat, $attribute, &$sid)
|
protected function urlRule($ruleFormat, $attribute, &$sid)
|
||||||
{
|
{
|
||||||
// TODO in hindsight, an url should not be excluded given a host or domain name.
|
// TODO in hindsight, an url should not be excluded given a host or domain name.
|
||||||
//$hostpart = parse_url($attribute['value'], PHP_URL_HOST);
|
//$hostpart = parse_url($attribute['value'], PHP_URL_HOST);
|
||||||
|
@ -630,7 +595,7 @@ abstract class NidsExport
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function userAgentRule($ruleFormat, $attribute, &$sid)
|
protected function userAgentRule($ruleFormat, $attribute, &$sid)
|
||||||
{
|
{
|
||||||
$overruled = $this->checkWhitelist($attribute['value']);
|
$overruled = $this->checkWhitelist($attribute['value']);
|
||||||
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
||||||
|
@ -652,17 +617,17 @@ abstract class NidsExport
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function ja3Rule($ruleFormat, $attribute, &$sid)
|
protected function ja3Rule($ruleFormat, $attribute, &$sid)
|
||||||
{
|
{
|
||||||
//Empty because Snort doesn't support JA3 Rules
|
//Empty because Snort doesn't support JA3 Rules
|
||||||
}
|
}
|
||||||
|
|
||||||
public function ja3sRule($ruleFormat, $attribute, &$sid)
|
protected function ja3sRule($ruleFormat, $attribute, &$sid)
|
||||||
{
|
{
|
||||||
//Empty because Snort doesn't support JA3S Rules
|
//Empty because Snort doesn't support JA3S Rules
|
||||||
}
|
}
|
||||||
|
|
||||||
public function snortRule($ruleFormat, $attribute, &$sid, $ruleFormatMsg, $ruleFormatReference)
|
protected function snortRule($attribute, &$sid, $ruleFormatMsg, $ruleFormatReference)
|
||||||
{
|
{
|
||||||
// LATER nids - test using lots of snort rules, some rules don't contain all the necessary to be a valid rule.
|
// LATER nids - test using lots of snort rules, some rules don't contain all the necessary to be a valid rule.
|
||||||
|
|
||||||
|
@ -678,46 +643,46 @@ abstract class NidsExport
|
||||||
// tag - '/tag\s*:\s*.+?;/'
|
// tag - '/tag\s*:\s*.+?;/'
|
||||||
$replaceCount = array();
|
$replaceCount = array();
|
||||||
$tmpRule = preg_replace('/sid\s*:\s*[0-9]+\s*;/', 'sid:' . $sid . ';', $tmpRule, -1, $replaceCount['sid']);
|
$tmpRule = preg_replace('/sid\s*:\s*[0-9]+\s*;/', 'sid:' . $sid . ';', $tmpRule, -1, $replaceCount['sid']);
|
||||||
if (null == $tmpRule) {
|
if (null === $tmpRule) {
|
||||||
return false;
|
return false;
|
||||||
} // don't output the rule on error with the regex
|
} // don't output the rule on error with the regex
|
||||||
$tmpRule = preg_replace('/rev\s*:\s*[0-9]+\s*;/', 'rev:1;', $tmpRule, -1, $replaceCount['rev']);
|
$tmpRule = preg_replace('/rev\s*:\s*[0-9]+\s*;/', 'rev:1;', $tmpRule, -1, $replaceCount['rev']);
|
||||||
if (null == $tmpRule) {
|
if (null === $tmpRule) {
|
||||||
return false;
|
return false;
|
||||||
} // don't output the rule on error with the regex
|
} // don't output the rule on error with the regex
|
||||||
$tmpRule = preg_replace('/classtype:[a-zA-Z_-]+;/', 'classtype:' . $this->classtype . ';', $tmpRule, -1, $replaceCount['classtype']);
|
$tmpRule = preg_replace('/classtype:[a-zA-Z_-]+;/', 'classtype:' . $this->classtype . ';', $tmpRule, -1, $replaceCount['classtype']);
|
||||||
if (null == $tmpRule) {
|
if (null === $tmpRule) {
|
||||||
return false;
|
return false;
|
||||||
} // don't output the rule on error with the regex
|
} // don't output the rule on error with the regex
|
||||||
$tmpRule = preg_replace('/msg\s*:\s*"(.*?)"\s*;/', sprintf($ruleFormatMsg, 'snort-rule | $1') . ';', $tmpRule, -1, $replaceCount['msg']);
|
$tmpRule = preg_replace('/msg\s*:\s*"(.*?)"\s*;/', sprintf($ruleFormatMsg, 'snort-rule | $1') . ';', $tmpRule, -1, $replaceCount['msg']);
|
||||||
if (null == $tmpRule) {
|
if (null === $tmpRule) {
|
||||||
return false;
|
return false;
|
||||||
} // don't output the rule on error with the regex
|
} // don't output the rule on error with the regex
|
||||||
$tmpRule = preg_replace('/reference\s*:\s*.+?;/', $ruleFormatReference . ';', $tmpRule, -1, $replaceCount['reference']);
|
$tmpRule = preg_replace('/reference\s*:\s*.+?;/', $ruleFormatReference . ';', $tmpRule, -1, $replaceCount['reference']);
|
||||||
if (null == $tmpRule) {
|
if (null === $tmpRule) {
|
||||||
return false;
|
return false;
|
||||||
} // don't output the rule on error with the regex
|
} // don't output the rule on error with the regex
|
||||||
$tmpRule = preg_replace('/reference\s*:\s*.+?;/', $ruleFormatReference . ';', $tmpRule, -1, $replaceCount['reference']);
|
$tmpRule = preg_replace('/reference\s*:\s*.+?;/', $ruleFormatReference . ';', $tmpRule, -1, $replaceCount['reference']);
|
||||||
if (null == $tmpRule) {
|
if (null === $tmpRule) {
|
||||||
return false;
|
return false;
|
||||||
} // don't output the rule on error with the regex
|
} // don't output the rule on error with the regex
|
||||||
// FIXME nids - implement priority overwriting
|
// FIXME nids - implement priority overwriting
|
||||||
|
|
||||||
// some values were not replaced, so we need to add them ourselves, and insert them in the rule
|
// some values were not replaced, so we need to add them ourselves, and insert them in the rule
|
||||||
$extraForRule = "";
|
$extraForRule = "";
|
||||||
if (0 == $replaceCount['sid']) {
|
if (0 === $replaceCount['sid']) {
|
||||||
$extraForRule .= 'sid:' . $sid . ';';
|
$extraForRule .= 'sid:' . $sid . ';';
|
||||||
}
|
}
|
||||||
if (0 == $replaceCount['rev']) {
|
if (0 === $replaceCount['rev']) {
|
||||||
$extraForRule .= 'rev:1;';
|
$extraForRule .= 'rev:1;';
|
||||||
}
|
}
|
||||||
if (0 == $replaceCount['classtype']) {
|
if (0 === $replaceCount['classtype']) {
|
||||||
$extraForRule .= 'classtype:' . $this->classtype . ';';
|
$extraForRule .= 'classtype:' . $this->classtype . ';';
|
||||||
}
|
}
|
||||||
if (0 == $replaceCount['msg']) {
|
if (0 === $replaceCount['msg']) {
|
||||||
$extraForRule .= $tmpMessage . ';';
|
$extraForRule .= $ruleFormatMsg . ';';
|
||||||
}
|
}
|
||||||
if (0 == $replaceCount['reference']) {
|
if (0 === $replaceCount['reference']) {
|
||||||
$extraForRule .= $ruleFormatReference . ';';
|
$extraForRule .= $ruleFormatReference . ';';
|
||||||
}
|
}
|
||||||
$tmpRule = preg_replace('/;\s*\)/', '; ' . $extraForRule . ')', $tmpRule);
|
$tmpRule = preg_replace('/;\s*\)/', '; ' . $extraForRule . ')', $tmpRule);
|
||||||
|
@ -734,7 +699,7 @@ abstract class NidsExport
|
||||||
* @param string $type the type of dns name - domain (default) or hostname
|
* @param string $type the type of dns name - domain (default) or hostname
|
||||||
* @return string raw snort compatible format of the dns name
|
* @return string raw snort compatible format of the dns name
|
||||||
*/
|
*/
|
||||||
public static function dnsNameToRawFormat($name, $type='domain')
|
protected static function dnsNameToRawFormat($name, $type='domain')
|
||||||
{
|
{
|
||||||
$rawName = "";
|
$rawName = "";
|
||||||
if ('hostname' == $type) {
|
if ('hostname' == $type) {
|
||||||
|
@ -747,7 +712,7 @@ abstract class NidsExport
|
||||||
// count the length of the part, and add |length| before
|
// count the length of the part, and add |length| before
|
||||||
$length = strlen($explodedName);
|
$length = strlen($explodedName);
|
||||||
if ($length > 255) {
|
if ($length > 255) {
|
||||||
log('WARNING: DNS name is too long for RFC: '.$name);
|
CakeLog::notice('WARNING: DNS name is too long for RFC: '.$name);
|
||||||
}
|
}
|
||||||
$hexLength = dechex($length);
|
$hexLength = dechex($length);
|
||||||
if (1 == strlen($hexLength)) {
|
if (1 == strlen($hexLength)) {
|
||||||
|
@ -768,7 +733,7 @@ abstract class NidsExport
|
||||||
* @param string $name dns name to be converted
|
* @param string $name dns name to be converted
|
||||||
* @return string raw snort compatible format of the dns name
|
* @return string raw snort compatible format of the dns name
|
||||||
*/
|
*/
|
||||||
public static function dnsNameToMSDNSLogFormat($name)
|
protected static function dnsNameToMSDNSLogFormat($name)
|
||||||
{
|
{
|
||||||
$rawName = "";
|
$rawName = "";
|
||||||
// in MS DNS log format we can't use (0) to distinguish between hostname and domain (including subdomains)
|
// in MS DNS log format we can't use (0) to distinguish between hostname and domain (including subdomains)
|
||||||
|
@ -779,7 +744,7 @@ abstract class NidsExport
|
||||||
// count the length of the part, and add |length| before
|
// count the length of the part, and add |length| before
|
||||||
$length = strlen($explodedName);
|
$length = strlen($explodedName);
|
||||||
if ($length > 255) {
|
if ($length > 255) {
|
||||||
log('WARNING: DNS name is too long for RFC: '.$name);
|
CakeLog::notice('WARNING: DNS name is too long for RFC: '.$name);
|
||||||
}
|
}
|
||||||
$hexLength = dechex($length);
|
$hexLength = dechex($length);
|
||||||
$rawName .= '(' . $hexLength . ')' . $explodedName;
|
$rawName .= '(' . $hexLength . ')' . $explodedName;
|
||||||
|
@ -793,9 +758,9 @@ abstract class NidsExport
|
||||||
/**
|
/**
|
||||||
* Replaces characters that are not allowed in a signature.
|
* Replaces characters that are not allowed in a signature.
|
||||||
* example: " is converted to |22|
|
* example: " is converted to |22|
|
||||||
* @param unknown_type $value
|
* @param string $value
|
||||||
*/
|
*/
|
||||||
public static function replaceIllegalChars($value)
|
protected static function replaceIllegalChars($value)
|
||||||
{
|
{
|
||||||
$replace_pairs = array(
|
$replace_pairs = array(
|
||||||
'|' => '|7c|', // Needs to stay on top !
|
'|' => '|7c|', // Needs to stay on top !
|
||||||
|
@ -808,19 +773,17 @@ abstract class NidsExport
|
||||||
return strtr($value, $replace_pairs);
|
return strtr($value, $replace_pairs);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function checkWhitelist($value)
|
/**
|
||||||
|
* @deprecated
|
||||||
|
* @param $value
|
||||||
|
* @return false
|
||||||
|
*/
|
||||||
|
protected function checkWhitelist($value)
|
||||||
{
|
{
|
||||||
if ($this->checkWhitelist && is_array($this->whitelist)) {
|
|
||||||
foreach ($this->whitelist as $wlitem) {
|
|
||||||
if (preg_match($wlitem, $value)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static function getProtocolPort($protocol, $customPort)
|
protected static function getProtocolPort($protocol, $customPort)
|
||||||
{
|
{
|
||||||
if ($customPort == null) {
|
if ($customPort == null) {
|
||||||
switch ($protocol) {
|
switch ($protocol) {
|
||||||
|
@ -840,7 +803,7 @@ abstract class NidsExport
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static function getCustomIP($customIP)
|
protected static function getCustomIP($customIP)
|
||||||
{
|
{
|
||||||
if (filter_var($customIP, FILTER_VALIDATE_IP)) {
|
if (filter_var($customIP, FILTER_VALIDATE_IP)) {
|
||||||
return $customIP;
|
return $customIP;
|
||||||
|
@ -853,7 +816,7 @@ abstract class NidsExport
|
||||||
* @param array $attribute
|
* @param array $attribute
|
||||||
* @return array|string[]
|
* @return array|string[]
|
||||||
*/
|
*/
|
||||||
public static function getIpPort($attribute)
|
protected static function getIpPort($attribute)
|
||||||
{
|
{
|
||||||
if (strpos($attribute['type'], 'port') !== false) {
|
if (strpos($attribute['type'], 'port') !== false) {
|
||||||
return explode('|', $attribute['value']);
|
return explode('|', $attribute['value']);
|
||||||
|
|
|
@ -4,11 +4,5 @@ App::uses('NidsExport', 'Export');
|
||||||
|
|
||||||
class NidsSnortExport extends NidsExport
|
class NidsSnortExport extends NidsExport
|
||||||
{
|
{
|
||||||
public function export($items, $startSid, $format = "suricata", $continue = false)
|
protected $format = 'snort';
|
||||||
{
|
|
||||||
// set the specific format
|
|
||||||
$this->format = 'snort';
|
|
||||||
// call the generic function
|
|
||||||
return parent::export($items, $startSid, $format, $continue);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,16 +3,10 @@ App::uses('NidsExport', 'Export');
|
||||||
|
|
||||||
class NidsSuricataExport extends NidsExport
|
class NidsSuricataExport extends NidsExport
|
||||||
{
|
{
|
||||||
public function export($items, $startSid, $format = "suricata", $continue = false)
|
protected $format = "suricata";
|
||||||
{
|
|
||||||
// set the specific format
|
|
||||||
$this->format = "suricata";
|
|
||||||
// call the generic function
|
|
||||||
return parent::export($items, $startSid, $format, $continue);
|
|
||||||
}
|
|
||||||
|
|
||||||
// below overwrite functions from NidsExport
|
// below overwrite functions from NidsExport
|
||||||
public function hostnameRule($ruleFormat, $attribute, &$sid)
|
protected function hostnameRule($ruleFormat, $attribute, &$sid)
|
||||||
{
|
{
|
||||||
$overruled = $this->checkWhitelist($attribute['value']);
|
$overruled = $this->checkWhitelist($attribute['value']);
|
||||||
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
||||||
|
@ -53,7 +47,7 @@ class NidsSuricataExport extends NidsExport
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function domainRule($ruleFormat, $attribute, &$sid)
|
protected function domainRule($ruleFormat, $attribute, &$sid)
|
||||||
{
|
{
|
||||||
$overruled = $this->checkWhitelist($attribute['value']);
|
$overruled = $this->checkWhitelist($attribute['value']);
|
||||||
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
||||||
|
@ -94,7 +88,7 @@ class NidsSuricataExport extends NidsExport
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function urlRule($ruleFormat, $attribute, &$sid)
|
protected function urlRule($ruleFormat, $attribute, &$sid)
|
||||||
{
|
{
|
||||||
$createRule = true;
|
$createRule = true;
|
||||||
$overruled = $this->checkWhitelist($attribute['value']);
|
$overruled = $this->checkWhitelist($attribute['value']);
|
||||||
|
@ -207,7 +201,7 @@ class NidsSuricataExport extends NidsExport
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public function userAgentRule($ruleFormat, $attribute, &$sid)
|
protected function userAgentRule($ruleFormat, $attribute, &$sid)
|
||||||
{
|
{
|
||||||
$overruled = $this->checkWhitelist($attribute['value']);
|
$overruled = $this->checkWhitelist($attribute['value']);
|
||||||
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
||||||
|
@ -230,7 +224,7 @@ class NidsSuricataExport extends NidsExport
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function ja3Rule($ruleFormat, $attribute, &$sid)
|
protected function ja3Rule($ruleFormat, $attribute, &$sid)
|
||||||
{
|
{
|
||||||
$overruled = $this->checkWhitelist($attribute['value']);
|
$overruled = $this->checkWhitelist($attribute['value']);
|
||||||
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
||||||
|
@ -253,7 +247,7 @@ class NidsSuricataExport extends NidsExport
|
||||||
}
|
}
|
||||||
|
|
||||||
// For Future use once JA3S Hash Attribute type is created
|
// For Future use once JA3S Hash Attribute type is created
|
||||||
public function ja3sRule($ruleFormat, $attribute, &$sid)
|
protected function ja3sRule($ruleFormat, $attribute, &$sid)
|
||||||
{
|
{
|
||||||
$overruled = $this->checkWhitelist($attribute['value']);
|
$overruled = $this->checkWhitelist($attribute['value']);
|
||||||
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
$attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule
|
||||||
|
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
class RPZExport
|
class RPZExport
|
||||||
{
|
{
|
||||||
private $__policies = array(
|
const POLICIES = array(
|
||||||
'Local-Data' => array(
|
'Local-Data' => array(
|
||||||
'explanation' => 'returns the defined alternate location.',
|
'explanation' => 'returns the defined alternate location.',
|
||||||
'action' => '$walled_garden',
|
'action' => '$walled_garden',
|
||||||
|
@ -35,19 +35,17 @@ class RPZExport
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
private $__items = array();
|
private $items = array();
|
||||||
|
|
||||||
public $additional_params = array(
|
public $additional_params = array(
|
||||||
'flatten' => 1
|
'flatten' => 1
|
||||||
);
|
);
|
||||||
|
|
||||||
private $__rpzSettings = array();
|
private $rpzSettings = array();
|
||||||
|
|
||||||
private $__valid_policies = array('NXDOMAIN', 'NODATA', 'DROP', 'Local-Data', 'PASSTHRU', 'TCP-only');
|
|
||||||
|
|
||||||
private $__server = null;
|
private $__server = null;
|
||||||
|
|
||||||
public $validTypes = array(
|
const VALID_TYPES = array(
|
||||||
'ip-src' => array(
|
'ip-src' => array(
|
||||||
'value' => 'ip'
|
'value' => 'ip'
|
||||||
),
|
),
|
||||||
|
@ -69,40 +67,39 @@ class RPZExport
|
||||||
public function handler($data, $options = array())
|
public function handler($data, $options = array())
|
||||||
{
|
{
|
||||||
if ($options['scope'] === 'Attribute') {
|
if ($options['scope'] === 'Attribute') {
|
||||||
return $this->__attributeHandler($data, $options);
|
$this->attributeHandler($data);
|
||||||
} else {
|
} else {
|
||||||
return $this->__eventHandler($data, $options);
|
$this->eventHandler($data);
|
||||||
}
|
}
|
||||||
|
return '';
|
||||||
}
|
}
|
||||||
|
|
||||||
private function __eventHandler($event, $options = array()) {
|
private function eventHandler($event)
|
||||||
|
{
|
||||||
foreach ($event['Attribute'] as $attribute) {
|
foreach ($event['Attribute'] as $attribute) {
|
||||||
if (isset($this->validTypes[$attribute['type']])) {
|
if (isset(self::VALID_TYPES[$attribute['type']])) {
|
||||||
if ($attribute['type'] == 'domain|ip') {
|
if ($attribute['type'] === 'domain|ip') {
|
||||||
$temp = explode('|', $attribute['value']);
|
$temp = explode('|', $attribute['value']);
|
||||||
$attribute['value1'] = $temp[0];
|
$attribute['value1'] = $temp[0];
|
||||||
$attribute['value2'] = $temp[1];
|
$attribute['value2'] = $temp[1];
|
||||||
}
|
}
|
||||||
$this->__attributeHandler(array('Attribute' => $attribute, $options));
|
$this->attributeHandler(array('Attribute' => $attribute));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return '';
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private function __attributeHandler($attribute, $options = array())
|
private function attributeHandler($attribute)
|
||||||
{
|
{
|
||||||
if (isset($attribute['Attribute'])) {
|
if (isset($attribute['Attribute'])) {
|
||||||
$attribute = $attribute['Attribute'];
|
$attribute = $attribute['Attribute'];
|
||||||
}
|
}
|
||||||
if (isset($this->validTypes[$attribute['type']])) {
|
if (isset(self::VALID_TYPES[$attribute['type']])) {
|
||||||
foreach ($this->validTypes[$attribute['type']] as $field => $mapping) {
|
foreach (self::VALID_TYPES[$attribute['type']] as $field => $mapping) {
|
||||||
// get rid of the in_array check
|
if (!isset($this->items[$mapping][$attribute[$field]])) {
|
||||||
if (empty($this->__items[$mapping]) || !isset($this->__items[$mapping][$attribute[$field]])) {
|
$this->items[$mapping][$attribute[$field]] = true;
|
||||||
$this->__items[$mapping][$attribute[$field]] = true;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return '';
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public function header($options = array())
|
public function header($options = array())
|
||||||
|
@ -117,16 +114,16 @@ class RPZExport
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (isset($options['filters'][$v])) {
|
if (isset($options['filters'][$v])) {
|
||||||
$this->__rpzSettings[$v] = $options['filters'][$v];
|
$this->rpzSettings[$v] = $options['filters'][$v];
|
||||||
} else {
|
} else {
|
||||||
$tempSetting = Configure::read('Plugin.RPZ_' . $v);
|
$tempSetting = Configure::read('Plugin.RPZ_' . $v);
|
||||||
if (isset($tempSetting)) {
|
if (isset($tempSetting)) {
|
||||||
$this->__rpzSettings[$v] = Configure::read('Plugin.RPZ_' . $v);
|
$this->rpzSettings[$v] = $tempSetting;
|
||||||
} else {
|
} else {
|
||||||
if (empty($this->__server)) {
|
if (empty($this->__server)) {
|
||||||
$this->__server = ClassRegistry::init('Server');
|
$this->__server = ClassRegistry::init('Server');
|
||||||
}
|
}
|
||||||
$this->__rpzSettings[$v] = $this->__server->serverSettings['Plugin']['RPZ_' . $v]['value'];
|
$this->rpzSettings[$v] = $this->__server->serverSettings['Plugin']['RPZ_' . $v]['value'];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -135,10 +132,7 @@ class RPZExport
|
||||||
|
|
||||||
public function footer($options = array())
|
public function footer($options = array())
|
||||||
{
|
{
|
||||||
foreach ($this->__items as $k => $v) {
|
return $this->export($this->items, $this->rpzSettings);
|
||||||
$this->__items[$k] = array_keys($this->__items[$k]);
|
|
||||||
}
|
|
||||||
return $this->export($this->__items, $this->__rpzSettings);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public function separator()
|
public function separator()
|
||||||
|
@ -146,39 +140,32 @@ class RPZExport
|
||||||
return '';
|
return '';
|
||||||
}
|
}
|
||||||
|
|
||||||
public function getPolicyById($id)
|
private function getPolicyById($id)
|
||||||
{
|
{
|
||||||
foreach ($this->__policies as $k => $v) {
|
foreach (self::POLICIES as $k => $v) {
|
||||||
if ($id == $v['setting_id']) {
|
if ($id === $v['setting_id']) {
|
||||||
return $k;
|
return $k;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
public function getIdByPolicy($policy)
|
private function getIdByPolicy($policy)
|
||||||
{
|
{
|
||||||
return $this->__policies[$policy]['setting_id'];
|
return self::POLICIES[$policy]['setting_id'];
|
||||||
}
|
}
|
||||||
|
|
||||||
public function explain($type, $policy)
|
private function explain($type, $policy)
|
||||||
{
|
{
|
||||||
$explanations = array(
|
$explanations = array(
|
||||||
'ip' => '; The following list of IP addresses will ',
|
'ip' => '; The following list of IP addresses will ',
|
||||||
'domain' => '; The following domain names and all of their sub-domains will ',
|
'domain' => '; The following domain names and all of their sub-domains will ',
|
||||||
'hostname' => '; The following hostnames will '
|
'hostname' => '; The following hostnames will '
|
||||||
);
|
);
|
||||||
$policy_explanations = array(
|
return $explanations[$type] . self::POLICIES[$policy]['explanation'] . PHP_EOL;
|
||||||
'Local-Data' => 'returns the defined alternate location.',
|
|
||||||
'NXDOMAIN' => 'return NXDOMAIN (name does not exist) irrespective of actual result received.',
|
|
||||||
'NODATA' => 'returns NODATA (name exists but no answers returned) irrespective of actual result received.',
|
|
||||||
'DROP' => 'timeout.',
|
|
||||||
'PASSTHRU' => 'lets queries through, but allows for logging the hits (useful for testing).',
|
|
||||||
'TCP-only' => 'force the client to use TCP.',
|
|
||||||
);
|
|
||||||
return $explanations[$type] . $this->__policies[$policy]['explanation'] . PHP_EOL;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public function buildHeader($rpzSettings)
|
private function buildHeader(array $rpzSettings)
|
||||||
{
|
{
|
||||||
$rpzSettings['serial'] = str_replace('$date', date('Ymd'), $rpzSettings['serial']);
|
$rpzSettings['serial'] = str_replace('$date', date('Ymd'), $rpzSettings['serial']);
|
||||||
$rpzSettings['serial'] = str_replace('$time', time(), $rpzSettings['serial']);
|
$rpzSettings['serial'] = str_replace('$time', time(), $rpzSettings['serial']);
|
||||||
|
@ -196,55 +183,55 @@ class RPZExport
|
||||||
return $header;
|
return $header;
|
||||||
}
|
}
|
||||||
|
|
||||||
public function export($items, $rpzSettings)
|
private function export(array $items, array $rpzSettings)
|
||||||
{
|
{
|
||||||
$result = $this->buildHeader($rpzSettings);
|
$result = $this->buildHeader($rpzSettings);
|
||||||
$policy = $this->getPolicyById($rpzSettings['policy']);
|
$policy = $this->getPolicyById($rpzSettings['policy']);
|
||||||
$action = $this->__policies[$policy]['action'];
|
$action = self::POLICIES[$policy]['action'];
|
||||||
if ($policy == 'Local-Data') {
|
if ($policy === 'Local-Data') {
|
||||||
$action = str_replace('$walled_garden', $rpzSettings['walled_garden'], $action);
|
$action = str_replace('$walled_garden', $rpzSettings['walled_garden'], $action);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isset($items['ip'])) {
|
if (isset($items['ip'])) {
|
||||||
$result .= $this->explain('ip', $policy);
|
$result .= $this->explain('ip', $policy);
|
||||||
foreach ($items['ip'] as $item) {
|
foreach ($items['ip'] as $item => $foo) {
|
||||||
$result .= $this->__convertIP($item, $action);
|
$result .= $this->convertIp($item, $action);
|
||||||
}
|
}
|
||||||
$result .= PHP_EOL;
|
$result .= PHP_EOL;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isset($items['domain'])) {
|
if (isset($items['domain'])) {
|
||||||
$result .= $this->explain('domain', $policy);
|
$result .= $this->explain('domain', $policy);
|
||||||
foreach ($items['domain'] as $item) {
|
foreach ($items['domain'] as $item => $foo) {
|
||||||
$result .= $this->__convertdomain($item, $action);
|
$result .= $this->convertDomain($item, $action);
|
||||||
}
|
}
|
||||||
$result .= PHP_EOL;
|
$result .= PHP_EOL;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isset($items['hostname'])) {
|
if (isset($items['hostname'])) {
|
||||||
$result .= $this->explain('hostname', $policy);
|
$result .= $this->explain('hostname', $policy);
|
||||||
foreach ($items['hostname'] as $item) {
|
foreach ($items['hostname'] as $item => $foo) {
|
||||||
$result .= $this->__converthostname($item, $action);
|
$result .= $this->convertHostname($item, $action);
|
||||||
}
|
}
|
||||||
$result .= PHP_EOL;
|
$result .= PHP_EOL;
|
||||||
}
|
}
|
||||||
return $result;
|
return $result;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function __convertdomain($input, $action)
|
private function convertDomain($input, $action)
|
||||||
{
|
{
|
||||||
return $input . ' CNAME ' . $action . PHP_EOL . '*.' . $input . ' CNAME ' . $action . PHP_EOL;
|
return $input . ' CNAME ' . $action . PHP_EOL . '*.' . $input . ' CNAME ' . $action . PHP_EOL;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function __converthostname($input, $action)
|
private function convertHostname($input, $action)
|
||||||
{
|
{
|
||||||
return $input . ' CNAME ' . $action . PHP_EOL;
|
return $input . ' CNAME ' . $action . PHP_EOL;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function __convertIP($input, $action)
|
private function convertIp($input, $action)
|
||||||
{
|
{
|
||||||
$type = filter_var($input, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) ? 'ipv6' : 'ipv4';
|
$isIpv6 = filter_var($input, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6);
|
||||||
if ($type == 'ipv6') {
|
if ($isIpv6) {
|
||||||
$prefix = '128';
|
$prefix = '128';
|
||||||
} else {
|
} else {
|
||||||
$prefix = '32';
|
$prefix = '32';
|
||||||
|
@ -252,7 +239,8 @@ class RPZExport
|
||||||
if (strpos($input, '/')) {
|
if (strpos($input, '/')) {
|
||||||
list($input, $prefix) = explode('/', $input);
|
list($input, $prefix) = explode('/', $input);
|
||||||
}
|
}
|
||||||
return $prefix . '.' . $this->{'__' . $type}($input) . '.rpz-ip CNAME ' . $action . PHP_EOL;
|
$converted = $isIpv6 ? $this->__ipv6($input) : $this->__ipv4($input);
|
||||||
|
return $prefix . '.' . $converted . '.rpz-ip CNAME ' . $action . PHP_EOL;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function __ipv6($input)
|
private function __ipv6($input)
|
||||||
|
|
|
@ -0,0 +1,198 @@
|
||||||
|
<?php
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
class ApcuCacheTool implements \Psr\SimpleCache\CacheInterface
|
||||||
|
{
|
||||||
|
/** @var string */
|
||||||
|
private $prefix;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $prefix
|
||||||
|
*/
|
||||||
|
public function __construct(string $prefix)
|
||||||
|
{
|
||||||
|
$this->prefix = $prefix;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetches a value from the cache.
|
||||||
|
*
|
||||||
|
* @param string $key The unique key of this item in the cache.
|
||||||
|
* @param mixed $default Default value to return if the key does not exist.
|
||||||
|
*
|
||||||
|
* @return mixed The value of the item from the cache, or $default in case of cache miss.
|
||||||
|
*
|
||||||
|
* @throws \Psr\SimpleCache\InvalidArgumentException
|
||||||
|
* MUST be thrown if the $key string is not a legal value.
|
||||||
|
*/
|
||||||
|
public function get($key, $default = null)
|
||||||
|
{
|
||||||
|
$value = \apcu_fetch("$this->prefix:$key", $success);
|
||||||
|
if ($success) {
|
||||||
|
return $value;
|
||||||
|
}
|
||||||
|
return $default;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Persists data in the cache, uniquely referenced by a key with an optional expiration TTL time.
|
||||||
|
*
|
||||||
|
* @param string $key The key of the item to store.
|
||||||
|
* @param mixed $value The value of the item to store, must be serializable.
|
||||||
|
* @param null|int|\DateInterval $ttl Optional. The TTL value of this item. If no value is sent and
|
||||||
|
* the driver supports TTL then the library may set a default value
|
||||||
|
* for it or let the driver take care of that.
|
||||||
|
*
|
||||||
|
* @return bool True on success and false on failure.
|
||||||
|
*
|
||||||
|
* @throws \Psr\SimpleCache\InvalidArgumentException
|
||||||
|
* MUST be thrown if the $key string is not a legal value.
|
||||||
|
*/
|
||||||
|
public function set($key, $value, $ttl = null)
|
||||||
|
{
|
||||||
|
return \apcu_store("$this->prefix:$key", $value, $this->tllToInt($ttl));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete an item from the cache by its unique key.
|
||||||
|
*
|
||||||
|
* @param string $key The unique cache key of the item to delete.
|
||||||
|
*
|
||||||
|
* @return bool True if the item was successfully removed. False if there was an error.
|
||||||
|
*
|
||||||
|
* @throws \Psr\SimpleCache\InvalidArgumentException
|
||||||
|
* MUST be thrown if the $key string is not a legal value.
|
||||||
|
*/
|
||||||
|
public function delete($key)
|
||||||
|
{
|
||||||
|
return \apcu_delete("$this->prefix:$key");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wipes clean the entire cache's keys.
|
||||||
|
*
|
||||||
|
* @return bool True on success and false on failure.
|
||||||
|
*/
|
||||||
|
public function clear()
|
||||||
|
{
|
||||||
|
$iterator = new APCUIterator(
|
||||||
|
'/^' . preg_quote($this->prefix . ':', '/') . '/',
|
||||||
|
APC_ITER_NONE
|
||||||
|
);
|
||||||
|
return \apcu_delete($iterator);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Obtains multiple cache items by their unique keys.
|
||||||
|
*
|
||||||
|
* @param iterable $keys A list of keys that can obtained in a single operation.
|
||||||
|
* @param mixed $default Default value to return for keys that do not exist.
|
||||||
|
*
|
||||||
|
* @return iterable A list of key => value pairs. Cache keys that do not exist or are stale will have $default as value.
|
||||||
|
*
|
||||||
|
* @throws \Psr\SimpleCache\InvalidArgumentException
|
||||||
|
* MUST be thrown if $keys is neither an array nor a Traversable,
|
||||||
|
* or if any of the $keys are not a legal value.
|
||||||
|
*/
|
||||||
|
public function getMultiple($keys, $default = null)
|
||||||
|
{
|
||||||
|
$keysToFetch = $this->keysToFetch($keys);
|
||||||
|
$values = \apcu_fetch($keysToFetch);
|
||||||
|
foreach ($keysToFetch as $keyToFetch) {
|
||||||
|
if (!isset($values[$keyToFetch])) {
|
||||||
|
$values[$keyToFetch] = $default;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return $values;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Persists a set of key => value pairs in the cache, with an optional TTL.
|
||||||
|
*
|
||||||
|
* @param iterable $values A list of key => value pairs for a multiple-set operation.
|
||||||
|
* @param null|int|\DateInterval $ttl Optional. The TTL value of this item. If no value is sent and
|
||||||
|
* the driver supports TTL then the library may set a default value
|
||||||
|
* for it or let the driver take care of that.
|
||||||
|
*
|
||||||
|
* @return bool True on success and false on failure.
|
||||||
|
*
|
||||||
|
* @throws \Psr\SimpleCache\InvalidArgumentException
|
||||||
|
* MUST be thrown if $values is neither an array nor a Traversable,
|
||||||
|
* or if any of the $values are not a legal value.
|
||||||
|
*/
|
||||||
|
public function setMultiple($values, $ttl = null)
|
||||||
|
{
|
||||||
|
$dataToSave = [];
|
||||||
|
foreach ($values as $key => $value) {
|
||||||
|
$dataToSave["$this->prefix:$key"] = $value;
|
||||||
|
}
|
||||||
|
return \apcu_store($dataToSave, null, $this->tllToInt($ttl));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deletes multiple cache items in a single operation.
|
||||||
|
*
|
||||||
|
* @param iterable $keys A list of string-based keys to be deleted.
|
||||||
|
*
|
||||||
|
* @return bool True if the items were successfully removed. False if there was an error.
|
||||||
|
*
|
||||||
|
* @throws \Psr\SimpleCache\InvalidArgumentException
|
||||||
|
* MUST be thrown if $keys is neither an array nor a Traversable,
|
||||||
|
* or if any of the $keys are not a legal value.
|
||||||
|
*/
|
||||||
|
public function deleteMultiple($keys)
|
||||||
|
{
|
||||||
|
$keysToDelete = $this->keysToFetch($keys);
|
||||||
|
return \apcu_delete($keysToDelete);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determines whether an item is present in the cache.
|
||||||
|
*
|
||||||
|
* NOTE: It is recommended that has() is only to be used for cache warming type purposes
|
||||||
|
* and not to be used within your live applications operations for get/set, as this method
|
||||||
|
* is subject to a race condition where your has() will return true and immediately after,
|
||||||
|
* another script can remove it making the state of your app out of date.
|
||||||
|
*
|
||||||
|
* @param string $key The cache item key.
|
||||||
|
*
|
||||||
|
* @return bool
|
||||||
|
*
|
||||||
|
* @throws \Psr\SimpleCache\InvalidArgumentException
|
||||||
|
* MUST be thrown if the $key string is not a legal value.
|
||||||
|
*/
|
||||||
|
public function has($key)
|
||||||
|
{
|
||||||
|
return \apcu_exists("$this->prefix:$key");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param iterable $keys
|
||||||
|
* @return array
|
||||||
|
*/
|
||||||
|
private function keysToFetch(iterable $keys): array
|
||||||
|
{
|
||||||
|
$keysToFetch = [];
|
||||||
|
foreach ($keys as $key) {
|
||||||
|
$keysToFetch[] = "$this->prefix:$key";
|
||||||
|
}
|
||||||
|
return $keysToFetch;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param null|int|\DateInterval $ttl
|
||||||
|
* @return int
|
||||||
|
*/
|
||||||
|
private function tllToInt($ttl = null): int
|
||||||
|
{
|
||||||
|
if ($ttl === null) {
|
||||||
|
return 0;
|
||||||
|
} elseif (is_int($ttl)) {
|
||||||
|
return $ttl;
|
||||||
|
} elseif ($ttl instanceof \DateInterval) {
|
||||||
|
return $ttl->days * 86400 + $ttl->h * 3600 + $ttl->i * 60 + $ttl->s;
|
||||||
|
} else {
|
||||||
|
throw new \Psr\SimpleCache\InvalidArgumentException("Invalid ttl value '$ttl' provided.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -41,7 +41,7 @@ class AttributeValidationTool
|
||||||
switch ($type) {
|
switch ($type) {
|
||||||
case 'ip-src':
|
case 'ip-src':
|
||||||
case 'ip-dst':
|
case 'ip-dst':
|
||||||
return self::compressIpv6($value);
|
return self::normalizeIp($value);
|
||||||
case 'md5':
|
case 'md5':
|
||||||
case 'sha1':
|
case 'sha1':
|
||||||
case 'sha224':
|
case 'sha224':
|
||||||
|
@ -98,7 +98,7 @@ class AttributeValidationTool
|
||||||
$parts[0] = $punyCode;
|
$parts[0] = $punyCode;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
$parts[1] = self::compressIpv6($parts[1]);
|
$parts[1] = self::normalizeIp($parts[1]);
|
||||||
return "$parts[0]|$parts[1]";
|
return "$parts[0]|$parts[1]";
|
||||||
case 'filename|md5':
|
case 'filename|md5':
|
||||||
case 'filename|sha1':
|
case 'filename|sha1':
|
||||||
|
@ -175,7 +175,7 @@ class AttributeValidationTool
|
||||||
} else {
|
} else {
|
||||||
return $value;
|
return $value;
|
||||||
}
|
}
|
||||||
return self::compressIpv6($parts[0]) . '|' . $parts[1];
|
return self::normalizeIp($parts[0]) . '|' . $parts[1];
|
||||||
case 'mac-address':
|
case 'mac-address':
|
||||||
case 'mac-eui-64':
|
case 'mac-eui-64':
|
||||||
$value = str_replace(array('.', ':', '-', ' '), '', strtolower($value));
|
$value = str_replace(array('.', ':', '-', ' '), '', strtolower($value));
|
||||||
|
@ -700,11 +700,30 @@ class AttributeValidationTool
|
||||||
* @param string $value
|
* @param string $value
|
||||||
* @return string
|
* @return string
|
||||||
*/
|
*/
|
||||||
private static function compressIpv6($value)
|
private static function normalizeIp($value)
|
||||||
{
|
{
|
||||||
|
// If IP is a CIDR
|
||||||
|
if (strpos($value, '/')) {
|
||||||
|
list($ip, $range) = explode('/', $value, 2);
|
||||||
|
|
||||||
|
// Compress IPv6
|
||||||
|
if (strpos($ip, ':') && $converted = inet_pton($ip)) {
|
||||||
|
$ip = inet_ntop($converted);
|
||||||
|
}
|
||||||
|
|
||||||
|
// If IP is in CIDR format, but the network is 32 for IPv4 or 128 for IPv6, normalize to non CIDR type
|
||||||
|
if (($range === '32' && strpos($value, '.')) || ($range === '128' && strpos($value, ':'))) {
|
||||||
|
return $ip;
|
||||||
|
}
|
||||||
|
|
||||||
|
return "$ip/$range";
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compress IPv6
|
||||||
if (strpos($value, ':') && $converted = inet_pton($value)) {
|
if (strpos($value, ':') && $converted = inet_pton($value)) {
|
||||||
return inet_ntop($converted);
|
return inet_ntop($converted);
|
||||||
}
|
}
|
||||||
|
|
||||||
return $value;
|
return $value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -66,8 +66,9 @@ class BackgroundJob implements JsonSerializable
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Run the job command
|
* Run the job command
|
||||||
|
* @param callable|null $runningCallback
|
||||||
*/
|
*/
|
||||||
public function run(): void
|
public function run(callable $runningCallback = null): void
|
||||||
{
|
{
|
||||||
$descriptorSpec = [
|
$descriptorSpec = [
|
||||||
1 => ["pipe", "w"], // stdout
|
1 => ["pipe", "w"], // stdout
|
||||||
|
@ -88,7 +89,7 @@ class BackgroundJob implements JsonSerializable
|
||||||
['BACKGROUND_JOB_ID' => $this->id]
|
['BACKGROUND_JOB_ID' => $this->id]
|
||||||
);
|
);
|
||||||
|
|
||||||
$this->pool($process, $pipes);
|
$this->pool($process, $pipes, $runningCallback);
|
||||||
|
|
||||||
if ($this->returnCode === 0 && empty($stderr)) {
|
if ($this->returnCode === 0 && empty($stderr)) {
|
||||||
$this->setStatus(BackgroundJob::STATUS_COMPLETED);
|
$this->setStatus(BackgroundJob::STATUS_COMPLETED);
|
||||||
|
@ -98,7 +99,13 @@ class BackgroundJob implements JsonSerializable
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private function pool($process, array $pipes)
|
/**
|
||||||
|
* @param resource $process
|
||||||
|
* @param array $pipes
|
||||||
|
* @param callable|null $runningCallback
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
private function pool($process, array $pipes, callable $runningCallback = null)
|
||||||
{
|
{
|
||||||
stream_set_blocking($pipes[1], false);
|
stream_set_blocking($pipes[1], false);
|
||||||
stream_set_blocking($pipes[2], false);
|
stream_set_blocking($pipes[2], false);
|
||||||
|
@ -106,6 +113,14 @@ class BackgroundJob implements JsonSerializable
|
||||||
$this->output = '';
|
$this->output = '';
|
||||||
$this->error = '';
|
$this->error = '';
|
||||||
|
|
||||||
|
if ($runningCallback) {
|
||||||
|
$status = proc_get_status($process);
|
||||||
|
if ($status === false) {
|
||||||
|
throw new RuntimeException("Could not get process status");
|
||||||
|
}
|
||||||
|
$runningCallback($status);
|
||||||
|
}
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
$read = [$pipes[1], $pipes[2]];
|
$read = [$pipes[1], $pipes[2]];
|
||||||
$write = null;
|
$write = null;
|
||||||
|
@ -118,6 +133,12 @@ class BackgroundJob implements JsonSerializable
|
||||||
$this->error .= stream_get_contents($pipes[2]);
|
$this->error .= stream_get_contents($pipes[2]);
|
||||||
}
|
}
|
||||||
$status = proc_get_status($process);
|
$status = proc_get_status($process);
|
||||||
|
if ($status === false) {
|
||||||
|
throw new RuntimeException("Could not get process status");
|
||||||
|
}
|
||||||
|
if ($runningCallback) {
|
||||||
|
$runningCallback($status);
|
||||||
|
}
|
||||||
if (!$status['running']) {
|
if (!$status['running']) {
|
||||||
// Just in case read rest data from stream
|
// Just in case read rest data from stream
|
||||||
$this->output .= stream_get_contents($pipes[1]);
|
$this->output .= stream_get_contents($pipes[1]);
|
||||||
|
@ -153,6 +174,9 @@ class BackgroundJob implements JsonSerializable
|
||||||
return ['id', 'command', 'args', 'createdAt', 'updatedAt', 'status', 'output', 'error', 'metadata'];
|
return ['id', 'command', 'args', 'createdAt', 'updatedAt', 'status', 'output', 'error', 'metadata'];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return string Background job ID in UUID format
|
||||||
|
*/
|
||||||
public function id(): string
|
public function id(): string
|
||||||
{
|
{
|
||||||
return $this->id;
|
return $this->id;
|
||||||
|
|
|
@ -65,7 +65,7 @@ class Worker implements JsonSerializable
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
public function pid(): ?int
|
public function pid(): int
|
||||||
{
|
{
|
||||||
return $this->pid;
|
return $this->pid;
|
||||||
}
|
}
|
||||||
|
|
|
@ -91,7 +91,8 @@ class BackgroundJobsTool
|
||||||
];
|
];
|
||||||
|
|
||||||
const JOB_STATUS_PREFIX = 'job_status',
|
const JOB_STATUS_PREFIX = 'job_status',
|
||||||
DATA_CONTENT_PREFIX = 'data_content';
|
DATA_CONTENT_PREFIX = 'data_content',
|
||||||
|
RUNNING_JOB_PREFIX = 'running';
|
||||||
|
|
||||||
/** @var array */
|
/** @var array */
|
||||||
private $settings;
|
private $settings;
|
||||||
|
@ -277,6 +278,54 @@ class BackgroundJobsTool
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param Worker $worker
|
||||||
|
* @param BackgroundJob $job
|
||||||
|
* @param int|null $pid
|
||||||
|
* @return void
|
||||||
|
* @throws RedisException
|
||||||
|
*/
|
||||||
|
public function markAsRunning(Worker $worker, BackgroundJob $job, $pid = null)
|
||||||
|
{
|
||||||
|
$key = self::RUNNING_JOB_PREFIX . ':' . $worker->queue() . ':' . $job->id();
|
||||||
|
$this->RedisConnection->setex($key, 60, [
|
||||||
|
'worker_pid' => $worker->pid(),
|
||||||
|
'process_pid' => $pid,
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param Worker $worker
|
||||||
|
* @param BackgroundJob $job
|
||||||
|
* @return void
|
||||||
|
* @throws RedisException
|
||||||
|
*/
|
||||||
|
public function removeFromRunning(Worker $worker, BackgroundJob $job)
|
||||||
|
{
|
||||||
|
$key = self::RUNNING_JOB_PREFIX . ':' . $worker->queue() . ':' . $job->id();
|
||||||
|
$this->RedisConnection->del($key);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return current running jobs
|
||||||
|
* @return array
|
||||||
|
* @throws RedisException
|
||||||
|
*/
|
||||||
|
public function runningJobs(): array
|
||||||
|
{
|
||||||
|
$pattern = $this->RedisConnection->_prefix(self::RUNNING_JOB_PREFIX . ':*');
|
||||||
|
$keys = RedisTool::keysByPattern($this->RedisConnection, $pattern);
|
||||||
|
|
||||||
|
$jobIds = [];
|
||||||
|
foreach ($keys as $key) {
|
||||||
|
$parts = explode(':', $key);
|
||||||
|
$queue = $parts[2];
|
||||||
|
$jobId = $parts[3];
|
||||||
|
$jobIds[$queue][$jobId] = $this->RedisConnection->get(self::RUNNING_JOB_PREFIX . ":$queue:$jobId");
|
||||||
|
}
|
||||||
|
return $jobIds;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the job status.
|
* Get the job status.
|
||||||
*
|
*
|
||||||
|
@ -500,19 +549,6 @@ class BackgroundJobsTool
|
||||||
$this->getSupervisor()->startProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart);
|
$this->getSupervisor()->startProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Purge queue
|
|
||||||
*
|
|
||||||
* @param string $queue
|
|
||||||
* @return void
|
|
||||||
*/
|
|
||||||
public function purgeQueue(string $queue)
|
|
||||||
{
|
|
||||||
$this->validateQueue($queue);
|
|
||||||
|
|
||||||
$this->RedisConnection->del($queue);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Return Background Jobs status
|
* Return Background Jobs status
|
||||||
*
|
*
|
||||||
|
@ -728,8 +764,7 @@ class BackgroundJobsTool
|
||||||
*
|
*
|
||||||
* @param integer $pid
|
* @param integer $pid
|
||||||
* @return \Supervisor\Process
|
* @return \Supervisor\Process
|
||||||
*
|
* @throws NotFoundException|Exception
|
||||||
* @throws NotFoundException
|
|
||||||
*/
|
*/
|
||||||
private function getProcessByPid(int $pid): \Supervisor\Process
|
private function getProcessByPid(int $pid): \Supervisor\Process
|
||||||
{
|
{
|
||||||
|
|
|
@ -52,10 +52,10 @@ class BetterCakeEventManager extends CakeEventManager
|
||||||
$result = [];
|
$result = [];
|
||||||
foreach ($priorities as $priority) {
|
foreach ($priorities as $priority) {
|
||||||
if (isset($globalListeners[$priority])) {
|
if (isset($globalListeners[$priority])) {
|
||||||
$result = array_merge($result, $globalListeners[$priority]);
|
array_push($result, ...$globalListeners[$priority]);
|
||||||
}
|
}
|
||||||
if (isset($localListeners[$priority])) {
|
if (isset($localListeners[$priority])) {
|
||||||
$result = array_merge($result, $localListeners[$priority]);
|
array_push($result, ...$localListeners[$priority]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return $result;
|
return $result;
|
||||||
|
|
|
@ -7,8 +7,8 @@ class BetterSecurity
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param string $plain
|
* @param string $plain
|
||||||
* @param string $key
|
* @param string $key Encryption key
|
||||||
* @return string
|
* @return string Cipher text with IV and tag
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
public static function encrypt($plain, $key)
|
public static function encrypt($plain, $key)
|
||||||
|
@ -33,17 +33,17 @@ class BetterSecurity
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param string $cipher
|
* @param string $cipherText Cipher text with IV and tag
|
||||||
* @param string $key
|
* @param string $key Decryption key
|
||||||
* @return string
|
* @return string
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
public static function decrypt($cipher, $key)
|
public static function decrypt($cipherText, $key)
|
||||||
{
|
{
|
||||||
if (strlen($key) < 32) {
|
if (strlen($key) < 32) {
|
||||||
throw new Exception('Invalid key, key must be at least 256 bits (32 bytes) long.');
|
throw new Exception('Invalid key, key must be at least 256 bits (32 bytes) long.');
|
||||||
}
|
}
|
||||||
if (empty($cipher)) {
|
if (empty($cipherText)) {
|
||||||
throw new Exception('The data to decrypt cannot be empty.');
|
throw new Exception('The data to decrypt cannot be empty.');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -52,12 +52,18 @@ class BetterSecurity
|
||||||
|
|
||||||
$ivSize = openssl_cipher_iv_length(self::METHOD);
|
$ivSize = openssl_cipher_iv_length(self::METHOD);
|
||||||
|
|
||||||
// Split out hmac for comparison
|
if (strlen($cipherText) < $ivSize + self::TAG_SIZE) {
|
||||||
$iv = substr($cipher, 0, $ivSize);
|
$length = strlen($cipherText);
|
||||||
$tag = substr($cipher, $ivSize, self::TAG_SIZE);
|
$minLength = $ivSize + self::TAG_SIZE;
|
||||||
$cipher = substr($cipher, $ivSize + self::TAG_SIZE);
|
throw new Exception("Provided cipher text is too short, $length bytes provided, expected at least $minLength bytes.");
|
||||||
|
}
|
||||||
|
|
||||||
$decrypted = openssl_decrypt($cipher, self::METHOD, $key, true, $iv, $tag);
|
// Split out hmac for comparison
|
||||||
|
$iv = substr($cipherText, 0, $ivSize);
|
||||||
|
$tag = substr($cipherText, $ivSize, self::TAG_SIZE);
|
||||||
|
$cipherText = substr($cipherText, $ivSize + self::TAG_SIZE);
|
||||||
|
|
||||||
|
$decrypted = openssl_decrypt($cipherText, self::METHOD, $key, OPENSSL_RAW_DATA, $iv, $tag);
|
||||||
if ($decrypted === false) {
|
if ($decrypted === false) {
|
||||||
throw new Exception('Could not decrypt. Maybe invalid encryption key?');
|
throw new Exception('Could not decrypt. Maybe invalid encryption key?');
|
||||||
}
|
}
|
||||||
|
|
|
@ -308,14 +308,13 @@ class ComplexTypeTool
|
||||||
*/
|
*/
|
||||||
private function parseFreetext($input)
|
private function parseFreetext($input)
|
||||||
{
|
{
|
||||||
$input = str_replace("\xc2\xa0", ' ', $input); // non breaking space to normal space
|
// convert non breaking space to normal space and all unicode chars from "other" category
|
||||||
$input = preg_replace('/\p{C}+/u', ' ', $input);
|
$input = preg_replace("/\p{C}+|\xc2\xa0/u", ' ', $input);
|
||||||
$iocArray = preg_split("/\r\n|\n|\r|\s|\s+|,|\<|\>|;/", $input);
|
$iocArray = preg_split("/\r\n|\n|\r|\s|\s+|,|<|>|;/", $input);
|
||||||
|
|
||||||
preg_match_all('/\"([^\"]+)\"/', $input, $matches);
|
preg_match_all('/\"([^\"]+)\"/', $input, $matches);
|
||||||
foreach ($matches[1] as $match) {
|
array_push($iocArray, ...$matches[1]);
|
||||||
$iocArray[] = $match;
|
|
||||||
}
|
|
||||||
return $iocArray;
|
return $iocArray;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,377 @@
|
||||||
|
<?php
|
||||||
|
App::uses('HttpSocketExtended', 'Tools');
|
||||||
|
|
||||||
|
class CurlClient extends HttpSocketExtended
|
||||||
|
{
|
||||||
|
/** @var resource */
|
||||||
|
private $ch;
|
||||||
|
|
||||||
|
/** @var int */
|
||||||
|
private $timeout = 30;
|
||||||
|
|
||||||
|
/** @var string|null */
|
||||||
|
private $caFile;
|
||||||
|
|
||||||
|
/** @var string|null */
|
||||||
|
private $localCert;
|
||||||
|
|
||||||
|
/** @var int */
|
||||||
|
private $cryptoMethod;
|
||||||
|
|
||||||
|
/** @var bool */
|
||||||
|
private $allowSelfSigned;
|
||||||
|
|
||||||
|
/** @var bool */
|
||||||
|
private $verifyPeer;
|
||||||
|
|
||||||
|
/** @var bool */
|
||||||
|
private $compress = true;
|
||||||
|
|
||||||
|
/** @var array */
|
||||||
|
private $proxy = [];
|
||||||
|
|
||||||
|
/** @var array */
|
||||||
|
private $defaultOptions;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param array $params
|
||||||
|
* @noinspection PhpMissingParentConstructorInspection
|
||||||
|
*/
|
||||||
|
public function __construct(array $params)
|
||||||
|
{
|
||||||
|
if (isset($params['timeout'])) {
|
||||||
|
$this->timeout = $params['timeout'];
|
||||||
|
}
|
||||||
|
if (isset($params['ssl_cafile'])) {
|
||||||
|
$this->caFile = $params['ssl_cafile'];
|
||||||
|
}
|
||||||
|
if (isset($params['ssl_local_cert'])) {
|
||||||
|
$this->localCert = $params['ssl_local_cert'];
|
||||||
|
}
|
||||||
|
if (isset($params['compress'])) {
|
||||||
|
$this->compress = $params['compress'];
|
||||||
|
}
|
||||||
|
if (isset($params['ssl_crypto_method'])) {
|
||||||
|
$this->cryptoMethod = $this->convertCryptoMethod($params['ssl_crypto_method']);
|
||||||
|
}
|
||||||
|
if (isset($params['ssl_allow_self_signed'])) {
|
||||||
|
$this->allowSelfSigned = $params['ssl_allow_self_signed'];
|
||||||
|
}
|
||||||
|
if (isset($params['ssl_verify_peer'])) {
|
||||||
|
$this->verifyPeer = $params['ssl_verify_peer'];
|
||||||
|
}
|
||||||
|
$this->defaultOptions = $this->generateDefaultOptions();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $uri
|
||||||
|
* @param array $query
|
||||||
|
* @param array $request
|
||||||
|
* @return HttpSocketResponseExtended
|
||||||
|
*/
|
||||||
|
public function head($uri = null, $query = [], $request = [])
|
||||||
|
{
|
||||||
|
return $this->internalRequest('HEAD', $uri, $query, $request);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $uri
|
||||||
|
* @param array $query
|
||||||
|
* @param array $request
|
||||||
|
* @return HttpSocketResponseExtended
|
||||||
|
*/
|
||||||
|
public function get($uri = null, $query = [], $request = [])
|
||||||
|
{
|
||||||
|
return $this->internalRequest('GET', $uri, $query, $request);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $uri
|
||||||
|
* @param array $data
|
||||||
|
* @param array $request
|
||||||
|
* @return HttpSocketResponseExtended
|
||||||
|
*/
|
||||||
|
public function post($uri = null, $data = [], $request = [])
|
||||||
|
{
|
||||||
|
return $this->internalRequest('POST', $uri, $data, $request);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $uri
|
||||||
|
* @param array$data
|
||||||
|
* @param $request
|
||||||
|
* @return HttpSocketResponseExtended
|
||||||
|
*/
|
||||||
|
public function put($uri = null, $data = [], $request = [])
|
||||||
|
{
|
||||||
|
return $this->internalRequest('PUT', $uri, $data, $request);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $uri
|
||||||
|
* @param array $data
|
||||||
|
* @param array $request
|
||||||
|
* @return HttpSocketResponseExtended
|
||||||
|
*/
|
||||||
|
public function patch($uri = null, $data = [], $request = [])
|
||||||
|
{
|
||||||
|
return $this->internalRequest('PATCH', $uri, $data, $request);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $uri
|
||||||
|
* @param array $data
|
||||||
|
* @param array $request
|
||||||
|
* @return HttpSocketResponseExtended
|
||||||
|
*/
|
||||||
|
public function delete($uri = null, $data = array(), $request = array())
|
||||||
|
{
|
||||||
|
return $this->internalRequest('DELETE', $uri, $data, $request);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function url($url = null, $uriTemplate = null)
|
||||||
|
{
|
||||||
|
throw new Exception('Not implemented');
|
||||||
|
}
|
||||||
|
|
||||||
|
public function request($request = array())
|
||||||
|
{
|
||||||
|
throw new Exception('Not implemented');
|
||||||
|
}
|
||||||
|
|
||||||
|
public function setContentResource($resource)
|
||||||
|
{
|
||||||
|
throw new Exception('Not implemented');
|
||||||
|
}
|
||||||
|
|
||||||
|
public function getMetaData()
|
||||||
|
{
|
||||||
|
return null; // not supported by curl extension
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $host
|
||||||
|
* @param int $port
|
||||||
|
* @param string $method
|
||||||
|
* @param string $user
|
||||||
|
* @param string $pass
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
public function configProxy($host, $port = 3128, $method = null, $user = null, $pass = null)
|
||||||
|
{
|
||||||
|
if (empty($host)) {
|
||||||
|
$this->proxy = [];
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (is_array($host)) {
|
||||||
|
$this->proxy = $host + ['host' => null];
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
$this->proxy = compact('host', 'port', 'method', 'user', 'pass');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $method
|
||||||
|
* @param string $url
|
||||||
|
* @param array|string $query
|
||||||
|
* @param array $request
|
||||||
|
* @return HttpSocketResponseExtended
|
||||||
|
*/
|
||||||
|
private function internalRequest($method, $url, $query, $request)
|
||||||
|
{
|
||||||
|
if (empty($url)) {
|
||||||
|
throw new InvalidArgumentException("No URL provided.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!$this->ch) {
|
||||||
|
// Share handle between requests to allow keep connection alive between requests
|
||||||
|
$this->ch = curl_init();
|
||||||
|
if (!$this->ch) {
|
||||||
|
throw new \RuntimeException("Could not initialize curl");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Reset options, so we can do another request
|
||||||
|
curl_reset($this->ch);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (($method === 'GET' || $method === 'HEAD') && !empty($query)) {
|
||||||
|
$url .= '?' . http_build_query($query, '', '&', PHP_QUERY_RFC3986);
|
||||||
|
}
|
||||||
|
|
||||||
|
$options = $this->defaultOptions;
|
||||||
|
$options[CURLOPT_URL] = $url;
|
||||||
|
$options[CURLOPT_CUSTOMREQUEST] = $method;
|
||||||
|
|
||||||
|
if (($method === 'POST' || $method === 'DELETE' || $method === 'PUT' || $method === 'PATCH') && !empty($query)) {
|
||||||
|
$options[CURLOPT_POSTFIELDS] = $query;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!empty($request['header'])) {
|
||||||
|
$headers = [];
|
||||||
|
foreach ($request['header'] as $key => $value) {
|
||||||
|
if (is_array($value)) {
|
||||||
|
$value = implode(', ', $value);
|
||||||
|
}
|
||||||
|
$headers[] = "$key: $value";
|
||||||
|
}
|
||||||
|
$options[CURLOPT_HTTPHEADER] = $headers;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse response headers
|
||||||
|
$responseHeaders = [];
|
||||||
|
$options[CURLOPT_HEADERFUNCTION] = function ($curl, $header) use (&$responseHeaders){
|
||||||
|
$len = strlen($header);
|
||||||
|
$header = explode(':', $header, 2);
|
||||||
|
if (count($header) < 2) { // ignore invalid headers
|
||||||
|
return $len;
|
||||||
|
}
|
||||||
|
$key = strtolower(trim($header[0]));
|
||||||
|
$value = trim($header[1]);
|
||||||
|
|
||||||
|
if (isset($responseHeaders[$key])) {
|
||||||
|
$responseHeaders[$key] = array_merge((array)$responseHeaders[$key], [$value]);
|
||||||
|
} else {
|
||||||
|
$responseHeaders[$key] = $value;
|
||||||
|
}
|
||||||
|
return $len;
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!curl_setopt_array($this->ch, $options)) {
|
||||||
|
throw new \RuntimeException('curl error: Could not set options');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download the given URL, and return output
|
||||||
|
$output = curl_exec($this->ch);
|
||||||
|
|
||||||
|
if ($output === false) {
|
||||||
|
$errorCode = curl_errno($this->ch);
|
||||||
|
$errorMessage = curl_error($this->ch);
|
||||||
|
if (!empty($errorMessage)) {
|
||||||
|
$errorMessage = ": $errorMessage";
|
||||||
|
}
|
||||||
|
throw new SocketException("curl error $errorCode '" . curl_strerror($errorCode) . "'" . $errorMessage);
|
||||||
|
}
|
||||||
|
|
||||||
|
$code = curl_getinfo($this->ch, CURLINFO_HTTP_CODE);
|
||||||
|
return $this->constructResponse($output, $responseHeaders, $code);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function disconnect()
|
||||||
|
{
|
||||||
|
if ($this->ch) {
|
||||||
|
curl_close($this->ch);
|
||||||
|
$this->ch = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $body
|
||||||
|
* @param array $headers
|
||||||
|
* @param int $code
|
||||||
|
* @return HttpSocketResponseExtended
|
||||||
|
*/
|
||||||
|
private function constructResponse($body, array $headers, $code)
|
||||||
|
{
|
||||||
|
if (isset($responseHeaders['content-encoding']) && $responseHeaders['content-encoding'] === 'zstd') {
|
||||||
|
if (!function_exists('zstd_uncompress')) {
|
||||||
|
throw new SocketException('Response is zstd encoded, but PHP do not support zstd decoding.');
|
||||||
|
}
|
||||||
|
$body = zstd_uncompress($body);
|
||||||
|
if ($body === false) {
|
||||||
|
throw new SocketException('Could not decode zstd encoded response.');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$response = new HttpSocketResponseExtended();
|
||||||
|
$response->code = $code;
|
||||||
|
$response->body = $body;
|
||||||
|
$response->headers = $headers;
|
||||||
|
return $response;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param int $cryptoMethod
|
||||||
|
* @return int
|
||||||
|
*/
|
||||||
|
private function convertCryptoMethod($cryptoMethod)
|
||||||
|
{
|
||||||
|
switch ($cryptoMethod) {
|
||||||
|
case STREAM_CRYPTO_METHOD_TLSv1_0_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_1_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
|
||||||
|
return CURL_SSLVERSION_TLSv1;
|
||||||
|
case STREAM_CRYPTO_METHOD_TLSv1_1_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
|
||||||
|
return CURL_SSLVERSION_TLSv1_1;
|
||||||
|
case STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
|
||||||
|
return CURL_SSLVERSION_TLSv1_2;
|
||||||
|
case STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
|
||||||
|
return CURL_SSLVERSION_TLSv1_3;
|
||||||
|
default:
|
||||||
|
throw new InvalidArgumentException("Unsupported crypto method value $cryptoMethod");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return array
|
||||||
|
*/
|
||||||
|
private function generateDefaultOptions()
|
||||||
|
{
|
||||||
|
$options = [
|
||||||
|
CURLOPT_FOLLOWLOCATION => true, // Allows to follow redirect
|
||||||
|
CURLOPT_MAXREDIRS => 10,
|
||||||
|
CURLOPT_RETURNTRANSFER => true, // Should cURL return or print out the data? (true = return, false = print)
|
||||||
|
CURLOPT_HEADER => false, // Include header in result?
|
||||||
|
CURLOPT_TIMEOUT => $this->timeout, // Timeout in seconds
|
||||||
|
CURLOPT_PROTOCOLS => CURLPROTO_HTTPS | CURLPROTO_HTTP, // be sure that only HTTP and HTTPS protocols are enabled,
|
||||||
|
];
|
||||||
|
|
||||||
|
if ($this->caFile) {
|
||||||
|
$options[CURLOPT_CAINFO] = $this->caFile;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($this->localCert) {
|
||||||
|
$options[CURLOPT_SSLCERT] = $this->localCert;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($this->cryptoMethod) {
|
||||||
|
$options[CURLOPT_SSLVERSION] = $this->cryptoMethod;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($this->compress) {
|
||||||
|
$options[CURLOPT_ACCEPT_ENCODING] = $this->supportedEncodings();
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($this->allowSelfSigned) {
|
||||||
|
$options[CURLOPT_SSL_VERIFYPEER] = $this->verifyPeer;
|
||||||
|
$options[CURLOPT_SSL_VERIFYHOST] = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!empty($this->proxy)) {
|
||||||
|
$options[CURLOPT_PROXY] = "{$this->proxy['host']}:{$this->proxy['port']}";
|
||||||
|
if (!empty($this->proxy['method']) && isset($this->proxy['user'], $this->proxy['pass'])) {
|
||||||
|
$options[CURLOPT_PROXYUSERPWD] = "{$this->proxy['user']}:{$this->proxy['pass']}";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return $options;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return string
|
||||||
|
*/
|
||||||
|
private function supportedEncodings()
|
||||||
|
{
|
||||||
|
$encodings = [];
|
||||||
|
// zstd is not supported by curl itself, but add support if PHP zstd extension is installed
|
||||||
|
if (function_exists('zstd_uncompress')) {
|
||||||
|
$encodings[] = 'zstd';
|
||||||
|
}
|
||||||
|
// brotli and gzip is supported by curl itself if it is compiled with these features
|
||||||
|
$info = curl_version();
|
||||||
|
if (defined('CURL_VERSION_BROTLI') && $info['features'] & CURL_VERSION_BROTLI) {
|
||||||
|
$encodings[] = 'br';
|
||||||
|
}
|
||||||
|
if ($info['features'] & CURL_VERSION_LIBZ) {
|
||||||
|
$encodings[] = 'gzip, deflate';
|
||||||
|
}
|
||||||
|
return implode(', ', $encodings);
|
||||||
|
}
|
||||||
|
}
|
|
@ -12,11 +12,7 @@ class GitTool
|
||||||
public static function getLatestTags(HttpSocketExtended $HttpSocket)
|
public static function getLatestTags(HttpSocketExtended $HttpSocket)
|
||||||
{
|
{
|
||||||
$url = 'https://api.github.com/repos/MISP/MISP/tags?per_page=10';
|
$url = 'https://api.github.com/repos/MISP/MISP/tags?per_page=10';
|
||||||
$response = $HttpSocket->get($url);
|
return self::gitHubRequest($HttpSocket, $url);
|
||||||
if (!$response->isOk()) {
|
|
||||||
throw new HttpSocketHttpException($response, $url);
|
|
||||||
}
|
|
||||||
return $response->json();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -28,11 +24,7 @@ class GitTool
|
||||||
public static function getLatestCommit(HttpSocketExtended $HttpSocket)
|
public static function getLatestCommit(HttpSocketExtended $HttpSocket)
|
||||||
{
|
{
|
||||||
$url = 'https://api.github.com/repos/MISP/MISP/commits?per_page=1';
|
$url = 'https://api.github.com/repos/MISP/MISP/commits?per_page=1';
|
||||||
$response = $HttpSocket->get($url);
|
$data = self::gitHubRequest($HttpSocket, $url);
|
||||||
if (!$response->isOk()) {
|
|
||||||
throw new HttpSocketHttpException($response, $url);
|
|
||||||
}
|
|
||||||
$data = $response->json();
|
|
||||||
if (!isset($data[0]['sha'])) {
|
if (!isset($data[0]['sha'])) {
|
||||||
throw new Exception("Response do not contains requested data.");
|
throw new Exception("Response do not contains requested data.");
|
||||||
}
|
}
|
||||||
|
@ -40,20 +32,49 @@ class GitTool
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
* @param HttpSocketExtended $HttpSocket
|
||||||
|
* @param string $url
|
||||||
|
* @return array
|
||||||
|
* @throws HttpSocketHttpException
|
||||||
|
* @throws HttpSocketJsonException
|
||||||
|
*/
|
||||||
|
private static function gitHubRequest(HttpSocketExtended $HttpSocket, $url)
|
||||||
|
{
|
||||||
|
$response = $HttpSocket->get($url, [], ['header' => ['User-Agent' => 'MISP']]);
|
||||||
|
if (!$response->isOk()) {
|
||||||
|
throw new HttpSocketHttpException($response, $url);
|
||||||
|
}
|
||||||
|
return $response->json();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns current SHA1 hash of current commit
|
||||||
* `git rev-parse HEAD`
|
* `git rev-parse HEAD`
|
||||||
|
* @param string $repoPath
|
||||||
* @return string
|
* @return string
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
public static function currentCommit()
|
public static function currentCommit($repoPath)
|
||||||
{
|
{
|
||||||
$head = rtrim(FileAccessTool::readFromFile(ROOT . '/.git/HEAD'));
|
if (is_file($repoPath . '/.git')) {
|
||||||
|
$fileContent = FileAccessTool::readFromFile($repoPath . '/.git');
|
||||||
|
if (substr($fileContent, 0, 8) === 'gitdir: ') {
|
||||||
|
$gitDir = $repoPath . '/' . trim(substr($fileContent, 8)) . '/';
|
||||||
|
} else {
|
||||||
|
throw new Exception("$repoPath/.git is file, but contains non expected content $fileContent");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
$gitDir = $repoPath . '/.git/';
|
||||||
|
}
|
||||||
|
|
||||||
|
$head = rtrim(FileAccessTool::readFromFile($gitDir . 'HEAD'));
|
||||||
if (substr($head, 0, 5) === 'ref: ') {
|
if (substr($head, 0, 5) === 'ref: ') {
|
||||||
$path = substr($head, 5);
|
$path = substr($head, 5);
|
||||||
return rtrim(FileAccessTool::readFromFile(ROOT . '/.git/' . $path));
|
return rtrim(FileAccessTool::readFromFile($gitDir . $path));
|
||||||
} else if (strlen($head) === 40) {
|
} else if (strlen($head) === 40) {
|
||||||
return $head;
|
return $head;
|
||||||
} else {
|
} else {
|
||||||
throw new Exception("Invalid head $head");
|
throw new Exception("Invalid head '$head' in $gitDir/HEAD");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -94,30 +115,18 @@ class GitTool
|
||||||
return $output;
|
return $output;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* @param string $submodule Path to Git repo
|
|
||||||
* @return string|null
|
|
||||||
*/
|
|
||||||
public static function submoduleCurrentCommit($submodule)
|
|
||||||
{
|
|
||||||
try {
|
|
||||||
$commit = ProcessTool::execute(['git', 'rev-parse', 'HEAD'], $submodule);
|
|
||||||
} catch (ProcessException $e) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return rtrim($commit);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param string $commit
|
* @param string $commit
|
||||||
* @param string|null $submodule Path to Git repo
|
* @param string|null $submodule Path to Git repo
|
||||||
* @return int|null
|
* @return int|null
|
||||||
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
public static function commitTimestamp($commit, $submodule = null)
|
public static function commitTimestamp($commit, $submodule = null)
|
||||||
{
|
{
|
||||||
try {
|
try {
|
||||||
$timestamp = ProcessTool::execute(['git', 'show', '-s', '--pretty=format:%ct', $commit], $submodule);
|
$timestamp = ProcessTool::execute(['git', 'show', '-s', '--pretty=format:%ct', $commit], $submodule);
|
||||||
} catch (ProcessException $e) {
|
} catch (ProcessException $e) {
|
||||||
|
CakeLog::notice("Could not get Git commit timestamp for $submodule: {$e->getMessage()}");
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
return (int)rtrim($timestamp);
|
return (int)rtrim($timestamp);
|
||||||
|
|
|
@ -18,10 +18,15 @@ class HttpSocketHttpException extends Exception
|
||||||
{
|
{
|
||||||
$this->response = $response;
|
$this->response = $response;
|
||||||
$this->url = $url;
|
$this->url = $url;
|
||||||
|
|
||||||
$message = "Remote server returns HTTP error code $response->code";
|
$message = "Remote server returns HTTP error code $response->code";
|
||||||
if ($url) {
|
if ($url) {
|
||||||
$message .= " for URL $url";
|
$message .= " for URL $url";
|
||||||
}
|
}
|
||||||
|
if ($response->body) {
|
||||||
|
$message .= ': ' . substr($response->body, 0, 100);
|
||||||
|
}
|
||||||
|
|
||||||
parent::__construct($message, (int)$response->code);
|
parent::__construct($message, (int)$response->code);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -153,7 +153,7 @@ class JSONConverterTool
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
yield '{"Event":{';
|
yield '{"Event":{';
|
||||||
$firstKey = key($event['Event']);
|
$firstKey = array_key_first($event['Event']);
|
||||||
foreach ($event['Event'] as $key => $value) {
|
foreach ($event['Event'] as $key => $value) {
|
||||||
if ($key === 'Attribute' || $key === 'Object') { // Encode every object or attribute separately
|
if ($key === 'Attribute' || $key === 'Object') { // Encode every object or attribute separately
|
||||||
yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":[";
|
yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":[";
|
||||||
|
|
|
@ -9,10 +9,7 @@ class JsonTool
|
||||||
*/
|
*/
|
||||||
public static function encode($value, $prettyPrint = false)
|
public static function encode($value, $prettyPrint = false)
|
||||||
{
|
{
|
||||||
$flags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE;
|
$flags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR;
|
||||||
if (defined('JSON_THROW_ON_ERROR')) {
|
|
||||||
$flags |= JSON_THROW_ON_ERROR; // Throw exception on error if supported
|
|
||||||
}
|
|
||||||
if ($prettyPrint) {
|
if ($prettyPrint) {
|
||||||
$flags |= JSON_PRETTY_PRINT;
|
$flags |= JSON_PRETTY_PRINT;
|
||||||
}
|
}
|
||||||
|
@ -34,16 +31,8 @@ class JsonTool
|
||||||
} catch (SimdJsonException $e) {
|
} catch (SimdJsonException $e) {
|
||||||
throw new JsonException($e->getMessage(), $e->getCode(), $e);
|
throw new JsonException($e->getMessage(), $e->getCode(), $e);
|
||||||
}
|
}
|
||||||
} elseif (defined('JSON_THROW_ON_ERROR')) {
|
}
|
||||||
// JSON_THROW_ON_ERROR is supported since PHP 7.3
|
|
||||||
return json_decode($value, true, 512, JSON_THROW_ON_ERROR);
|
return json_decode($value, true, 512, JSON_THROW_ON_ERROR);
|
||||||
} else {
|
|
||||||
$decoded = json_decode($value, true);
|
|
||||||
if ($decoded === null) {
|
|
||||||
throw new UnexpectedValueException('Could not parse JSON: ' . json_last_error_msg(), json_last_error());
|
|
||||||
}
|
|
||||||
return $decoded;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -78,4 +67,39 @@ class JsonTool
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @see https://www.php.net/manual/en/function.array-is-list.php
|
||||||
|
* @param array $array
|
||||||
|
* @return bool
|
||||||
|
*/
|
||||||
|
public static function arrayIsList(array $array)
|
||||||
|
{
|
||||||
|
if (function_exists('array_is_list')) {
|
||||||
|
return array_is_list($array);
|
||||||
|
}
|
||||||
|
|
||||||
|
$i = -1;
|
||||||
|
foreach ($array as $k => $v) {
|
||||||
|
++$i;
|
||||||
|
if ($k !== $i) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* JSON supports just unicode strings. This helper method converts non unicode chars to Unicode Replacement Character U+FFFD (UTF-8)
|
||||||
|
* @param string $string
|
||||||
|
* @return string
|
||||||
|
*/
|
||||||
|
public static function escapeNonUnicode($string)
|
||||||
|
{
|
||||||
|
if (mb_check_encoding($string, 'UTF-8')) {
|
||||||
|
return $string; // string is valid unicode
|
||||||
|
}
|
||||||
|
|
||||||
|
return htmlspecialchars_decode(htmlspecialchars($string, ENT_SUBSTITUTE, 'UTF-8'));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,14 +8,14 @@ class ProcessException extends Exception
|
||||||
private $stdout;
|
private $stdout;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param string|array $command
|
* @param array $command
|
||||||
* @param int $returnCode
|
* @param int $returnCode
|
||||||
* @param string $stderr
|
* @param string $stderr
|
||||||
* @param string $stdout
|
* @param string $stdout
|
||||||
*/
|
*/
|
||||||
public function __construct($command, $returnCode, $stderr, $stdout)
|
public function __construct(array $command, $returnCode, $stderr, $stdout)
|
||||||
{
|
{
|
||||||
$commandForException = is_array($command) ? implode(' ', $command) : $command;
|
$commandForException = implode(' ', $command);
|
||||||
$message = "Command '$commandForException' finished with error code $returnCode.\nSTDERR: '$stderr'\nSTDOUT: '$stdout'";
|
$message = "Command '$commandForException' finished with error code $returnCode.\nSTDERR: '$stderr'\nSTDOUT: '$stdout'";
|
||||||
$this->stderr = $stderr;
|
$this->stderr = $stderr;
|
||||||
$this->stdout = $stdout;
|
$this->stdout = $stdout;
|
||||||
|
@ -56,11 +56,6 @@ class ProcessTool
|
||||||
self::logMessage('Running command ' . implode(' ', $command));
|
self::logMessage('Running command ' . implode(' ', $command));
|
||||||
}
|
}
|
||||||
|
|
||||||
// PHP older than 7.4 do not support proc_open with array, so we need to convert values to string manually
|
|
||||||
if (PHP_VERSION_ID < 70400) {
|
|
||||||
$command = array_map('escapeshellarg', $command);
|
|
||||||
$command = implode(' ', $command);
|
|
||||||
}
|
|
||||||
$process = proc_open($command, $descriptorSpec, $pipes, $cwd);
|
$process = proc_open($command, $descriptorSpec, $pipes, $cwd);
|
||||||
if (!$process) {
|
if (!$process) {
|
||||||
$commandForException = self::commandFormat($command);
|
$commandForException = self::commandFormat($command);
|
||||||
|
@ -136,8 +131,8 @@ class ProcessTool
|
||||||
* @param array|string $command
|
* @param array|string $command
|
||||||
* @return string
|
* @return string
|
||||||
*/
|
*/
|
||||||
private static function commandFormat($command)
|
private static function commandFormat(array $command)
|
||||||
{
|
{
|
||||||
return is_array($command) ? implode(' ', $command) : $command;
|
return implode(' ', $command);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -178,8 +178,12 @@ class PubSubTool
|
||||||
|
|
||||||
public function killService()
|
public function killService()
|
||||||
{
|
{
|
||||||
if ($this->checkIfRunning()) {
|
|
||||||
$settings = $this->getSetSettings();
|
$settings = $this->getSetSettings();
|
||||||
|
if ($settings['supervisor_managed']) {
|
||||||
|
throw new RuntimeException('ZeroMQ server is managed by supervisor, it is not possible to restart it.');
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($this->checkIfRunning()) {
|
||||||
$redis = $this->createRedisConnection($settings);
|
$redis = $this->createRedisConnection($settings);
|
||||||
$redis->rPush('command', 'kill');
|
$redis->rPush('command', 'kill');
|
||||||
sleep(1);
|
sleep(1);
|
||||||
|
@ -213,12 +217,16 @@ class PubSubTool
|
||||||
|
|
||||||
public function restartServer()
|
public function restartServer()
|
||||||
{
|
{
|
||||||
|
$settings = $this->getSetSettings();
|
||||||
|
if ($settings['supervisor_managed']) {
|
||||||
|
throw new RuntimeException('ZeroMQ server is managed by supervisor, it is not possible to restart it.');
|
||||||
|
}
|
||||||
|
|
||||||
if (!$this->checkIfRunning()) {
|
if (!$this->checkIfRunning()) {
|
||||||
if (!$this->killService()) {
|
if (!$this->killService()) {
|
||||||
return 'Could not kill the previous instance of the ZeroMQ script.';
|
return 'Could not kill the previous instance of the ZeroMQ script.';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
$settings = $this->getSetSettings();
|
|
||||||
$this->setupPubServer($settings);
|
$this->setupPubServer($settings);
|
||||||
if ($this->checkIfRunning() === false) {
|
if ($this->checkIfRunning() === false) {
|
||||||
return 'Failed starting the ZeroMQ script.';
|
return 'Failed starting the ZeroMQ script.';
|
||||||
|
@ -226,12 +234,22 @@ class PubSubTool
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function createConfigFile()
|
||||||
|
{
|
||||||
|
$settings = $this->getSetSettings();
|
||||||
|
$this->saveSettingToFile($settings);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param array $settings
|
* @param array $settings
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
private function setupPubServer(array $settings)
|
private function setupPubServer(array $settings)
|
||||||
{
|
{
|
||||||
|
if ($settings['supervisor_managed']) {
|
||||||
|
return; // server is managed by supervisor, we don't need to check if is running or start it when not
|
||||||
|
}
|
||||||
|
|
||||||
if ($this->checkIfRunning() === false) {
|
if ($this->checkIfRunning() === false) {
|
||||||
if ($this->checkIfRunning(self::OLD_PID_LOCATION)) {
|
if ($this->checkIfRunning(self::OLD_PID_LOCATION)) {
|
||||||
// Old version is running, kill it and start again new one.
|
// Old version is running, kill it and start again new one.
|
||||||
|
@ -250,6 +268,7 @@ class PubSubTool
|
||||||
* @param string|array $data
|
* @param string|array $data
|
||||||
* @return bool
|
* @return bool
|
||||||
* @throws JsonException
|
* @throws JsonException
|
||||||
|
* @throws RedisException
|
||||||
*/
|
*/
|
||||||
private function pushToRedis($ns, $data)
|
private function pushToRedis($ns, $data)
|
||||||
{
|
{
|
||||||
|
@ -295,9 +314,12 @@ class PubSubTool
|
||||||
FileAccessTool::writeToFile($settingFilePath, JsonTool::encode($settings));
|
FileAccessTool::writeToFile($settingFilePath, JsonTool::encode($settings));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return array
|
||||||
|
*/
|
||||||
private function getSetSettings()
|
private function getSetSettings()
|
||||||
{
|
{
|
||||||
$settings = array(
|
$settings = [
|
||||||
'redis_host' => 'localhost',
|
'redis_host' => 'localhost',
|
||||||
'redis_port' => 6379,
|
'redis_port' => 6379,
|
||||||
'redis_password' => '',
|
'redis_password' => '',
|
||||||
|
@ -307,7 +329,8 @@ class PubSubTool
|
||||||
'port' => '50000',
|
'port' => '50000',
|
||||||
'username' => null,
|
'username' => null,
|
||||||
'password' => null,
|
'password' => null,
|
||||||
);
|
'supervisor_managed' => false,
|
||||||
|
];
|
||||||
|
|
||||||
$pluginConfig = Configure::read('Plugin');
|
$pluginConfig = Configure::read('Plugin');
|
||||||
foreach ($settings as $key => $setting) {
|
foreach ($settings as $key => $setting) {
|
||||||
|
|
|
@ -57,25 +57,34 @@ class RedisTool
|
||||||
/**
|
/**
|
||||||
* @param Redis $redis
|
* @param Redis $redis
|
||||||
* @param string|array $pattern
|
* @param string|array $pattern
|
||||||
* @return int|Redis Number of deleted keys or instance of Redis if used in MULTI mode
|
* @return Generator<string>
|
||||||
* @throws RedisException
|
* @throws RedisException
|
||||||
*/
|
*/
|
||||||
public static function deleteKeysByPattern(Redis $redis, $pattern)
|
public static function keysByPattern(Redis $redis, $pattern)
|
||||||
{
|
{
|
||||||
if (is_string($pattern)) {
|
if (is_string($pattern)) {
|
||||||
$pattern = [$pattern];
|
$pattern = [$pattern];
|
||||||
}
|
}
|
||||||
|
|
||||||
$allKeys = [];
|
|
||||||
foreach ($pattern as $p) {
|
foreach ($pattern as $p) {
|
||||||
$iterator = null;
|
$iterator = null;
|
||||||
while (false !== ($keys = $redis->scan($iterator, $p, 1000))) {
|
while (false !== ($keys = $redis->scan($iterator, $p, 1000))) {
|
||||||
foreach ($keys as $key) {
|
foreach ($keys as $key) {
|
||||||
$allKeys[] = $key;
|
yield $key;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param Redis $redis
|
||||||
|
* @param string|array $pattern
|
||||||
|
* @return int|Redis Number of deleted keys or instance of Redis if used in MULTI mode
|
||||||
|
* @throws RedisException
|
||||||
|
*/
|
||||||
|
public static function deleteKeysByPattern(Redis $redis, $pattern)
|
||||||
|
{
|
||||||
|
$allKeys = iterator_to_array(self::keysByPattern($redis, $pattern));
|
||||||
if (empty($allKeys)) {
|
if (empty($allKeys)) {
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
@ -153,11 +162,7 @@ class RedisTool
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (self::$serializer === null) {
|
if ($string[0] === "\x00") {
|
||||||
self::$serializer = Configure::read('MISP.redis_serializer') ?: false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (self::$serializer === 'igbinary') {
|
|
||||||
return igbinary_unserialize($string);
|
return igbinary_unserialize($string);
|
||||||
} else {
|
} else {
|
||||||
return JsonTool::decode($string);
|
return JsonTool::decode($string);
|
||||||
|
|
|
@ -355,6 +355,14 @@ class ServerSyncTool
|
||||||
return $this->server['Server']['id'];
|
return $this->server['Server']['id'];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return string
|
||||||
|
*/
|
||||||
|
public function serverName()
|
||||||
|
{
|
||||||
|
return $this->server['Server']['name'];
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return array
|
* @return array
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -1,5 +1,4 @@
|
||||||
<?php
|
<?php
|
||||||
|
|
||||||
class SyncTool
|
class SyncTool
|
||||||
{
|
{
|
||||||
|
|
||||||
|
@ -84,8 +83,14 @@ class SyncTool
|
||||||
$params['ssl_crypto_method'] = $version;
|
$params['ssl_crypto_method'] = $version;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (function_exists('curl_init')) {
|
||||||
|
App::uses('CurlClient', 'Tools');
|
||||||
|
$HttpSocket = new CurlClient($params);
|
||||||
|
} else {
|
||||||
App::uses('HttpSocketExtended', 'Tools');
|
App::uses('HttpSocketExtended', 'Tools');
|
||||||
$HttpSocket = new HttpSocketExtended($params);
|
$HttpSocket = new HttpSocketExtended($params);
|
||||||
|
}
|
||||||
|
|
||||||
$proxy = Configure::read('Proxy');
|
$proxy = Configure::read('Proxy');
|
||||||
if (empty($params['skip_proxy']) && isset($proxy['host']) && !empty($proxy['host'])) {
|
if (empty($params['skip_proxy']) && isset($proxy['host']) && !empty($proxy['host'])) {
|
||||||
$HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']);
|
$HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']);
|
||||||
|
|
|
@ -71,12 +71,6 @@ class AccessLog extends AppModel
|
||||||
{
|
{
|
||||||
$accessLog = &$this->data['AccessLog'];
|
$accessLog = &$this->data['AccessLog'];
|
||||||
|
|
||||||
$this->externalLog($accessLog);
|
|
||||||
|
|
||||||
if (Configure::read('MISP.log_paranoid_skip_db')) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Truncate
|
// Truncate
|
||||||
foreach (['request_id', 'user_agent', 'url'] as $field) {
|
foreach (['request_id', 'user_agent', 'url'] as $field) {
|
||||||
if (isset($accessLog[$field]) && strlen($accessLog[$field]) > 255) {
|
if (isset($accessLog[$field]) && strlen($accessLog[$field]) > 255) {
|
||||||
|
@ -202,7 +196,7 @@ class AccessLog extends AppModel
|
||||||
|
|
||||||
if ($includeSqlQueries && !empty($sqlLog['log'])) {
|
if ($includeSqlQueries && !empty($sqlLog['log'])) {
|
||||||
foreach ($sqlLog['log'] as &$log) {
|
foreach ($sqlLog['log'] as &$log) {
|
||||||
$log['query'] = $this->escapeNonUnicode($log['query']);
|
$log['query'] = JsonTool::escapeNonUnicode($log['query']);
|
||||||
unset($log['affected']); // affected is the same as numRows
|
unset($log['affected']); // affected is the same as numRows
|
||||||
unset($log['params']); // no need to save for your use case
|
unset($log['params']); // no need to save for your use case
|
||||||
}
|
}
|
||||||
|
@ -214,6 +208,12 @@ class AccessLog extends AppModel
|
||||||
$data['query_count'] = $queryCount;
|
$data['query_count'] = $queryCount;
|
||||||
$data['duration'] = (int)((microtime(true) - $requestTime->format('U.u')) * 1000); // in milliseconds
|
$data['duration'] = (int)((microtime(true) - $requestTime->format('U.u')) * 1000); // in milliseconds
|
||||||
|
|
||||||
|
$this->externalLog($data);
|
||||||
|
|
||||||
|
if (Configure::read('MISP.log_paranoid_skip_db')) {
|
||||||
|
return true; // do not save access log to database
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
return $this->save($data, ['atomic' => false]);
|
return $this->save($data, ['atomic' => false]);
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
|
@ -226,7 +226,7 @@ class AccessLog extends AppModel
|
||||||
* @param array $data
|
* @param array $data
|
||||||
* @return void
|
* @return void
|
||||||
*/
|
*/
|
||||||
public function externalLog(array $data)
|
private function externalLog(array $data)
|
||||||
{
|
{
|
||||||
if ($this->pubToZmq('audit')) {
|
if ($this->pubToZmq('audit')) {
|
||||||
$this->getPubSubTool()->publish($data, 'audit', 'log');
|
$this->getPubSubTool()->publish($data, 'audit', 'log');
|
||||||
|
@ -310,36 +310,4 @@ class AccessLog extends AppModel
|
||||||
}
|
}
|
||||||
return $data;
|
return $data;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* @param $string
|
|
||||||
* @return string
|
|
||||||
*/
|
|
||||||
private function escapeNonUnicode($string)
|
|
||||||
{
|
|
||||||
if (json_encode($string, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_LINE_TERMINATORS) !== false) {
|
|
||||||
return $string; // string is valid unicode
|
|
||||||
}
|
|
||||||
|
|
||||||
if (function_exists('mb_str_split')) {
|
|
||||||
$result = mb_str_split($string);
|
|
||||||
} else {
|
|
||||||
$result = [];
|
|
||||||
$length = mb_strlen($string);
|
|
||||||
for ($i = 0; $i < $length; $i++) {
|
|
||||||
$result[] = mb_substr($string, $i, 1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$string = '';
|
|
||||||
foreach ($result as $char) {
|
|
||||||
if (strlen($char) === 1 && !preg_match('/[[:print:]]/', $char)) {
|
|
||||||
$string .= '\x' . bin2hex($char);
|
|
||||||
} else {
|
|
||||||
$string .= $char;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return $string;
|
|
||||||
}
|
|
||||||
}
|
}
|
|
@ -90,6 +90,7 @@ class AdminSetting extends AppModel
|
||||||
$time = time();
|
$time = time();
|
||||||
$this->__deleteScriptTmpFiles($time);
|
$this->__deleteScriptTmpFiles($time);
|
||||||
$this->__deleteTaxiiTmpFiles($time);
|
$this->__deleteTaxiiTmpFiles($time);
|
||||||
|
$this->__deleteCachedExportFiles($time);
|
||||||
}
|
}
|
||||||
|
|
||||||
private function __deleteScriptTmpFiles($time) {
|
private function __deleteScriptTmpFiles($time) {
|
||||||
|
@ -107,6 +108,29 @@ class AdminSetting extends AppModel
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private function __deleteCachedExportFiles($time) {
|
||||||
|
$cache_path = APP . 'tmp/cached_exports';
|
||||||
|
$cache_dir = new Folder($cache_path);
|
||||||
|
$cache_data = $cache_dir->read(false, false);
|
||||||
|
if (!empty($cache_data[0])) {
|
||||||
|
foreach ($cache_data[0] as $cache_export_dir) {
|
||||||
|
$tmp_dir = new Folder($cache_path . '/' . $cache_export_dir);
|
||||||
|
$cache_export_dir_contents = $tmp_dir->read(false, false);
|
||||||
|
if (!empty(count($cache_export_dir_contents[1]))) {
|
||||||
|
$files_count = count($cache_export_dir_contents[1]);
|
||||||
|
$files_removed = 0;
|
||||||
|
foreach ($cache_export_dir_contents[1] as $tmp_file) {
|
||||||
|
$tmp_file = new File($cache_path . '/' . $cache_export_dir . '/' . $tmp_file);
|
||||||
|
if ($time > $tmp_file->lastChange() + 3600) {
|
||||||
|
$tmp_file->delete();
|
||||||
|
$files_removed += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private function __deleteTaxiiTmpFiles($time) {
|
private function __deleteTaxiiTmpFiles($time) {
|
||||||
$taxii_path = APP . 'files/scripts/tmp/Taxii';
|
$taxii_path = APP . 'files/scripts/tmp/Taxii';
|
||||||
$taxii_dir = new Folder($taxii_path);
|
$taxii_dir = new Folder($taxii_path);
|
||||||
|
@ -114,13 +138,13 @@ class AdminSetting extends AppModel
|
||||||
if (!empty($taxii_contents[0])) {
|
if (!empty($taxii_contents[0])) {
|
||||||
foreach ($taxii_contents[0] as $taxii_temp_dir) {
|
foreach ($taxii_contents[0] as $taxii_temp_dir) {
|
||||||
if (preg_match('/^[a-zA-Z0-9]{12}$/', $taxii_temp_dir)) {
|
if (preg_match('/^[a-zA-Z0-9]{12}$/', $taxii_temp_dir)) {
|
||||||
$tmp_dir = new Folder($taxii_path . $taxii_temp_dir);
|
$tmp_dir = new Folder($taxii_path . '/' .$taxii_temp_dir);
|
||||||
$taxii_temp_dir_contents = $tmp_dir->read(false, false);
|
$taxii_temp_dir_contents = $tmp_dir->read(false, false);
|
||||||
if (!empty(count($taxii_temp_dir_contents[1]))) {
|
if (!empty(count($taxii_temp_dir_contents[1]))) {
|
||||||
$files_count = count($taxii_temp_dir_contents[1]);
|
$files_count = count($taxii_temp_dir_contents[1]);
|
||||||
$files_removed = 0;
|
$files_removed = 0;
|
||||||
foreach ($taxii_temp_dir_contents[1] as $tmp_file) {
|
foreach ($taxii_temp_dir_contents[1] as $tmp_file) {
|
||||||
$tmp_file = new File($taxii_path . $taxii_temp_dir . '/' . $tmp_file);
|
$tmp_file = new File($taxii_path . '/' . $taxii_temp_dir . '/' . $tmp_file);
|
||||||
if ($time > $tmp_file->lastChange() + 3600) {
|
if ($time > $tmp_file->lastChange() + 3600) {
|
||||||
$tmp_file->delete();
|
$tmp_file->delete();
|
||||||
$files_removed += 1;
|
$files_removed += 1;
|
||||||
|
|
|
@ -89,6 +89,9 @@ class Allowedlist extends AppModel
|
||||||
if ($isAttributeArray) {
|
if ($isAttributeArray) {
|
||||||
// loop through each attribute and unset the ones that are allowedlisted
|
// loop through each attribute and unset the ones that are allowedlisted
|
||||||
foreach ($data as $k => $attribute) {
|
foreach ($data as $k => $attribute) {
|
||||||
|
if (empty($attribute['Attribute'])) {
|
||||||
|
$attribute = ['Attribute' => $attribute];
|
||||||
|
}
|
||||||
// loop through each allowedlist item and run a preg match against the attribute value. If it matches, unset the attribute
|
// loop through each allowedlist item and run a preg match against the attribute value. If it matches, unset the attribute
|
||||||
foreach ($allowedlists as $wlitem) {
|
foreach ($allowedlists as $wlitem) {
|
||||||
if (preg_match($wlitem, $attribute['Attribute']['value'])) {
|
if (preg_match($wlitem, $attribute['Attribute']['value'])) {
|
||||||
|
|
|
@ -2374,9 +2374,9 @@ class AppModel extends Model
|
||||||
}
|
}
|
||||||
|
|
||||||
// alternative to the build in notempty/notblank validation functions, compatible with cakephp <= 2.6 and cakephp and cakephp >= 2.7
|
// alternative to the build in notempty/notblank validation functions, compatible with cakephp <= 2.6 and cakephp and cakephp >= 2.7
|
||||||
public function valueNotEmpty($value)
|
public function valueNotEmpty(array $value)
|
||||||
{
|
{
|
||||||
$field = array_keys($value)[0];
|
$field = array_key_first($value);
|
||||||
$value = trim($value[$field]);
|
$value = trim($value[$field]);
|
||||||
if (!empty($value)) {
|
if (!empty($value)) {
|
||||||
return true;
|
return true;
|
||||||
|
@ -2384,27 +2384,27 @@ class AppModel extends Model
|
||||||
return ucfirst($field) . ' cannot be empty.';
|
return ucfirst($field) . ' cannot be empty.';
|
||||||
}
|
}
|
||||||
|
|
||||||
public function valueIsJson($value)
|
public function valueIsJson(array $value)
|
||||||
{
|
{
|
||||||
$value = array_values($value)[0];
|
$value = current($value);
|
||||||
if (!JsonTool::isValid($value)) {
|
if (!JsonTool::isValid($value)) {
|
||||||
return __('Invalid JSON.');
|
return __('Invalid JSON.');
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
public function valueIsID($value)
|
public function valueIsID(array $value)
|
||||||
{
|
{
|
||||||
$field = array_keys($value)[0];
|
$field = array_key_first($value);
|
||||||
if (!is_numeric($value[$field]) || $value[$field] < 0) {
|
if (!is_numeric($value[$field]) || $value[$field] < 0) {
|
||||||
return 'Invalid ' . ucfirst($field) . ' ID';
|
return 'Invalid ' . ucfirst($field) . ' ID';
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
public function stringNotEmpty($value)
|
public function stringNotEmpty(array $value)
|
||||||
{
|
{
|
||||||
$field = array_keys($value)[0];
|
$field = array_key_first($value);
|
||||||
$value = trim($value[$field]);
|
$value = trim($value[$field]);
|
||||||
if (!isset($value) || ($value == false && $value !== "0")) {
|
if (!isset($value) || ($value == false && $value !== "0")) {
|
||||||
return ucfirst($field) . ' cannot be empty.';
|
return ucfirst($field) . ' cannot be empty.';
|
||||||
|
@ -3267,14 +3267,13 @@ class AppModel extends Model
|
||||||
* Returns MISP version from VERSION.json file as array with major, minor and hotfix keys.
|
* Returns MISP version from VERSION.json file as array with major, minor and hotfix keys.
|
||||||
*
|
*
|
||||||
* @return array
|
* @return array
|
||||||
* @throws JsonException
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
public function checkMISPVersion()
|
public function checkMISPVersion()
|
||||||
{
|
{
|
||||||
static $versionArray;
|
static $versionArray;
|
||||||
if ($versionArray === null) {
|
if ($versionArray === null) {
|
||||||
$content = FileAccessTool::readFromFile(ROOT . DS . 'VERSION.json');
|
$versionArray = FileAccessTool::readJsonFromFile(ROOT . DS . 'VERSION.json', true);
|
||||||
$versionArray = JsonTool::decode($content);
|
|
||||||
}
|
}
|
||||||
return $versionArray;
|
return $versionArray;
|
||||||
}
|
}
|
||||||
|
@ -3290,7 +3289,7 @@ class AppModel extends Model
|
||||||
if ($commit === null) {
|
if ($commit === null) {
|
||||||
App::uses('GitTool', 'Tools');
|
App::uses('GitTool', 'Tools');
|
||||||
try {
|
try {
|
||||||
$commit = GitTool::currentCommit();
|
$commit = GitTool::currentCommit(ROOT);
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
$this->logException('Could not get current git commit', $e, LOG_NOTICE);
|
$this->logException('Could not get current git commit', $e, LOG_NOTICE);
|
||||||
$commit = false;
|
$commit = false;
|
||||||
|
@ -3714,7 +3713,7 @@ class AppModel extends Model
|
||||||
if (!$isRule) {
|
if (!$isRule) {
|
||||||
$args = func_get_args();
|
$args = func_get_args();
|
||||||
$fields = $args[1];
|
$fields = $args[1];
|
||||||
$or = isset($args[2]) ? $args[2] : true;
|
$or = $args[2] ?? true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!is_array($fields)) {
|
if (!is_array($fields)) {
|
||||||
|
@ -3859,8 +3858,7 @@ class AppModel extends Model
|
||||||
protected function isMysql()
|
protected function isMysql()
|
||||||
{
|
{
|
||||||
$dataSource = ConnectionManager::getDataSource('default');
|
$dataSource = ConnectionManager::getDataSource('default');
|
||||||
$dataSourceName = $dataSource->config['datasource'];
|
return $dataSource instanceof Mysql;
|
||||||
return $dataSourceName === 'Database/Mysql' || $dataSourceName === 'Database/MysqlObserver' || $dataSourceName === 'Database/MysqlExtended' || $dataSource instanceof Mysql;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -3996,21 +3994,21 @@ class AppModel extends Model
|
||||||
");
|
");
|
||||||
}
|
}
|
||||||
|
|
||||||
public function findOrder($order, $order_model, $valid_order_fields)
|
public function findOrder($order, $orderModel, $validOrderFields)
|
||||||
{
|
{
|
||||||
if (!is_array($order)) {
|
if (!is_array($order)) {
|
||||||
$order_rules = explode(' ', strtolower($order));
|
$orderRules = explode(' ', strtolower($order));
|
||||||
$order_field = explode('.', $order_rules[0]);
|
$orderField = explode('.', $orderRules[0]);
|
||||||
$order_field = end($order_field);
|
$orderField = end($orderField);
|
||||||
if (in_array($order_field, $valid_order_fields)) {
|
if (in_array($orderField, $validOrderFields, true)) {
|
||||||
$direction = 'asc';
|
$direction = 'asc';
|
||||||
if (!empty($order_rules[1]) && trim($order_rules[1]) === 'desc') {
|
if (!empty($orderRules[1]) && trim($orderRules[1]) === 'desc') {
|
||||||
$direction = 'desc';
|
$direction = 'desc';
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
return $order_model . '.' . $order_field . ' ' . $direction;
|
return $orderModel . '.' . $orderField . ' ' . $direction;
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
|
@ -189,6 +189,7 @@ class AttachmentScan extends AppModel
|
||||||
/** @var Job $job */
|
/** @var Job $job */
|
||||||
$job = ClassRegistry::init('Job');
|
$job = ClassRegistry::init('Job');
|
||||||
if ($jobId && !$job->exists($jobId)) {
|
if ($jobId && !$job->exists($jobId)) {
|
||||||
|
$this->log("Job with ID $jobId not found in database", LOG_NOTICE);
|
||||||
$jobId = null;
|
$jobId = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -252,12 +253,12 @@ class AttachmentScan extends AppModel
|
||||||
$infected = $this->scanAttachment($type, $attribute[$type], $moduleInfo);
|
$infected = $this->scanAttachment($type, $attribute[$type], $moduleInfo);
|
||||||
if ($infected === true) {
|
if ($infected === true) {
|
||||||
$virusFound++;
|
$virusFound++;
|
||||||
}
|
|
||||||
$scanned++;
|
$scanned++;
|
||||||
} catch (NotFoundException $e) {
|
} else if ($infected === false) {
|
||||||
// skip if file doesn't exists
|
$scanned++;
|
||||||
|
}
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
$this->logException("Could not scan attachment for $type {$attribute['Attribute']['id']}", $e);
|
$this->logException("Could not scan attachment for $type {$attribute['Attribute']['id']}", $e, LOG_WARNING);
|
||||||
$fails++;
|
$fails++;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -297,14 +298,14 @@ class AttachmentScan extends AppModel
|
||||||
$job = ClassRegistry::init('Job');
|
$job = ClassRegistry::init('Job');
|
||||||
$jobId = $job->createJob(
|
$jobId = $job->createJob(
|
||||||
'SYSTEM',
|
'SYSTEM',
|
||||||
Job::WORKER_DEFAULT,
|
Job::WORKER_PRIO,
|
||||||
'virus_scan',
|
'virus_scan',
|
||||||
($type === self::TYPE_ATTRIBUTE ? 'Attribute: ' : 'Shadow attribute: ') . $attribute['id'],
|
($type === self::TYPE_ATTRIBUTE ? 'Attribute: ' : 'Shadow attribute: ') . $attribute['id'],
|
||||||
'Scanning...'
|
'Scanning...'
|
||||||
);
|
);
|
||||||
|
|
||||||
$this->getBackgroundJobsTool()->enqueue(
|
$this->getBackgroundJobsTool()->enqueue(
|
||||||
BackgroundJobsTool::DEFAULT_QUEUE,
|
BackgroundJobsTool::PRIO_QUEUE,
|
||||||
BackgroundJobsTool::CMD_ADMIN,
|
BackgroundJobsTool::CMD_ADMIN,
|
||||||
[
|
[
|
||||||
'scanAttachment',
|
'scanAttachment',
|
||||||
|
@ -319,10 +320,12 @@ class AttachmentScan extends AppModel
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
* Return true if attachment is infected, null if attachment was not scanned and false if attachment is OK
|
||||||
|
*
|
||||||
* @param string $type
|
* @param string $type
|
||||||
* @param array $attribute
|
* @param array $attribute
|
||||||
* @param array $moduleInfo
|
* @param array $moduleInfo
|
||||||
* @return bool|null Return true if attachment is infected.
|
* @return bool|null
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
private function scanAttachment($type, array $attribute, array $moduleInfo)
|
private function scanAttachment($type, array $attribute, array $moduleInfo)
|
||||||
|
@ -351,11 +354,10 @@ class AttachmentScan extends AppModel
|
||||||
return false; // empty file is automatically considered as not infected
|
return false; // empty file is automatically considered as not infected
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if ($fileSize > 25 * 1024 * 1024) {
|
||||||
/* if ($file->size() > 50 * 1024 * 1024) {
|
$this->log("File '$file->path' is bigger than 25 MB, will be not scanned.", LOG_NOTICE);
|
||||||
$this->log("File '$file->path' is bigger than 50 MB, will be not scanned.", LOG_NOTICE);
|
return null;
|
||||||
return false;
|
}
|
||||||
}*/
|
|
||||||
|
|
||||||
$fileContent = $file->read();
|
$fileContent = $file->read();
|
||||||
if ($fileContent === false) {
|
if ($fileContent === false) {
|
||||||
|
|
|
@ -434,7 +434,7 @@ class Attribute extends AppModel
|
||||||
public function afterSave($created, $options = array())
|
public function afterSave($created, $options = array())
|
||||||
{
|
{
|
||||||
// Passing event in `parentEvent` field will speed up correlation
|
// Passing event in `parentEvent` field will speed up correlation
|
||||||
$passedEvent = isset($options['parentEvent']) ? $options['parentEvent'] : false;
|
$passedEvent = $options['parentEvent'] ?? false;
|
||||||
|
|
||||||
$attribute = $this->data['Attribute'];
|
$attribute = $this->data['Attribute'];
|
||||||
|
|
||||||
|
@ -545,6 +545,28 @@ class Attribute extends AppModel
|
||||||
return $result;
|
return $result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method is called after all data are successfully saved into database
|
||||||
|
* @return void
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
private function afterDatabaseSave(array $data)
|
||||||
|
{
|
||||||
|
$attribute = $data['Attribute'];
|
||||||
|
if (isset($attribute['type']) && $this->typeIsAttachment($attribute['type'])) {
|
||||||
|
$this->loadAttachmentScan()->backgroundScan(AttachmentScan::TYPE_ATTRIBUTE, $attribute);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public function save($data = null, $validate = true, $fieldList = array())
|
||||||
|
{
|
||||||
|
$result = parent::save($data, $validate, $fieldList);
|
||||||
|
if ($result) {
|
||||||
|
$this->afterDatabaseSave($result);
|
||||||
|
}
|
||||||
|
return $result;
|
||||||
|
}
|
||||||
|
|
||||||
public function beforeDelete($cascade = true)
|
public function beforeDelete($cascade = true)
|
||||||
{
|
{
|
||||||
// delete attachments from the disk
|
// delete attachments from the disk
|
||||||
|
@ -786,7 +808,7 @@ class Attribute extends AppModel
|
||||||
// check whether the variable is null or datetime
|
// check whether the variable is null or datetime
|
||||||
public function datetimeOrNull($fields)
|
public function datetimeOrNull($fields)
|
||||||
{
|
{
|
||||||
$seen = array_values($fields)[0];
|
$seen = current($fields);
|
||||||
if ($seen === null) {
|
if ($seen === null) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
@ -881,7 +903,6 @@ class Attribute extends AppModel
|
||||||
}
|
}
|
||||||
$result = $this->loadAttachmentTool()->save($attribute['event_id'], $attribute['id'], $attribute['data']);
|
$result = $this->loadAttachmentTool()->save($attribute['event_id'], $attribute['id'], $attribute['data']);
|
||||||
if ($result) {
|
if ($result) {
|
||||||
$this->loadAttachmentScan()->backgroundScan(AttachmentScan::TYPE_ATTRIBUTE, $attribute);
|
|
||||||
// Clean thumbnail cache
|
// Clean thumbnail cache
|
||||||
if ($this->isImage($attribute) && Configure::read('MISP.thumbnail_in_redis')) {
|
if ($this->isImage($attribute) && Configure::read('MISP.thumbnail_in_redis')) {
|
||||||
$redis = RedisTool::init();
|
$redis = RedisTool::init();
|
||||||
|
@ -1224,30 +1245,59 @@ class Attribute extends AppModel
|
||||||
$this->Correlation->purgeCorrelations($eventId);
|
$this->Correlation->purgeCorrelations($eventId);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function reportValidationIssuesAttributes($eventId)
|
/**
|
||||||
|
* This method is useful if you want to iterate all attributes sorted by ID
|
||||||
|
* @param array $conditions
|
||||||
|
* @param array $fields
|
||||||
|
* @param bool|string $callbacks
|
||||||
|
* @return Generator<array>|void
|
||||||
|
*/
|
||||||
|
public function fetchAttributesInChunks(array $conditions = [], array $fields = [], $callbacks = true)
|
||||||
|
{
|
||||||
|
$query = [
|
||||||
|
'recursive' => -1,
|
||||||
|
'conditions' => $conditions,
|
||||||
|
'limit' => 500,
|
||||||
|
'order' => ['Attribute.id'],
|
||||||
|
'fields' => $fields,
|
||||||
|
'callbacks' => $callbacks,
|
||||||
|
];
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
$attributes = $this->find('all', $query);
|
||||||
|
foreach ($attributes as $attribute) {
|
||||||
|
yield $attribute;
|
||||||
|
}
|
||||||
|
$count = count($attributes);
|
||||||
|
if ($count < 500) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
$lastAttribute = $attributes[$count - 1];
|
||||||
|
$query['conditions']['Attribute.id >'] = $lastAttribute['Attribute']['id'];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param int|null $eventId
|
||||||
|
* @return Generator
|
||||||
|
*/
|
||||||
|
public function reportValidationIssuesAttributes($eventId = null)
|
||||||
{
|
{
|
||||||
$conditions = array();
|
$conditions = array();
|
||||||
if ($eventId && is_numeric($eventId)) {
|
if ($eventId && is_numeric($eventId)) {
|
||||||
$conditions = array('event_id' => $eventId);
|
$conditions = array('event_id' => $eventId);
|
||||||
}
|
}
|
||||||
|
|
||||||
$attributeIds = $this->find('column', array(
|
$attributes = $this->fetchAttributesInChunks($conditions);
|
||||||
'fields' => array('id'),
|
|
||||||
'conditions' => $conditions
|
|
||||||
));
|
|
||||||
$chunks = array_chunk($attributeIds, 500);
|
|
||||||
|
|
||||||
$result = array();
|
|
||||||
foreach ($chunks as $chunk) {
|
|
||||||
$attributes = $this->find('all', array('recursive' => -1, 'conditions' => array('id' => $chunk)));
|
|
||||||
foreach ($attributes as $attribute) {
|
foreach ($attributes as $attribute) {
|
||||||
$this->set($attribute);
|
$this->set($attribute);
|
||||||
if (!$this->validates()) {
|
if (!$this->validates()) {
|
||||||
$resultErrors = array();
|
$resultErrors = [];
|
||||||
foreach ($this->validationErrors as $field => $error) {
|
foreach ($this->validationErrors as $field => $error) {
|
||||||
$resultErrors[$field] = array('value' => $attribute['Attribute'][$field], 'error' => $error[0]);
|
$resultErrors[$field] = ['value' => $attribute['Attribute'][$field], 'error' => $error[0]];
|
||||||
}
|
}
|
||||||
$result[] = [
|
yield [
|
||||||
'id' => $attribute['Attribute']['id'],
|
'id' => $attribute['Attribute']['id'],
|
||||||
'error' => $resultErrors,
|
'error' => $resultErrors,
|
||||||
'details' => 'Event ID: [' . $attribute['Attribute']['event_id'] . "] - Category: [" . $attribute['Attribute']['category'] . "] - Type: [" . $attribute['Attribute']['type'] . "] - Value: [" . $attribute['Attribute']['value'] . ']',
|
'details' => 'Event ID: [' . $attribute['Attribute']['event_id'] . "] - Category: [" . $attribute['Attribute']['category'] . "] - Type: [" . $attribute['Attribute']['type'] . "] - Value: [" . $attribute['Attribute']['value'] . ']',
|
||||||
|
@ -1255,7 +1305,36 @@ class Attribute extends AppModel
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return $result;
|
|
||||||
|
/**
|
||||||
|
* @param bool $dryRun If true, no changes will be made to
|
||||||
|
* @return Generator
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
public function normalizeIpAddress($dryRun = false)
|
||||||
|
{
|
||||||
|
$attributes = $this->fetchAttributesInChunks([
|
||||||
|
'Attribute.type' => ['ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'domain|ip'],
|
||||||
|
]);
|
||||||
|
|
||||||
|
foreach ($attributes as $attribute) {
|
||||||
|
$value = $attribute['Attribute']['value'];
|
||||||
|
$normalizedValue = AttributeValidationTool::modifyBeforeValidation($attribute['Attribute']['type'], $value);
|
||||||
|
if ($value !== $normalizedValue) {
|
||||||
|
if (!$dryRun) {
|
||||||
|
$attribute['Attribute']['value'] = $normalizedValue;
|
||||||
|
$this->save($attribute, true, ['value1', 'value2']);
|
||||||
|
}
|
||||||
|
|
||||||
|
yield [
|
||||||
|
'id' => (int) $attribute['Attribute']['id'],
|
||||||
|
'event_id' => (int) $attribute['Attribute']['event_id'],
|
||||||
|
'type' => $attribute['Attribute']['type'],
|
||||||
|
'value' => $value,
|
||||||
|
'normalized_value' => $normalizedValue,
|
||||||
|
];
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -1610,6 +1689,7 @@ class Attribute extends AppModel
|
||||||
* @param array $user
|
* @param array $user
|
||||||
* @param array $options
|
* @param array $options
|
||||||
* @param int|false $result_count If false, count is not fetched
|
* @param int|false $result_count If false, count is not fetched
|
||||||
|
* @param bool $real_count
|
||||||
* @return array
|
* @return array
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
|
@ -3096,8 +3176,7 @@ class Attribute extends AppModel
|
||||||
$exportTool->additional_params
|
$exportTool->additional_params
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
ClassRegistry::init('ConnectionManager');
|
|
||||||
$db = ConnectionManager::getDataSource('default');
|
|
||||||
$tmpfile = new TmpFileTool();
|
$tmpfile = new TmpFileTool();
|
||||||
$tmpfile->write($exportTool->header($exportToolParams));
|
$tmpfile->write($exportTool->header($exportToolParams));
|
||||||
$loop = false;
|
$loop = false;
|
||||||
|
@ -3673,7 +3752,7 @@ class Attribute extends AppModel
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private function findAttributeByValue($attribute)
|
private function findAttributeByValue(array $attribute)
|
||||||
{
|
{
|
||||||
$type = $attribute['type'];
|
$type = $attribute['type'];
|
||||||
$conditions = [
|
$conditions = [
|
||||||
|
|
|
@ -207,7 +207,7 @@ class AuthKey extends AppModel
|
||||||
*/
|
*/
|
||||||
private function updateUniqueIp(array $authkey)
|
private function updateUniqueIp(array $authkey)
|
||||||
{
|
{
|
||||||
if (Configure::read("MISP.disable_seen_ips_authkeys")) {
|
if (PHP_SAPI === 'cli' || Configure::read("MISP.disable_seen_ips_authkeys")) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -153,7 +153,7 @@ class Correlation extends AppModel
|
||||||
if (!empty($eventIds)) {
|
if (!empty($eventIds)) {
|
||||||
$eventCount = count($eventIds);
|
$eventCount = count($eventIds);
|
||||||
foreach ($eventIds as $j => $currentEventId) {
|
foreach ($eventIds as $j => $currentEventId) {
|
||||||
$attributeCount += $this->__iteratedCorrelation(
|
$attributeCount += $this->iteratedCorrelation(
|
||||||
$jobId,
|
$jobId,
|
||||||
$full,
|
$full,
|
||||||
$attributeId,
|
$attributeId,
|
||||||
|
@ -179,7 +179,7 @@ class Correlation extends AppModel
|
||||||
* @return int
|
* @return int
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
private function __iteratedCorrelation(
|
private function iteratedCorrelation(
|
||||||
$jobId = false,
|
$jobId = false,
|
||||||
$full = false,
|
$full = false,
|
||||||
$attributeId = null,
|
$attributeId = null,
|
||||||
|
@ -215,30 +215,14 @@ class Correlation extends AppModel
|
||||||
if ($attributeId) {
|
if ($attributeId) {
|
||||||
$attributeConditions['Attribute.id'] = $attributeId;
|
$attributeConditions['Attribute.id'] = $attributeId;
|
||||||
}
|
}
|
||||||
$query = [
|
|
||||||
'recursive' => -1,
|
$attributes = $this->Attribute->fetchAttributesInChunks($attributeConditions, $this->getFieldRules(), false);
|
||||||
'conditions' => $attributeConditions,
|
|
||||||
// fetch just necessary fields to save memory
|
|
||||||
'fields' => $this->getFieldRules(),
|
|
||||||
'order' => 'Attribute.id',
|
|
||||||
'limit' => 5000,
|
|
||||||
'callbacks' => false, // memory leak fix
|
|
||||||
];
|
|
||||||
$attributeCount = 0;
|
$attributeCount = 0;
|
||||||
do {
|
|
||||||
$attributes = $this->Attribute->find('all', $query);
|
|
||||||
foreach ($attributes as $attribute) {
|
foreach ($attributes as $attribute) {
|
||||||
$this->afterSaveCorrelation($attribute['Attribute'], $full, $event);
|
$this->afterSaveCorrelation($attribute['Attribute'], $full, $event);
|
||||||
|
++$attributeCount;
|
||||||
}
|
}
|
||||||
$fetchedAttributes = count($attributes);
|
|
||||||
unset($attributes);
|
|
||||||
$attributeCount += $fetchedAttributes;
|
|
||||||
if ($fetchedAttributes === 5000) { // maximum number of attributes fetched, continue in next loop
|
|
||||||
$query['conditions']['Attribute.id >'] = $attribute['Attribute']['id'];
|
|
||||||
} else {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
} while (true);
|
|
||||||
|
|
||||||
// Generating correlations can take long time, so clear caches after each event to refresh them
|
// Generating correlations can take long time, so clear caches after each event to refresh them
|
||||||
$this->cidrListCache = null;
|
$this->cidrListCache = null;
|
||||||
|
|
|
@ -7,6 +7,14 @@ App::uses('Mysql', 'Model/Datasource/Database');
|
||||||
*/
|
*/
|
||||||
class MysqlExtended extends Mysql
|
class MysqlExtended extends Mysql
|
||||||
{
|
{
|
||||||
|
const PDO_MAP = [
|
||||||
|
'integer' => PDO::PARAM_INT,
|
||||||
|
'float' => PDO::PARAM_STR,
|
||||||
|
'boolean' => PDO::PARAM_BOOL,
|
||||||
|
'string' => PDO::PARAM_STR,
|
||||||
|
'text' => PDO::PARAM_STR
|
||||||
|
];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Output MD5 as binary, that is faster and uses less memory
|
* Output MD5 as binary, that is faster and uses less memory
|
||||||
* @param string $value
|
* @param string $value
|
||||||
|
@ -157,15 +165,9 @@ class MysqlExtended extends Mysql
|
||||||
public function insertMulti($table, $fields, $values)
|
public function insertMulti($table, $fields, $values)
|
||||||
{
|
{
|
||||||
$table = $this->fullTableName($table);
|
$table = $this->fullTableName($table);
|
||||||
$holder = implode(',', array_fill(0, count($fields), '?'));
|
$holder = substr(str_repeat('?,', count($fields)), 0, -1);
|
||||||
$fields = implode(',', array_map([$this, 'name'], $fields));
|
$fields = implode(',', array_map([$this, 'name'], $fields));
|
||||||
$pdoMap = [
|
|
||||||
'integer' => PDO::PARAM_INT,
|
|
||||||
'float' => PDO::PARAM_STR,
|
|
||||||
'boolean' => PDO::PARAM_BOOL,
|
|
||||||
'string' => PDO::PARAM_STR,
|
|
||||||
'text' => PDO::PARAM_STR
|
|
||||||
];
|
|
||||||
$columnMap = [];
|
$columnMap = [];
|
||||||
foreach ($values[key($values)] as $key => $val) {
|
foreach ($values[key($values)] as $key => $val) {
|
||||||
if (is_int($val)) {
|
if (is_int($val)) {
|
||||||
|
@ -174,21 +176,21 @@ class MysqlExtended extends Mysql
|
||||||
$columnMap[$key] = PDO::PARAM_BOOL;
|
$columnMap[$key] = PDO::PARAM_BOOL;
|
||||||
} else {
|
} else {
|
||||||
$type = $this->introspectType($val);
|
$type = $this->introspectType($val);
|
||||||
$columnMap[$key] = $pdoMap[$type];
|
$columnMap[$key] = self::PDO_MAP[$type];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
$sql = "INSERT INTO $table ($fields) VALUES ";
|
$sql = "INSERT INTO $table ($fields) VALUES ";
|
||||||
$sql .= implode(',', array_fill(0, count($values), "($holder)"));
|
$sql .= substr(str_repeat("($holder),", count($values)), 0, -1);
|
||||||
$statement = $this->_connection->prepare($sql);
|
$statement = $this->_connection->prepare($sql);
|
||||||
$valuesList = array();
|
$valuesList = array();
|
||||||
$i = 1;
|
$i = 0;
|
||||||
foreach ($values as $value) {
|
foreach ($values as $value) {
|
||||||
foreach ($value as $col => $val) {
|
foreach ($value as $col => $val) {
|
||||||
if ($this->fullDebug) {
|
if ($this->fullDebug) {
|
||||||
$valuesList[] = $val;
|
$valuesList[] = $val;
|
||||||
}
|
}
|
||||||
$statement->bindValue($i++, $val, $columnMap[$col]);
|
$statement->bindValue(++$i, $val, $columnMap[$col]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
$result = $statement->execute();
|
$result = $statement->execute();
|
||||||
|
|
|
@ -3743,7 +3743,10 @@ class Event extends AppModel
|
||||||
unset($this->Attribute->validate['value']['uniqueValue']); // unset this - we are saving a new event, there are no values to compare against and event_id is not set in the attributes
|
unset($this->Attribute->validate['value']['uniqueValue']); // unset this - we are saving a new event, there are no values to compare against and event_id is not set in the attributes
|
||||||
}
|
}
|
||||||
unset($data['Event']['id']);
|
unset($data['Event']['id']);
|
||||||
if (isset($data['Event']['published']) && $data['Event']['published'] && $user['Role']['perm_publish'] == 0) {
|
if (
|
||||||
|
(Configure::read('MISP.block_publishing_for_same_creator', false) && !$user['Role']['perm_sync']) ||
|
||||||
|
(isset($data['Event']['published']) && $data['Event']['published'] && $user['Role']['perm_publish'] == 0)
|
||||||
|
) {
|
||||||
$data['Event']['published'] = 0;
|
$data['Event']['published'] = 0;
|
||||||
}
|
}
|
||||||
if (isset($data['Event']['uuid'])) {
|
if (isset($data['Event']['uuid'])) {
|
||||||
|
@ -4059,7 +4062,10 @@ class Event extends AppModel
|
||||||
} else {
|
} else {
|
||||||
return array('error' => 'Event could not be saved: Could not find the local event.');
|
return array('error' => 'Event could not be saved: Could not find the local event.');
|
||||||
}
|
}
|
||||||
if (!empty($data['Event']['published']) && !$user['Role']['perm_publish']) {
|
if (
|
||||||
|
(Configure::read('MISP.block_publishing_for_same_creator', false) && !$user['Role']['perm_sync'] && $user['id'] == $existingEvent['Event']['user_id']) ||
|
||||||
|
(!empty($data['Event']['published']) && !$user['Role']['perm_publish'])
|
||||||
|
) {
|
||||||
$data['Event']['published'] = 0;
|
$data['Event']['published'] = 0;
|
||||||
}
|
}
|
||||||
if (!isset($data['Event']['published'])) {
|
if (!isset($data['Event']['published'])) {
|
||||||
|
@ -4190,7 +4196,7 @@ class Event extends AppModel
|
||||||
if ((true != Configure::read('MISP.disablerestalert')) && (empty($server) || empty($server['Server']['publish_without_email']))) {
|
if ((true != Configure::read('MISP.disablerestalert')) && (empty($server) || empty($server['Server']['publish_without_email']))) {
|
||||||
$this->sendAlertEmailRouter($id, $user, $existingEvent['Event']['publish_timestamp']);
|
$this->sendAlertEmailRouter($id, $user, $existingEvent['Event']['publish_timestamp']);
|
||||||
}
|
}
|
||||||
$this->publish($existingEvent['Event']['id']);
|
$this->publish($existingEvent['Event']['id'], $passAlong);
|
||||||
}
|
}
|
||||||
if ($jobId) {
|
if ($jobId) {
|
||||||
$eventLock->deleteBackgroundJobLock($data['Event']['id'], $jobId);
|
$eventLock->deleteBackgroundJobLock($data['Event']['id'], $jobId);
|
||||||
|
@ -5952,8 +5958,10 @@ class Event extends AppModel
|
||||||
$this->add_original_file($decoded['original'], $originalFile, $created_id, $stixVersion);
|
$this->add_original_file($decoded['original'], $originalFile, $created_id, $stixVersion);
|
||||||
}
|
}
|
||||||
if ($publish && $user['Role']['perm_publish']) {
|
if ($publish && $user['Role']['perm_publish']) {
|
||||||
|
if (!Configure::read('MISP.block_publishing_for_same_creator', false) || $user['Role']['perm_sync']) {
|
||||||
$this->publish($created_id);
|
$this->publish($created_id);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
return $created_id;
|
return $created_id;
|
||||||
} else if (is_numeric($result)) {
|
} else if (is_numeric($result)) {
|
||||||
return __('Event with the same UUID already exists.');
|
return __('Event with the same UUID already exists.');
|
||||||
|
|
|
@ -2062,6 +2062,7 @@ class Feed extends AppModel
|
||||||
$contentType = $response->getHeader('content-type');
|
$contentType = $response->getHeader('content-type');
|
||||||
if ($contentType === 'application/zip') {
|
if ($contentType === 'application/zip') {
|
||||||
$zipFilePath = FileAccessTool::writeToTempFile($response->body);
|
$zipFilePath = FileAccessTool::writeToTempFile($response->body);
|
||||||
|
unset($response->body); // cleanup variable to reduce memory usage
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$response->body = $this->unzipFirstFile($zipFilePath);
|
$response->body = $this->unzipFirstFile($zipFilePath);
|
||||||
|
@ -2198,7 +2199,7 @@ class Feed extends AppModel
|
||||||
ZipArchive::ER_READ => 'read error',
|
ZipArchive::ER_READ => 'read error',
|
||||||
ZipArchive::ER_SEEK => 'seek error',
|
ZipArchive::ER_SEEK => 'seek error',
|
||||||
];
|
];
|
||||||
$message = isset($errorCodes[$result]) ? $errorCodes[$result] : 'error ' . $result;
|
$message = $errorCodes[$result] ?? 'error ' . $result;
|
||||||
throw new Exception("Remote server returns ZIP file, that cannot be open ($message)");
|
throw new Exception("Remote server returns ZIP file, that cannot be open ($message)");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -9,38 +9,32 @@ class FuzzyCorrelateSsdeep extends AppModel
|
||||||
|
|
||||||
public function ssdeep_prepare($hash)
|
public function ssdeep_prepare($hash)
|
||||||
{
|
{
|
||||||
list($block_size, $hash) = explode(':', $hash, 2);
|
list($blockSize, $hash) = explode(':', $hash, 2);
|
||||||
|
|
||||||
|
$uniqueChars = array_unique(str_split($hash), SORT_REGULAR);
|
||||||
|
|
||||||
$chars = array();
|
|
||||||
for ($i = 0; $i < strlen($hash); $i++) {
|
|
||||||
if (!in_array($hash[$i], $chars, true)) {
|
|
||||||
$chars[] = $hash[$i];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$search = true;
|
$search = true;
|
||||||
while ($search) {
|
while ($search) {
|
||||||
$search = false;
|
$search = false;
|
||||||
foreach ($chars as $c) {
|
foreach ($uniqueChars as $c) {
|
||||||
if (strpos($hash, $c . $c . $c . $c)) {
|
if (strpos($hash, $c . $c . $c . $c)) {
|
||||||
$hash = str_replace($c . $c . $c . $c, $c . $c . $c, $hash);
|
$hash = str_replace($c . $c . $c . $c, $c . $c . $c, $hash);
|
||||||
$search = true;
|
$search = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
$hash = explode(':', $hash);
|
|
||||||
$block_data = $hash[0];
|
|
||||||
$double_block_data = $hash[1];
|
|
||||||
// (struct.unpack("<Q", base64.b64decode(h[i:i + 7] + "=") + "\x00\x00\x00")[0] for i in range(len(h) - 6)))
|
|
||||||
|
|
||||||
$result = array(
|
$hash = explode(':', $hash);
|
||||||
$block_size,
|
list($block_data, $double_block_data) = $hash;
|
||||||
$this->get_all_7_char_chunks($block_data),
|
|
||||||
$this->get_all_7_char_chunks($double_block_data)
|
return [
|
||||||
);
|
$blockSize,
|
||||||
return $result;
|
$this->getAll7CharChunks($block_data),
|
||||||
|
$this->getAll7CharChunks($double_block_data)
|
||||||
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
public function get_all_7_char_chunks($hash)
|
private function getAll7CharChunks($hash)
|
||||||
{
|
{
|
||||||
$results = array();
|
$results = array();
|
||||||
for ($i = 0; $i < strlen($hash) - 6; $i++) {
|
for ($i = 0; $i < strlen($hash) - 6; $i++) {
|
||||||
|
@ -56,16 +50,22 @@ class FuzzyCorrelateSsdeep extends AppModel
|
||||||
return $results;
|
return $results;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $hash
|
||||||
|
* @param int $attributeId
|
||||||
|
* @return array
|
||||||
|
*/
|
||||||
public function query_ssdeep_chunks($hash, $attributeId)
|
public function query_ssdeep_chunks($hash, $attributeId)
|
||||||
{
|
{
|
||||||
$chunks = $this->ssdeep_prepare($hash);
|
$chunks = $this->ssdeep_prepare($hash);
|
||||||
|
$bothPartChunks = array_merge($chunks[1], $chunks[2]);
|
||||||
|
|
||||||
// Original algo from article https://www.virusbulletin.com/virusbulletin/2015/11/optimizing-ssdeep-use-scale
|
// Original algo from article https://www.virusbulletin.com/virusbulletin/2015/11/optimizing-ssdeep-use-scale
|
||||||
// also propose to insert chunk size to database, but current database schema doesn't contain that column.
|
// also propose to insert chunk size to database, but current database schema doesn't contain that column.
|
||||||
// This optimisation can be add in future versions.
|
// This optimisation can be add in future versions.
|
||||||
$result = $this->find('column', array(
|
$result = $this->find('column', array(
|
||||||
'conditions' => array(
|
'conditions' => array(
|
||||||
'FuzzyCorrelateSsdeep.chunk' => array_merge($chunks[1], $chunks[2]),
|
'FuzzyCorrelateSsdeep.chunk' => $bothPartChunks,
|
||||||
),
|
),
|
||||||
'fields' => array('FuzzyCorrelateSsdeep.attribute_id'),
|
'fields' => array('FuzzyCorrelateSsdeep.attribute_id'),
|
||||||
'unique' => true,
|
'unique' => true,
|
||||||
|
@ -73,15 +73,11 @@ class FuzzyCorrelateSsdeep extends AppModel
|
||||||
|
|
||||||
$toSave = [];
|
$toSave = [];
|
||||||
$attributeId = (int) $attributeId;
|
$attributeId = (int) $attributeId;
|
||||||
foreach (array(1, 2) as $type) {
|
foreach ($bothPartChunks as $chunk) {
|
||||||
foreach ($chunks[$type] as $chunk) {
|
|
||||||
$toSave[] = [$attributeId, $chunk];
|
$toSave[] = [$attributeId, $chunk];
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if (!empty($toSave)) {
|
|
||||||
$db = $this->getDataSource();
|
$db = $this->getDataSource();
|
||||||
$db->insertMulti($this->table, ['attribute_id', 'chunk'], $toSave);
|
$db->insertMulti($this->table, ['attribute_id', 'chunk'], $toSave);
|
||||||
}
|
|
||||||
return $result;
|
return $result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -264,7 +264,7 @@ class Galaxy extends AppModel
|
||||||
$fields = array('galaxy_cluster_id', 'key', 'value');
|
$fields = array('galaxy_cluster_id', 'key', 'value');
|
||||||
$db->insertMulti('galaxy_elements', $fields, $elements);
|
$db->insertMulti('galaxy_elements', $fields, $elements);
|
||||||
}
|
}
|
||||||
$allRelations = array_merge($allRelations, $relations);
|
array_push($allRelations, ...$relations);
|
||||||
}
|
}
|
||||||
// Save relation as last part when all clusters are created
|
// Save relation as last part when all clusters are created
|
||||||
if (!empty($allRelations)) {
|
if (!empty($allRelations)) {
|
||||||
|
@ -287,24 +287,42 @@ class Galaxy extends AppModel
|
||||||
if (empty($galaxy['uuid'])) {
|
if (empty($galaxy['uuid'])) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
$existingGalaxy = $this->find('first', array(
|
|
||||||
|
$existingGalaxy = $this->find('first', [
|
||||||
'recursive' => -1,
|
'recursive' => -1,
|
||||||
'conditions' => array('Galaxy.uuid' => $galaxy['uuid'])
|
'conditions' => ['Galaxy.uuid' => $galaxy['uuid']],
|
||||||
));
|
]);
|
||||||
if (empty($existingGalaxy)) {
|
|
||||||
if ($user['Role']['perm_site_admin'] || $user['Role']['perm_galaxy_editor']) {
|
|
||||||
$this->create();
|
|
||||||
unset($galaxy['id']);
|
unset($galaxy['id']);
|
||||||
$this->save($galaxy);
|
if (!empty($existingGalaxy)) {
|
||||||
$existingGalaxy = $this->find('first', array(
|
// check if provided galaxy has the same fields as galaxy that are saved in database
|
||||||
'recursive' => -1,
|
$fieldsToSave = [];
|
||||||
'conditions' => array('Galaxy.id' => $this->id)
|
foreach (array_keys(array_intersect_key($existingGalaxy, $galaxy)) as $key) {
|
||||||
));
|
if ($existingGalaxy['Galaxy'][$key] != $galaxy[$key]) {
|
||||||
|
$fieldsToSave[$key] = $galaxy[$key];
|
||||||
|
}
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
return false;
|
$fieldsToSave = $galaxy;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (empty($fieldsToSave) && !empty($existingGalaxy)) {
|
||||||
|
return $existingGalaxy; // galaxy already exists and galaxy fields are the same
|
||||||
}
|
}
|
||||||
return $existingGalaxy;
|
|
||||||
|
if (!$user['Role']['perm_site_admin'] && !$user['Role']['perm_galaxy_editor']) {
|
||||||
|
return false; // user has no permission to modify galaxy
|
||||||
|
}
|
||||||
|
|
||||||
|
if (empty($existingGalaxy)) {
|
||||||
|
$this->create();
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->save($fieldsToSave);
|
||||||
|
return $this->find('first', [
|
||||||
|
'recursive' => -1,
|
||||||
|
'conditions' => ['Galaxy.id' => $this->id],
|
||||||
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -48,37 +48,6 @@ class GalaxyElement extends AppModel
|
||||||
$this->saveMany($tempElements);
|
$this->saveMany($tempElements);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function update($galaxy_id, $oldClusters, $newClusters)
|
|
||||||
{
|
|
||||||
$elementsToSave = array();
|
|
||||||
// Since we are dealing with flat files as the end all be all content, we are safe to just drop all of the old clusters and recreate them.
|
|
||||||
foreach ($oldClusters as $oldCluster) {
|
|
||||||
$this->deleteAll(array('GalaxyElement.galaxy_cluster_id' => $oldCluster['GalaxyCluster']['id']));
|
|
||||||
}
|
|
||||||
foreach ($newClusters as $newCluster) {
|
|
||||||
$tempCluster = array();
|
|
||||||
foreach ($newCluster as $key => $value) {
|
|
||||||
// Don't store the reserved fields as elements
|
|
||||||
if ($key == 'description' || $key == 'value') {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (is_array($value)) {
|
|
||||||
foreach ($value as $arrayElement) {
|
|
||||||
$tempCluster[] = array('key' => $key, 'value' => $arrayElement);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
$tempCluster[] = array('key' => $key, 'value' => $value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach ($tempCluster as $key => $value) {
|
|
||||||
$tempCluster[$key]['galaxy_cluster_id'] = $oldCluster['GalaxyCluster']['id'];
|
|
||||||
}
|
|
||||||
$elementsToSave = array_merge($elementsToSave, $tempCluster);
|
|
||||||
}
|
|
||||||
$this->saveMany($elementsToSave);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function captureElements($user, $elements, $clusterId)
|
public function captureElements($user, $elements, $clusterId)
|
||||||
{
|
{
|
||||||
$tempElements = array();
|
$tempElements = array();
|
||||||
|
|
|
@ -153,6 +153,8 @@ class MispObject extends AppModel
|
||||||
'object_name' => array('function' => 'set_filter_object_name'),
|
'object_name' => array('function' => 'set_filter_object_name'),
|
||||||
'object_template_uuid' => array('function' => 'set_filter_object_template_uuid'),
|
'object_template_uuid' => array('function' => 'set_filter_object_template_uuid'),
|
||||||
'object_template_version' => array('function' => 'set_filter_object_template_version'),
|
'object_template_version' => array('function' => 'set_filter_object_template_version'),
|
||||||
|
'first_seen' => array('function' => 'set_filter_seen'),
|
||||||
|
'last_seen' => array('function' => 'set_filter_seen'),
|
||||||
'deleted' => array('function' => 'set_filter_deleted')
|
'deleted' => array('function' => 'set_filter_deleted')
|
||||||
),
|
),
|
||||||
'Event' => array(
|
'Event' => array(
|
||||||
|
@ -181,8 +183,8 @@ class MispObject extends AppModel
|
||||||
'deleted' => array('function' => 'set_filter_deleted'),
|
'deleted' => array('function' => 'set_filter_deleted'),
|
||||||
'timestamp' => array('function' => 'set_filter_timestamp'),
|
'timestamp' => array('function' => 'set_filter_timestamp'),
|
||||||
'attribute_timestamp' => array('function' => 'set_filter_timestamp'),
|
'attribute_timestamp' => array('function' => 'set_filter_timestamp'),
|
||||||
'first_seen' => array('function' => 'set_filter_seen'),
|
//'first_seen' => array('function' => 'set_filter_seen'),
|
||||||
'last_seen' => array('function' => 'set_filter_seen'),
|
//'last_seen' => array('function' => 'set_filter_seen'),
|
||||||
'to_ids' => array('function' => 'set_filter_to_ids'),
|
'to_ids' => array('function' => 'set_filter_to_ids'),
|
||||||
'comment' => array('function' => 'set_filter_comment')
|
'comment' => array('function' => 'set_filter_comment')
|
||||||
)
|
)
|
||||||
|
@ -1678,7 +1680,9 @@ class MispObject extends AppModel
|
||||||
$results = $this->Sightingdb->attachToObjects($results, $user);
|
$results = $this->Sightingdb->attachToObjects($results, $user);
|
||||||
}
|
}
|
||||||
$params['page'] += 1;
|
$params['page'] += 1;
|
||||||
$results = $this->Allowedlist->removeAllowedlistedFromArray($results, true);
|
foreach ($results as $k => $result) {
|
||||||
|
$results[$k]['Attribute'] = $this->Allowedlist->removeAllowedlistedFromArray($result['Attribute'], true);
|
||||||
|
}
|
||||||
$results = array_values($results);
|
$results = array_values($results);
|
||||||
$i = 0;
|
$i = 0;
|
||||||
foreach ($results as $object) {
|
foreach ($results as $object) {
|
||||||
|
|
|
@ -50,6 +50,8 @@ class Module extends AppModel
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
private $httpSocket = [];
|
||||||
|
|
||||||
public function validateIPField($value)
|
public function validateIPField($value)
|
||||||
{
|
{
|
||||||
if (!filter_var($value, FILTER_VALIDATE_IP) === false) {
|
if (!filter_var($value, FILTER_VALIDATE_IP) === false) {
|
||||||
|
@ -309,16 +311,9 @@ class Module extends AppModel
|
||||||
if (!$serverUrl) {
|
if (!$serverUrl) {
|
||||||
throw new Exception("Module type $moduleFamily is not enabled.");
|
throw new Exception("Module type $moduleFamily is not enabled.");
|
||||||
}
|
}
|
||||||
App::uses('HttpSocketExtended', 'Tools');
|
|
||||||
$httpSocketSetting = ['timeout' => $timeout];
|
$httpSocket = $this->initHttpSocket($moduleFamily, $timeout);
|
||||||
$sslSettings = array('ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_verify_peer', 'ssl_cafile');
|
|
||||||
foreach ($sslSettings as $sslSetting) {
|
|
||||||
$value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
|
|
||||||
if ($value && $value !== '') {
|
|
||||||
$httpSocketSetting[$sslSetting] = $value;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$httpSocket = new HttpSocketExtended($httpSocketSetting);
|
|
||||||
$request = [];
|
$request = [];
|
||||||
if ($moduleFamily === 'Cortex') {
|
if ($moduleFamily === 'Cortex') {
|
||||||
if (!empty(Configure::read('Plugin.' . $moduleFamily . '_authkey'))) {
|
if (!empty(Configure::read('Plugin.' . $moduleFamily . '_authkey'))) {
|
||||||
|
@ -422,4 +417,37 @@ class Module extends AppModel
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $moduleFamily
|
||||||
|
* @param int $timeout
|
||||||
|
* @return HttpSocketExtended|CurlClient
|
||||||
|
*/
|
||||||
|
private function initHttpSocket($moduleFamily, $timeout)
|
||||||
|
{
|
||||||
|
$unique = "$moduleFamily:$timeout";
|
||||||
|
|
||||||
|
if (isset($this->httpSocket[$unique])) {
|
||||||
|
return $this->httpSocket[$unique];
|
||||||
|
}
|
||||||
|
|
||||||
|
$httpSocketSetting = ['timeout' => $timeout];
|
||||||
|
$sslSettings = ['ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_cafile'];
|
||||||
|
foreach ($sslSettings as $sslSetting) {
|
||||||
|
$value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
|
||||||
|
if ($value && $value !== '') {
|
||||||
|
$httpSocketSetting[$sslSetting] = $value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (function_exists('curl_init')) {
|
||||||
|
App::uses('CurlClient', 'Tools');
|
||||||
|
$httpSocket = new CurlClient($httpSocketSetting);
|
||||||
|
} else {
|
||||||
|
App::uses('HttpSocketExtended', 'Tools');
|
||||||
|
$httpSocket = new HttpSocketExtended($httpSocketSetting);
|
||||||
|
}
|
||||||
|
|
||||||
|
return $this->httpSocket[$unique] = $httpSocket;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -76,14 +76,28 @@ class Organisation extends AppModel
|
||||||
);
|
);
|
||||||
|
|
||||||
const ORGANISATION_ASSOCIATIONS = array(
|
const ORGANISATION_ASSOCIATIONS = array(
|
||||||
|
'AccessLog' => array('table' => 'access_logs', 'fields' => array('org_id')),
|
||||||
|
'AuditLog' => array('table' => 'audit_logs', 'fields' => array('org_id')),
|
||||||
'Correlation' => array('table' => 'correlations', 'fields' => array('org_id')),
|
'Correlation' => array('table' => 'correlations', 'fields' => array('org_id')),
|
||||||
|
'Cerebrate' => array('table' => 'cerebrates', 'fields' => array('org_id')),
|
||||||
|
'Dashboard' => array('table' => 'dashboards', 'fields' => array('restrict_to_org_id')),
|
||||||
'Event' => array('table' => 'events', 'fields' => array('org_id', 'orgc_id')),
|
'Event' => array('table' => 'events', 'fields' => array('org_id', 'orgc_id')),
|
||||||
|
'EventGraph' => array('table' => 'event_graph', 'fields' => array('org_id')),
|
||||||
|
'Feed' => array('table' => 'feeds', 'fields' => array('orgc_id')),
|
||||||
|
'GalaxyCluster' => array('table' => 'galaxy_clusters', 'fields' => array('org_id', 'orgc_id')),
|
||||||
|
'ObjectTemplate' => array('table' => 'object_templates', 'fields' => array('org_id')),
|
||||||
'Job' => array('table' => 'jobs', 'fields' => array('org_id')),
|
'Job' => array('table' => 'jobs', 'fields' => array('org_id')),
|
||||||
|
'RestClientHistory' => array('table' => 'rest_client_histories', 'fields' => array('org_id')),
|
||||||
'Server' => array('table' => 'servers', 'fields' => array('org_id', 'remote_org_id')),
|
'Server' => array('table' => 'servers', 'fields' => array('org_id', 'remote_org_id')),
|
||||||
'ShadowAttribute' => array('table' => 'shadow_attributes', 'fields' => array('org_id', 'event_org_id')),
|
'ShadowAttribute' => array('table' => 'shadow_attributes', 'fields' => array('org_id', 'event_org_id')),
|
||||||
'SharingGroup' => array('table' => 'sharing_groups', 'fields' => array('org_id')),
|
'SharingGroup' => array('table' => 'sharing_groups', 'fields' => array('org_id')),
|
||||||
'SharingGroupOrg' => array('table' => 'sharing_group_orgs', 'fields' => array('org_id')),
|
'SharingGroupOrg' => array('table' => 'sharing_group_orgs', 'fields' => array('org_id')),
|
||||||
|
'SharingGroupBlueprint' => array('table' => 'sharing_group_blueprints', 'fields' => array('org_id')),
|
||||||
|
'Sighting' => array('table' => 'sightings', 'fields' => array('org_id')),
|
||||||
|
'SightingdbOrg' => array('table' => 'sightingdb_orgs', 'fields' => array('org_id')),
|
||||||
'Thread' => array('table' => 'threads', 'fields' => array('org_id')),
|
'Thread' => array('table' => 'threads', 'fields' => array('org_id')),
|
||||||
|
'Tag' => array('table' => 'tags', 'fields' => array('org_id')),
|
||||||
|
'TagCollection' => array('table' => 'tag_collections', 'fields' => array('org_id')),
|
||||||
'User' => array('table' => 'users', 'fields' => array('org_id'))
|
'User' => array('table' => 'users', 'fields' => array('org_id'))
|
||||||
);
|
);
|
||||||
|
|
||||||
|
@ -287,6 +301,9 @@ class Organisation extends AppModel
|
||||||
public function orgMerge($id, $request, $user)
|
public function orgMerge($id, $request, $user)
|
||||||
{
|
{
|
||||||
$currentOrg = $this->find('first', array('recursive' => -1, 'conditions' => array('Organisation.id' => $id)));
|
$currentOrg = $this->find('first', array('recursive' => -1, 'conditions' => array('Organisation.id' => $id)));
|
||||||
|
if (isset($currentOrg['Organisation']['restricted_to_domain'])) {
|
||||||
|
$currentOrg['Organisation']['restricted_to_domain'] = json_encode($currentOrg['Organisation']['restricted_to_domain']);
|
||||||
|
}
|
||||||
$currentOrgUserCount = $this->User->find('count', array(
|
$currentOrgUserCount = $this->User->find('count', array(
|
||||||
'conditions' => array('User.org_id' => $id)
|
'conditions' => array('User.org_id' => $id)
|
||||||
));
|
));
|
||||||
|
|
|
@ -472,7 +472,21 @@ class Server extends AppModel
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, Event $eventModel, $server, $user, $jobId, $force = false, $headers = false, $body = false)
|
/**
|
||||||
|
* @param array $event
|
||||||
|
* @param int|string $eventId
|
||||||
|
* @param array $successes
|
||||||
|
* @param array $fails
|
||||||
|
* @param Event $eventModel
|
||||||
|
* @param array $server
|
||||||
|
* @param array $user
|
||||||
|
* @param int $jobId
|
||||||
|
* @param bool $force
|
||||||
|
* @param HttpSocketResponseExtended $response
|
||||||
|
* @return false|void
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, Event $eventModel, $server, $user, $jobId, $force = false, $response)
|
||||||
{
|
{
|
||||||
// check if the event already exist (using the uuid)
|
// check if the event already exist (using the uuid)
|
||||||
$existingEvent = $eventModel->find('first', [
|
$existingEvent = $eventModel->find('first', [
|
||||||
|
@ -485,7 +499,7 @@ class Server extends AppModel
|
||||||
if (!$existingEvent) {
|
if (!$existingEvent) {
|
||||||
// add data for newly imported events
|
// add data for newly imported events
|
||||||
if (isset($event['Event']['protected']) && $event['Event']['protected']) {
|
if (isset($event['Event']['protected']) && $event['Event']['protected']) {
|
||||||
if (!$eventModel->CryptographicKey->validateProtectedEvent($body, $user, $headers['x-pgp-signature'], $event)) {
|
if (!$eventModel->CryptographicKey->validateProtectedEvent($response->body, $user, $response->getHeader('x-pgp-signature'), $event)) {
|
||||||
$fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.');
|
$fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.');
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
@ -505,7 +519,7 @@ class Server extends AppModel
|
||||||
$fails[$eventId] = __('Blocked an edit to an event that was created locally. This can happen if a synchronised event that was created on this instance was modified by an administrator on the remote side.');
|
$fails[$eventId] = __('Blocked an edit to an event that was created locally. This can happen if a synchronised event that was created on this instance was modified by an administrator on the remote side.');
|
||||||
} else {
|
} else {
|
||||||
if ($existingEvent['Event']['protected']) {
|
if ($existingEvent['Event']['protected']) {
|
||||||
if (!$eventModel->CryptographicKey->validateProtectedEvent($body, $user, $headers['x-pgp-signature'], $existingEvent)) {
|
if (!$eventModel->CryptographicKey->validateProtectedEvent($response->body, $user, $response->getHeader('x-pgp-signature'), $existingEvent)) {
|
||||||
$fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.');
|
$fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -549,12 +563,10 @@ class Server extends AppModel
|
||||||
$params['excludeLocalTags'] = 1;
|
$params['excludeLocalTags'] = 1;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
$event = $serverSync->fetchEvent($eventId, $params);
|
$response = $serverSync->fetchEvent($eventId, $params);
|
||||||
$headers = $event->headers;
|
$event = $response->json();
|
||||||
$body = $event->body;
|
|
||||||
$event = $event->json();
|
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
$this->logException("Failed downloading the event $eventId from remote server {$serverSync->serverId()}", $e);
|
$this->logException("Failed to download the event $eventId from remote server {$serverSync->serverId()} '{$serverSync->serverName()}'", $e);
|
||||||
$fails[$eventId] = __('failed downloading the event');
|
$fails[$eventId] = __('failed downloading the event');
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
@ -568,7 +580,7 @@ class Server extends AppModel
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $headers, $body);
|
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $response);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2359,23 +2371,21 @@ class Server extends AppModel
|
||||||
return $setting;
|
return $setting;
|
||||||
}
|
}
|
||||||
|
|
||||||
public function serverSettingsEditValue(array $user, array $setting, $value, $forceSave = false)
|
/**
|
||||||
|
* @param array|string $user
|
||||||
|
* @param array $setting
|
||||||
|
* @param mixed $value
|
||||||
|
* @param bool $forceSave
|
||||||
|
* @return mixed|string|true|null
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
public function serverSettingsEditValue($user, array $setting, $value, $forceSave = false)
|
||||||
{
|
{
|
||||||
if (isset($setting['beforeHook'])) {
|
if (isset($setting['beforeHook'])) {
|
||||||
$beforeResult = call_user_func_array(array($this, $setting['beforeHook']), array($setting['name'], $value));
|
$beforeResult = $this->{$setting['beforeHook']}($setting['name'], $value);
|
||||||
if ($beforeResult !== true) {
|
if ($beforeResult !== true) {
|
||||||
$this->Log = ClassRegistry::init('Log');
|
$change = 'There was an issue witch changing ' . $setting['name'] . ' to ' . $value . '. The error message returned is: ' . $beforeResult . 'No changes were made.';
|
||||||
$this->Log->create();
|
$this->loadLog()->createLogEntry($user, 'serverSettingsEdit', 'Server', 0, 'Server setting issue', $change);
|
||||||
$this->Log->saveOrFailSilently(array(
|
|
||||||
'org' => $user['Organisation']['name'],
|
|
||||||
'model' => 'Server',
|
|
||||||
'model_id' => 0,
|
|
||||||
'email' => $user['email'],
|
|
||||||
'action' => 'serverSettingsEdit',
|
|
||||||
'user_id' => $user['id'],
|
|
||||||
'title' => 'Server setting issue',
|
|
||||||
'change' => 'There was an issue witch changing ' . $setting['name'] . ' to ' . $value . '. The error message returned is: ' . $beforeResult . 'No changes were made.',
|
|
||||||
));
|
|
||||||
return $beforeResult;
|
return $beforeResult;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -2384,7 +2394,7 @@ class Server extends AppModel
|
||||||
if ($setting['type'] === 'boolean') {
|
if ($setting['type'] === 'boolean') {
|
||||||
$value = (bool)$value;
|
$value = (bool)$value;
|
||||||
} else if ($setting['type'] === 'numeric') {
|
} else if ($setting['type'] === 'numeric') {
|
||||||
$value = (int)($value);
|
$value = (int)$value;
|
||||||
}
|
}
|
||||||
if (isset($setting['test'])) {
|
if (isset($setting['test'])) {
|
||||||
if ($setting['test'] instanceof Closure) {
|
if ($setting['test'] instanceof Closure) {
|
||||||
|
@ -2425,7 +2435,7 @@ class Server extends AppModel
|
||||||
if ($setting['afterHook'] instanceof Closure) {
|
if ($setting['afterHook'] instanceof Closure) {
|
||||||
$afterResult = $setting['afterHook']($setting['name'], $value, $oldValue);
|
$afterResult = $setting['afterHook']($setting['name'], $value, $oldValue);
|
||||||
} else {
|
} else {
|
||||||
$afterResult = call_user_func_array(array($this, $setting['afterHook']), array($setting['name'], $value, $oldValue));
|
$afterResult = $this->{$setting['afterHook']}($setting['name'], $value, $oldValue);
|
||||||
}
|
}
|
||||||
if ($afterResult !== true) {
|
if ($afterResult !== true) {
|
||||||
$change = 'There was an issue after setting a new setting. The error message returned is: ' . $afterResult;
|
$change = 'There was an issue after setting a new setting. The error message returned is: ' . $afterResult;
|
||||||
|
@ -2434,9 +2444,8 @@ class Server extends AppModel
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
} else {
|
|
||||||
return __('Something went wrong. MISP tried to save a malformed config file. Setting change reverted.');
|
|
||||||
}
|
}
|
||||||
|
return __('Something went wrong. MISP tried to save a malformed config file or you dont have permission to write to config file. Setting change reverted.');
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -2539,10 +2548,10 @@ class Server extends AppModel
|
||||||
'name' => __('Organisation logos'),
|
'name' => __('Organisation logos'),
|
||||||
'description' => __('The logo used by an organisation on the event index, event view, discussions, proposals, etc. Make sure that the filename is in the org.png format, where org is the case-sensitive organisation name.'),
|
'description' => __('The logo used by an organisation on the event index, event view, discussions, proposals, etc. Make sure that the filename is in the org.png format, where org is the case-sensitive organisation name.'),
|
||||||
'expected' => array(),
|
'expected' => array(),
|
||||||
'valid_format' => __('48x48 pixel .png files'),
|
'valid_format' => __('48x48 pixel .png files or .svg file'),
|
||||||
'path' => APP . 'webroot' . DS . 'img' . DS . 'orgs',
|
'path' => APP . 'webroot' . DS . 'img' . DS . 'orgs',
|
||||||
'regex' => '.*\.(png|PNG)$',
|
'regex' => '.*\.(png|svg)$',
|
||||||
'regex_error' => __('Filename must be in the following format: *.png'),
|
'regex_error' => __('Filename must be in the following format: *.png or *.svg'),
|
||||||
'files' => array(),
|
'files' => array(),
|
||||||
),
|
),
|
||||||
'img' => array(
|
'img' => array(
|
||||||
|
@ -2578,6 +2587,7 @@ class Server extends AppModel
|
||||||
'read' => $f->isReadable(),
|
'read' => $f->isReadable(),
|
||||||
'write' => $f->isWritable(),
|
'write' => $f->isWritable(),
|
||||||
'execute' => $f->isExecutable(),
|
'execute' => $f->isExecutable(),
|
||||||
|
'link' => $f->isLink(),
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -4155,12 +4165,13 @@ class Server extends AppModel
|
||||||
private function checkRemoteVersion($HttpSocket)
|
private function checkRemoteVersion($HttpSocket)
|
||||||
{
|
{
|
||||||
try {
|
try {
|
||||||
$json_decoded_tags = GitTool::getLatestTags($HttpSocket);
|
$tags = GitTool::getLatestTags($HttpSocket);
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
|
$this->logException('Could not retrieve latest tags from GitHub', $e, LOG_NOTICE);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
// find the latest version tag in the v[major].[minor].[hotfix] format
|
// find the latest version tag in the v[major].[minor].[hotfix] format
|
||||||
foreach ($json_decoded_tags as $tag) {
|
foreach ($tags as $tag) {
|
||||||
if (preg_match('/^v[0-9]+\.[0-9]+\.[0-9]+$/', $tag['name'])) {
|
if (preg_match('/^v[0-9]+\.[0-9]+\.[0-9]+$/', $tag['name'])) {
|
||||||
return $this->checkVersion($tag['name']);
|
return $this->checkVersion($tag['name']);
|
||||||
}
|
}
|
||||||
|
@ -4182,7 +4193,7 @@ class Server extends AppModel
|
||||||
try {
|
try {
|
||||||
$latestCommit = GitTool::getLatestCommit($HttpSocket);
|
$latestCommit = GitTool::getLatestCommit($HttpSocket);
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
$latestCommit = false;
|
$this->logException('Could not retrieve version from GitHub', $e, LOG_NOTICE);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -4202,6 +4213,7 @@ class Server extends AppModel
|
||||||
try {
|
try {
|
||||||
return GitTool::currentBranch();
|
return GitTool::currentBranch();
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
|
$this->logException('Could not retrieve current Git branch', $e, LOG_NOTICE);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -4252,38 +4264,38 @@ class Server extends AppModel
|
||||||
'app/files/scripts/misp-opendata',
|
'app/files/scripts/misp-opendata',
|
||||||
'app/files/scripts/python-maec',
|
'app/files/scripts/python-maec',
|
||||||
'app/files/scripts/python-stix',
|
'app/files/scripts/python-stix',
|
||||||
|
|
||||||
);
|
);
|
||||||
return in_array($submodule, $accepted_submodules_names, true);
|
return in_array($submodule, $accepted_submodules_names, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param string $submodule_name
|
* @param string $submoduleName
|
||||||
* @param string $superproject_submodule_commit_id
|
* @param string $superprojectSubmoduleCommitId
|
||||||
* @return array
|
* @return array
|
||||||
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
private function getSubmoduleGitStatus($submodule_name, $superproject_submodule_commit_id)
|
private function getSubmoduleGitStatus($submoduleName, $superprojectSubmoduleCommitId)
|
||||||
{
|
{
|
||||||
$path = APP . '../' . $submodule_name;
|
$path = APP . '../' . $submoduleName;
|
||||||
$submodule_name = (strpos($submodule_name, '/') >= 0 ? explode('/', $submodule_name) : $submodule_name);
|
$submoduleName = (strpos($submoduleName, '/') >= 0 ? explode('/', $submoduleName) : $submoduleName);
|
||||||
$submodule_name = end($submodule_name);
|
$submoduleName = end($submoduleName);
|
||||||
|
|
||||||
$submoduleCurrentCommitId = GitTool::submoduleCurrentCommit($path);
|
$submoduleCurrentCommitId = GitTool::currentCommit($path);
|
||||||
|
|
||||||
$currentTimestamp = GitTool::commitTimestamp($submoduleCurrentCommitId, $path);
|
$currentTimestamp = GitTool::commitTimestamp($submoduleCurrentCommitId, $path);
|
||||||
if ($submoduleCurrentCommitId !== $superproject_submodule_commit_id) {
|
if ($submoduleCurrentCommitId !== $superprojectSubmoduleCommitId) {
|
||||||
$remoteTimestamp = GitTool::commitTimestamp($superproject_submodule_commit_id, $path);
|
$remoteTimestamp = GitTool::commitTimestamp($superprojectSubmoduleCommitId, $path);
|
||||||
} else {
|
} else {
|
||||||
$remoteTimestamp = $currentTimestamp;
|
$remoteTimestamp = $currentTimestamp;
|
||||||
}
|
}
|
||||||
|
|
||||||
$status = array(
|
$status = array(
|
||||||
'moduleName' => $submodule_name,
|
'moduleName' => $submoduleName,
|
||||||
'current' => $submoduleCurrentCommitId,
|
'current' => $submoduleCurrentCommitId,
|
||||||
'currentTimestamp' => $currentTimestamp,
|
'currentTimestamp' => $currentTimestamp,
|
||||||
'remote' => $superproject_submodule_commit_id,
|
'remote' => $superprojectSubmoduleCommitId,
|
||||||
'remoteTimestamp' => $remoteTimestamp,
|
'remoteTimestamp' => $remoteTimestamp,
|
||||||
'upToDate' => '',
|
'upToDate' => 'error',
|
||||||
'isReadable' => is_readable($path) && is_readable($path . '/.git'),
|
'isReadable' => is_readable($path) && is_readable($path . '/.git'),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
@ -4295,15 +4307,11 @@ class Server extends AppModel
|
||||||
} else {
|
} else {
|
||||||
$status['upToDate'] = 'younger';
|
$status['upToDate'] = 'younger';
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
$status['upToDate'] = 'error';
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($status['isReadable'] && !empty($status['remoteTimestamp']) && !empty($status['currentTimestamp'])) {
|
if ($status['isReadable'] && !empty($status['remoteTimestamp']) && !empty($status['currentTimestamp'])) {
|
||||||
$date1 = new DateTime();
|
$date1 = new DateTime("@{$status['remoteTimestamp']}");
|
||||||
$date1->setTimestamp($status['remoteTimestamp']);
|
$date2 = new DateTime("@{$status['currentTimestamp']}");
|
||||||
$date2 = new DateTime();
|
|
||||||
$date2->setTimestamp($status['currentTimestamp']);
|
|
||||||
$status['timeDiff'] = $date1->diff($date2);
|
$status['timeDiff'] = $date1->diff($date2);
|
||||||
} else {
|
} else {
|
||||||
$status['upToDate'] = 'error';
|
$status['upToDate'] = 'error';
|
||||||
|
@ -4793,11 +4801,11 @@ class Server extends AppModel
|
||||||
|
|
||||||
$results = [
|
$results = [
|
||||||
__('User') => $user['User']['email'],
|
__('User') => $user['User']['email'],
|
||||||
__('Role name') => isset($user['Role']['name']) ? $user['Role']['name'] : __('Unknown, outdated instance'),
|
__('Role name') => $user['Role']['name'] ?? __('Unknown, outdated instance'),
|
||||||
__('Sync flag') => isset($user['Role']['perm_sync']) ? ($user['Role']['perm_sync'] ? __('Yes') : __('No')) : __('Unknown, outdated instance'),
|
__('Sync flag') => isset($user['Role']['perm_sync']) ? ($user['Role']['perm_sync'] ? __('Yes') : __('No')) : __('Unknown, outdated instance'),
|
||||||
];
|
];
|
||||||
if (isset($response->headers['X-Auth-Key-Expiration'])) {
|
if ($response->getHeader('X-Auth-Key-Expiration')) {
|
||||||
$date = new DateTime($response->headers['X-Auth-Key-Expiration']);
|
$date = new DateTime($response->getHeader('X-Auth-Key-Expiration'));
|
||||||
$results[__('Auth key expiration')] = $date->format('Y-m-d H:i:s');
|
$results[__('Auth key expiration')] = $date->format('Y-m-d H:i:s');
|
||||||
}
|
}
|
||||||
return $results;
|
return $results;
|
||||||
|
@ -4935,6 +4943,28 @@ class Server extends AppModel
|
||||||
return $this->saveMany($toSave, ['validate' => false, 'fields' => ['authkey']]);
|
return $this->saveMany($toSave, ['validate' => false, 'fields' => ['authkey']]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $encryptionKey
|
||||||
|
* @return bool
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
public function isEncryptionKeyValid($encryptionKey)
|
||||||
|
{
|
||||||
|
$servers = $this->find('list', [
|
||||||
|
'fields' => ['Server.id', 'Server.authkey'],
|
||||||
|
]);
|
||||||
|
foreach ($servers as $id => $authkey) {
|
||||||
|
if (EncryptedValue::isEncrypted($authkey)) {
|
||||||
|
try {
|
||||||
|
BetterSecurity::decrypt(substr($authkey, 2), $encryptionKey);
|
||||||
|
} catch (Exception $e) {
|
||||||
|
throw new Exception("Could not decrypt auth key for server #$id", 0, $e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Return all Attribute and Object types
|
* Return all Attribute and Object types
|
||||||
*/
|
*/
|
||||||
|
@ -6143,6 +6173,14 @@ class Server extends AppModel
|
||||||
'type' => 'boolean',
|
'type' => 'boolean',
|
||||||
'null' => true,
|
'null' => true,
|
||||||
],
|
],
|
||||||
|
'block_publishing_for_same_creator' => [
|
||||||
|
'level' => self::SETTING_OPTIONAL,
|
||||||
|
'description' => __('Enabling this setting will make MISP block event publishing in the case of the publisher being the same user as the event creator.'),
|
||||||
|
'value' => false,
|
||||||
|
'test' => 'testBool',
|
||||||
|
'type' => 'boolean',
|
||||||
|
'null' => true,
|
||||||
|
],
|
||||||
'self_update' => [
|
'self_update' => [
|
||||||
'level' => self::SETTING_CRITICAL,
|
'level' => self::SETTING_CRITICAL,
|
||||||
'description' => __('Enable the GUI button for MISP self-update on the Diagnostics page.'),
|
'description' => __('Enable the GUI button for MISP self-update on the Diagnostics page.'),
|
||||||
|
|
|
@ -1017,16 +1017,16 @@ class Sighting extends AppModel
|
||||||
* @return TmpFileTool
|
* @return TmpFileTool
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
public function restSearch(array $user, $returnFormat, $filters)
|
public function restSearch(array $user, $returnFormat, array $filters)
|
||||||
{
|
{
|
||||||
$allowedContext = array('event', 'attribute');
|
$allowedContext = array('event', 'attribute');
|
||||||
// validate context
|
// validate context
|
||||||
if (isset($filters['context']) && !in_array($filters['context'], $allowedContext, true)) {
|
if (isset($filters['context']) && !in_array($filters['context'], $allowedContext, true)) {
|
||||||
throw new MethodNotAllowedException(__('Invalid context %s.', $filters['context']));
|
throw new BadRequestException(__('Invalid context %s.', $filters['context']));
|
||||||
}
|
}
|
||||||
// ensure that an id or uuid is provided if context is set
|
// ensure that an id or uuid is provided if context is set
|
||||||
if (!empty($filters['context']) && !(isset($filters['id']) || isset($filters['uuid'])) ) {
|
if (!empty($filters['context']) && !(isset($filters['id']) || isset($filters['uuid'])) ) {
|
||||||
throw new MethodNotAllowedException(__('An ID or UUID must be provided if the context is set.'));
|
throw new BadRequestException(__('An ID or UUID must be provided if the context is set.'));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!isset($this->validFormats[$returnFormat][1])) {
|
if (!isset($this->validFormats[$returnFormat][1])) {
|
||||||
|
@ -1396,7 +1396,7 @@ class Sighting extends AppModel
|
||||||
try {
|
try {
|
||||||
$sightings = $serverSync->fetchSightingsForEvents($chunk);
|
$sightings = $serverSync->fetchSightingsForEvents($chunk);
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
$this->logException("Failed downloading the sightings from {$serverSync->server()['Server']['name']}.", $e);
|
$this->logException("Failed to download sightings from {$serverSync->server()['Server']['name']}.", $e);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -46,7 +46,7 @@ class SystemSetting extends AppModel
|
||||||
{
|
{
|
||||||
/** @var self $systemSetting */
|
/** @var self $systemSetting */
|
||||||
$systemSetting = ClassRegistry::init('SystemSetting');
|
$systemSetting = ClassRegistry::init('SystemSetting');
|
||||||
if (!$systemSetting->databaseExists()) {
|
if (!$systemSetting->tableExists()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
$settings = $systemSetting->getSettings();
|
$settings = $systemSetting->getSettings();
|
||||||
|
@ -58,7 +58,7 @@ class SystemSetting extends AppModel
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public function databaseExists()
|
private function tableExists()
|
||||||
{
|
{
|
||||||
$tables = ConnectionManager::getDataSource($this->useDbConfig)->listSources();
|
$tables = ConnectionManager::getDataSource($this->useDbConfig)->listSources();
|
||||||
return in_array('system_settings', $tables, true);
|
return in_array('system_settings', $tables, true);
|
||||||
|
@ -154,6 +154,32 @@ class SystemSetting extends AppModel
|
||||||
return $this->saveMany($toSave);
|
return $this->saveMany($toSave);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if provided encryption key is valid for all encrypted settings
|
||||||
|
* @param string $encryptionKey
|
||||||
|
* @return bool
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
public function isEncryptionKeyValid($encryptionKey)
|
||||||
|
{
|
||||||
|
$settings = $this->find('list', [
|
||||||
|
'fields' => ['SystemSetting.setting', 'SystemSetting.value'],
|
||||||
|
]);
|
||||||
|
foreach ($settings as $setting => $value) {
|
||||||
|
if (!self::isSensitive($setting)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (EncryptedValue::isEncrypted($value)) {
|
||||||
|
try {
|
||||||
|
BetterSecurity::decrypt(substr($value, 2), $encryptionKey);
|
||||||
|
} catch (Exception $e) {
|
||||||
|
throw new Exception("Could not decrypt `$setting` setting.", 0, $e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Sensitive setting are passwords or api keys.
|
* Sensitive setting are passwords or api keys.
|
||||||
* @param string $setting Setting name
|
* @param string $setting Setting name
|
||||||
|
|
|
@ -659,21 +659,18 @@ class User extends AppModel
|
||||||
public function getUserById($id)
|
public function getUserById($id)
|
||||||
{
|
{
|
||||||
if (empty($id)) {
|
if (empty($id)) {
|
||||||
throw new NotFoundException('Invalid user ID.');
|
throw new InvalidArgumentException('Invalid user ID.');
|
||||||
}
|
}
|
||||||
return $this->find(
|
return $this->find('first', [
|
||||||
'first',
|
'conditions' => ['User.id' => $id],
|
||||||
array(
|
|
||||||
'conditions' => array('User.id' => $id),
|
|
||||||
'recursive' => -1,
|
'recursive' => -1,
|
||||||
'contain' => array(
|
'contain' => [
|
||||||
'Organisation',
|
'Organisation',
|
||||||
'Role',
|
'Role',
|
||||||
'Server',
|
'Server',
|
||||||
'UserSetting',
|
'UserSetting',
|
||||||
)
|
]
|
||||||
)
|
]);
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -740,7 +737,7 @@ class User extends AppModel
|
||||||
],
|
],
|
||||||
]);
|
]);
|
||||||
if (empty($user)) {
|
if (empty($user)) {
|
||||||
return $user;
|
return null;
|
||||||
}
|
}
|
||||||
return $this->rearrangeToAuthForm($user);
|
return $this->rearrangeToAuthForm($user);
|
||||||
}
|
}
|
||||||
|
@ -861,6 +858,10 @@ class User extends AppModel
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!isset($user['User'])) {
|
||||||
|
throw new InvalidArgumentException("Invalid user model provided.");
|
||||||
|
}
|
||||||
|
|
||||||
if ($user['User']['disabled'] || !$this->checkIfUserIsValid($user['User'])) {
|
if ($user['User']['disabled'] || !$this->checkIfUserIsValid($user['User'])) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
@ -937,6 +938,11 @@ class User extends AppModel
|
||||||
*/
|
*/
|
||||||
public function describeAuthFields()
|
public function describeAuthFields()
|
||||||
{
|
{
|
||||||
|
static $fields; // generate array just once
|
||||||
|
if ($fields) {
|
||||||
|
return $fields;
|
||||||
|
}
|
||||||
|
|
||||||
$fields = $this->schema();
|
$fields = $this->schema();
|
||||||
// Do not include keys, because they are big and usually not necessary
|
// Do not include keys, because they are big and usually not necessary
|
||||||
unset($fields['gpgkey']);
|
unset($fields['gpgkey']);
|
||||||
|
@ -1105,13 +1111,18 @@ class User extends AppModel
|
||||||
return $hashed;
|
return $hashed;
|
||||||
}
|
}
|
||||||
|
|
||||||
public function createInitialUser($org_id)
|
/**
|
||||||
|
* @param int $orgId
|
||||||
|
* @return string User auth key
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
public function createInitialUser($orgId)
|
||||||
{
|
{
|
||||||
$authKey = $this->generateAuthKey();
|
$authKey = $this->generateAuthKey();
|
||||||
$admin = array('User' => array(
|
$admin = array('User' => array(
|
||||||
'id' => 1,
|
'id' => 1,
|
||||||
'email' => 'admin@admin.test',
|
'email' => 'admin@admin.test',
|
||||||
'org_id' => $org_id,
|
'org_id' => $orgId,
|
||||||
'password' => 'admin',
|
'password' => 'admin',
|
||||||
'confirm_password' => 'admin',
|
'confirm_password' => 'admin',
|
||||||
'authkey' => $authKey,
|
'authkey' => $authKey,
|
||||||
|
@ -1123,7 +1134,6 @@ class User extends AppModel
|
||||||
$this->validator()->remove('password'); // password is too simple, remove validation
|
$this->validator()->remove('password'); // password is too simple, remove validation
|
||||||
$this->save($admin);
|
$this->save($admin);
|
||||||
if (!empty(Configure::read("Security.advanced_authkeys"))) {
|
if (!empty(Configure::read("Security.advanced_authkeys"))) {
|
||||||
$this->AuthKey = ClassRegistry::init('AuthKey');
|
|
||||||
$newKey = [
|
$newKey = [
|
||||||
'authkey' => $authKey,
|
'authkey' => $authKey,
|
||||||
'user_id' => 1,
|
'user_id' => 1,
|
||||||
|
@ -2068,12 +2078,10 @@ class User extends AppModel
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
$cutoff = $redis->get('misp:session_destroy:' . $id);
|
list($cutoff, $allcutoff) = $redis->mGet(['misp:session_destroy:' . $id, 'misp:session_destroy:all']);
|
||||||
$allcutoff = $redis->get('misp:session_destroy:all');
|
|
||||||
if (
|
if (
|
||||||
empty($cutoff) ||
|
empty($cutoff) ||
|
||||||
(
|
(
|
||||||
!empty($cutoff) &&
|
|
||||||
!empty($allcutoff) &&
|
!empty($allcutoff) &&
|
||||||
$allcutoff < $cutoff
|
$allcutoff < $cutoff
|
||||||
)
|
)
|
||||||
|
@ -2156,7 +2164,7 @@ class User extends AppModel
|
||||||
if (!ctype_alnum($token)) {
|
if (!ctype_alnum($token)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
$redis = $this->setupRedis();
|
$redis = RedisTool::init();
|
||||||
$userId = $redis->get('misp:forgot:' . $token);
|
$userId = $redis->get('misp:forgot:' . $token);
|
||||||
if (empty($userId)) {
|
if (empty($userId)) {
|
||||||
return false;
|
return false;
|
||||||
|
@ -2167,8 +2175,78 @@ class User extends AppModel
|
||||||
|
|
||||||
public function purgeForgetToken($token)
|
public function purgeForgetToken($token)
|
||||||
{
|
{
|
||||||
$redis = $this->setupRedis();
|
$redis = RedisTool::init();
|
||||||
$userId = $redis->del('misp:forgot:' . $token);
|
$redis->del('misp:forgot:' . $token);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create default Role, Organisation and User
|
||||||
|
* @return string|null Created user auth key
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
public function init()
|
||||||
|
{
|
||||||
|
if (!$this->Role->hasAny()) {
|
||||||
|
$siteAdmin = ['Role' => [
|
||||||
|
'id' => 1,
|
||||||
|
'name' => 'Site Admin',
|
||||||
|
'permission' => 3,
|
||||||
|
'perm_add' => 1,
|
||||||
|
'perm_modify' => 1,
|
||||||
|
'perm_modify_org' => 1,
|
||||||
|
'perm_publish' => 1,
|
||||||
|
'perm_sync' => 1,
|
||||||
|
'perm_admin' => 1,
|
||||||
|
'perm_audit' => 1,
|
||||||
|
'perm_auth' => 1,
|
||||||
|
'perm_site_admin' => 1,
|
||||||
|
'perm_regexp_access' => 1,
|
||||||
|
'perm_sharing_group' => 1,
|
||||||
|
'perm_template' => 1,
|
||||||
|
'perm_tagger' => 1,
|
||||||
|
]];
|
||||||
|
$this->Role->save($siteAdmin);
|
||||||
|
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
|
||||||
|
if (!$this->isMysql()) {
|
||||||
|
$sql = "SELECT setval('roles_id_seq', (SELECT MAX(id) FROM roles));";
|
||||||
|
$this->Role->query($sql);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!$this->Organisation->hasAny(['Organisation.local' => true])) {
|
||||||
|
$this->runUpdates();
|
||||||
|
$org = ['Organisation' => [
|
||||||
|
'id' => 1,
|
||||||
|
'name' => !empty(Configure::read('MISP.org')) ? Configure::read('MISP.org') : 'ADMIN',
|
||||||
|
'description' => 'Automatically generated admin organisation',
|
||||||
|
'type' => 'ADMIN',
|
||||||
|
'date_created' => date('Y-m-d H:i:s'),
|
||||||
|
'local' => 1,
|
||||||
|
]];
|
||||||
|
$this->Organisation->save($org);
|
||||||
|
// PostgreSQL: update value of auto incremented serial primary key after setting the column by force
|
||||||
|
if (!$this->isMysql()) {
|
||||||
|
$sql = "SELECT setval('organisations_id_seq', (SELECT MAX(id) FROM organisations));";
|
||||||
|
$this->Organisation->query($sql);
|
||||||
|
}
|
||||||
|
$orgId = $this->Organisation->id;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!$this->hasAny()) {
|
||||||
|
if (!isset($orgId)) {
|
||||||
|
$hostOrg = $this->Organisation->find('first', array('conditions' => array('Organisation.name' => Configure::read('MISP.org'), 'Organisation.local' => true), 'recursive' => -1));
|
||||||
|
if (!empty($hostOrg)) {
|
||||||
|
$orgId = $hostOrg['Organisation']['id'];
|
||||||
|
} else {
|
||||||
|
$firstOrg = $this->Organisation->find('first', array('conditions' => array('Organisation.local' => true), 'order' => 'Organisation.id ASC'));
|
||||||
|
$orgId = $firstOrg['Organisation']['id'];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$this->runUpdates();
|
||||||
|
return $this->createInitialUser($orgId);
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -36,22 +36,55 @@ class UserLoginProfile extends AppModel
|
||||||
];
|
];
|
||||||
|
|
||||||
const BROWSER_CACHE_DIR = APP . DS . 'tmp' . DS . 'browscap';
|
const BROWSER_CACHE_DIR = APP . DS . 'tmp' . DS . 'browscap';
|
||||||
const BROWSER_INI_FILE = APP . DS . 'files' . DS . 'browscap'. DS . 'browscap.ini'; // Browscap file managed by MISP - https://browscap.org/stream?q=Lite_PHP_BrowsCapINI
|
const BROWSER_INI_FILE = APP . DS . 'files' . DS . 'browscap'. DS . 'browscap.ini.gz'; // Browscap file managed by MISP - https://browscap.org/stream?q=Lite_PHP_BrowsCapINI
|
||||||
const GEOIP_DB_FILE = APP . DS . 'files' . DS . 'geo-open' . DS . 'GeoOpen-Country.mmdb'; // GeoIP file managed by MISP - https://data.public.lu/en/datasets/geo-open-ip-address-geolocation-per-country-in-mmdb-format/
|
const GEOIP_DB_FILE = APP . DS . 'files' . DS . 'geo-open' . DS . 'GeoOpen-Country.mmdb'; // GeoIP file managed by MISP - https://data.public.lu/en/datasets/geo-open-ip-address-geolocation-per-country-in-mmdb-format/
|
||||||
|
|
||||||
private $userProfile;
|
private $userProfile;
|
||||||
|
|
||||||
private $knownUserProfiles = [];
|
private $knownUserProfiles = [];
|
||||||
|
|
||||||
private function _buildBrowscapCache()
|
private function browscapGetBrowser()
|
||||||
{
|
{
|
||||||
$this->log("Browscap - building new cache from browscap.ini file.", LOG_INFO);
|
$logger = new \Monolog\Logger('name');
|
||||||
|
|
||||||
|
if (function_exists('apcu_fetch')) {
|
||||||
|
App::uses('ApcuCacheTool', 'Tools');
|
||||||
|
$cache = new ApcuCacheTool('misp:browscap');
|
||||||
|
} else {
|
||||||
$fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR);
|
$fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR);
|
||||||
$cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache);
|
$cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache);
|
||||||
|
}
|
||||||
|
|
||||||
$logger = new \Monolog\Logger('name');
|
try {
|
||||||
$bc = new \BrowscapPHP\BrowscapUpdater($cache, $logger);
|
$bc = new \BrowscapPHP\Browscap($cache, $logger);
|
||||||
$bc->convertFile(UserLoginProfile::BROWSER_INI_FILE);
|
return $bc->getBrowser();
|
||||||
|
} catch (\BrowscapPHP\Exception $e) {
|
||||||
|
$this->log("Browscap - building new cache from browscap.ini file.", LOG_INFO);
|
||||||
|
$bcUpdater = new \BrowscapPHP\BrowscapUpdater($cache, $logger);
|
||||||
|
$bcUpdater->convertString(FileAccessTool::readCompressedFile(UserLoginProfile::BROWSER_INI_FILE));
|
||||||
|
}
|
||||||
|
|
||||||
|
$bc = new \BrowscapPHP\Browscap($cache, $logger);
|
||||||
|
return $bc->getBrowser();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $ip
|
||||||
|
* @return string|null
|
||||||
|
*/
|
||||||
|
public function countryByIp($ip)
|
||||||
|
{
|
||||||
|
if (class_exists('GeoIp2\Database\Reader')) {
|
||||||
|
$geoDbReader = new GeoIp2\Database\Reader(UserLoginProfile::GEOIP_DB_FILE);
|
||||||
|
try {
|
||||||
|
$record = $geoDbReader->country($ip);
|
||||||
|
return $record->country->isoCode;
|
||||||
|
} catch (InvalidArgumentException $e) {
|
||||||
|
$this->logException("Could not get country code for IP address", $e, LOG_NOTICE);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
public function beforeSave($options = [])
|
public function beforeSave($options = [])
|
||||||
|
@ -76,16 +109,7 @@ class UserLoginProfile extends AppModel
|
||||||
if (!$this->userProfile) {
|
if (!$this->userProfile) {
|
||||||
// below uses https://github.com/browscap/browscap-php
|
// below uses https://github.com/browscap/browscap-php
|
||||||
if (class_exists('\BrowscapPHP\Browscap')) {
|
if (class_exists('\BrowscapPHP\Browscap')) {
|
||||||
try {
|
$browser = $this->browscapGetBrowser();
|
||||||
$fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR);
|
|
||||||
$cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache);
|
|
||||||
$logger = new \Monolog\Logger('name');
|
|
||||||
$bc = new \BrowscapPHP\Browscap($cache, $logger);
|
|
||||||
$browser = $bc->getBrowser();
|
|
||||||
} catch (\BrowscapPHP\Exception $e) {
|
|
||||||
$this->_buildBrowscapCache();
|
|
||||||
return $this->_getUserProfile();
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
// a primitive OS & browser extraction capability
|
// a primitive OS & browser extraction capability
|
||||||
$ua = $_SERVER['HTTP_USER_AGENT'] ?? null;
|
$ua = $_SERVER['HTTP_USER_AGENT'] ?? null;
|
||||||
|
@ -100,18 +124,7 @@ class UserLoginProfile extends AppModel
|
||||||
$browser->browser = "browser";
|
$browser->browser = "browser";
|
||||||
}
|
}
|
||||||
$ip = $this->_remoteIp();
|
$ip = $this->_remoteIp();
|
||||||
if (class_exists('GeoIp2\Database\Reader')) {
|
$country = $this->countryByIp($ip) ?? 'None';
|
||||||
try {
|
|
||||||
$geoDbReader = new GeoIp2\Database\Reader(UserLoginProfile::GEOIP_DB_FILE);
|
|
||||||
$record = $geoDbReader->country($ip);
|
|
||||||
$country = $record->country->isoCode;
|
|
||||||
} catch (InvalidArgumentException $e) {
|
|
||||||
$this->logException("Could not get country code for IP address", $e);
|
|
||||||
$country = 'None';
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
$country = 'None';
|
|
||||||
}
|
|
||||||
$this->userProfile = [
|
$this->userProfile = [
|
||||||
'user_agent' => $_SERVER['HTTP_USER_AGENT'] ?? null,
|
'user_agent' => $_SERVER['HTTP_USER_AGENT'] ?? null,
|
||||||
'ip' => $ip,
|
'ip' => $ip,
|
||||||
|
@ -247,13 +260,13 @@ class UserLoginProfile extends AppModel
|
||||||
public function emailNewLogin(array $user)
|
public function emailNewLogin(array $user)
|
||||||
{
|
{
|
||||||
if (!Configure::read('MISP.disable_emailing')) {
|
if (!Configure::read('MISP.disable_emailing')) {
|
||||||
$date_time = date('c');
|
$user = $this->User->getUserById($user['id']); // fetch in database format
|
||||||
|
$datetime = date('c'); // ISO 8601 date
|
||||||
$body = new SendEmailTemplate('userloginprofile_newlogin');
|
$body = new SendEmailTemplate('userloginprofile_newlogin');
|
||||||
$body->set('userLoginProfile', $this->User->UserLoginProfile->_getUserProfile());
|
$body->set('userLoginProfile', $this->User->UserLoginProfile->_getUserProfile());
|
||||||
$body->set('baseurl', Configure::read('MISP.baseurl'));
|
$body->set('baseurl', Configure::read('MISP.baseurl'));
|
||||||
$body->set('misp_org', Configure::read('MISP.org'));
|
$body->set('misp_org', Configure::read('MISP.org'));
|
||||||
$body->set('date_time', $date_time);
|
$body->set('date_time', $datetime);
|
||||||
// Fetch user that contains also PGP or S/MIME keys for e-mail encryption
|
// Fetch user that contains also PGP or S/MIME keys for e-mail encryption
|
||||||
$this->User->sendEmail($user, $body, false, "[" . Configure::read('MISP.org') . " MISP] New sign in.");
|
$this->User->sendEmail($user, $body, false, "[" . Configure::read('MISP.org') . " MISP] New sign in.");
|
||||||
}
|
}
|
||||||
|
|
|
@ -390,8 +390,7 @@ class Warninglist extends AppModel
|
||||||
$warninglistId = (int)$this->id;
|
$warninglistId = (int)$this->id;
|
||||||
$result = true;
|
$result = true;
|
||||||
|
|
||||||
$keys = array_keys($list['list']);
|
if (JsonTool::arrayIsList($list['list'])) {
|
||||||
if ($keys === array_keys($keys)) {
|
|
||||||
foreach (array_chunk($list['list'], 1000) as $chunk) {
|
foreach (array_chunk($list['list'], 1000) as $chunk) {
|
||||||
$valuesToInsert = [];
|
$valuesToInsert = [];
|
||||||
foreach ($chunk as $value) {
|
foreach ($chunk as $value) {
|
||||||
|
|
|
@ -0,0 +1,276 @@
|
||||||
|
<?php
|
||||||
|
require_once CAKE_CORE_INCLUDE_PATH . '/Cake/Cache/CacheEngine.php';
|
||||||
|
require_once CAKE_CORE_INCLUDE_PATH . '/Cake/Cache/Engine/FileEngine.php';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is faster version of FileEngine cache engine
|
||||||
|
* - stores file in binary format, so no need to change line endings
|
||||||
|
* - uses igbinary if supported for faster serialization/deserialization and smaller cache files
|
||||||
|
* - default file mask is set to 0660, so cache files are not readable by other users
|
||||||
|
* - optimised file saving and fetching
|
||||||
|
*/
|
||||||
|
class BinaryFileEngine extends FileEngine
|
||||||
|
{
|
||||||
|
const BINARY_CACHE_TIME_LENGTH = 8;
|
||||||
|
|
||||||
|
private $useIgbinary = false;
|
||||||
|
|
||||||
|
public function init($settings = [])
|
||||||
|
{
|
||||||
|
$settings += [
|
||||||
|
'engine' => 'BinaryFile',
|
||||||
|
'path' => CACHE,
|
||||||
|
'prefix' => 'cake_',
|
||||||
|
'serialize' => true,
|
||||||
|
'mask' => 0660,
|
||||||
|
];
|
||||||
|
CacheEngine::init($settings);
|
||||||
|
|
||||||
|
$this->useIgbinary = function_exists('igbinary_serialize');
|
||||||
|
if (substr($this->settings['path'], -1) !== DS) {
|
||||||
|
$this->settings['path'] .= DS;
|
||||||
|
}
|
||||||
|
if (!empty($this->_groupPrefix)) {
|
||||||
|
$this->_groupPrefix = str_replace('_', DS, $this->_groupPrefix);
|
||||||
|
}
|
||||||
|
return $this->_active();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $key
|
||||||
|
* @param mixed $data
|
||||||
|
* @param int $duration
|
||||||
|
* @return bool
|
||||||
|
*/
|
||||||
|
public function write($key, $data, $duration)
|
||||||
|
{
|
||||||
|
if (!$this->_init) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
$fileInfo = $this->cacheFilePath($key);
|
||||||
|
$resource = $this->createFile($fileInfo);
|
||||||
|
if (!$resource) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!empty($this->settings['serialize'])) {
|
||||||
|
if ($this->useIgbinary) {
|
||||||
|
$data = igbinary_serialize($data);
|
||||||
|
if ($data === null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
$data = serialize($data);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$expires = pack("q", time() + $duration);
|
||||||
|
|
||||||
|
flock($resource, LOCK_EX);
|
||||||
|
|
||||||
|
ftruncate($resource, 0);
|
||||||
|
|
||||||
|
$result = fwrite($resource, $expires);
|
||||||
|
if ($result !== self::BINARY_CACHE_TIME_LENGTH) {
|
||||||
|
$this->handleWriteError($fileInfo);
|
||||||
|
fclose($resource);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
$result = fwrite($resource, $data);
|
||||||
|
if ($result !== strlen($data)) {
|
||||||
|
$this->handleWriteError($fileInfo);
|
||||||
|
fclose($resource);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
fclose($resource);
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $key
|
||||||
|
* @return false|mixed|string
|
||||||
|
*/
|
||||||
|
public function read($key)
|
||||||
|
{
|
||||||
|
if (!$this->_init) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
$fileInfo = $this->cacheFilePath($key);
|
||||||
|
|
||||||
|
$exists = file_exists($fileInfo->getPathname());
|
||||||
|
if (!$exists) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
$resource = $this->openFile($fileInfo);
|
||||||
|
if (!$resource) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
$time = time();
|
||||||
|
|
||||||
|
flock($resource, LOCK_SH);
|
||||||
|
|
||||||
|
$cacheTimeBinary = fread($resource, self::BINARY_CACHE_TIME_LENGTH);
|
||||||
|
if (!$cacheTimeBinary) {
|
||||||
|
fclose($resource);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
$cacheTime = $this->unpackCacheTime($cacheTimeBinary);
|
||||||
|
if ($cacheTime < $time || ($time + $this->settings['duration']) < $cacheTime) {
|
||||||
|
fclose($resource);
|
||||||
|
return false; // already expired
|
||||||
|
}
|
||||||
|
|
||||||
|
$data = stream_get_contents($resource, null, self::BINARY_CACHE_TIME_LENGTH);
|
||||||
|
fclose($resource);
|
||||||
|
|
||||||
|
if (!empty($this->settings['serialize'])) {
|
||||||
|
if ($this->useIgbinary) {
|
||||||
|
$data = igbinary_unserialize($data);
|
||||||
|
} else {
|
||||||
|
$data = unserialize($data);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return $data;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $path
|
||||||
|
* @param int $now
|
||||||
|
* @param int $threshold
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
protected function _clearDirectory($path, $now, $threshold)
|
||||||
|
{
|
||||||
|
$prefixLength = strlen($this->settings['prefix']);
|
||||||
|
|
||||||
|
if (!is_dir($path)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
$dir = dir($path);
|
||||||
|
if ($dir === false) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
while (($entry = $dir->read()) !== false) {
|
||||||
|
if (substr($entry, 0, $prefixLength) !== $this->settings['prefix']) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
$file = new SplFileObject($path . $entry, 'rb');
|
||||||
|
} catch (Exception $e) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($threshold) {
|
||||||
|
$mtime = $file->getMTime();
|
||||||
|
if ($mtime > $threshold) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$expires = $this->unpackCacheTime($file->fread(self::BINARY_CACHE_TIME_LENGTH));
|
||||||
|
if ($expires > $now) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ($file->isFile()) {
|
||||||
|
$filePath = $file->getRealPath();
|
||||||
|
$file = null;
|
||||||
|
@unlink($filePath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param SplFileInfo $fileInfo
|
||||||
|
* @return false|resource
|
||||||
|
*/
|
||||||
|
private function createFile(SplFileInfo $fileInfo)
|
||||||
|
{
|
||||||
|
$exists = file_exists($fileInfo->getPathname());
|
||||||
|
if (!$exists) {
|
||||||
|
$resource = $this->openFile($fileInfo, 'cb');
|
||||||
|
if ($resource && !chmod($fileInfo->getPathname(), (int)$this->settings['mask'])) {
|
||||||
|
trigger_error(__d(
|
||||||
|
'cake_dev', 'Could not apply permission mask "%s" on cache file "%s"',
|
||||||
|
[$fileInfo->getPathname(), $this->settings['mask']]), E_USER_WARNING);
|
||||||
|
}
|
||||||
|
return $resource;
|
||||||
|
}
|
||||||
|
|
||||||
|
return $this->openFile($fileInfo, 'cb');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param SplFileInfo $fileInfo
|
||||||
|
* @param string $mode
|
||||||
|
* @return false|resource
|
||||||
|
*/
|
||||||
|
private function openFile(SplFileInfo $fileInfo, $mode = 'rb')
|
||||||
|
{
|
||||||
|
$resource = fopen($fileInfo->getPathname(), $mode);
|
||||||
|
if (!$resource) {
|
||||||
|
trigger_error(__d(
|
||||||
|
'cake_dev', 'Could not open file %s',
|
||||||
|
array($fileInfo->getPathname())), E_USER_WARNING);
|
||||||
|
}
|
||||||
|
return $resource;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $key
|
||||||
|
* @return SplFileInfo
|
||||||
|
*/
|
||||||
|
private function cacheFilePath(string $key): SplFileInfo
|
||||||
|
{
|
||||||
|
$groups = null;
|
||||||
|
if (!empty($this->_groupPrefix)) {
|
||||||
|
$groups = vsprintf($this->_groupPrefix, $this->groups());
|
||||||
|
}
|
||||||
|
$dir = $this->settings['path'] . $groups;
|
||||||
|
|
||||||
|
if (!is_dir($dir)) {
|
||||||
|
mkdir($dir, 0775, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
$suffix = '.bin';
|
||||||
|
if ($this->settings['serialize'] && $this->useIgbinary) {
|
||||||
|
$suffix = '.igbin';
|
||||||
|
}
|
||||||
|
|
||||||
|
return new SplFileInfo($dir . $key . $suffix);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param SplFileInfo $fileInfo
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
private function handleWriteError(SplFileInfo $fileInfo)
|
||||||
|
{
|
||||||
|
unlink($fileInfo->getPathname()); // delete file in case file was just partially written
|
||||||
|
trigger_error(__d(
|
||||||
|
'cake_dev', 'Could not write to file %s',
|
||||||
|
array($fileInfo->getPathname())), E_USER_WARNING);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $cacheTimeBinary
|
||||||
|
* @return int
|
||||||
|
*/
|
||||||
|
private function unpackCacheTime($cacheTimeBinary)
|
||||||
|
{
|
||||||
|
if ($cacheTimeBinary === false || strlen($cacheTimeBinary) !== self::BINARY_CACHE_TIME_LENGTH) {
|
||||||
|
throw new InvalidArgumentException("Invalid cache time in binary format provided '$cacheTimeBinary'");
|
||||||
|
}
|
||||||
|
return unpack("q", $cacheTimeBinary)[1];
|
||||||
|
}
|
||||||
|
}
|
|
@ -57,7 +57,7 @@ class EcsLog implements CakeLogInterface
|
||||||
'log' => [
|
'log' => [
|
||||||
'level' => $type,
|
'level' => $type,
|
||||||
],
|
],
|
||||||
'message' => $message,
|
'message' => JsonTool::escapeNonUnicode($message),
|
||||||
];
|
];
|
||||||
|
|
||||||
static::writeMessage($message);
|
static::writeMessage($message);
|
||||||
|
|
|
@ -28,7 +28,6 @@ class Oidc
|
||||||
$claims = $oidc->getVerifiedClaims();
|
$claims = $oidc->getVerifiedClaims();
|
||||||
|
|
||||||
$mispUsername = $claims->email ?? $oidc->requestUserInfo('email');
|
$mispUsername = $claims->email ?? $oidc->requestUserInfo('email');
|
||||||
|
|
||||||
if (empty($mispUsername)) {
|
if (empty($mispUsername)) {
|
||||||
$sub = $claims->sub ?? 'UNKNOWN';
|
$sub = $claims->sub ?? 'UNKNOWN';
|
||||||
throw new Exception("OIDC user $sub doesn't have email address, that is required by MISP.");
|
throw new Exception("OIDC user $sub doesn't have email address, that is required by MISP.");
|
||||||
|
@ -66,13 +65,13 @@ class Oidc
|
||||||
$roleProperty = $this->getConfig('roles_property', 'roles');
|
$roleProperty = $this->getConfig('roles_property', 'roles');
|
||||||
$roles = $claims->{$roleProperty} ?? $oidc->requestUserInfo($roleProperty);
|
$roles = $claims->{$roleProperty} ?? $oidc->requestUserInfo($roleProperty);
|
||||||
if ($roles === null) {
|
if ($roles === null) {
|
||||||
$this->log($mispUsername, "Role property `$roleProperty` is missing in claims.", LOG_WARNING);
|
$this->log($mispUsername, "Role property `$roleProperty` is missing in claims, access prohibited.", LOG_WARNING);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
$roleId = $this->getUserRole($roles, $mispUsername);
|
$roleId = $this->getUserRole($roles, $mispUsername);
|
||||||
if ($roleId === null) {
|
if ($roleId === null) {
|
||||||
$this->log($mispUsername, 'No role was assigned.');
|
$this->log($mispUsername, 'No role was assigned, access prohibited.', LOG_WARNING);
|
||||||
if ($user) {
|
if ($user) {
|
||||||
$this->block($user);
|
$this->block($user);
|
||||||
}
|
}
|
||||||
|
|
|
@ -124,6 +124,16 @@ class AttributeValidationToolTest extends TestCase
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function testRemoveCidrFromIp(): void
|
||||||
|
{
|
||||||
|
$this->assertEquals('127.0.0.1', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1/32'));
|
||||||
|
$this->assertEquals('127.0.0.1/31', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1/31'));
|
||||||
|
$this->assertEquals('example.com|1234:fd2:5621:1:89::4500', AttributeValidationTool::modifyBeforeValidation('domain|ip', 'example.com|1234:0fd2:5621:0001:0089:0000:0000:4500/128'));
|
||||||
|
$this->assertEquals('1234:fd2:5621:1:89::4500|80', AttributeValidationTool::modifyBeforeValidation('ip-src|port', '1234:0fd2:5621:0001:0089:0000:0000:4500/128|80'));
|
||||||
|
$this->assertEquals('1234:fd2:5621:1:89::4500/127|80', AttributeValidationTool::modifyBeforeValidation('ip-src|port', '1234:0fd2:5621:0001:0089:0000:0000:4500/127|80'));
|
||||||
|
$this->assertEquals('127.0.0.1', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1'));
|
||||||
|
}
|
||||||
|
|
||||||
public function testCompressIpv6(): void
|
public function testCompressIpv6(): void
|
||||||
{
|
{
|
||||||
$this->assertEquals('1234:fd2:5621:1:89::4500', AttributeValidationTool::modifyBeforeValidation('ip-src', '1234:0fd2:5621:0001:0089:0000:0000:4500'));
|
$this->assertEquals('1234:fd2:5621:1:89::4500', AttributeValidationTool::modifyBeforeValidation('ip-src', '1234:0fd2:5621:0001:0089:0000:0000:4500'));
|
||||||
|
|
|
@ -527,10 +527,31 @@ EOT;
|
||||||
public function testCheckFreeTextNonBreakableSpace(): void
|
public function testCheckFreeTextNonBreakableSpace(): void
|
||||||
{
|
{
|
||||||
$complexTypeTool = new ComplexTypeTool();
|
$complexTypeTool = new ComplexTypeTool();
|
||||||
|
|
||||||
$results = $complexTypeTool->checkFreeText("127.0.0.1\xc2\xa0127.0.0.2");
|
$results = $complexTypeTool->checkFreeText("127.0.0.1\xc2\xa0127.0.0.2");
|
||||||
$this->assertCount(2, $results);
|
$this->assertCount(2, $results);
|
||||||
$this->assertEquals('127.0.0.1', $results[0]['value']);
|
$this->assertEquals('127.0.0.1', $results[0]['value']);
|
||||||
$this->assertEquals('ip-dst', $results[0]['default_type']);
|
$this->assertEquals('ip-dst', $results[0]['default_type']);
|
||||||
|
|
||||||
|
$results = $complexTypeTool->checkFreeText("127.0.0.1\xc2\xa0\xc2\xa0127.0.0.2");
|
||||||
|
$this->assertCount(2, $results);
|
||||||
|
$this->assertEquals('127.0.0.1', $results[0]['value']);
|
||||||
|
$this->assertEquals('ip-dst', $results[0]['default_type']);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testCheckFreeTextControlCharToSpace(): void
|
||||||
|
{
|
||||||
|
$complexTypeTool = new ComplexTypeTool();
|
||||||
|
|
||||||
|
$results = $complexTypeTool->checkFreeText("127.0.0.1\x1d127.0.0.2");
|
||||||
|
$this->assertCount(2, $results);
|
||||||
|
$this->assertEquals('127.0.0.1', $results[0]['value']);
|
||||||
|
$this->assertEquals('ip-dst', $results[0]['default_type']);
|
||||||
|
|
||||||
|
$results = $complexTypeTool->checkFreeText("127.0.0.1\x1d\x1d127.0.0.2");
|
||||||
|
$this->assertCount(2, $results);
|
||||||
|
$this->assertEquals('127.0.0.1', $results[0]['value']);
|
||||||
|
$this->assertEquals('ip-dst', $results[0]['default_type']);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function testCheckFreeTextQuoted(): void
|
public function testCheckFreeTextQuoted(): void
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
<?php
|
<?php
|
||||||
|
require_once __DIR__ . '/../Lib/Tools/JsonTool.php';
|
||||||
require_once __DIR__ . '/../Lib/Tools/JSONConverterTool.php';
|
require_once __DIR__ . '/../Lib/Tools/JSONConverterTool.php';
|
||||||
|
|
||||||
use PHPUnit\Framework\TestCase;
|
use PHPUnit\Framework\TestCase;
|
||||||
|
@ -57,11 +58,6 @@ class JSONConverterToolTest extends TestCase
|
||||||
$jsonNormalWithoutSpaces = preg_replace("/\s+/", "", JSONConverterTool::convert($event));
|
$jsonNormalWithoutSpaces = preg_replace("/\s+/", "", JSONConverterTool::convert($event));
|
||||||
$this->assertEquals($jsonNormalWithoutSpaces, $jsonStreamWithoutSpaces);
|
$this->assertEquals($jsonNormalWithoutSpaces, $jsonStreamWithoutSpaces);
|
||||||
|
|
||||||
if (defined('JSON_THROW_ON_ERROR')) {
|
$this->assertTrue(JsonTool::isValid($json));
|
||||||
json_decode($json, true, 512, JSON_THROW_ON_ERROR);
|
|
||||||
$this->assertTrue(true);
|
|
||||||
} else {
|
|
||||||
$this->assertNotNull(json_decode($json));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,12 +3,12 @@ $keyUsageCsv = null;
|
||||||
if (isset($keyUsage)) {
|
if (isset($keyUsage)) {
|
||||||
$todayString = date('Y-m-d');
|
$todayString = date('Y-m-d');
|
||||||
$today = strtotime($todayString);
|
$today = strtotime($todayString);
|
||||||
$startDate = key($keyUsage); // oldest date for sparkline
|
$startDate = array_key_first($keyUsage); // oldest date for sparkline
|
||||||
$startDate = strtotime($startDate) - (3600 * 24 * 3);
|
$startDate = strtotime($startDate) - (3600 * 24 * 3);
|
||||||
$keyUsageCsv = 'Date,Close\n';
|
$keyUsageCsv = 'Date,Close\n';
|
||||||
for ($date = $startDate; $date <= $today; $date += (3600 * 24)) {
|
for ($date = $startDate; $date <= $today; $date += (3600 * 24)) {
|
||||||
$dateAsString = date('Y-m-d', $date);
|
$dateAsString = date('Y-m-d', $date);
|
||||||
$keyUsageCsv .= $dateAsString . ',' . (isset($keyUsage[$dateAsString]) ? $keyUsage[$dateAsString] : 0) . '\n';
|
$keyUsageCsv .= $dateAsString . ',' . ($keyUsage[$dateAsString] ?? '0') . '\n';
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
$lastUsed = null;
|
$lastUsed = null;
|
||||||
|
|
|
@ -277,11 +277,6 @@
|
||||||
'url' => $baseurl . '/servers/createSync',
|
'url' => $baseurl . '/servers/createSync',
|
||||||
'requirement' => $isAclSync && !$isSiteAdmin
|
'requirement' => $isAclSync && !$isSiteAdmin
|
||||||
),
|
),
|
||||||
array(
|
|
||||||
'text' => __('Import Server Settings'),
|
|
||||||
'url' => $baseurl . '/servers/import',
|
|
||||||
'requirement' => $this->Acl->canAccess('servers', 'import'),
|
|
||||||
),
|
|
||||||
array(
|
array(
|
||||||
'text' => __('Remote Servers'),
|
'text' => __('Remote Servers'),
|
||||||
'url' => $baseurl . '/servers/index',
|
'url' => $baseurl . '/servers/index',
|
||||||
|
@ -292,11 +287,6 @@
|
||||||
'url' => $baseurl . '/feeds/index',
|
'url' => $baseurl . '/feeds/index',
|
||||||
'requirement' => $this->Acl->canAccess('feeds', 'index'),
|
'requirement' => $this->Acl->canAccess('feeds', 'index'),
|
||||||
),
|
),
|
||||||
array(
|
|
||||||
'text' => __('Search Feed Caches'),
|
|
||||||
'url' => $baseurl . '/feeds/searchCaches',
|
|
||||||
'requirement' => $this->Acl->canAccess('feeds', 'searchCaches'),
|
|
||||||
),
|
|
||||||
array(
|
array(
|
||||||
'text' => __('SightingDB'),
|
'text' => __('SightingDB'),
|
||||||
'url' => $baseurl . '/sightingdb/index',
|
'url' => $baseurl . '/sightingdb/index',
|
||||||
|
@ -313,7 +303,7 @@
|
||||||
'requirement' => $this->Acl->canAccess('cerebrates', 'index'),
|
'requirement' => $this->Acl->canAccess('cerebrates', 'index'),
|
||||||
),
|
),
|
||||||
array(
|
array(
|
||||||
'text' => __('List Taxii Servers'),
|
'text' => __('TAXII Servers'),
|
||||||
'url' => $baseurl . '/TaxiiServers/index',
|
'url' => $baseurl . '/TaxiiServers/index',
|
||||||
'requirement' => $this->Acl->canAccess('taxiiServers', 'index'),
|
'requirement' => $this->Acl->canAccess('taxiiServers', 'index'),
|
||||||
),
|
),
|
||||||
|
|
|
@ -1,3 +1,10 @@
|
||||||
|
<?php
|
||||||
|
$humanReadableFilesize = function ($bytes, $dec = 2) {
|
||||||
|
$size = ['B', 'kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
|
||||||
|
$factor = floor((strlen($bytes) - 1) / 3);
|
||||||
|
return sprintf("%.{$dec}f %s", ($bytes / (1024 ** $factor)), $size[$factor]);
|
||||||
|
};
|
||||||
|
?>
|
||||||
<div style="border:1px solid #dddddd; margin-top:1px; width:100%; padding:10px">
|
<div style="border:1px solid #dddddd; margin-top:1px; width:100%; padding:10px">
|
||||||
<p><?php echo __('Below you will find a list of the uploaded files based on type.');?></p>
|
<p><?php echo __('Below you will find a list of the uploaded files based on type.');?></p>
|
||||||
<?php
|
<?php
|
||||||
|
@ -5,13 +12,13 @@
|
||||||
?>
|
?>
|
||||||
<h3><?php echo h($file['name']); ?></h3>
|
<h3><?php echo h($file['name']); ?></h3>
|
||||||
<div>
|
<div>
|
||||||
<b><?php echo __('Description');?></b>: <?php echo $file['description']; ?><br />
|
<b><?php echo __('Description');?></b>: <?php echo $file['description']; ?><br>
|
||||||
<b><?php echo __('Expected Format');?></b>: <?php echo h($file['valid_format']);?><br />
|
<b><?php echo __('Expected Format');?></b>: <?php echo h($file['valid_format']);?><br>
|
||||||
<b><?php echo __('Path');?></b>: <?php echo h($file['path']);?><br />
|
<b><?php echo __('Path');?></b>: <?php echo h($file['path']);?><br>
|
||||||
<?php
|
<?php
|
||||||
if (!empty($file['expected'])):
|
if (!empty($file['expected'])):
|
||||||
?>
|
?>
|
||||||
<b><?php echo __('Files set for each relevant setting');?>:</b><br />
|
<b><?php echo __('Files set for each relevant setting');?>:</b><br>
|
||||||
<ul>
|
<ul>
|
||||||
<?php foreach ($file['expected'] as $expectedKey => $expectedValue):
|
<?php foreach ($file['expected'] as $expectedKey => $expectedValue):
|
||||||
$colour = 'red';
|
$colour = 'red';
|
||||||
|
@ -24,7 +31,7 @@
|
||||||
endif;
|
endif;
|
||||||
?>
|
?>
|
||||||
</div>
|
</div>
|
||||||
<table class="table table-striped table-hover table-condensed" style="width:600px;">
|
<table class="table table-striped table-hover table-condensed" style="width:700px">
|
||||||
<tr>
|
<tr>
|
||||||
<th><?php echo __('Filename');?></th>
|
<th><?php echo __('Filename');?></th>
|
||||||
<th><?php echo __('Used by');?></th>
|
<th><?php echo __('Used by');?></th>
|
||||||
|
@ -35,19 +42,10 @@
|
||||||
<?php
|
<?php
|
||||||
foreach ($file['files'] as $f):
|
foreach ($file['files'] as $f):
|
||||||
$permission = "";
|
$permission = "";
|
||||||
|
if ($f['link']) $permission .= "l";
|
||||||
if ($f['read']) $permission .= "r";
|
if ($f['read']) $permission .= "r";
|
||||||
if ($f['write']) $permission .= "w";
|
if ($f['write']) $permission .= "w";
|
||||||
if ($f['execute']) $permission .= "x";
|
if ($f['execute']) $permission .= "x";
|
||||||
$sizeUnit = "B";
|
|
||||||
if (($f['filesize'] / 1024) > 1) {
|
|
||||||
$f['filesize'] = $f['filesize'] / 1024;
|
|
||||||
$sizeUnit = "KB";
|
|
||||||
if (($f['filesize'] / 1024) > 1) {
|
|
||||||
$f['filesize'] = $f['filesize'] / 1024;
|
|
||||||
$sizeUnit = "MB";
|
|
||||||
}
|
|
||||||
$f['filesize'] = round($f['filesize'], 1);
|
|
||||||
}
|
|
||||||
?>
|
?>
|
||||||
<tr>
|
<tr>
|
||||||
<td><?php echo h($f['filename']);?></td>
|
<td><?php echo h($f['filename']);?></td>
|
||||||
|
@ -55,7 +53,7 @@
|
||||||
<?php
|
<?php
|
||||||
if ($k != 'orgs'):
|
if ($k != 'orgs'):
|
||||||
foreach ($file['expected'] as $ek => $ev):
|
foreach ($file['expected'] as $ek => $ev):
|
||||||
if ($f['filename'] == $ev) echo h($ek) . "<br />";
|
if ($f['filename'] == $ev) echo h($ek) . "<br>";
|
||||||
endforeach;
|
endforeach;
|
||||||
else:
|
else:
|
||||||
echo __('N/A');
|
echo __('N/A');
|
||||||
|
@ -63,7 +61,7 @@
|
||||||
?>
|
?>
|
||||||
</td>
|
</td>
|
||||||
<td width="75px;">
|
<td width="75px;">
|
||||||
<?php echo h($f['filesize']) . ' ' . $sizeUnit;?>
|
<?= $humanReadableFilesize($f['filesize'], 1) ?>
|
||||||
</td>
|
</td>
|
||||||
<td class="short">
|
<td class="short">
|
||||||
<?php echo $permission;?>
|
<?php echo $permission;?>
|
||||||
|
@ -93,5 +91,4 @@
|
||||||
echo $this->Form->end();
|
echo $this->Form->end();
|
||||||
endforeach;
|
endforeach;
|
||||||
?>
|
?>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
|
@ -1,11 +1,3 @@
|
||||||
<?php
|
|
||||||
if (!$isSiteAdmin) exit();
|
|
||||||
?>
|
|
||||||
<div class="actions">
|
|
||||||
<ol class="nav nav-list">
|
|
||||||
|
|
||||||
</ol>
|
|
||||||
</div>
|
|
||||||
<div class="index">
|
<div class="index">
|
||||||
<h2><?php echo __('Administrative actions');?></h2>
|
<h2><?php echo __('Administrative actions');?></h2>
|
||||||
<ul>
|
<ul>
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
<?php echo $this->Flash->render(); ?>
|
<?php echo $this->Flash->render(); ?>
|
||||||
<?php
|
<?php
|
||||||
$detailsHtml = __("To enable TOTP for your account, scan the following QR code with your TOTP application and validate the token.");;
|
$detailsHtml = __("To enable TOTP for your account, scan the following QR code with your TOTP application (for example Google authenticator or KeepassXC) and validate the token.");;
|
||||||
$secretHtml = __("Alternatively you can enter the following secret in your TOTP application: ") . "<pre>" . $secret . "</pre>";
|
$secretHtml = __("Alternatively you can enter the following secret in your TOTP application. This can be particularly handy in case you don't have a supported application in your working environment. Once the verification is done you'll also get 50 \"paper-based\" login tokens so you don't have to use a TOTP application each time: ") . "<pre>" . $secret . "</pre>";
|
||||||
|
|
||||||
echo $this->element('/genericElements/Form/genericForm', array(
|
echo $this->element('/genericElements/Form/genericForm', array(
|
||||||
"form" => $this->Form,
|
"form" => $this->Form,
|
||||||
|
|
|
@ -36,6 +36,7 @@
|
||||||
"ext-rdkafka": "Required for publishing events to Kafka broker",
|
"ext-rdkafka": "Required for publishing events to Kafka broker",
|
||||||
"ext-apcu": "To cache data in memory instead of file system",
|
"ext-apcu": "To cache data in memory instead of file system",
|
||||||
"ext-simdjson": "To decode JSON structures faster",
|
"ext-simdjson": "To decode JSON structures faster",
|
||||||
|
"ext-curl": "For faster remote requests",
|
||||||
"elasticsearch/elasticsearch": "For logging to elasticsearch",
|
"elasticsearch/elasticsearch": "For logging to elasticsearch",
|
||||||
"aws/aws-sdk-php": "To upload samples to S3",
|
"aws/aws-sdk-php": "To upload samples to S3",
|
||||||
"jakub-onderka/openid-connect-php": "For OIDC authentication",
|
"jakub-onderka/openid-connect-php": "For OIDC authentication",
|
||||||
|
|
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
@ -4,15 +4,18 @@ from zmq.auth.thread import ThreadAuthenticator
|
||||||
from zmq.utils.monitor import recv_monitor_message
|
from zmq.utils.monitor import recv_monitor_message
|
||||||
import sys
|
import sys
|
||||||
import redis
|
import redis
|
||||||
import json
|
|
||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
import threading
|
import threading
|
||||||
import logging
|
import logging
|
||||||
|
import typing
|
||||||
|
import argparse
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
try:
|
||||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s:%(levelname)s:%(name)s:%(message)s")
|
import orjson as json
|
||||||
|
except ImportError:
|
||||||
|
import json
|
||||||
|
|
||||||
|
|
||||||
def check_pid(pid):
|
def check_pid(pid):
|
||||||
|
@ -55,10 +58,11 @@ class MispZmq:
|
||||||
socket = None
|
socket = None
|
||||||
pidfile = None
|
pidfile = None
|
||||||
|
|
||||||
r: redis.StrictRedis
|
redis: redis.StrictRedis
|
||||||
namespace: str
|
namespace: str
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self, debug=False):
|
||||||
|
logging.basicConfig(level=logging.DEBUG if debug else logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
|
||||||
self._logger = logging.getLogger()
|
self._logger = logging.getLogger()
|
||||||
|
|
||||||
self.tmp_location = Path(__file__).parent.parent / "tmp"
|
self.tmp_location = Path(__file__).parent.parent / "tmp"
|
||||||
|
@ -67,7 +71,7 @@ class MispZmq:
|
||||||
with open(self.pidfile.as_posix()) as f:
|
with open(self.pidfile.as_posix()) as f:
|
||||||
pid = f.read()
|
pid = f.read()
|
||||||
if check_pid(pid):
|
if check_pid(pid):
|
||||||
raise Exception("mispzmq already running on PID {}".format(pid))
|
raise Exception(f"mispzmq already running on PID {pid}")
|
||||||
else:
|
else:
|
||||||
# Cleanup
|
# Cleanup
|
||||||
self.pidfile.unlink()
|
self.pidfile.unlink()
|
||||||
|
@ -77,17 +81,18 @@ class MispZmq:
|
||||||
raise Exception("The settings file is missing.")
|
raise Exception("The settings file is missing.")
|
||||||
|
|
||||||
def _setup(self):
|
def _setup(self):
|
||||||
with open((self.tmp_location / "mispzmq_settings.json").as_posix()) as settings_file:
|
with open((self.tmp_location / "mispzmq_settings.json").as_posix(), 'rb') as settings_file:
|
||||||
self.settings = json.load(settings_file)
|
self.settings = json.loads(settings_file.read())
|
||||||
|
|
||||||
self.namespace = self.settings["redis_namespace"]
|
self.namespace = self.settings["redis_namespace"]
|
||||||
# Check if TLS is being used with Redis host
|
# Check if TLS is being used with Redis host
|
||||||
redis_host = self.settings["redis_host"]
|
redis_host = self.settings["redis_host"]
|
||||||
redis_ssl = redis_host.startswith("tls://")
|
redis_ssl = redis_host.startswith("tls://")
|
||||||
if redis_host.startswith("tls://"):
|
if redis_host.startswith("tls://"):
|
||||||
redis_host = redis_host[6:]
|
redis_host = redis_host[6:]
|
||||||
self.r = redis.StrictRedis(host=redis_host, db=self.settings["redis_database"],
|
self.redis = redis.StrictRedis(host=redis_host, db=self.settings["redis_database"],
|
||||||
password=self.settings["redis_password"], port=self.settings["redis_port"],
|
password=self.settings["redis_password"], port=self.settings["redis_port"],
|
||||||
decode_responses=True, ssl=redis_ssl)
|
ssl=redis_ssl)
|
||||||
self.timestamp_settings = time.time()
|
self.timestamp_settings = time.time()
|
||||||
self._logger.debug("Connected to Redis {}:{}/{}".format(self.settings["redis_host"], self.settings["redis_port"],
|
self._logger.debug("Connected to Redis {}:{}/{}".format(self.settings["redis_host"], self.settings["redis_port"],
|
||||||
self.settings["redis_database"]))
|
self.settings["redis_database"]))
|
||||||
|
@ -122,34 +127,38 @@ class MispZmq:
|
||||||
self.socket.disable_monitor()
|
self.socket.disable_monitor()
|
||||||
self.monitor_thread = None
|
self.monitor_thread = None
|
||||||
|
|
||||||
def _handle_command(self, command):
|
def _handle_command(self, command: bytes):
|
||||||
if command == "kill":
|
if command == b"kill":
|
||||||
self._logger.info("Kill command received, shutting down.")
|
self._logger.info("Kill command received, shutting down.")
|
||||||
self.clean()
|
self.clean()
|
||||||
sys.exit()
|
sys.exit()
|
||||||
|
|
||||||
elif command == "reload":
|
elif command == b"reload":
|
||||||
self._logger.info("Reload command received, reloading settings from file.")
|
self._logger.info("Reload command received, reloading settings from file.")
|
||||||
self._setup()
|
self._setup()
|
||||||
self._setup_zmq()
|
self._setup_zmq()
|
||||||
|
|
||||||
elif command == "status":
|
elif command == b"status":
|
||||||
self._logger.info("Status command received, responding with latest stats.")
|
self._logger.info("Status command received, responding with latest stats.")
|
||||||
self.r.delete("{}:status".format(self.namespace))
|
self.redis.delete(f"{self.namespace}:status")
|
||||||
self.r.lpush("{}:status".format(self.namespace),
|
self.redis.lpush(f"{self.namespace}:status",
|
||||||
json.dumps({"timestamp": time.time(),
|
json.dumps({"timestamp": time.time(),
|
||||||
"timestampSettings": self.timestamp_settings,
|
"timestampSettings": self.timestamp_settings,
|
||||||
"publishCount": self.publish_count,
|
"publishCount": self.publish_count,
|
||||||
"messageCount": self.message_count}))
|
"messageCount": self.message_count}))
|
||||||
else:
|
else:
|
||||||
self._logger.warning("Received invalid command '{}'.".format(command))
|
self._logger.warning(f"Received invalid command '{command}'.")
|
||||||
|
|
||||||
def _create_pid_file(self):
|
def _create_pid_file(self):
|
||||||
with open(self.pidfile.as_posix(), "w") as f:
|
with open(self.pidfile.as_posix(), "w") as f:
|
||||||
f.write(str(os.getpid()))
|
f.write(str(os.getpid()))
|
||||||
|
|
||||||
def _pub_message(self, topic, data):
|
def _pub_message(self, topic: bytes, data: typing.Union[str, bytes]):
|
||||||
self.socket.send_string("{} {}".format(topic, data))
|
data_to_send = bytearray()
|
||||||
|
data_to_send.extend(topic)
|
||||||
|
data_to_send.extend(b" ")
|
||||||
|
data_to_send.extend(data.encode("utf-8") if isinstance(data, str) else data)
|
||||||
|
self.socket.send(bytes(data_to_send))
|
||||||
|
|
||||||
def clean(self):
|
def clean(self):
|
||||||
if self.monitor_thread:
|
if self.monitor_thread:
|
||||||
|
@ -179,12 +188,14 @@ class MispZmq:
|
||||||
"misp_json_tag", "misp_json_warninglist", "misp_json_workflow"
|
"misp_json_tag", "misp_json_warninglist", "misp_json_workflow"
|
||||||
]
|
]
|
||||||
|
|
||||||
lists = ["{}:command".format(self.namespace)]
|
lists = [f"{self.namespace}:command"]
|
||||||
for topic in topics:
|
for topic in topics:
|
||||||
lists.append("{}:data:{}".format(self.namespace, topic))
|
lists.append(f"{self.namespace}:data:{topic}")
|
||||||
|
|
||||||
|
key_prefix = f"{self.namespace}:".encode("utf-8")
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
data = self.r.blpop(lists, timeout=10)
|
data = self.redis.blpop(lists, timeout=10)
|
||||||
|
|
||||||
if data is None:
|
if data is None:
|
||||||
# redis timeout expired
|
# redis timeout expired
|
||||||
|
@ -195,26 +206,30 @@ class MispZmq:
|
||||||
"status": status_array[status_entry],
|
"status": status_array[status_entry],
|
||||||
"uptime": current_time - int(self.timestamp_settings)
|
"uptime": current_time - int(self.timestamp_settings)
|
||||||
}
|
}
|
||||||
self._pub_message("misp_json_self", json.dumps(status_message))
|
self._pub_message(b"misp_json_self", json.dumps(status_message))
|
||||||
self._logger.debug("No message received for 10 seconds, sending ZMQ status message.")
|
self._logger.debug("No message received from Redis for 10 seconds, sending ZMQ status message.")
|
||||||
else:
|
else:
|
||||||
key, value = data
|
key, value = data
|
||||||
key = key.replace("{}:".format(self.namespace), "")
|
key = key.replace(key_prefix, b"")
|
||||||
if key == "command":
|
if key == b"command":
|
||||||
self._handle_command(value)
|
self._handle_command(value)
|
||||||
elif key.startswith("data:"):
|
elif key.startswith(b"data:"):
|
||||||
topic = key.split(":")[1]
|
topic = key.split(b":", 1)[1]
|
||||||
self._logger.debug("Received data for topic '{}', sending to ZMQ.".format(topic))
|
self._logger.debug("Received data for topic %s, sending to ZMQ.", topic)
|
||||||
self._pub_message(topic, value)
|
self._pub_message(topic, value)
|
||||||
self.message_count += 1
|
self.message_count += 1
|
||||||
if topic == "misp_json":
|
if topic == b"misp_json":
|
||||||
self.publish_count += 1
|
self.publish_count += 1
|
||||||
else:
|
else:
|
||||||
self._logger.warning("Received invalid message '{}'.".format(key))
|
self._logger.warning("Received invalid message type %s.", key)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
mzq = MispZmq()
|
arg_parser = argparse.ArgumentParser(description="MISP ZeroMQ PUB server")
|
||||||
|
arg_parser.add_argument("--debug", action="store_true", help="Enable debugging messages")
|
||||||
|
parsed = arg_parser.parse_args()
|
||||||
|
|
||||||
|
mzq = MispZmq(parsed.debug)
|
||||||
try:
|
try:
|
||||||
mzq.main()
|
mzq.main()
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
|
|
|
@ -0,0 +1,29 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import sys
|
||||||
|
import zmq
|
||||||
|
import argparse
|
||||||
|
|
||||||
|
|
||||||
|
def main(port: int):
|
||||||
|
context = zmq.Context()
|
||||||
|
|
||||||
|
print("Connecting to MISP ZeroMQ server…", file=sys.stderr)
|
||||||
|
socket = context.socket(zmq.SUB)
|
||||||
|
socket.connect(f"tcp://localhost:{port}")
|
||||||
|
socket.setsockopt(zmq.SUBSCRIBE, b"misp_")
|
||||||
|
print(f"Connected to tcp://localhost:{port}", file=sys.stderr)
|
||||||
|
|
||||||
|
while True:
|
||||||
|
message = socket.recv()
|
||||||
|
print(message)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
parser = argparse.ArgumentParser(description="Example Python client for MISP ZMQ")
|
||||||
|
parser.add_argument("--port", default=50000, type=int)
|
||||||
|
parsed = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
main(parsed.port)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
pass
|
|
@ -60,7 +60,7 @@
|
||||||
class DATABASE_CONFIG {
|
class DATABASE_CONFIG {
|
||||||
|
|
||||||
public $default = array(
|
public $default = array(
|
||||||
'datasource' => 'Database/Mysql',
|
'datasource' => 'Database/MysqlExtended',
|
||||||
'persistent' => false,
|
'persistent' => false,
|
||||||
'host' => '127.0.0.1',
|
'host' => '127.0.0.1',
|
||||||
'login' => 'misp',
|
'login' => 'misp',
|
|
@ -1,7 +1,7 @@
|
||||||
<?php
|
<?php
|
||||||
class EmailConfig {
|
class EmailConfig {
|
||||||
public $default = array(
|
public $default = [
|
||||||
'transport' => 'Debug',
|
'transport' => 'Debug',
|
||||||
'log' => true
|
'log' => true,
|
||||||
);
|
];
|
||||||
}
|
}
|
|
@ -1,15 +0,0 @@
|
||||||
<VirtualHost misp.local>
|
|
||||||
ServerAdmin me@me.local
|
|
||||||
ServerName misp.local
|
|
||||||
DocumentRoot %TRAVIS_BUILD_DIR%/app/webroot
|
|
||||||
<Directory %TRAVIS_BUILD_DIR%/app/webroot>
|
|
||||||
Options -Indexes
|
|
||||||
AllowOverride all
|
|
||||||
Require all granted
|
|
||||||
</Directory>
|
|
||||||
|
|
||||||
LogLevel warn
|
|
||||||
ErrorLog /var/log/apache2/misp.local_error.log
|
|
||||||
CustomLog /var/log/apache2/misp.local_access.log combined
|
|
||||||
ServerSignature Off
|
|
||||||
</VirtualHost>
|
|
|
@ -9,16 +9,15 @@ python ./../app/files/scripts/mispzmq/mispzmqtest.py
|
||||||
python ./../app/files/scripts/generate_file_objects.py -c | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if len([i for i in data.values() if i is not False]) == 0 else 1)'
|
python ./../app/files/scripts/generate_file_objects.py -c | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if len([i for i in data.values() if i is not False]) == 0 else 1)'
|
||||||
|
|
||||||
# Try to extract data from file
|
# Try to extract data from file
|
||||||
python ./../app/files/scripts/generate_file_objects.py -p /bin/ls
|
|
||||||
python ./../app/files/scripts/generate_file_objects.py -p /bin/ls | python3 -c 'import sys, json; data = json.load(sys.stdin); sys.exit(0 if "objects" in data else 1)'
|
python ./../app/files/scripts/generate_file_objects.py -p /bin/ls | python3 -c 'import sys, json; data = json.load(sys.stdin); sys.exit(0 if "objects" in data else 1)'
|
||||||
|
|
||||||
# Test converting stix1 to MISP format
|
# Test converting stix1 to MISP format
|
||||||
curl https://stixproject.github.io/documentation/idioms/c2-indicator/indicator-for-c2-ip-address.xml > ./../app/files/scripts/tmp/test-stix1.xml
|
curl -sS --compressed https://stixproject.github.io/documentation/idioms/c2-indicator/indicator-for-c2-ip-address.xml > ./../app/files/scripts/tmp/test-stix1.xml
|
||||||
python ./../app/files/scripts/stix2misp.py test-stix1.xml 1 1 ./../app/files/scripts/synonymsToTagNames.json | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if data["success"] == 1 else 1)'
|
python ./../app/files/scripts/stix2misp.py test-stix1.xml 1 1 ./../app/files/scripts/synonymsToTagNames.json | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if data["success"] == 1 else 1)'
|
||||||
rm -f ./../app/files/scripts/tmp/{test-stix1.xml,test-stix1.xml.json}
|
rm -f ./../app/files/scripts/tmp/{test-stix1.xml,test-stix1.xml.json}
|
||||||
|
|
||||||
# Test converting stix2 to MISP format
|
# Test converting stix2 to MISP format
|
||||||
curl https://raw.githubusercontent.com/oasis-open/cti-stix2-json-schemas/master/examples/indicator-for-c2-ip-address.json > ./../app/files/scripts/tmp/test-stix2.json
|
curl -sS --compressed https://raw.githubusercontent.com/oasis-open/cti-stix2-json-schemas/master/examples/indicator-for-c2-ip-address.json > ./../app/files/scripts/tmp/test-stix2.json
|
||||||
python ./../app/files/scripts/stix2/stix2misp.py -i ./../app/files/scripts/tmp/test-stix2.json --distribution 1 | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if data["success"] == 1 else 1)'
|
python ./../app/files/scripts/stix2/stix2misp.py -i ./../app/files/scripts/tmp/test-stix2.json --distribution 1 | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if data["success"] == 1 else 1)'
|
||||||
rm -f ./../app/files/scripts/tmp/{test-stix2.json,test-stix2.json.stix2}
|
rm -f ./../app/files/scripts/tmp/{test-stix2.json,test-stix2.json.stix2}
|
||||||
|
|
||||||
|
|
|
@ -1,15 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
set -e
|
|
||||||
set -x
|
|
||||||
|
|
||||||
AUTH="$1"
|
|
||||||
|
|
||||||
# Check if user is logged
|
|
||||||
curl -i -H "Authorization: $AUTH" -H "Accept: application/json" -X GET http://misp.local/servers/getVersion
|
|
||||||
curl -i -H "Accept: application/json" -H "content-type: application/json" -H "Authorization: $AUTH" --data "@event.json" -X POST http://misp.local/events
|
|
||||||
curl -H "Authorization: $AUTH" -X GET http://misp.local/events/csv/download/1/ignore:1 | sed -e 's/^M//g' | cut -d, -f2 --complement | sort > 1.csv
|
|
||||||
cat 1.csv
|
|
||||||
cut -d, -f2 --complement event.csv | sort > compare.csv
|
|
||||||
diff compare.csv 1.csv
|
|
||||||
curl -i -H "Accept: application/json" -H "content-type: application/json" -H "Authorization: $AUTH" -X POST http://misp.local/events/delete/1
|
|
|
@ -7,13 +7,13 @@ AUTH="$1"
|
||||||
HOST="$2"
|
HOST="$2"
|
||||||
|
|
||||||
# Check if user is logged
|
# Check if user is logged
|
||||||
curl -i -H "Authorization: $AUTH" -H "Accept: application/json" -X GET http://${HOST}/servers/getVersion
|
curl -sS -i -H "Authorization: $AUTH" -H "Accept: application/json" -X GET http://${HOST}/servers/getVersion
|
||||||
curl -i -H "Accept: application/json" -H "content-type: application/json" -H "Authorization: $AUTH" --data "@event.json" -X POST http://${HOST}/events > /dev/null
|
curl -sS -i -H "Accept: application/json" -H "content-type: application/json" -H "Authorization: $AUTH" --data "@event.json" -X POST http://${HOST}/events > /dev/null
|
||||||
curl -H "Authorization: $AUTH" -X GET http://${HOST}/events/csv/download/1/ignore:1 | sed -e 's/^M//g' | cut -d, -f2 --complement | sort > 1.csv
|
curl -sS -H "Authorization: $AUTH" -X GET http://${HOST}/events/csv/download/1/ignore:1 | sed -e 's/^M//g' | cut -d, -f2 --complement | sort > 1.csv
|
||||||
cat 1.csv
|
cat 1.csv
|
||||||
cut -d, -f2 --complement event.csv | sort > compare.csv
|
cut -d, -f2 --complement event.csv | sort > compare.csv
|
||||||
diff compare.csv 1.csv
|
diff compare.csv 1.csv
|
||||||
# Test alert email generating
|
# Test alert email generating
|
||||||
sudo -u www-data ../app/Console/cake Event testEventNotificationEmail 1 1 > /dev/null
|
sudo -u www-data ../app/Console/cake Event testEventNotificationEmail 1 1 > /dev/null
|
||||||
# Delete created event
|
# Delete created event
|
||||||
curl -i -H "Accept: application/json" -H "content-type: application/json" -H "Authorization: $AUTH" -X POST http://${HOST}/events/delete/1
|
curl -sS -i -H "Accept: application/json" -H "content-type: application/json" -H "Authorization: $AUTH" -X POST http://${HOST}/events/delete/1
|
||||||
|
|
|
@ -10,9 +10,10 @@ if (!isset($argv[2])) {
|
||||||
if (!in_array($argv[1], ['modify', 'replace'], true)) {
|
if (!in_array($argv[1], ['modify', 'replace'], true)) {
|
||||||
fail(1, "Invalid argument '{$argv[1]}', it must be 'modify' or 'replace'.");
|
fail(1, "Invalid argument '{$argv[1]}', it must be 'modify' or 'replace'.");
|
||||||
}
|
}
|
||||||
$newConfig = json_decode($argv[2], true);
|
try {
|
||||||
if ($newConfig === null) {
|
$newConfig = json_decode($argv[2], true, JSON_THROW_ON_ERROR);
|
||||||
fail(2, "Could not decode new config, it is not JSON: " . json_last_error_msg());
|
} catch (Exception $e) {
|
||||||
|
fail(2, "Could not decode new config, it is not JSON: " . $e->getMessage());
|
||||||
}
|
}
|
||||||
if (!is_array($newConfig)) {
|
if (!is_array($newConfig)) {
|
||||||
fail(2, "Provided new config is not array, `" . gettype($newConfig) . "` given.");
|
fail(2, "Provided new config is not array, `" . gettype($newConfig) . "` given.");
|
||||||
|
@ -41,4 +42,4 @@ if ($argv[1] === 'modify') {
|
||||||
file_put_contents($configFile, "<?php\n\$config = " . var_export($merged, true) . ';', LOCK_EX);
|
file_put_contents($configFile, "<?php\n\$config = " . var_export($merged, true) . ';', LOCK_EX);
|
||||||
|
|
||||||
// Returns config file before modification
|
// Returns config file before modification
|
||||||
echo json_encode($config);
|
echo json_encode($config, JSON_THROW_ON_ERROR | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
|
||||||
|
|
|
@ -2,6 +2,7 @@
|
||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
import uuid
|
import uuid
|
||||||
|
import logging
|
||||||
import inspect
|
import inspect
|
||||||
import subprocess
|
import subprocess
|
||||||
import unittest
|
import unittest
|
||||||
|
@ -11,21 +12,19 @@ from xml.etree import ElementTree as ET
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
import urllib3 # type: ignore
|
import urllib3 # type: ignore
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
|
from typing import Union
|
||||||
import logging
|
|
||||||
logging.disable(logging.CRITICAL)
|
|
||||||
logger = logging.getLogger('pymisp')
|
|
||||||
|
|
||||||
|
|
||||||
from pymisp import PyMISP, MISPOrganisation, MISPUser, MISPRole, MISPSharingGroup, MISPEvent, MISPLog, MISPSighting, Distribution, ThreatLevel, Analysis, MISPEventReport, MISPServerError
|
from pymisp import PyMISP, MISPOrganisation, MISPUser, MISPRole, MISPSharingGroup, MISPEvent, MISPLog, MISPSighting, Distribution, ThreatLevel, Analysis, MISPEventReport, MISPServerError
|
||||||
from pymisp.tools import DomainIPObject
|
from pymisp.tools import DomainIPObject
|
||||||
|
from pymisp.api import get_uuid_or_id_from_abstract_misp
|
||||||
|
|
||||||
|
logging.disable(logging.CRITICAL)
|
||||||
|
logger = logging.getLogger('pymisp')
|
||||||
|
urllib3.disable_warnings()
|
||||||
|
|
||||||
# Load access information for env variables
|
# Load access information for env variables
|
||||||
url = "http://" + os.environ["HOST"]
|
url = "http://" + os.environ["HOST"]
|
||||||
key = os.environ["AUTH"]
|
key = os.environ["AUTH"]
|
||||||
|
|
||||||
urllib3.disable_warnings()
|
|
||||||
|
|
||||||
|
|
||||||
def create_simple_event() -> MISPEvent:
|
def create_simple_event() -> MISPEvent:
|
||||||
caller_name = inspect.stack()[1].function
|
caller_name = inspect.stack()[1].function
|
||||||
|
@ -52,13 +51,19 @@ def request(pymisp: PyMISP, request_type: str, url: str, data: dict = {}) -> dic
|
||||||
return pymisp._check_json_response(response)
|
return pymisp._check_json_response(response)
|
||||||
|
|
||||||
|
|
||||||
|
def publish_immediately(pymisp: PyMISP, event: Union[MISPEvent, int, str, uuid.UUID], with_email: bool = False):
|
||||||
|
event_id = get_uuid_or_id_from_abstract_misp(event)
|
||||||
|
action = "alert" if with_email else "publish"
|
||||||
|
return check_response(request(pymisp, 'POST', f'events/{action}/{event_id}/disable_background_processing:1'))
|
||||||
|
|
||||||
|
|
||||||
class MISPSetting:
|
class MISPSetting:
|
||||||
def __init__(self, admin_connector: PyMISP, new_setting: dict):
|
def __init__(self, admin_connector: PyMISP, new_setting: dict):
|
||||||
self.admin_connector = admin_connector
|
self.admin_connector = admin_connector
|
||||||
self.new_setting = new_setting
|
self.new_setting = new_setting
|
||||||
|
|
||||||
def __enter__(self):
|
def __enter__(self):
|
||||||
self.original = self.__run("modify", json.dumps(self.new_setting))
|
self.original = self.__run("modify", json.dumps(self.new_setting).encode("utf-8"))
|
||||||
# Try to reset config cache
|
# Try to reset config cache
|
||||||
self.admin_connector.get_server_setting("MISP.live")
|
self.admin_connector.get_server_setting("MISP.live")
|
||||||
|
|
||||||
|
@ -68,12 +73,12 @@ class MISPSetting:
|
||||||
self.admin_connector.get_server_setting("MISP.live")
|
self.admin_connector.get_server_setting("MISP.live")
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def __run(command: str, data: str) -> str:
|
def __run(command: str, data: bytes) -> bytes:
|
||||||
dir_path = os.path.dirname(os.path.realpath(__file__))
|
dir_path = os.path.dirname(os.path.realpath(__file__))
|
||||||
r = subprocess.run(["php", dir_path + "/modify_config.php", command, data], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
r = subprocess.run(["php", dir_path + "/modify_config.php", command, data], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
if r.returncode != 0:
|
if r.returncode != 0:
|
||||||
raise Exception([r.returncode, r.stdout, r.stderr])
|
raise Exception([r.returncode, r.stdout, r.stderr])
|
||||||
return r.stdout.decode("utf-8")
|
return r.stdout
|
||||||
|
|
||||||
|
|
||||||
class TestComprehensive(unittest.TestCase):
|
class TestComprehensive(unittest.TestCase):
|
||||||
|
@ -465,8 +470,6 @@ class TestComprehensive(unittest.TestCase):
|
||||||
check_response(self.admin_misp_connector.delete_event(event))
|
check_response(self.admin_misp_connector.delete_event(event))
|
||||||
|
|
||||||
def test_publish_alert_filter(self):
|
def test_publish_alert_filter(self):
|
||||||
check_response(self.admin_misp_connector.set_server_setting('MISP.background_jobs', 0, force=True))
|
|
||||||
|
|
||||||
first = create_simple_event()
|
first = create_simple_event()
|
||||||
first.add_tag('test_publish_filter')
|
first.add_tag('test_publish_filter')
|
||||||
first.threat_level_id = ThreatLevel.medium
|
first.threat_level_id = ThreatLevel.medium
|
||||||
|
@ -499,7 +502,7 @@ class TestComprehensive(unittest.TestCase):
|
||||||
|
|
||||||
# Publish events
|
# Publish events
|
||||||
for event in (first, second, third, four):
|
for event in (first, second, third, four):
|
||||||
check_response(self.admin_misp_connector.publish(event, alert=True))
|
publish_immediately(self.admin_misp_connector, event, with_email=True)
|
||||||
|
|
||||||
# Email notification should be send just to first event
|
# Email notification should be send just to first event
|
||||||
mail_logs = self.admin_misp_connector.search_logs(model='User', action='email')
|
mail_logs = self.admin_misp_connector.search_logs(model='User', action='email')
|
||||||
|
@ -516,8 +519,6 @@ class TestComprehensive(unittest.TestCase):
|
||||||
check_response(self.admin_misp_connector.update_user(self.admin_misp_connector._current_user))
|
check_response(self.admin_misp_connector.update_user(self.admin_misp_connector._current_user))
|
||||||
# Delete filter
|
# Delete filter
|
||||||
self.admin_misp_connector.delete_user_setting('publish_alert_filter')
|
self.admin_misp_connector.delete_user_setting('publish_alert_filter')
|
||||||
# Reenable background jobs
|
|
||||||
check_response(self.admin_misp_connector.set_server_setting('MISP.background_jobs', 1, force=True))
|
|
||||||
# Delete events
|
# Delete events
|
||||||
for event in (first, second, third, four):
|
for event in (first, second, third, four):
|
||||||
check_response(self.admin_misp_connector.delete_event(event))
|
check_response(self.admin_misp_connector.delete_event(event))
|
||||||
|
@ -740,20 +741,27 @@ class TestComprehensive(unittest.TestCase):
|
||||||
event.add_attribute("ip-src", "1.2.4.5", to_ids=True)
|
event.add_attribute("ip-src", "1.2.4.5", to_ids=True)
|
||||||
event = check_response(self.admin_misp_connector.add_event(event))
|
event = check_response(self.admin_misp_connector.add_event(event))
|
||||||
|
|
||||||
result = self._search({'returnFormat': "openioc", 'eventid': event.id, "published": [0, 1]})
|
result = self._search_event({'returnFormat': "openioc", 'eventid': event.id, "published": [0, 1]})
|
||||||
ET.fromstring(result) # check if result is valid XML
|
ET.fromstring(result) # check if result is valid XML
|
||||||
self.assertTrue("1.2.4.5" in result, result)
|
self.assertTrue("1.2.4.5" in result, result)
|
||||||
|
|
||||||
result = self._search({'returnFormat': "yara", 'eventid': event.id, "published": [0, 1]})
|
result = self._search_event({'returnFormat': "yara", 'eventid': event.id, "published": [0, 1]})
|
||||||
self.assertTrue("1.2.4.5" in result, result)
|
self.assertTrue("1.2.4.5" in result, result)
|
||||||
self.assertTrue("GENERATED" in result, result)
|
self.assertTrue("GENERATED" in result, result)
|
||||||
self.assertTrue("AS-IS" in result, result)
|
self.assertTrue("AS-IS" in result, result)
|
||||||
|
|
||||||
result = self._search({'returnFormat': "yara-json", 'eventid': event.id, "published": [0, 1]})
|
result = self._search_event({'returnFormat': "yara-json", 'eventid': event.id, "published": [0, 1]})
|
||||||
self.assertIn("generated", result)
|
self.assertIn("generated", result)
|
||||||
self.assertEqual(len(result["generated"]), 1, result)
|
self.assertEqual(len(result["generated"]), 1, result)
|
||||||
self.assertIn("as-is", result)
|
self.assertIn("as-is", result)
|
||||||
|
|
||||||
|
# RPZ
|
||||||
|
result = self._search_event({'returnFormat': "rpz", 'eventid': event.id, "published": [0, 1]})
|
||||||
|
self.assertTrue("32.5.4.2.1" in result, result)
|
||||||
|
|
||||||
|
result = self._search_attribute({'returnFormat': "rpz", 'eventid': event.id, "published": [0, 1]})
|
||||||
|
self.assertTrue("32.5.4.2.1" in result, result)
|
||||||
|
|
||||||
check_response(self.admin_misp_connector.delete_event(event))
|
check_response(self.admin_misp_connector.delete_event(event))
|
||||||
|
|
||||||
def test_event_report_empty_name(self):
|
def test_event_report_empty_name(self):
|
||||||
|
@ -916,16 +924,18 @@ class TestComprehensive(unittest.TestCase):
|
||||||
def test_search_snort_suricata(self):
|
def test_search_snort_suricata(self):
|
||||||
event = create_simple_event()
|
event = create_simple_event()
|
||||||
event.add_attribute('ip-src', '8.8.8.8', to_ids=True)
|
event.add_attribute('ip-src', '8.8.8.8', to_ids=True)
|
||||||
event = self.user_misp_connector.add_event(event)
|
event.add_attribute('snort', 'alert tcp 192.168.1.0/24 any -> 131.171.127.1 25 (content: "hacking"; msg: "malicious packet"; sid:2000001;)', to_ids=True)
|
||||||
check_response(event)
|
# Snort rule without msg, test for #9515
|
||||||
|
event.add_attribute('snort', 'alert tcp 192.168.1.0/24 any -> 131.171.127.1 25 (content: "hacking"; sid:2000001;)', to_ids=True)
|
||||||
|
event = check_response(self.user_misp_connector.add_event(event))
|
||||||
|
|
||||||
self.admin_misp_connector.publish(event, alert=False)
|
publish_immediately(self.admin_misp_connector, event)
|
||||||
time.sleep(6)
|
|
||||||
snort = self._search({'returnFormat': 'snort', 'eventid': event.id})
|
snort = self._search_event({'returnFormat': 'snort', 'eventid': event.id})
|
||||||
self.assertIsInstance(snort, str)
|
self.assertIsInstance(snort, str)
|
||||||
self.assertIn('8.8.8.8', snort)
|
self.assertIn('8.8.8.8', snort)
|
||||||
|
|
||||||
suricata = self._search({'returnFormat': 'suricata', 'eventid': event.id})
|
suricata = self._search_event({'returnFormat': 'suricata', 'eventid': event.id})
|
||||||
self.assertIsInstance(suricata, str)
|
self.assertIsInstance(suricata, str)
|
||||||
self.assertIn('8.8.8.8', suricata)
|
self.assertIn('8.8.8.8', suricata)
|
||||||
|
|
||||||
|
@ -962,7 +972,38 @@ class TestComprehensive(unittest.TestCase):
|
||||||
|
|
||||||
self.admin_misp_connector.delete_event(event)
|
self.admin_misp_connector.delete_event(event)
|
||||||
|
|
||||||
def _search(self, query: dict):
|
def test_restsearch_sightings(self):
|
||||||
|
# Create test event
|
||||||
|
event = create_simple_event()
|
||||||
|
event = self.admin_misp_connector.add_event(event)
|
||||||
|
check_response(event)
|
||||||
|
|
||||||
|
# Add sighting
|
||||||
|
sighting = MISPSighting()
|
||||||
|
sighting.value = 'test'
|
||||||
|
sighting.source = 'Testcases'
|
||||||
|
sighting.type = '1'
|
||||||
|
|
||||||
|
response = self.admin_misp_connector.add_sighting(sighting, event.attributes[0])
|
||||||
|
check_response(response)
|
||||||
|
self.assertEqual(response.source, 'Testcases')
|
||||||
|
|
||||||
|
# Try to find sighting by event UUID, this is the same type of request when doing sync
|
||||||
|
search_result = self._search_sighting('event', {
|
||||||
|
'returnFormat': 'json',
|
||||||
|
'last': 0,
|
||||||
|
'includeUuid': True,
|
||||||
|
'uuid': [event.uuid],
|
||||||
|
})
|
||||||
|
self.assertEqual(len(search_result), 1, search_result)
|
||||||
|
sighting = search_result[0]["Sighting"]
|
||||||
|
self.assertIn("attribute_uuid", sighting)
|
||||||
|
self.assertIn("event_uuid", sighting)
|
||||||
|
self.assertEqual(sighting["event_uuid"], event.uuid, search_result)
|
||||||
|
|
||||||
|
self.admin_misp_connector.delete_event(event)
|
||||||
|
|
||||||
|
def _search_event(self, query: dict):
|
||||||
response = self.admin_misp_connector._prepare_request('POST', 'events/restSearch', data=query)
|
response = self.admin_misp_connector._prepare_request('POST', 'events/restSearch', data=query)
|
||||||
response = self.admin_misp_connector._check_response(response)
|
response = self.admin_misp_connector._check_response(response)
|
||||||
check_response(response)
|
check_response(response)
|
||||||
|
@ -974,6 +1015,12 @@ class TestComprehensive(unittest.TestCase):
|
||||||
check_response(response)
|
check_response(response)
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
def _search_sighting(self, context: str, query: dict):
|
||||||
|
response = self.admin_misp_connector._prepare_request('POST', f'sightings/restSearch/{context}', data=query)
|
||||||
|
response = self.admin_misp_connector._check_response(response)
|
||||||
|
check_response(response)
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
class TestLastPwChange(unittest.TestCase):
|
class TestLastPwChange(unittest.TestCase):
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|
|
@ -33,8 +33,14 @@ def main():
|
||||||
global unpublish_event_on_remote
|
global unpublish_event_on_remote
|
||||||
parser = argparse.ArgumentParser(Path(__file__).name)
|
parser = argparse.ArgumentParser(Path(__file__).name)
|
||||||
parser.add_argument('-c', '--config', default='config.json', help='The JSON config file to use')
|
parser.add_argument('-c', '--config', default='config.json', help='The JSON config file to use')
|
||||||
|
parser.add_argument('-l', '--log', default='INFO', help='The loglevel to use.', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR',])
|
||||||
options = parser.parse_args()
|
options = parser.parse_args()
|
||||||
|
|
||||||
|
numeric_log_level = getattr(logging, options.log.upper(), None)
|
||||||
|
if not isinstance(numeric_log_level, int):
|
||||||
|
raise ValueError('Invalid log level: %s' % numeric_log_level)
|
||||||
|
logger.setLevel(numeric_log_level)
|
||||||
|
|
||||||
config = {}
|
config = {}
|
||||||
config_file_path = Path(__file__).parent / options.config
|
config_file_path = Path(__file__).parent / options.config
|
||||||
with open(config_file_path, 'r') as f:
|
with open(config_file_path, 'r') as f:
|
||||||
|
@ -74,7 +80,7 @@ def main():
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
# Collect events from source
|
# Collect events from source
|
||||||
logger.debug('Collecting events from source...')
|
logger.debug('Collecting *published* events from source...')
|
||||||
try:
|
try:
|
||||||
events_on_source = collect_events_from_source(source_instance, filters, incremental_sync)
|
events_on_source = collect_events_from_source(source_instance, filters, incremental_sync)
|
||||||
except Exception as err:
|
except Exception as err:
|
||||||
|
@ -185,6 +191,7 @@ def update_event_for_push(event: dict) -> dict:
|
||||||
elif attribute['distribution'] == 2:
|
elif attribute['distribution'] == 2:
|
||||||
event['Attribute'][i]['distribution'] = 1
|
event['Attribute'][i]['distribution'] = 1
|
||||||
|
|
||||||
|
if 'Tag' in attribute:
|
||||||
for t, tag in enumerate(attribute['Tag']):
|
for t, tag in enumerate(attribute['Tag']):
|
||||||
if tag['local']:
|
if tag['local']:
|
||||||
event['Attribute'][i]['Tag'].pop(t)
|
event['Attribute'][i]['Tag'].pop(t)
|
||||||
|
@ -201,6 +208,7 @@ def update_event_for_push(event: dict) -> dict:
|
||||||
elif attribute['distribution'] == 2:
|
elif attribute['distribution'] == 2:
|
||||||
event['Object'][i]['Attribute'][j]['distribution'] = 1
|
event['Object'][i]['Attribute'][j]['distribution'] = 1
|
||||||
|
|
||||||
|
if 'Tag' in attribute:
|
||||||
for t, tag in enumerate(attribute['Tag']):
|
for t, tag in enumerate(attribute['Tag']):
|
||||||
if tag['local']:
|
if tag['local']:
|
||||||
event['Object'][i]['Attribute'][j]['Tag'].pop(t)
|
event['Object'][i]['Attribute'][j]['Tag'].pop(t)
|
||||||
|
@ -222,7 +230,7 @@ def collect_events_from_source(source_instance: MISPInstance, filters: dict, inc
|
||||||
}
|
}
|
||||||
last_timestamp = get_last_sync_timestamp()
|
last_timestamp = get_last_sync_timestamp()
|
||||||
if incremental_sync and last_timestamp is not None:
|
if incremental_sync and last_timestamp is not None:
|
||||||
logger.debug('Using timestamp from last synchronisation %s (%s)', last_timestamp, datetime.fromtimestamp(last_timestamp))
|
logger.info('Using timestamp from last synchronisation %s (%s)', last_timestamp, datetime.fromtimestamp(last_timestamp))
|
||||||
sync_filters['timestamp'] = last_timestamp # type: ignore
|
sync_filters['timestamp'] = last_timestamp # type: ignore
|
||||||
sync_filters.update(filters)
|
sync_filters.update(filters)
|
||||||
events = source_instance.POST('/events/index', payload=sync_filters)
|
events = source_instance.POST('/events/index', payload=sync_filters)
|
||||||
|
|
|
@ -1,154 +0,0 @@
|
||||||
<?php
|
|
||||||
/**
|
|
||||||
* This file is loaded automatically by the app/webroot/index.php file after core.php
|
|
||||||
*
|
|
||||||
* This file should load/create any application wide configuration settings, such as
|
|
||||||
* Caching, Logging, loading additional configuration files.
|
|
||||||
*
|
|
||||||
* You should also use this file to include any files that provide global functions/constants
|
|
||||||
* that your application uses.
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Cache Engine Configuration
|
|
||||||
* Default settings provided below
|
|
||||||
*
|
|
||||||
* File storage engine.
|
|
||||||
*
|
|
||||||
* Cache::config('default', array(
|
|
||||||
* 'engine' => 'File', //[required]
|
|
||||||
* 'duration'=> 3600, //[optional]
|
|
||||||
* 'probability'=> 100, //[optional]
|
|
||||||
* 'path' => CACHE, //[optional] use system tmp directory - remember to use absolute path
|
|
||||||
* 'prefix' => 'cake_', //[optional] prefix every cache file with this string
|
|
||||||
* 'lock' => false, //[optional] use file locking
|
|
||||||
* 'serialize' => true, // [optional]
|
|
||||||
* 'mask' => 0666, // [optional] permission mask to use when creating cache files
|
|
||||||
* ));
|
|
||||||
*
|
|
||||||
* APC (http://pecl.php.net/package/APC)
|
|
||||||
*
|
|
||||||
* Cache::config('default', array(
|
|
||||||
* 'engine' => 'Apc', //[required]
|
|
||||||
* 'duration'=> 3600, //[optional]
|
|
||||||
* 'probability'=> 100, //[optional]
|
|
||||||
* 'prefix' => Inflector::slug(APP_DIR) . '_', //[optional] prefix every cache file with this string
|
|
||||||
* ));
|
|
||||||
*
|
|
||||||
* Xcache (http://xcache.lighttpd.net/)
|
|
||||||
*
|
|
||||||
* Cache::config('default', array(
|
|
||||||
* 'engine' => 'Xcache', //[required]
|
|
||||||
* 'duration'=> 3600, //[optional]
|
|
||||||
* 'probability'=> 100, //[optional]
|
|
||||||
* 'prefix' => Inflector::slug(APP_DIR) . '_', //[optional] prefix every cache file with this string
|
|
||||||
* 'user' => 'user', //user from xcache.admin.user settings
|
|
||||||
* 'password' => 'password', //plaintext password (xcache.admin.pass)
|
|
||||||
* ));
|
|
||||||
*
|
|
||||||
* Memcache (http://memcached.org/)
|
|
||||||
*
|
|
||||||
* Cache::config('default', array(
|
|
||||||
* 'engine' => 'Memcache', //[required]
|
|
||||||
* 'duration'=> 3600, //[optional]
|
|
||||||
* 'probability'=> 100, //[optional]
|
|
||||||
* 'prefix' => Inflector::slug(APP_DIR) . '_', //[optional] prefix every cache file with this string
|
|
||||||
* 'servers' => array(
|
|
||||||
* '127.0.0.1:11211' // localhost, default port 11211
|
|
||||||
* ), //[optional]
|
|
||||||
* 'persistent' => true, // [optional] set this to false for non-persistent connections
|
|
||||||
* 'compress' => false, // [optional] compress data in Memcache (slower, but uses less memory)
|
|
||||||
* ));
|
|
||||||
*
|
|
||||||
* Wincache (http://php.net/wincache)
|
|
||||||
*
|
|
||||||
* Cache::config('default', array(
|
|
||||||
* 'engine' => 'Wincache', //[required]
|
|
||||||
* 'duration'=> 3600, //[optional]
|
|
||||||
* 'probability'=> 100, //[optional]
|
|
||||||
* 'prefix' => Inflector::slug(APP_DIR) . '_', //[optional] prefix every cache file with this string
|
|
||||||
* ));
|
|
||||||
*
|
|
||||||
* Redis (http://http://redis.io/)
|
|
||||||
*
|
|
||||||
* Cache::config('default', array(
|
|
||||||
* 'engine' => 'Redis', //[required]
|
|
||||||
* 'duration'=> 3600, //[optional]
|
|
||||||
* 'probability'=> 100, //[optional]
|
|
||||||
* 'prefix' => Inflector::slug(APP_DIR) . '_', //[optional] prefix every cache file with this string
|
|
||||||
* 'server' => '127.0.0.1' // localhost
|
|
||||||
* 'port' => 6379 // default port 6379
|
|
||||||
* 'timeout' => 0 // timeout in seconds, 0 = unlimited
|
|
||||||
* 'persistent' => true, // [optional] set this to false for non-persistent connections
|
|
||||||
* ));
|
|
||||||
*/
|
|
||||||
Cache::config('default', array('engine' => 'File'));
|
|
||||||
Configure::load('config');
|
|
||||||
|
|
||||||
if (!Configure::read('MISP.baseurl')) {
|
|
||||||
if (!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off' || $_SERVER['SERVER_PORT'] == 443) {
|
|
||||||
Configure::write('MISP.baseurl', sprintf('https://%s:%d', $_SERVER['SERVER_ADDR'], $_SERVER['SERVER_PORT']));
|
|
||||||
} else {
|
|
||||||
Configure::write('MISP.baseurl', sprintf('http://%s:%d', $_SERVER['SERVER_ADDR'], $_SERVER['SERVER_PORT']));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Plugins need to be loaded manually, you can either load them one by one or all of them in a single call
|
|
||||||
* Uncomment one of the lines below, as you need. make sure you read the documentation on CakePlugin to use more
|
|
||||||
* advanced ways of loading plugins
|
|
||||||
*
|
|
||||||
* CakePlugin::loadAll(); // Loads all plugins at once
|
|
||||||
* CakePlugin::load('DebugKit'); //Loads a single plugin named DebugKit
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
|
|
||||||
CakePlugin::load('SysLog');
|
|
||||||
CakePlugin::load('Assets'); // having Logable
|
|
||||||
CakePlugin::load('SysLogLogable');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Uncomment the following line to enable client SSL certificate authentication.
|
|
||||||
* It's also necessary to configure the plugin — for more information, please read app/Plugin/CertAuth/reame.md
|
|
||||||
*/
|
|
||||||
// CakePlugin::load('CertAuth');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* You can attach event listeners to the request lifecyle as Dispatcher Filter . By Default CakePHP bundles two filters:
|
|
||||||
*
|
|
||||||
* - AssetDispatcher filter will serve your asset files (css, images, js, etc) from your themes and plugins
|
|
||||||
* - CacheDispatcher filter will read the Cache.check configure variable and try to serve cached content generated from controllers
|
|
||||||
*
|
|
||||||
* Feel free to remove or add filters as you see fit for your application. A few examples:
|
|
||||||
*
|
|
||||||
* Configure::write('Dispatcher.filters', array(
|
|
||||||
* 'MyCacheFilter', // will use MyCacheFilter class from the Routing/Filter package in your app.
|
|
||||||
* 'MyPlugin.MyFilter', // will use MyFilter class from the Routing/Filter package in MyPlugin plugin.
|
|
||||||
* array('callable' => $aFunction, 'on' => 'before', 'priority' => 9), // A valid PHP callback type to be called on beforeDispatch
|
|
||||||
* array('callable' => $anotherMethod, 'on' => 'after'), // A valid PHP callback type to be called on afterDispatch
|
|
||||||
*
|
|
||||||
* ));
|
|
||||||
*/
|
|
||||||
Configure::write('Dispatcher.filters', array(
|
|
||||||
'AssetDispatcher',
|
|
||||||
'CacheDispatcher'
|
|
||||||
));
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Configures default file logging options
|
|
||||||
*/
|
|
||||||
App::uses('CakeLog', 'Log');
|
|
||||||
CakeLog::config('debug', array(
|
|
||||||
'engine' => 'FileLog',
|
|
||||||
'types' => array('notice', 'info', 'debug'),
|
|
||||||
'file' => 'debug',
|
|
||||||
));
|
|
||||||
CakeLog::config('error', array(
|
|
||||||
'engine' => 'FileLog',
|
|
||||||
'types' => array('warning', 'error', 'critical', 'alert', 'emergency'),
|
|
||||||
'file' => 'error',
|
|
||||||
));
|
|
||||||
|
|
||||||
CakePlugin::loadAll(array(
|
|
||||||
'CakeResque' => array('bootstrap' => true)
|
|
||||||
));
|
|
|
@ -1,81 +0,0 @@
|
||||||
<?php
|
|
||||||
$config = array (
|
|
||||||
'debug' => 1,
|
|
||||||
'Security' =>
|
|
||||||
array (
|
|
||||||
'level' => 'medium',
|
|
||||||
'salt' => 'Rooraenietu8Eeyo<Qu2eeNfterd-dd+',
|
|
||||||
'cipherSeed' => '',
|
|
||||||
//'auth'=>array('CertAuth.Certificate'), // additional authentication methods
|
|
||||||
),
|
|
||||||
'MISP' =>
|
|
||||||
array (
|
|
||||||
'baseurl' => 'http://misp.local',
|
|
||||||
'footerpart1' => 'Powered by MISP',
|
|
||||||
'footerpart2' => '© Belgian Defense CERT & NCIRC',
|
|
||||||
'org' => 'ORGNAME',
|
|
||||||
'showorg' => true,
|
|
||||||
'background_jobs' => false,
|
|
||||||
'email' => 'email@address.com',
|
|
||||||
'contact' => 'email@address.com',
|
|
||||||
'cveurl' => 'http://web.nvd.nist.gov/view/vuln/detail?vulnId=',
|
|
||||||
'disablerestalert' => false,
|
|
||||||
'default_event_distribution' => '0',
|
|
||||||
'default_attribute_distribution' => 'event',
|
|
||||||
'tagging' => true,
|
|
||||||
'full_tags_on_event_index' => true,
|
|
||||||
'footer_logo' => '',
|
|
||||||
'take_ownership_xml_import' => false,
|
|
||||||
'unpublishedprivate' => false,
|
|
||||||
),
|
|
||||||
'GnuPG' =>
|
|
||||||
array (
|
|
||||||
'onlyencrypted' => false,
|
|
||||||
'email' => '',
|
|
||||||
'homedir' => '',
|
|
||||||
'password' => '',
|
|
||||||
'bodyonlyencrypted' => false,
|
|
||||||
),
|
|
||||||
'Proxy' =>
|
|
||||||
array (
|
|
||||||
'host' => '',
|
|
||||||
'port' => '',
|
|
||||||
'method' => '',
|
|
||||||
'user' => '',
|
|
||||||
'password' => '',
|
|
||||||
),
|
|
||||||
'SecureAuth' =>
|
|
||||||
array (
|
|
||||||
'amount' => 5,
|
|
||||||
'expire' => 300,
|
|
||||||
),
|
|
||||||
// Uncomment the following to enable client SSL certificate authentication
|
|
||||||
/*
|
|
||||||
'CertAuth' =>
|
|
||||||
array(
|
|
||||||
'ca' => array( 'FIRST.Org' ), // allowed CAs
|
|
||||||
'caId' => 'O', // which attribute will be used to verify the CA
|
|
||||||
'userModel' => 'User', // name of the User class to check if user exists
|
|
||||||
'userModelKey' => 'nids_sid', // User field that will be used for querying
|
|
||||||
'map' => array( // maps client certificate attributes to User properties
|
|
||||||
'O' => 'org',
|
|
||||||
'emailAddress'=>'email',
|
|
||||||
),
|
|
||||||
'syncUser' => true, // should the User be synchronized with an external REST API
|
|
||||||
'userDefaults'=> array( // default user attributes, only used when creating new users
|
|
||||||
'role_id' => 4,
|
|
||||||
),
|
|
||||||
'restApi' => array( // API parameters
|
|
||||||
'url' => 'https://example.com/data/users', // URL to query
|
|
||||||
'headers' => array(), // additional headers, used for authentication
|
|
||||||
'param' => array( 'email' => 'email'), // query parameters to add to the URL, mapped to USer properties
|
|
||||||
'map' => array( // maps REST result to the User properties
|
|
||||||
'uid' => 'nids_sid',
|
|
||||||
'team' => 'org',
|
|
||||||
'email' => 'email',
|
|
||||||
'pgp_public'=> 'gpgkey',
|
|
||||||
),
|
|
||||||
),
|
|
||||||
),
|
|
||||||
*/
|
|
||||||
);
|
|
285
travis/core.php
285
travis/core.php
|
@ -1,285 +0,0 @@
|
||||||
<?php
|
|
||||||
/**
|
|
||||||
* This is core configuration file.
|
|
||||||
*
|
|
||||||
* Use it to configure core behavior of Cake.
|
|
||||||
*
|
|
||||||
* PHP 5
|
|
||||||
*
|
|
||||||
* CakePHP(tm) : Rapid Development Framework (http://cakephp.org)
|
|
||||||
* Copyright 2005-2012, Cake Software Foundation, Inc. (http://cakefoundation.org)
|
|
||||||
*
|
|
||||||
* Licensed under The MIT License
|
|
||||||
* Redistributions of files must retain the above copyright notice.
|
|
||||||
*
|
|
||||||
* @copyright Copyright 2005-2012, Cake Software Foundation, Inc. (http://cakefoundation.org)
|
|
||||||
* @link http://cakephp.org CakePHP(tm) Project
|
|
||||||
* @package app.Config
|
|
||||||
* @since CakePHP(tm) v 0.2.9
|
|
||||||
* @license MIT License (http://www.opensource.org/licenses/mit-license.php)
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* CakePHP Debug Level:
|
|
||||||
*
|
|
||||||
* Production Mode:
|
|
||||||
* 0: No error messages, errors, or warnings shown. Flash messages redirect.
|
|
||||||
*
|
|
||||||
* Development Mode:
|
|
||||||
* 1: Errors and warnings shown, model caches refreshed, flash messages halted.
|
|
||||||
* 2: As in 1, but also with full debug messages and SQL output.
|
|
||||||
*
|
|
||||||
* In production mode, flash messages redirect after a time interval.
|
|
||||||
* In development mode, you need to click the flash message to continue.
|
|
||||||
*/
|
|
||||||
Configure::write('debug', 2); // 0 = for production, 2 = full debug mode
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Configure the Error handler used to handle errors for your application. By default
|
|
||||||
* ErrorHandler::handleError() is used. It will display errors using Debugger, when debug > 0
|
|
||||||
* and log errors with CakeLog when debug = 0.
|
|
||||||
*
|
|
||||||
* Options:
|
|
||||||
*
|
|
||||||
* - `handler` - callback - The callback to handle errors. You can set this to any callable type,
|
|
||||||
* including anonymous functions.
|
|
||||||
* - `level` - int - The level of errors you are interested in capturing.
|
|
||||||
* - `trace` - boolean - Include stack traces for errors in log files.
|
|
||||||
*
|
|
||||||
* @see ErrorHandler for more information on error handling and configuration.
|
|
||||||
*/
|
|
||||||
Configure::write('Error', array(
|
|
||||||
'handler' => 'ErrorHandler::handleError',
|
|
||||||
'level' => E_ALL & ~E_DEPRECATED,
|
|
||||||
'trace' => true
|
|
||||||
));
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Configure the Exception handler used for uncaught exceptions. By default,
|
|
||||||
* ErrorHandler::handleException() is used. It will display a HTML page for the exception, and
|
|
||||||
* while debug > 0, framework errors like Missing Controller will be displayed. When debug = 0,
|
|
||||||
* framework errors will be coerced into generic HTTP errors.
|
|
||||||
*
|
|
||||||
* Options:
|
|
||||||
*
|
|
||||||
* - `handler` - callback - The callback to handle exceptions. You can set this to any callback type,
|
|
||||||
* including anonymous functions.
|
|
||||||
* - `renderer` - string - The class responsible for rendering uncaught exceptions. If you choose a custom class you
|
|
||||||
* should place the file for that class in app/Lib/Error. This class needs to implement a render method.
|
|
||||||
* - `log` - boolean - Should Exceptions be logged?
|
|
||||||
*
|
|
||||||
* @see ErrorHandler for more information on exception handling and configuration.
|
|
||||||
*/
|
|
||||||
Configure::write('Exception', array(
|
|
||||||
'handler' => 'ErrorHandler::handleException',
|
|
||||||
'renderer' => 'ExceptionRenderer',
|
|
||||||
'log' => true,
|
|
||||||
'skipLog' => array(
|
|
||||||
'NotFoundException',
|
|
||||||
)
|
|
||||||
));
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Application wide charset encoding
|
|
||||||
*/
|
|
||||||
Configure::write('App.encoding', 'UTF-8');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* To configure CakePHP *not* to use mod_rewrite and to
|
|
||||||
* use CakePHP pretty URLs, remove these .htaccess
|
|
||||||
* files:
|
|
||||||
*
|
|
||||||
* /.htaccess
|
|
||||||
* /app/.htaccess
|
|
||||||
* /app/webroot/.htaccess
|
|
||||||
*
|
|
||||||
* And uncomment the App.baseUrl below:
|
|
||||||
*/
|
|
||||||
//Configure::write('App.baseUrl', env('SCRIPT_NAME'));
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Uncomment the define below to use CakePHP prefix routes.
|
|
||||||
*
|
|
||||||
* The value of the define determines the names of the routes
|
|
||||||
* and their associated controller actions:
|
|
||||||
*
|
|
||||||
* Set to an array of prefixes you want to use in your application. Use for
|
|
||||||
* admin or other prefixed routes.
|
|
||||||
*
|
|
||||||
* Routing.prefixes = array('admin', 'manager');
|
|
||||||
*
|
|
||||||
* Enables:
|
|
||||||
* `admin_index()` and `/admin/controller/index`
|
|
||||||
* `manager_index()` and `/manager/controller/index`
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
Configure::write('Routing.prefixes', array('admin'));
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Turn off all caching application-wide.
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
Configure::write('Cache.disable', false);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Enable cache checking.
|
|
||||||
*
|
|
||||||
* If set to true, for view caching you must still use the controller
|
|
||||||
* public $cacheAction inside your controllers to define caching settings.
|
|
||||||
* You can either set it controller-wide by setting public $cacheAction = true,
|
|
||||||
* or in each action using $this->cacheAction = true.
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
//Configure::write('Cache.check', true);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Defines the default error type when using the log() function. Used for
|
|
||||||
* differentiating error logging and debugging. Currently PHP supports LOG_DEBUG.
|
|
||||||
*/
|
|
||||||
define('LOG_ERROR', LOG_ERR);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Session configuration.
|
|
||||||
*
|
|
||||||
* Contains an array of settings to use for session configuration. The defaults key is
|
|
||||||
* used to define a default preset to use for sessions, any settings declared here will override
|
|
||||||
* the settings of the default config.
|
|
||||||
*
|
|
||||||
* ## Options
|
|
||||||
*
|
|
||||||
* - `Session.cookie` - The name of the cookie to use. Defaults to 'CAKEPHP'
|
|
||||||
* - `Session.timeout` - The number of minutes you want sessions to live for. This timeout is handled by CakePHP
|
|
||||||
* - `Session.cookieTimeout` - The number of minutes you want session cookies to live for.
|
|
||||||
* - `Session.checkAgent` - Do you want the user agent to be checked when starting sessions? You might want to set the
|
|
||||||
* value to false, when dealing with older versions of IE, Chrome Frame or certain web-browsing devices and AJAX
|
|
||||||
* - `Session.defaults` - The default configuration set to use as a basis for your session.
|
|
||||||
* There are four builtins: php, cake, cache, database.
|
|
||||||
* - `Session.handler` - Can be used to enable a custom session handler. Expects an array of callables,
|
|
||||||
* that can be used with `session_save_handler`. Using this option will automatically add `session.save_handler`
|
|
||||||
* to the ini array.
|
|
||||||
* - `Session.autoRegenerate` - Enabling this setting, turns on automatic renewal of sessions, and
|
|
||||||
* sessionids that change frequently. See CakeSession::$requestCountdown.
|
|
||||||
* - `Session.ini` - An associative array of additional ini values to set.
|
|
||||||
*
|
|
||||||
* The built in defaults are:
|
|
||||||
*
|
|
||||||
* - 'php' - Uses settings defined in your php.ini.
|
|
||||||
* - 'cake' - Saves session files in CakePHP's /tmp directory.
|
|
||||||
* - 'database' - Uses CakePHP's database sessions.
|
|
||||||
* - 'cache' - Use the Cache class to save sessions.
|
|
||||||
*
|
|
||||||
* To define a custom session handler, save it at /app/Model/Datasource/Session/<name>.php.
|
|
||||||
* Make sure the class implements `CakeSessionHandlerInterface` and set Session.handler to <name>
|
|
||||||
*
|
|
||||||
* To use database sessions, run the app/Config/Schema/sessions.php schema using
|
|
||||||
* the cake shell command: cake schema create Sessions
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
Configure::write('Session', array(
|
|
||||||
'timeout' => 60, // Session timeout, default is 1 hour
|
|
||||||
'defaults' => 'php'
|
|
||||||
));
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The level of CakePHP security.
|
|
||||||
*/
|
|
||||||
Configure::write('Security.level', 'medium');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A random string used in security hashing methods.
|
|
||||||
*/
|
|
||||||
Configure::write('Security.salt', 'Rooraenietu8Eeyo<Qu2eeNfterd-dd+');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A random numeric string (digits only) used to encrypt/decrypt strings.
|
|
||||||
*/
|
|
||||||
Configure::write('Security.cipherSeed', '395786739573056621429506834955');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Apply timestamps with the last modified time to static assets (js, css, images).
|
|
||||||
* Will append a querystring parameter containing the time the file was modified. This is
|
|
||||||
* useful for invalidating browser caches.
|
|
||||||
*
|
|
||||||
* Set to `true` to apply timestamps when debug > 0. Set to 'force' to always enable
|
|
||||||
* timestamping regardless of debug value.
|
|
||||||
*/
|
|
||||||
//Configure::write('Asset.timestamp', true);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compress CSS output by removing comments, whitespace, repeating tags, etc.
|
|
||||||
* This requires a/var/cache directory to be writable by the web server for caching.
|
|
||||||
* and /vendors/csspp/csspp.php
|
|
||||||
*
|
|
||||||
* To use, prefix the CSS link URL with '/ccss/' instead of '/css/' or use HtmlHelper::css().
|
|
||||||
*/
|
|
||||||
//Configure::write('Asset.filter.css', 'css.php');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Plug in your own custom JavaScript compressor by dropping a script in your webroot to handle the
|
|
||||||
* output, and setting the config below to the name of the script.
|
|
||||||
*
|
|
||||||
* To use, prefix your JavaScript link URLs with '/cjs/' instead of '/js/' or use JavaScriptHelper::link().
|
|
||||||
*/
|
|
||||||
//Configure::write('Asset.filter.js', 'custom_javascript_output_filter.php');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The classname and database used in CakePHP's
|
|
||||||
* access control lists.
|
|
||||||
*/
|
|
||||||
Configure::write('Acl.classname', 'DbAcl');
|
|
||||||
Configure::write('Acl.database', 'default');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Uncomment this line and correct your server timezone to fix
|
|
||||||
* any date & time related errors.
|
|
||||||
*/
|
|
||||||
//date_default_timezone_set('UTC');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Pick the caching engine to use. If APC is enabled use it.
|
|
||||||
* If running via cli - apc is disabled by default. ensure it's available and enabled in this case
|
|
||||||
*
|
|
||||||
* Note: 'default' and other application caches should be configured in app/Config/bootstrap.php.
|
|
||||||
* Please check the comments in boostrap.php for more info on the cache engines available
|
|
||||||
* and their setttings.
|
|
||||||
*/
|
|
||||||
$engine = 'File';
|
|
||||||
if (function_exists('apcu_dec') && (PHP_SAPI !== 'cli' || ini_get('apc.enable_cli'))) {
|
|
||||||
require_once APP . 'Plugin/ApcuCache/Engine/ApcuEngine.php'; // it is not possible to use plugin
|
|
||||||
$engine = 'Apcu';
|
|
||||||
}
|
|
||||||
|
|
||||||
// In development mode, caches should expire quickly.
|
|
||||||
$duration = '+999 days';
|
|
||||||
if (Configure::read('debug') >= 1) {
|
|
||||||
$duration = '+10 seconds';
|
|
||||||
}
|
|
||||||
|
|
||||||
// Prefix each application on the same server with a different string, to avoid Memcache and APC conflicts.
|
|
||||||
$prefix = 'myapp_';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Configure the cache used for general framework caching. Path information,
|
|
||||||
* object listings, and translation cache files are stored with this configuration.
|
|
||||||
*/
|
|
||||||
Cache::config('_cake_core_', array(
|
|
||||||
'engine' => $engine,
|
|
||||||
'prefix' => $prefix . 'cake_core_',
|
|
||||||
'path' => CACHE . 'persistent' . DS,
|
|
||||||
'serialize' => ($engine === 'File'),
|
|
||||||
'duration' => $duration
|
|
||||||
));
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Configure the cache for model and datasource caches. This cache configuration
|
|
||||||
* is used to store schema descriptions, and table listings in connections.
|
|
||||||
*/
|
|
||||||
Cache::config('_cake_model_', array(
|
|
||||||
'engine' => $engine,
|
|
||||||
'prefix' => $prefix . 'cake_model_',
|
|
||||||
'path' => CACHE . 'models' . DS,
|
|
||||||
'serialize' => ($engine === 'File'),
|
|
||||||
'duration' => $duration
|
|
||||||
));
|
|
||||||
|
|
||||||
//require_once dirname(__DIR__) . '/Vendor/autoload.php';
|
|
Loading…
Reference in New Issue