diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ef392fa02..d60373e3a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,9 +6,9 @@ name: misp # events but only for the 2.4 and develop branches on: push: - branches: [ 2.4, develop, misp-stix, taxii ] + branches: [ '2.4', develop, misp-stix, taxii ] pull_request: - branches: [ 2.4, develop, misp-stix ] + branches: [ '2.4', develop, misp-stix ] # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: @@ -62,186 +62,178 @@ jobs: php_version: ${{ matrix.php }} run: | sudo apt-get -y update - # Repo is missing for unknown reason - LC_ALL=C.UTF-8 sudo apt-add-repository ppa:ondrej/php -y - if [[ $php_version == "7.2" ]]; then - # hotfix due to: https://bugs.php.net/bug.php?id=81640 TODO: remove after libpcre2-8-0:10.36 gets to stable channel - sudo apt-get --fix-broken install - fi sudo apt-get -y install curl python3 python3-pip python3-virtualenv apache2 libapache2-mod-php$php_version # Runs a set of commands using the runners shell - name: Install deps run: | - sudo chown $USER:www-data $HOME/.composer - pushd app - sudo -H -u $USER composer config --no-plugins allow-plugins.composer/installers true - sudo -H -u $USER composer install --no-progress - popd - cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php - # Set perms - sudo chown -R $USER:www-data `pwd` - sudo chmod -R 775 `pwd` - sudo chmod -R g+ws `pwd`/app/tmp - sudo chmod -R g+ws `pwd`/app/tmp/cache - sudo chmod -R g+ws `pwd`/app/tmp/cache/persistent - sudo chmod -R g+ws `pwd`/app/tmp/cache/models - sudo chmod -R g+ws `pwd`/app/tmp/logs - sudo chmod -R g+ws `pwd`/app/files - sudo chmod -R g+ws `pwd`/app/files/scripts/tmp - sudo chown -R $USER:www-data `pwd` - # Resque perms - sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp - sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp - # install MySQL - sudo chmod -R 777 `pwd`/INSTALL - mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';" - mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant usage on *.* to misp@'%' identified by 'blah';" - mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant all privileges on misp.* to misp@'%';" - mysql -h 127.0.0.1 --port 3306 -u misp -pblah misp < INSTALL/MYSQL.sql - # configure apache virtual hosts - sudo chmod -R 777 `pwd`/build - sudo mkdir -p /etc/apache2/sites-available - sudo cp -f build/github-action-ci-apache /etc/apache2/sites-available/misp.conf - sudo sed -e "s?%GITHUB_WORKSPACE%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.conf - sudo sed -e "s?%HOST%?${HOST}?g" --in-place /etc/apache2/sites-available/misp.conf - sudo a2dissite 000-default - sudo a2ensite misp.conf - cat /etc/apache2/sites-enabled/misp.conf - sudo a2enmod rewrite - sudo systemctl restart apache2 - # MISP configuration - sudo chmod -R 777 `pwd`/travis - sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php - sudo cp travis/database.php app/Config/database.php - sudo cp app/Config/core.default.php app/Config/core.php - sudo cp app/Config/config.default.php app/Config/config.php - sudo cp travis/email.php app/Config/email.php - # Ensure the perms - sudo chown -R $USER:www-data `pwd`/app/Config - sudo chmod -R 777 `pwd`/app/Config - # GPG setup - sudo mkdir `pwd`/.gnupg - # /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS - sudo cp -a /dev/urandom /dev/random - sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/travis/gpg - sudo gpg --list-secret-keys --homedir `pwd`/.gnupg - # change perms - sudo chown -R $USER:www-data `pwd` - sudo chown -R www-data:www-data `pwd`/.gnupg - sudo chmod -R 700 `pwd`/.gnupg - sudo usermod -a -G www-data $USER - sudo chmod -R 777 `pwd`/app/Plugin/CakeResque/tmp/ - # Ensure the perms of config files - sudo chown -R $USER:www-data `pwd`/app/Config - sudo chmod -R 777 `pwd`/app/Config - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.server_settings_skip_backup_rotate" 1' - sudo chown -R $USER:www-data `pwd`/app/Config - sudo chmod -R 777 `pwd`/app/Config + sudo chown $USER:www-data $HOME/.composer + pushd app + composer config --no-plugins allow-plugins.composer/installers true + composer install --no-progress + popd + cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php - # fix perms (?) - namei -m /home/runner/work - sudo chmod +x /home/runner/work - sudo chmod +x /home/runner - sudo chmod +x /home - sudo chmod +x / + # Set perms + sudo chown -R $USER:www-data `pwd` + sudo chmod -R 775 `pwd` + sudo chmod -R g+ws `pwd`/app/tmp + sudo chmod -R g+ws `pwd`/app/tmp/cache + sudo chmod -R g+ws `pwd`/app/tmp/cache/persistent + sudo chmod -R g+ws `pwd`/app/tmp/cache/models + sudo chmod -R g+ws `pwd`/app/tmp/logs + sudo chmod -R g+ws `pwd`/app/files + sudo chmod -R g+ws `pwd`/app/files/scripts/tmp + sudo chown -R $USER:www-data `pwd` - - name: DB Update - run: | - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.osuser" $USER' - sudo -E su $USER -c 'app/Console/cake Admin runUpdates' - sudo -E su $USER -c 'app/Console/cake Admin schemaDiagnostics' + # Resque perms + sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp + sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp - - name: Configure MISP - run: | - sudo -u $USER app/Console/cake userInit -q | sudo tee ./key.txt - echo "AUTH=`cat key.txt`" >> $GITHUB_ENV - sudo -u $USER app/Console/cake Admin setSetting "Session.autoRegenerate" 0 - sudo -u $USER app/Console/cake Admin setSetting "Session.timeout" 600 - sudo -u $USER app/Console/cake Admin setSetting "Session.cookieTimeout" 3600 - sudo -u $USER app/Console/cake Admin setSetting "MISP.host_org_id" 1 - sudo -u $USER app/Console/cake Admin setSetting "MISP.email" "info@admin.test" - sudo -u $USER app/Console/cake Admin setSetting "MISP.disable_emailing" false - sudo -u $USER app/Console/cake Admin setSetting --force "debug" true - sudo -u $USER app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false - sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1" - sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_port" 6379 - sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_database" 13 - sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_password" "" - sudo -u $USER app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test" - sudo -u $USER app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg" - sudo -u $USER app/Console/cake Admin setSetting "GnuPG.password" "travistest" - sudo -u $USER app/Console/cake Admin setSetting "MISP.download_gpg_from_homedir" 1 + # Fill database with basic MISP schema + mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';" + mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant usage on *.* to misp@'%' identified by 'blah';" + mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant all privileges on misp.* to misp@'%';" + mysql -h 127.0.0.1 --port 3306 -u misp -pblah misp < INSTALL/MYSQL.sql - - name: Configure ZMQ - run: | - sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_host" "127.0.0.1" - sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_port" 6379 - sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_database" 1 - sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_password" "" - sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" 1 - sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_audit_notifications_enable" 1 + # configure apache virtual hosts + sudo mkdir -p /etc/apache2/sites-available + sudo cp -f build/github-action-ci-apache /etc/apache2/sites-available/misp.conf + sudo sed -e "s?%GITHUB_WORKSPACE%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.conf + sudo sed -e "s?%HOST%?${HOST}?g" --in-place /etc/apache2/sites-available/misp.conf + sudo a2dissite 000-default + sudo a2ensite misp.conf + cat /etc/apache2/sites-enabled/misp.conf + sudo a2enmod rewrite + sudo systemctl start --no-block apache2 - - name: Update Galaxies - run: sudo -E su $USER -c 'app/Console/cake Admin updateGalaxies' + # MISP configuration + sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php + sudo cp build/database.php app/Config/database.php + sudo cp app/Config/core.default.php app/Config/core.php + sudo cp app/Config/config.default.php app/Config/config.php + sudo cp build/email.php app/Config/email.php - - name: Update Taxonomies - run: sudo -E su $USER -c 'app/Console/cake Admin updateTaxonomies' + # GPG setup + sudo mkdir `pwd`/.gnupg + # /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS + sudo cp -a /dev/urandom /dev/random + sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/build/gpg + sudo gpg --list-secret-keys --homedir `pwd`/.gnupg - - name: Update Warninglists - run: sudo -E su $USER -c 'app/Console/cake Admin updateWarningLists --verbose' + # change perms + sudo chown -R $USER:www-data `pwd` + sudo chown -R www-data:www-data `pwd`/.gnupg + sudo chmod -R 700 `pwd`/.gnupg + sudo usermod -a -G www-data $USER + sudo chmod -R 777 `pwd`/app/Plugin/CakeResque/tmp/ + # Ensure the perms of config files + sudo chown -R $USER:www-data `pwd`/app/Config + sudo chmod -R 777 `pwd`/app/Config + app/Console/cake Admin setSetting "MISP.server_settings_skip_backup_rotate" 1 + sudo chown -R $USER:www-data `pwd`/app/Config + sudo chmod -R 777 `pwd`/app/Config - - name: Update Noticelists - run: sudo -E su $USER -c 'app/Console/cake Admin updateNoticeLists' - - - name: Update Object Templates - run: sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates 1' - - - name: Turn MISP live - run: sudo -E su $USER -c 'app/Console/cake Live 1' - - - name: Check if Redis is ready - run: sudo -E su $USER -c 'app/Console/cake Admin redisReady' - - - name: Start workers - run: | - sudo chmod +x app/Console/worker/start.sh - sudo -u www-data 'app/Console/worker/start.sh' + # fix perms (?) + namei -m /home/runner/work + sudo chmod +x /home/runner/work + sudo chmod +x /home/runner + sudo chmod +x /home + sudo chmod +x / - name: Python setup run: | - sudo chmod 777 ./key.txt - sudo chmod -R 777 ./tests - # Start workers - # Dirty install python stuff - python3 -m virtualenv -p python3 ./venv - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python"' - . ./venv/bin/activate - export PYTHONPATH=$PYTHONPATH:./app/files/scripts - pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara pytest - deactivate + # Dirty install python stuff + python3 -m virtualenv -p python3 ./venv + app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python" + . ./venv/bin/activate + export PYTHONPATH=$PYTHONPATH:./app/files/scripts + pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara pytest + deactivate + + - name: DB Update + run: | + app/Console/cake Admin setSetting "MISP.osuser" $USER + app/Console/cake Admin runUpdates + app/Console/cake Admin schemaDiagnostics + + - name: Configure MISP + run: | + app/Console/cake User init | sudo tee ./key.txt + echo "AUTH=`cat key.txt`" >> $GITHUB_ENV + app/Console/cake Admin setSetting "Session.autoRegenerate" 0 + app/Console/cake Admin setSetting "Session.timeout" 600 + app/Console/cake Admin setSetting "Session.cookieTimeout" 3600 + app/Console/cake Admin setSetting "MISP.host_org_id" 1 + app/Console/cake Admin setSetting "MISP.email" "info@admin.test" + app/Console/cake Admin setSetting "MISP.disable_emailing" false + app/Console/cake Admin setSetting --force "debug" true + app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false + app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1" + app/Console/cake Admin setSetting "MISP.redis_port" 6379 + app/Console/cake Admin setSetting "MISP.redis_database" 13 + app/Console/cake Admin setSetting "MISP.redis_password" "" + app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test" + app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg" + app/Console/cake Admin setSetting "GnuPG.password" "travistest" + app/Console/cake Admin setSetting "MISP.download_gpg_from_homedir" 1 + app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_host" "127.0.0.1" + app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_port" 6379 + app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_database" 1 + app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_password" "" + app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" 1 + app/Console/cake Admin setSetting "Plugin.ZeroMQ_audit_notifications_enable" 1 + + - name: Update Galaxies + run: app/Console/cake Admin updateGalaxies + + - name: Update Taxonomies + run: app/Console/cake Admin updateTaxonomies + + - name: Update Warninglists + run: app/Console/cake Admin updateWarningLists --verbose + + - name: Update Noticelists + run: app/Console/cake Admin updateNoticeLists + + - name: Update Object Templates + run: app/Console/cake Admin updateObjectTemplates 1 + + - name: Turn MISP live + run: app/Console/cake Admin live 1 + + - name: Check if Redis is ready + run: app/Console/cake Admin redisReady + + - name: Start workers + run: | + sudo chmod +x app/Console/worker/start.sh + sudo -u www-data 'app/Console/worker/start.sh' - name: Test if apache is working run: | - sudo systemctl status apache2 --no-pager -l - sudo apache2ctl -S - curl http://${HOST} - sudo chmod -R 777 PyMISP - pushd PyMISP - echo 'url = "http://'${HOST}'"' >> tests/keys.py - echo 'key = "'${AUTH}'"' >> tests/keys.py - cat tests/keys.py - popd - . ./venv/bin/activate - pushd tests - bash ./build-test.sh - popd - deactivate + sudo systemctl status apache2 --no-pager -l + sudo apache2ctl -S + curl -sS http://${HOST} + + - name: Check if dependencies working as expected + run: | + sudo chmod -R 777 PyMISP + pushd PyMISP + echo 'url = "http://'${HOST}'"' >> tests/keys.py + echo 'key = "'${AUTH}'"' >> tests/keys.py + cat tests/keys.py + popd + . ./venv/bin/activate + pushd tests + bash ./build-test.sh + popd + deactivate - name: Run PHP tests run: | - ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/ - sudo -u www-data ./app/Vendor/bin/phpunit app/Test/ + ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ -e php,ctp app/ + sudo -u www-data ./app/Vendor/bin/phpunit app/Test/ - name: Clone test files uses: actions/checkout@v4 @@ -249,31 +241,30 @@ jobs: repository: viper-framework/viper-test-files path: PyMISP/tests/viper-test-files - - name: Run tests run: | - pushd tests - ./curl_tests_GH.sh $AUTH $HOST - popd + pushd tests + ./curl_tests_GH.sh $AUTH $HOST + popd - sudo chmod -R g+ws `pwd`/app/tmp/logs + sudo chmod -R g+ws `pwd`/app/tmp/logs - . ./venv/bin/activate - pushd PyMISP - cp tests/keys.py . - python -m pytest -v --durations=0 tests/test_mispevent.py - python -m pytest -v --durations=0 tests/testlive_comprehensive.py - popd - python tests/testlive_security.py -v - python tests/testlive_sync.py - python tests/testlive_comprehensive_local.py -v - cp PyMISP/tests/keys.py PyMISP/examples/events/ - pushd PyMISP/examples/events/ - python ./create_massive_dummy_events.py -l 5 -a 30 - popd - pip install jsonschema - python tools/misp-feed/validate.py - deactivate + . ./venv/bin/activate + pushd PyMISP + cp tests/keys.py . + python -m pytest -v --durations=0 tests/test_mispevent.py + python -m pytest -v --durations=0 tests/testlive_comprehensive.py + popd + python tests/testlive_comprehensive_local.py -v + python tests/testlive_sync.py -v + python tests/testlive_security.py -v + cp PyMISP/tests/keys.py PyMISP/examples/events/ + pushd PyMISP/examples/events/ + python ./create_massive_dummy_events.py -l 5 -a 30 + popd + pip install jsonschema + python tools/misp-feed/validate.py + deactivate - name: Check requirements.txt run: python tests/check_requirements.py @@ -282,13 +273,13 @@ jobs: if: ${{ always() }} # update logs_test.sh when adding more logsources here run: | - tail -n +1 `pwd`/app/tmp/logs/* - tail -n +1 /var/log/apache2/*.log + tail -n +1 `pwd`/app/tmp/logs/* + tail -n +1 /var/log/apache2/*.log - sudo -u $USER app/Console/cake Log export /tmp/logs.json.gz --without-changes - zcat /tmp/logs.json.gz + app/Console/cake Log export /tmp/logs.json.gz --without-changes + zcat /tmp/logs.json.gz - name: Errors in Logs if: ${{ always() }} run: | - ./tests/logs_tests.sh \ No newline at end of file + ./tests/logs_tests.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 4378f9c8f..000000000 --- a/.travis.yml +++ /dev/null @@ -1,195 +0,0 @@ -language: php - -php: - - 7.2 - - 7.3 - - 7.4 - - nightly - -services: - - redis - -sudo: required -dist: bionic - -addons: - mariadb: '10.2' - hosts: - - misp.local - - localhost - -before_install: - - git config --global user.name "TravisCI" - - export PATH="$HOME/.local/bin:$PATH" - -install: - - date - - sudo apt-get -y update - # Install haveged, because Travis lacks entropy. - - sudo apt-get -y install haveged python3 python3-venv python3-pip python3-dev python3-nose python3-redis python3-lxml python3-dateutil python3-msgpack libxml2-dev libzmq3-dev zlib1g-dev apache2 curl php-mysql php-dev php-cli libapache2-mod-php libfuzzy-dev php-mbstring libonig4 php-json php-xml php-opcache php-readline php-redis php-gnupg php-gd - - sudo pip3 install --upgrade pip setuptools requests - - sudo pip3 install --upgrade -r requirements.txt - - sudo pip3 install --upgrade -r requirements-dev.txt - - pip3 install --user poetry - - phpenv rehash - - sudo mkdir $HOME/.composer ; sudo chown $USER:www-data $HOME/.composer - - pushd app - - sudo -H -u $USER php composer.phar install --no-progress - - sudo phpenmod redis - - sudo phpenmod gnupg - - popd - - cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php - # Set perms - - sudo chown -R $USER:www-data `pwd` - - sudo chmod -R 775 `pwd` - - sudo chmod -R g+ws `pwd`/app/tmp - - sudo chmod -R g+ws `pwd`/app/tmp/cache - - sudo chmod -R g+ws `pwd`/app/tmp/cache/persistent - - sudo chmod -R g+ws `pwd`/app/tmp/cache/models - - sudo chmod -R g+ws `pwd`/app/tmp/logs - - sudo chmod -R g+ws `pwd`/app/files - - sudo chmod -R g+ws `pwd`/app/files/scripts/tmp - - sudo chown -R $USER:www-data `pwd` - # Resque perms - - sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp - - sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp - # install MySQL - - sudo chmod -R 777 `pwd`/INSTALL - - mysql -u root -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';" - - mysql -u root -e 'create database misp;' - - mysql -u root -e "grant usage on *.* to misp@localhost identified by 'blah'"; - - mysql -u root -e "grant all privileges on misp.* to misp@localhost;" - - mysql -u misp -pblah misp < INSTALL/MYSQL.sql - # configure apache virtual hosts - - sudo chmod -R 777 `pwd`/build - - sudo mkdir -p /etc/apache2/sites-available - - sudo cp -f build/travis-ci-apache /etc/apache2/sites-available/misp.local.conf - - sudo sed -e "s?%TRAVIS_BUILD_DIR%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.local.conf - - sudo a2dissite 000-default - - sudo a2ensite misp.local.conf - - sudo a2enmod rewrite - - sudo service apache2 restart - # MISP configuration - - sudo chmod -R 777 `pwd`/travis - - sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php - - sudo cp travis/database.php app/Config/database.php - - sudo cp app/Config/core.default.php app/Config/core.php - - sudo cp app/Config/config.default.php app/Config/config.php - - sudo cp travis/email.php app/Config/email.php - # Ensure the perms - - sudo chown -R $USER:www-data `pwd`/app/Config - - sudo chmod -R 770 `pwd`/app/Config - # GPG setup - - sudo mkdir `pwd`/.gnupg - # /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS - - sudo cp -a /dev/urandom /dev/random - - sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/travis/gpg - - sudo gpg --list-secret-keys --homedir `pwd`/.gnupg - # change perms - - sudo chown -R $USER:www-data `pwd` - - sudo chmod +x /home/travis/build - - sudo chmod +x /home/travis - - sudo chmod +x /home - - sudo chmod -R 770 `pwd`/.gnupg - # Get authkey - - sudo usermod -a -G www-data $USER - - sudo -E su $USER -c 'app/Console/cake Admin runUpdates' - - sudo -E su $USER -c 'app/Console/cake userInit -q | sudo tee ./key.txt' - - sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.autoRegenerate" 0' - - sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.timeout" 600' - - sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.cookieTimeout" 3600' - - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.host_org_id" 1' - - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.email" "info@admin.test"' - - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.disable_emailing" false' - - sudo -E su $USER -c 'app/Console/cake Admin setSetting "debug" true' - - sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false' - - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1"' - - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_port" 6379' - - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_database" 13' - - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_password" ""' - - sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test"' - - sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg"' - - sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.password" "travistest"' - - sudo -E su $USER -c 'app/Console/cake Admin updateGalaxies' - - sudo -E su $USER -c 'app/Console/cake Admin updateTaxonomies' - - sudo -E su $USER -c 'app/Console/cake Admin updateWarningLists' - - sudo -E su $USER -c 'app/Console/cake Admin updateNoticeLists' - - sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates 1' - - sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" true' - - sudo -E su $USER -c 'app/Console/cake Live 1' - - sudo chmod 777 ./key.txt - - sudo chmod -R 777 ./tests - # Start workers - - sudo chmod +x app/Console/worker/start.sh - - sudo -E su $USER -c 'app/Console/worker/start.sh &' - - sleep 10 - # Dirty install python stuff - - virtualenv -p python3.6 ./venv - - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$TRAVIS_BUILD_DIR/venv/bin/python"' - - . ./venv/bin/activate - - pushd cti-python-stix2 - - pip install . - - popd - - pushd PyMISP - - pip install .[fileobjects] - - popd - - pip install stix zmq redis plyara - - deactivate - -before_script: - - curl http://misp.local - - AUTH=`cat key.txt` - - sudo chmod -R 777 PyMISP - - pushd PyMISP - - echo 'url = "http://misp.local"' >> tests/keys.py - - echo 'key = "'${AUTH}'"' >> tests/keys.py - - cat tests/keys.py - - popd - -script: - - ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/ - - ./app/Vendor/bin/phpunit app/Test/ComplexTypeToolTest.php - - ./app/Vendor/bin/phpunit app/Test/JSONConverterToolTest.php - # Ensure the perms - - sudo chown -R $USER:www-data `pwd`/app/Config - - sudo chmod -R 770 `pwd`/app/Config - - pushd tests - - ./curl_tests.sh $AUTH - - popd - - pushd PyMISP - - git submodule init - - git submodule update - - travis_retry poetry install -E fileobjects -E openioc -E virustotal -E docs -E pdfexport - - poetry run python tests/testlive_comprehensive.py - - poetry run python tests/test_mispevent.py - - popd - - cp PyMISP/tests/keys.py PyMISP/examples/events/ - - pushd PyMISP/examples/events/ - - poetry run python ./create_massive_dummy_events.py -l 5 -a 30 - - popd - - python3 tools/misp-feed/validate.py - -after_failure: - - curl http://misp.local - - cat /etc/apache2/sites-available/misp.local.conf - - sudo tail -n +1 `pwd`/app/tmp/logs/* - - sudo ls -l /var/log/apache2 - - sudo cat /var/log/apache2/error.log - - sudo cat /var/log/apache2/misp.local_error.log - - sudo cat /var/log/apache2/misp.local_access.log - - pwd - -notifications: - webhooks: - urls: - - https://webhooks.gitter.im/e/05e30284086a8e948d31 - on_success: change # options: [always|never|change] default: always - on_failure: always # options: [always|never|change] default: always - on_start: never # options: [always|never|change] default: always - -after_success: - - sudo tail -n +1 `pwd`/app/tmp/logs/* - - coveralls - - coverage report - - coverage xml - - codecov diff --git a/PyMISP b/PyMISP index 81f5d596a..685ef22d0 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit 81f5d596a7dd5cb1ca7213ac4fbdf07b402420b7 +Subproject commit 685ef22d0a1123c518093fba14dd2c1a0d153ad0 diff --git a/VERSION.json b/VERSION.json index 418741696..cd351f15e 100644 --- a/VERSION.json +++ b/VERSION.json @@ -1 +1 @@ -{"major":2, "minor":4, "hotfix":183} +{"major":2, "minor":4, "hotfix":184} diff --git a/app/Console/Command/AdminShell.php b/app/Console/Command/AdminShell.php index 9930e5fa5..544e3a54f 100644 --- a/app/Console/Command/AdminShell.php +++ b/app/Console/Command/AdminShell.php @@ -46,11 +46,11 @@ class AdminShell extends AppShell 'help' => __('Update the JSON definition of taxonomies.'), )); $parser->addSubcommand('setSetting', [ - 'help' => __('Set setting in PHP config file.'), + 'help' => __('Set setting in MISP config'), 'parser' => [ 'arguments' => [ 'name' => ['help' => __('Setting name'), 'required' => true], - 'value' => ['help' => __('Setting value'), 'required' => true], + 'value' => ['help' => __('Setting value')], ], 'options' => [ 'force' => [ @@ -72,7 +72,7 @@ class AdminShell extends AppShell 'help' => __('Set if MISP instance is live and accessible for users.'), 'parser' => [ 'arguments' => [ - 'state' => ['help' => __('Set Live state')], + 'state' => ['help' => __('Set Live state (boolean). If not provided, current state will be printed.')], ], ], ]); @@ -85,6 +85,14 @@ class AdminShell extends AppShell ], ], ]); + $parser->addSubcommand('isEncryptionKeyValid', [ + 'help' => __('Check if current encryption key is valid.'), + 'parser' => [ + 'options' => [ + 'encryptionKey' => ['help' => __('Encryption key to test. If not provided, current key will be used.')], + ], + ], + ]); $parser->addSubcommand('dumpCurrentDatabaseSchema', [ 'help' => __('Dump current database schema to JSON file.'), ]); @@ -109,6 +117,20 @@ class AdminShell extends AppShell $parser->addSubcommand('configLint', [ 'help' => __('Check if settings has correct value.'), ]); + $parser->addSubcommand('createZmqConfig', [ + 'help' => __('Create config file for ZeroMQ server.'), + ]); + $parser->addSubcommand('scanAttachment', [ + 'help' => __('Scan attachments with AV.'), + 'parser' => [ + 'arguments' => [ + 'type' => ['help' => __('all, Attribute or ShadowAttribute'), 'required' => true], + 'attributeId' => ['help' => __('ID to scan.')], + 'jobId' => ['help' => __('Job ID')], + + ], + ], + ]); return $parser; } @@ -485,32 +507,47 @@ class AdminShell extends AppShell } } } - echo json_encode($result, JSON_PRETTY_PRINT) . PHP_EOL; + $this->out($this->json($result)); } public function setSetting() { - list($setting_name, $value) = $this->args; - if ($value === 'false') { - $value = 0; - } elseif ($value === 'true') { - $value = 1; - } - if ($this->params['null']) { + list($settingName) = $this->args; + + if ($this->params['null'] && isset($this->args[1])) { + $this->error(__('Trying to set setting to null value, but value was provided.')); + } else if ($this->params['null']) { $value = null; + } elseif (isset($this->args[1])) { + $value = $this->args[1]; + } else { + $this->error(__('No setting value provided.')); } - $cli_user = array('id' => 0, 'email' => 'SYSTEM', 'Organisation' => array('name' => 'SYSTEM')); - if (empty($setting_name) || ($value === null && !$this->params['null'])) { - die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Set setting'] . PHP_EOL); - } - $setting = $this->Server->getSettingData($setting_name); + + $setting = $this->Server->getSettingData($settingName); if (empty($setting)) { - $message = 'Invalid setting "' . $setting_name . '". Please make sure that the setting that you are attempting to change exists and if a module parameter, the modules are running.' . PHP_EOL; + $message = 'Invalid setting "' . $settingName . '". Please make sure that the setting that you are attempting to change exists and if a module parameter, the modules are running.' . PHP_EOL; $this->error(__('Setting change rejected.'), $message); } - $result = $this->Server->serverSettingsEditValue($cli_user, $setting, $value, $this->params['force']); + + // Convert value to boolean or to int + if ($value !== null) { + if ($setting['type'] === 'boolean') { + $value = $this->toBoolean($value); + } else if ($setting['type'] === 'numeric') { + if (is_numeric($value)) { + $value = (int)$value; + } elseif ($value === 'true' || $value === 'false') { + $value = $value === 'true' ? 1 : 0; // special case for `debug` setting + } else { + $this->error(__('Setting "%s" change rejected.', $settingName), __('Provided value %s is not a number.', $value)); + } + } + } + + $result = $this->Server->serverSettingsEditValue('SYSTEM', $setting, $value, $this->params['force']); if ($result === true) { - $this->out(__('Setting "%s" changed to %s', $setting_name, is_string($value) ? '"' . $value . '"' : (string)$value)); + $this->out(__('Setting "%s" changed to %s', $settingName, is_string($value) ? '"' . $value . '"' : json_encode($value))); } else { $message = __("The setting change was rejected. MISP considers the requested setting value as invalid and would lead to the following error:\n\n\"%s\"\n\nIf you still want to force this change, please supply the --force argument.\n", $result); $this->error(__('Setting change rejected.'), $message); @@ -648,6 +685,8 @@ class AdminShell extends AppShell */ public function change_authkey() { + $this->deprecated('cake user change_authkey [user_id]'); + if (empty($this->args[0])) { echo 'MISP apikey command line tool' . PHP_EOL . 'To assign a new random API key for a user: ' . APP . 'Console/cake Admin change_authkey [user_email]' . PHP_EOL . 'To assign a fixed API key: ' . APP . 'Console/cake Admin change_authkey [user_email] [authkey]' . PHP_EOL; die(); @@ -787,6 +826,8 @@ class AdminShell extends AppShell */ public function UserIP() { + $this->deprecated('cake user user_ips [user_id]'); + if (empty($this->args[0])) { die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get IPs for user ID'] . PHP_EOL); } @@ -814,6 +855,8 @@ class AdminShell extends AppShell */ public function IPUser() { + $this->deprecated('cake user ip_user [ip]'); + if (empty($this->args[0])) { die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get user ID for user IP'] . PHP_EOL); } @@ -839,8 +882,8 @@ class AdminShell extends AppShell public function scanAttachment() { $input = $this->args[0]; - $attributeId = isset($this->args[1]) ? $this->args[1] : null; - $jobId = isset($this->args[2]) ? $this->args[2] : null; + $attributeId = $this->args[1] ?? null; + $jobId = $this->args[2] ?? null; $this->loadModel('AttachmentScan'); $result = $this->AttachmentScan->scan($input, $attributeId, $jobId); @@ -951,7 +994,7 @@ class AdminShell extends AppShell $newStatus = $this->toBoolean($this->args[0]); $overallSuccess = false; try { - $redis = $this->Server->setupRedisWithException(); + $redis = RedisTool::init(); if ($newStatus) { $redis->del('misp:live'); $this->out('Set live status to True in Redis.'); @@ -980,7 +1023,7 @@ class AdminShell extends AppShell } else { $this->out('Current status:'); $this->out('PHP Config file: ' . (Configure::read('MISP.live') ? 'True' : 'False')); - $newStatus = $this->Server->setupRedisWithException()->get('misp:live'); + $newStatus = RedisTool::init()->get('misp:live'); $this->out('Redis: ' . ($newStatus !== '0' ? 'True' : 'False')); } } @@ -1031,6 +1074,27 @@ class AdminShell extends AppShell $this->out(__('New encryption key "%s" saved into config file.', $new)); } + public function isEncryptionKeyValid() + { + $encryptionKey = $this->params['encryptionKey'] ?? null; + if ($encryptionKey === null) { + $encryptionKey = Configure::read('Security.encryption_key'); + } + if (!$encryptionKey) { + $this->error('No encryption key provided'); + } + + /** @var SystemSetting $systemSetting */ + $systemSetting = ClassRegistry::init('SystemSetting'); + + try { + $systemSetting->isEncryptionKeyValid($encryptionKey); + $this->Server->isEncryptionKeyValid($encryptionKey); + } catch (Exception $e) { + $this->error($e->getMessage(), __('Probably provided encryption key is invalid')); + } + } + public function redisMemoryUsage() { $redis = RedisTool::init(); @@ -1240,4 +1304,10 @@ class AdminShell extends AppShell $this->Job->saveField('message', __('Database truncated: ' . $table)); } } + + public function createZmqConfig() + { + $this->Server->getPubSubTool()->createConfigFile(); + $this->err("Config file created in " . PubSubTool::SCRIPTS_TMP); + } } diff --git a/app/Console/Command/AppShell.php b/app/Console/Command/AppShell.php index 58608115f..6d63a94f4 100644 --- a/app/Console/Command/AppShell.php +++ b/app/Console/Command/AppShell.php @@ -31,15 +31,13 @@ require_once dirname(__DIR__) . '/../Model/Attribute.php'; // FIXME workaround */ abstract class AppShell extends Shell { - public $tasks = array('ConfigLoad'); - /** @var BackgroundJobsTool */ private $BackgroundJobsTool; public function initialize() { - $this->ConfigLoad = $this->Tasks->load('ConfigLoad'); - $this->ConfigLoad->execute(); + $configLoad = $this->Tasks->load('ConfigLoad'); + $configLoad->execute(); parent::initialize(); } @@ -84,6 +82,15 @@ abstract class AppShell extends Shell } } + /** + * @param string $newCommand + * @return void + */ + protected function deprecated($newCommand) + { + $this->err("Warning: This method is deprecated. Next time please use `$newCommand`."); + } + /** * @return BackgroundJobsTool * @throws Exception diff --git a/app/Console/Command/AuthkeyShell.php b/app/Console/Command/AuthkeyShell.php index 1860ae266..b30d2e13f 100644 --- a/app/Console/Command/AuthkeyShell.php +++ b/app/Console/Command/AuthkeyShell.php @@ -12,7 +12,7 @@ class AuthkeyShell extends AppShell { public function main() { - $this->err('This method is deprecated. Next time please use `cake user change_authkey [user] [authkey]` command.'); + $this->deprecated('cake user change_authkey [user] [authkey]'); if (!isset($this->args[0]) || empty($this->args[0])) echo 'MISP authkey reset command line tool.' . PHP_EOL . 'To assign a new authkey for a user:' . PHP_EOL . APP . 'Console/cake Authkey [email] [auth_key | optional]' . PHP_EOL; else { diff --git a/app/Console/Command/BaseurlShell.php b/app/Console/Command/BaseurlShell.php index 6a5f06455..36c501b5e 100644 --- a/app/Console/Command/BaseurlShell.php +++ b/app/Console/Command/BaseurlShell.php @@ -11,7 +11,7 @@ class BaseurlShell extends AppShell { public function main() { - $this->err('This method is deprecated. Next time please use `cake admin setSetting MISP.baseurl [baseurl]` command.'); + $this->deprecated('cake admin setSetting MISP.baseurl [baseurl]'); $baseurl = $this->args[0]; $result = $this->Server->testBaseURL($baseurl); diff --git a/app/Console/Command/EventShell.php b/app/Console/Command/EventShell.php index 8940673d8..87965cff7 100644 --- a/app/Console/Command/EventShell.php +++ b/app/Console/Command/EventShell.php @@ -53,10 +53,21 @@ class EventShell extends AppShell $parser->addSubcommand('mergeTags', [ 'help' => __('Merge tags'), 'parser' => [ - 'arguments' => array( + 'arguments' => [ 'source' => ['help' => __('Source tag ID or name. Source tag will be deleted.'), 'required' => true], 'destination' => ['help' => __('Destination tag ID or name.'), 'required' => true], - ) + ], + ], + ]); + $parser->addSubcommand('reportValidationIssuesAttributes', [ + 'help' => __('Report validation issues on attributes'), + ]); + $parser->addSubcommand('normalizeIpAddress', [ + 'help' => __('Normalize IP address format in old events'), + 'parser' => [ + 'options' => [ + 'dry-run' => ['help' => __('Just show what changes will be made.'), 'boolean' => true], + ], ], ]); return $parser; @@ -636,18 +647,28 @@ class EventShell extends AppShell } } - /** - * @param int $userId - * @return array - */ - private function getUser($userId) + public function reportValidationIssuesAttributes() { - $user = $this->User->getAuthUser($userId, true); - if (empty($user)) { - $this->error("User with ID $userId does not exist."); + foreach ($this->Event->Attribute->reportValidationIssuesAttributes() as $validationIssue) { + echo $this->json($validationIssue) . "\n"; + } + } + + public function normalizeIpAddress() + { + $dryRun = $this->param('dry-run'); + + $count = 0; + foreach ($this->Event->Attribute->normalizeIpAddress($dryRun) as $attribute) { + $count++; + echo JsonTool::encode($attribute) . "\n"; + } + + if ($dryRun) { + $this->err(__n("%s attribute to fix", "%s attributes to fix", $count, $count)); + } else { + $this->err(__n("%s attribute fixed", "%s attributes fixed", $count, $count)); } - Configure::write('CurrentUserId', $user['id']); // for audit logging purposes - return $user; } public function generateTopCorrelations() @@ -668,4 +689,18 @@ class EventShell extends AppShell $this->Job->save($job); } } + + /** + * @param int $userId + * @return array + */ + private function getUser($userId) + { + $user = $this->User->getAuthUser($userId, true); + if (empty($user)) { + $this->error("User with ID $userId does not exist."); + } + Configure::write('CurrentUserId', $user['id']); // for audit logging purposes + return $user; + } } diff --git a/app/Console/Command/LiveShell.php b/app/Console/Command/LiveShell.php index 657fb7280..e4c15de0c 100644 --- a/app/Console/Command/LiveShell.php +++ b/app/Console/Command/LiveShell.php @@ -1,8 +1,9 @@ deprecated('cake admin live [0|1]'); + $live = $this->args[0]; if ($live != 0 && $live != 1) { echo 'Invalid parameters. Usage: /var/www/MISP/app/Console/cake Live [0|1]'; diff --git a/app/Console/Command/PasswordShell.php b/app/Console/Command/PasswordShell.php index 3a8586ad2..ff9358c33 100644 --- a/app/Console/Command/PasswordShell.php +++ b/app/Console/Command/PasswordShell.php @@ -12,7 +12,7 @@ class PasswordShell extends AppShell { public function main() { - $this->err('This method is deprecated. Next time please use `cake user change_pw [user] [password]` command.'); + $this->deprecated('cake user change_pw [user] [password]'); if (!isset($this->args[0]) || empty($this->args[0]) || !isset($this->args[1]) || empty($this->args[1])) echo 'MISP password reset command line tool.' . PHP_EOL . 'To assign a new password for a user:' . PHP_EOL . APP . 'Console/cake Password [email] [password]' . PHP_EOL; else { diff --git a/app/Console/Command/StartWorkerShell.php b/app/Console/Command/StartWorkerShell.php index 85590c33c..963089e09 100644 --- a/app/Console/Command/StartWorkerShell.php +++ b/app/Console/Command/StartWorkerShell.php @@ -37,24 +37,32 @@ class StartWorkerShell extends AppShell public function main() { + $pid = getmypid(); + if ($pid === false) { + throw new RuntimeException("Could not get current process ID"); + } + $this->worker = new Worker( [ - 'pid' => getmypid(), + 'pid' => $pid, 'queue' => $this->args[0], 'user' => ProcessTool::whoami(), ] ); $this->maxExecutionTime = (int)$this->params['maxExecutionTime']; + $queue = $this->worker->queue(); + $backgroundJobTool = $this->getBackgroundJobsTool(); - CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - starting to process background jobs..."); + CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$queue}] - starting to process background jobs..."); while (true) { $this->checkMaxExecutionTime(); - $job = $this->getBackgroundJobsTool()->dequeue($this->worker->queue()); + $job = $backgroundJobTool->dequeue($queue); if ($job) { $this->runJob($job); + $backgroundJobTool->removeFromRunning($this->worker, $job); } } } @@ -64,7 +72,7 @@ class StartWorkerShell extends AppShell */ private function runJob(BackgroundJob $job) { - CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}..."); + CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}"); try { $job->setStatus(BackgroundJob::STATUS_RUNNING); @@ -73,12 +81,16 @@ class StartWorkerShell extends AppShell CakeLog::info("[JOB ID: {$job->id()}] - started command `$command`."); $this->getBackgroundJobsTool()->update($job); - $job->run(); + $start = microtime(true); + $job->run(function (array $status) use ($job) { + $this->getBackgroundJobsTool()->markAsRunning($this->worker, $job, $status['pid']); + }); + $duration = number_format(microtime(true) - $start, 3, '.', ''); if ($job->status() === BackgroundJob::STATUS_COMPLETED) { - CakeLog::info("[JOB ID: {$job->id()}] - completed."); + CakeLog::info("[JOB ID: {$job->id()}] - successfully completed in $duration seconds."); } else { - CakeLog::error("[JOB ID: {$job->id()}] - failed with error code {$job->returnCode()}. STDERR: {$job->error()}. STDOUT: {$job->output()}."); + CakeLog::error("[JOB ID: {$job->id()}] - failed with error code {$job->returnCode()} after $duration seconds. STDERR: {$job->error()}. STDOUT: {$job->output()}."); } } catch (Exception $exception) { CakeLog::error("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - job ID: {$job->id()} failed with exception: {$exception->getMessage()}"); diff --git a/app/Console/Command/Task/ConfigLoadTask.php b/app/Console/Command/Task/ConfigLoadTask.php index b624dc573..fc3333c72 100644 --- a/app/Console/Command/Task/ConfigLoadTask.php +++ b/app/Console/Command/Task/ConfigLoadTask.php @@ -3,8 +3,6 @@ class ConfigLoadTask extends Shell { public function execute() { - Configure::load('config'); - if (Configure::read('MISP.system_setting_db')) { App::uses('SystemSetting', 'Model'); SystemSetting::setGlobalSetting(); diff --git a/app/Console/Command/UserInitShell.php b/app/Console/Command/UserInitShell.php index f0a0d7225..bc5238f69 100644 --- a/app/Console/Command/UserInitShell.php +++ b/app/Console/Command/UserInitShell.php @@ -1,7 +1,13 @@ deprecated('cake user init'); + if (!Configure::read('Security.salt')) { $this->loadModel('Server'); $this->Server->serverSettingsSaveValue('Security.salt', $this->User->generateRandomPassword(32)); diff --git a/app/Console/Command/UserShell.php b/app/Console/Command/UserShell.php index db0b5ffbd..d65a0409f 100644 --- a/app/Console/Command/UserShell.php +++ b/app/Console/Command/UserShell.php @@ -3,10 +3,11 @@ /** * @property User $User * @property Log $Log + * @property UserLoginProfile $UserLoginProfile */ class UserShell extends AppShell { - public $uses = ['User', 'Log']; + public $uses = ['User', 'Log', 'UserLoginProfile']; public function getOptionParser() { @@ -22,16 +23,24 @@ class UserShell extends AppShell ], ] ]); + $parser->addSubcommand('init', [ + 'help' => __('Create default role, organisation and user when not exists.'), + ]); $parser->addSubcommand('authkey', [ 'help' => __('Get information about given authkey.'), 'parser' => [ 'arguments' => [ - 'authkey' => ['help' => __('Authentication key. If not provide, it will be read from STDIN.')], + 'authkey' => ['help' => __('Authentication key. If not provided, it will be read from STDIN.')], ], ] ]); $parser->addSubcommand('authkey_valid', [ 'help' => __('Check if given authkey by STDIN is valid.'), + 'parser' => [ + 'options' => [ + 'disableStdLog' => ['help' => __('Do not show logs in STDOUT or STDERR.'), 'boolean' => true], + ], + ], ]); $parser->addSubcommand('block', [ 'help' => __('Immediately block user.'), @@ -104,6 +113,14 @@ class UserShell extends AppShell ], ], ]); + $parser->addSubcommand('ip_country', [ + 'help' => __('Get country for given IP address'), + 'parser' => [ + 'arguments' => [ + 'ip' => ['help' => __('IPv4 or IPv6 address.'), 'required' => true], + ] + ], + ]); $parser->addSubcommand('require_password_change_for_old_passwords', [ 'help' => __('Trigger forced password change on next login for users with an old (older than x days) password.'), 'parser' => [ @@ -121,7 +138,7 @@ class UserShell extends AppShell public function list() { - $userId = isset($this->args[0]) ? $this->args[0] : null; + $userId = $this->args[0] ?? null; if ($userId) { $conditions = ['OR' => [ 'User.id' => $userId, @@ -163,13 +180,24 @@ class UserShell extends AppShell } } + public function init() + { + if (!Configure::read('Security.salt')) { + $this->loadModel('Server'); + $this->Server->serverSettingsSaveValue('Security.salt', $this->User->generateRandomPassword(32)); + } + + $authKey = $this->User->init(); + if ($authKey === null) { + $this->err('Script aborted: MISP instance already initialised.'); + } else { + $this->out($authKey); + } + } + public function authkey() { - if (isset($this->args[0])) { - $authkey = $this->args[0]; - } else { - $authkey = fgets(STDIN); // read line from STDIN - } + $authkey = $this->args[0] ?? fgets(STDIN); $authkey = trim($authkey); if (strlen($authkey) !== 40) { $this->error('Authkey has not valid format.'); @@ -212,28 +240,37 @@ class UserShell extends AppShell */ public function authkey_valid() { + if ($this->params['disableStdLog']) { + $this->_useLogger(false); + } + $cache = []; $randomKey = random_bytes(16); - do { + $advancedAuthKeysEnabled = (bool)Configure::read('Security.advanced_authkeys'); + + while (true) { $authkey = fgets(STDIN); // read line from STDIN $authkey = trim($authkey); if (strlen($authkey) !== 40) { - fwrite(STDOUT, "0\n"); // authkey is not in valid format - $this->log("Authkey in incorrect format provided.", LOG_WARNING); + echo "0\n"; // authkey is not in valid format + $this->log("Authkey in incorrect format provided, expected 40 chars long string, $authkey provided.", LOG_WARNING); continue; } - $time = time(); + // Generate hash from authkey to not store raw authkey in memory $keyHash = sha1($authkey . $randomKey, true); + + // If authkey is in cache and is fresh, use info from cache + $time = time(); if (isset($cache[$keyHash]) && $cache[$keyHash][1] > $time) { - fwrite(STDOUT, $cache[$keyHash][0] ? "1\n" : "0\n"); + echo $cache[$keyHash][0] ? "1\n" : "0\n"; continue; } $user = false; for ($i = 0; $i < 5; $i++) { try { - if (Configure::read('Security.advanced_authkeys')) { + if ($advancedAuthKeysEnabled) { $user = $this->User->AuthKey->getAuthUserByAuthKey($authkey); } else { $user = $this->User->getAuthUserByAuthkey($authkey); @@ -251,18 +288,34 @@ class UserShell extends AppShell } } - $user = (bool)$user; if (!$user) { - $start = substr($authkey, 0, 4); - $end = substr($authkey, -4); - $authKeyToStore = $start . str_repeat('*', 32) . $end; - $this->log("Not valid authkey $authKeyToStore provided.", LOG_WARNING); + $valid = null; + } else if ($user['disabled']) { + $valid = false; + } else { + $valid = true; } - // Cache results for 5 seconds - $cache[$keyHash] = [$user, $time + 5]; - fwrite(STDOUT, $user ? "1\n" : "0\n"); - } while (true); + echo $valid ? "1\n" : "0\n"; + + if ($valid) { + // Cache results for 60 seconds if key is valid + $cache[$keyHash] = [true, $time + 60]; + } else { + // Cache results for 5 seconds if key is invalid + $cache[$keyHash] = [false, $time + 5]; + + $start = substr($authkey, 0, 4); + $end = substr($authkey, -4); + $authKeyForLog = $start . str_repeat('*', 32) . $end; + + if ($valid === false) { + $this->log("Authkey $authKeyForLog belongs to user {$user['id']} that is disabled.", LOG_WARNING); + } else { + $this->log("Authkey $authKeyForLog is invalid or expired.", LOG_WARNING); + } + } + } } public function block() @@ -305,7 +358,7 @@ class UserShell extends AppShell $conditions = ['User.disabled' => false]; // fetch just not disabled users - $userId = isset($this->args[0]) ? $this->args[0] : null; + $userId = $this->args[0] ?? null; if ($userId) { $conditions['OR'] = [ 'User.id' => $userId, @@ -364,7 +417,7 @@ class UserShell extends AppShell } $user = $this->getUser($userId); - # validate new authentication key if provided + // validate new authentication key if provided if (!empty($newkey) && (strlen($newkey) != 40 || !ctype_alnum($newkey))) { $this->error('The new auth key needs to be 40 characters long and only alphanumeric.'); } @@ -399,7 +452,7 @@ class UserShell extends AppShell $this->out('Storing user IP addresses is disabled.'); } - $ips = $this->User->setupRedisWithException()->smembers('misp:user_ip:' . $user['id']); + $ips = RedisTool::init()->smembers('misp:user_ip:' . $user['id']); if ($this->params['json']) { $this->out($this->json($ips)); @@ -422,36 +475,50 @@ class UserShell extends AppShell $this->out('Storing user IP addresses is disabled.'); } - $userId = $this->User->setupRedisWithException()->get('misp:ip_user:' . $ip); + $userId = RedisTool::init()->get('misp:ip_user:' . $ip); if (empty($userId)) { $this->out('No hits.'); $this->_stop(); } - $user = $this->User->find('first', array( + $user = $this->User->find('first', [ 'recursive' => -1, - 'conditions' => array('User.id' => $userId), + 'conditions' => ['User.id' => $userId], 'fields' => ['id', 'email'], - )); + ]); if (empty($user)) { $this->error("User with ID $userId doesn't exists anymore."); } + $ipCountry = $this->UserLoginProfile->countryByIp($ip); + if ($this->params['json']) { $this->out($this->json([ 'ip' => $ip, 'id' => $user['User']['id'], 'email' => $user['User']['email'], + 'country' => $ipCountry, ])); } else { - $this->out(sprintf( - '%s==============================%sIP: %s%s==============================%sUser #%s: %s%s==============================%s', - PHP_EOL, PHP_EOL, $ip, PHP_EOL, PHP_EOL, $user['User']['id'], $user['User']['email'], PHP_EOL, PHP_EOL - )); + $this->hr(); + $this->out("IP: $ip (country $ipCountry)"); + $this->hr(); + $this->out("User #{$user['User']['id']}: {$user['User']['email']}"); + $this->hr(); } } + public function ip_country() + { + list($ip) = $this->args; + if (!filter_var($ip, FILTER_VALIDATE_IP)) { + $this->error("IP `$ip` is not valid IPv4 or IPv6 address"); + } + + $this->out($this->UserLoginProfile->countryByIp($ip)); + } + public function require_password_change_for_old_passwords() { list($days) = $this->args; diff --git a/app/Console/Command/WorkerShell.php b/app/Console/Command/WorkerShell.php new file mode 100644 index 000000000..88d5a87ab --- /dev/null +++ b/app/Console/Command/WorkerShell.php @@ -0,0 +1,119 @@ +addSubcommand('showQueues', [ + 'help' => __('Show jobs in worker queues'), + ]); + $parser->addSubcommand('flushQueue', [ + 'help' => __('Flush jobs in given queue'), + 'parser' => [ + 'arguments' => [ + 'queue' => ['help' => __('Queue name'), 'required' => true], + ], + ], + ]); + $parser->addSubcommand('showJobStatus', [ + 'help' => __('Show job status'), + 'parser' => [ + 'arguments' => [ + 'job_id' => ['help' => __('Job ID (ID or UUID)'), 'required' => true], + ], + ], + ]); + return $parser; + } + + /** + * @throws RedisException + * @throws JsonException + */ + public function showQueues() + { + $tool = $this->getBackgroundJobsTool(); + $runningJobs = $tool->runningJobs(); + + foreach (BackgroundJobsTool::VALID_QUEUES as $queue) { + $this->out("{$queue}:\t{$tool->getQueueSize($queue)}"); + $queueJobs = $runningJobs[$queue] ?? []; + foreach ($queueJobs as $jobId => $data) { + $this->out(" - $jobId (" . JsonTool::encode($data) .")"); + } + } + } + + public function flushQueue() + { + $queue = $this->args[0]; + try { + $this->getBackgroundJobsTool()->clearQueue($queue); + } catch (InvalidArgumentException $e) { + $this->error($e->getMessage()); + } + } + + public function showJobStatus() + { + $processId = $this->args[0]; + if (is_numeric($processId)) { + $job = $this->Job->find('first', [ + 'conditions' => ['Job.id' => $processId], + 'recursive' => -1, + ]); + if (!$job) { + $this->error('Job not found', "Job with ID {$processId} not found"); + } + + $this->out($this->json($job['Job'])); + $processId = $job['Job']['process_id']; + } + + if (!Validation::uuid($processId)) { + $this->error('Job not found', "Job ID must be number or UUID, '$processId' given"); + } + + $jobStatus = $this->getBackgroundJobsTool()->getJob($processId); + if (!$jobStatus) { + $this->error('Job not found', "Job with UUID {$processId} not found"); + } + + $jobStatus = $jobStatus->jsonSerialize(); + + foreach (['createdAt', 'updatedAt'] as $timeField) { + if (isset($jobStatus[$timeField])) { + $jobStatus[$timeField] = date('c', $jobStatus[$timeField]); + } + } + + if (isset($jobStatus['status'])) { + $jobStatus['status'] = $this->jobStatusToString($jobStatus['status']); + } + + $this->out($this->json($jobStatus)); + } + + private function jobStatusToString(int $jobStatus) + { + switch ($jobStatus) { + case Job::STATUS_WAITING: + return 'waiting'; + case Job::STATUS_RUNNING: + return 'running'; + case Job::STATUS_FAILED: + return 'failed'; + case Job::STATUS_COMPLETED: + return 'completed'; + } + throw new InvalidArgumentException("Invalid job status $jobStatus"); + } +} \ No newline at end of file diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index de96c6014..640143d04 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -38,8 +38,6 @@ class AppController extends Controller public $phpmin = '7.2'; public $phprec = '7.4'; public $phptoonew = '8.0'; - public $pythonmin = '3.6'; - public $pythonrec = '3.7'; private $isApiAuthed = false; public $baseurl = ''; @@ -232,6 +230,10 @@ class AppController extends Controller $this->Security->csrfCheck = false; $loginByAuthKeyResult = $this->__loginByAuthKey(); if ($loginByAuthKeyResult === false || $this->Auth->user() === null) { + if ($this->IndexFilter->isXhr()) { + throw new ForbiddenException('Authentication failed.'); + } + if ($loginByAuthKeyResult === null) { $this->loadModel('Log'); $this->Log->createLogEntry('SYSTEM', 'auth_fail', 'User', 0, "Failed API authentication. No authkey was provided."); @@ -601,7 +603,7 @@ class AppController extends Controller if (!empty($user['allowed_ips'])) { App::uses('CidrTool', 'Tools'); $cidrTool = new CidrTool($user['allowed_ips']); - $remoteIp = $this->_remoteIp(); + $remoteIp = $this->User->_remoteIp(); if ($remoteIp === null) { $this->Auth->logout(); throw new ForbiddenException('Auth key is limited to IP address, but IP address not found'); @@ -694,7 +696,7 @@ class AppController extends Controller return; } - $remoteAddress = $this->_remoteIp(); + $remoteAddress = $this->User->_remoteIp(); $pipe = $redis->pipeline(); // keep for 30 days @@ -737,7 +739,7 @@ class AppController extends Controller $includeRequestBody = !empty(Configure::read('MISP.log_paranoid_include_post_body')) || $userMonitoringEnabled; /** @var AccessLog $accessLog */ $accessLog = ClassRegistry::init('AccessLog'); - $accessLog->logRequest($user, $this->_remoteIp(), $this->request, $includeRequestBody); + $accessLog->logRequest($user, $this->User->_remoteIp(), $this->request, $includeRequestBody); } if ( @@ -828,29 +830,34 @@ class AppController extends Controller private function __rateLimitCheck(array $user) { - $info = array(); $rateLimitCheck = $this->RateLimit->check( $user, $this->request->params['controller'], - $this->request->action, - $info, - $this->response->type() + $this->request->params['action'], ); - if (!empty($info)) { - $this->RestResponse->setHeader('X-Rate-Limit-Limit', $info['limit']); - $this->RestResponse->setHeader('X-Rate-Limit-Remaining', $info['remaining']); - $this->RestResponse->setHeader('X-Rate-Limit-Reset', $info['reset']); + + if ($rateLimitCheck) { + $headers = [ + 'X-Rate-Limit-Limit' => $rateLimitCheck['limit'], + 'X-Rate-Limit-Remaining' => $rateLimitCheck['remaining'], + 'X-Rate-Limit-Reset' => $rateLimitCheck['reset'], + ]; + + if ($rateLimitCheck['exceeded']) { + $response = $this->RestResponse->throwException( + 429, + __('Rate limit exceeded.'), + '/' . $this->request->params['controller'] . '/' . $this->request->params['action'], + false, + false, + $headers + ); + $response->send(); + $this->_stop(); + } else { + $this->RestResponse->headers = array_merge($this->RestResponse->headers, $headers); + } } - if ($rateLimitCheck !== true) { - $this->response->header('X-Rate-Limit-Limit', $info['limit']); - $this->response->header('X-Rate-Limit-Remaining', $info['remaining']); - $this->response->header('X-Rate-Limit-Reset', $info['reset']); - $this->response->body($rateLimitCheck); - $this->response->statusCode(429); - $this->response->send(); - $this->_stop(); - } - return true; } public function afterFilter() @@ -1143,14 +1150,14 @@ class AppController extends Controller $headerNamespace = ''; } if (isset($server[$headerNamespace . $header]) && !empty($server[$headerNamespace . $header])) { - if (Configure::read('Plugin.CustomAuth_only_allow_source') && Configure::read('Plugin.CustomAuth_only_allow_source') !== $this->_remoteIp()) { + if (Configure::read('Plugin.CustomAuth_only_allow_source') && Configure::read('Plugin.CustomAuth_only_allow_source') !== $this->User->_remoteIp()) { $this->Log = ClassRegistry::init('Log'); $this->Log->createLogEntry( 'SYSTEM', 'auth_fail', 'User', 0, - 'Failed authentication using external key (' . trim($server[$headerNamespace . $header]) . ') - the user has not arrived from the expected address. Instead the request came from: ' . $this->_remoteIp(), + 'Failed authentication using external key (' . trim($server[$headerNamespace . $header]) . ') - the user has not arrived from the expected address. Instead the request came from: ' . $this->User->_remoteIp(), null); $this->__preAuthException($authName . ' authentication failed. Contact your MISP support for additional information at: ' . Configure::read('MISP.contact')); } @@ -1310,7 +1317,7 @@ class AppController extends Controller $exception = false; $filters = $this->_harvestParameters($filterData, $exception, $this->_legacyParams); if (empty($filters) && $this->request->is('get')) { - throw new InvalidArgumentException(__('Restsearch queries using GET and no parameters are not allowed. If you have passed parameters via a JSON body, make sure you use POST requests.')); + throw new BadRequestException(__('Restsearch queries using GET and no parameters are not allowed. If you have passed parameters via a JSON body, make sure you use POST requests.')); } if (empty($filters['returnFormat'])) { $filters['returnFormat'] = 'json'; diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index d35c06e5d..2d6f13691 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -1917,7 +1917,7 @@ class AttributesController extends AppController public function reportValidationIssuesAttributes($eventId = false) { // search for validation problems in the attributes - $this->set('result', $this->Attribute->reportValidationIssuesAttributes($eventId)); + $this->set('result', iterator_to_array($this->Attribute->reportValidationIssuesAttributes($eventId))); } public function generateCorrelation() diff --git a/app/Controller/AuditLogsController.php b/app/Controller/AuditLogsController.php index 9c812ed6b..6ddee8443 100644 --- a/app/Controller/AuditLogsController.php +++ b/app/Controller/AuditLogsController.php @@ -91,21 +91,6 @@ class AuditLogsController extends AppController ]; } - private function __applyAuditACL(array $user) - { - $acl = []; - if (empty($user['Role']['perm_site_admin'])) { - if (!empty($user['Role']['perm_admin'])) { - // ORG admins can see their own org info - $acl = ['AuditLog.org_id' => $user['org_id']]; - } else { - // users can see their own info - $acl = ['AuditLog.user_id' => $user['id']]; - } - } - return $acl; - } - public function admin_index() { $this->paginate['fields'][] = 'ip'; @@ -134,7 +119,8 @@ class AuditLogsController extends AppController ]); $this->paginate['conditions'] = $this->__searchConditions($params); - $acl = $this->__applyAuditACL($this->Auth->user()); + $user = $this->Auth->user(); + $acl = $this->__applyAuditAcl($user); if ($acl) { $this->paginate['conditions']['AND'][] = $acl; } @@ -144,7 +130,7 @@ class AuditLogsController extends AppController return $this->RestResponse->viewData($list, 'json'); } - $list = $this->__appendModelLinks($list); + $list = $this->__appendModelLinks($user, $list); foreach ($list as $k => $item) { $list[$k]['AuditLog']['action_human'] = $this->actions[$item['AuditLog']['action']]; } @@ -222,7 +208,7 @@ class AuditLogsController extends AppController public function fullChange($id) { - $acl = $this->__applyAuditACL($this->Auth->user()); + $acl = $this->__applyAuditAcl($this->Auth->user()); $log = $this->AuditLog->find('first', [ 'conditions' => [ 'AND' => [ @@ -234,7 +220,7 @@ class AuditLogsController extends AppController 'fields' => ['change', 'action'], ]); if (empty($log)) { - throw new Exception('Log not found.'); + throw new NotFoundException('Log not found.'); } $this->set('log', $log); } @@ -252,6 +238,21 @@ class AuditLogsController extends AppController return $this->RestResponse->viewData($data, $this->response->type()); } + private function __applyAuditAcl(array $user) + { + $acl = []; + if (empty($user['Role']['perm_site_admin'])) { + if (!empty($user['Role']['perm_admin'])) { + // ORG admins can see their own org info + $acl = ['AuditLog.org_id' => $user['org_id']]; + } else { + // users can see their own info + $acl = ['AuditLog.user_id' => $user['id']]; + } + } + return $acl; + } + /** * @return array */ @@ -441,10 +442,11 @@ class AuditLogsController extends AppController /** * Generate link to model view if exists and use has permission to access it. + * @param array $user * @param array $auditLogs * @return array */ - private function __appendModelLinks(array $auditLogs) + private function __appendModelLinks(array $user, array $auditLogs) { $models = []; foreach ($auditLogs as $auditLog) { @@ -455,7 +457,7 @@ class AuditLogsController extends AppController } } - $eventIds = isset($models['Event']) ? $models['Event'] : []; + $eventIds = $models['Event'] ?? []; if (isset($models['ObjectReference'])) { $this->loadModel('ObjectReference'); @@ -467,11 +469,11 @@ class AuditLogsController extends AppController if (isset($models['Object']) || isset($objectReferences)) { $objectIds = array_unique(array_merge( - isset($models['Object']) ? $models['Object'] : [], + $models['Object'] ?? [], isset($objectReferences) ? array_values($objectReferences) : [] )); $this->loadModel('MispObject'); - $conditions = $this->MispObject->buildConditions($this->Auth->user()); + $conditions = $this->MispObject->buildConditions($user); $conditions['Object.id'] = $objectIds; $objects = $this->MispObject->find('all', [ 'conditions' => $conditions, @@ -479,22 +481,22 @@ class AuditLogsController extends AppController 'fields' => ['Object.id', 'Object.event_id', 'Object.uuid', 'Object.deleted'], ]); $objects = array_column(array_column($objects, 'Object'), null, 'id'); - $eventIds = array_merge($eventIds, array_column($objects, 'event_id')); + array_push($eventIds, ...array_column($objects, 'event_id')); } if (isset($models['Attribute'])) { $this->loadModel('Attribute'); - $attributes = $this->Attribute->fetchAttributesSimple($this->Auth->user(), [ + $attributes = $this->Attribute->fetchAttributesSimple($user, [ 'conditions' => ['Attribute.id' => array_unique($models['Attribute'])], 'fields' => ['Attribute.id', 'Attribute.event_id', 'Attribute.uuid', 'Attribute.deleted'], ]); $attributes = array_column(array_column($attributes, 'Attribute'), null, 'id'); - $eventIds = array_merge($eventIds, array_column($attributes, 'event_id')); + array_push($eventIds, ...array_column($attributes, 'event_id')); } if (isset($models['ShadowAttribute'])) { $this->loadModel('ShadowAttribute'); - $conditions = $this->ShadowAttribute->buildConditions($this->Auth->user()); + $conditions = $this->ShadowAttribute->buildConditions($user); $conditions['AND'][] = ['ShadowAttribute.id' => array_unique($models['ShadowAttribute'])]; $shadowAttributes = $this->ShadowAttribute->find('all', [ 'conditions' => $conditions, @@ -502,12 +504,12 @@ class AuditLogsController extends AppController 'contain' => ['Event', 'Attribute'], ]); $shadowAttributes = array_column(array_column($shadowAttributes, 'ShadowAttribute'), null, 'id'); - $eventIds = array_merge($eventIds, array_column($shadowAttributes, 'event_id')); + array_push($eventIds, ...array_column($shadowAttributes, 'event_id')); } if (!empty($eventIds)) { $this->loadModel('Event'); - $conditions = $this->Event->createEventConditions($this->Auth->user()); + $conditions = $this->Event->createEventConditions($user); $conditions['Event.id'] = array_unique($eventIds); $events = $this->Event->find('list', [ 'conditions' => $conditions, diff --git a/app/Controller/Component/IndexFilterComponent.php b/app/Controller/Component/IndexFilterComponent.php index 064a0bb87..318a7af47 100644 --- a/app/Controller/Component/IndexFilterComponent.php +++ b/app/Controller/Component/IndexFilterComponent.php @@ -8,7 +8,9 @@ class IndexFilterComponent extends Component { /** @var Controller */ public $Controller; - public $isRest = null; + + /** @var bool|null */ + private $isRest = null; // Used for isApiFunction(), a check that returns true if the controller & action combo matches an action that is a non-xml and non-json automation method // This is used to allow authentication via headers for methods not covered by _isRest() - as that only checks for JSON and XML formats @@ -93,6 +95,11 @@ class IndexFilterComponent extends Component } } + public function isXhr() + { + return $this->Controller->request->header('X-Requested-With') === 'XMLHttpRequest'; + } + public function isJson() { return $this->Controller->request->header('Accept') === 'application/json' || $this->Controller->RequestHandler->prefers() === 'json'; @@ -103,11 +110,6 @@ class IndexFilterComponent extends Component return $this->Controller->request->header('Accept') === 'text/csv' || $this->Controller->RequestHandler->prefers() === 'csv'; } - public function isXml() - { - - } - /** * @param string $controller * @param string $action diff --git a/app/Controller/Component/RateLimitComponent.php b/app/Controller/Component/RateLimitComponent.php index c3611749e..a957a1887 100644 --- a/app/Controller/Component/RateLimitComponent.php +++ b/app/Controller/Component/RateLimitComponent.php @@ -12,58 +12,60 @@ class RateLimitComponent extends Component ) ); - public $components = array('RestResponse'); - /** * @param array $user * @param string $controller * @param string $action - * @param array $info - * @param string $responseType - * @return bool + * @return array|null * @throws RedisException */ - public function check(array $user, $controller, $action, &$info = array(), $responseType) + public function check(array $user, $controller, $action) { - if (!empty($user['Role']['enforce_rate_limit']) && isset(self::LIMITED_FUNCTIONS[$controller][$action])) { - if ($user['Role']['rate_limit_count'] == 0) { - throw new MethodNotAllowedException(__('API searches are not allowed for this user role.')); - } - try { - $redis = RedisTool::init(); - } catch (Exception $e) { - return true; // redis is not available, allow access - } - $uuid = Configure::read('MISP.uuid') ?: 'no-uuid'; - $keyName = 'misp:' . $uuid . ':rate_limit:' . $user['id']; - $count = $redis->get($keyName); - if ($count !== false && $count >= $user['Role']['rate_limit_count']) { - $info = array( - 'limit' => $user['Role']['rate_limit_count'], - 'reset' => $redis->ttl($keyName), - 'remaining' => $user['Role']['rate_limit_count'] - $count, - ); - return $this->RestResponse->throwException( - 429, - __('Rate limit exceeded.'), - '/' . $controller . '/' . $action, - $responseType - ); - } else { - if ($count === false) { - $redis->setEx($keyName, 900, 1); - } else { - $redis->setEx($keyName, $redis->ttl($keyName), intval($count) + 1); - } - } - $count += 1; - $info = array( - 'limit' => $user['Role']['rate_limit_count'], - 'reset' => $redis->ttl($keyName), - 'remaining' => $user['Role']['rate_limit_count'] - $count - ); - + if (!isset(self::LIMITED_FUNCTIONS[$controller][$action])) { + return null; // no limit enforced for this controller action } - return true; + + if (empty($user['Role']['enforce_rate_limit'])) { + return null; // no limit enforced for this role + } + + $rateLimit = (int)$user['Role']['rate_limit_count']; + if ($rateLimit === 0) { + throw new MethodNotAllowedException(__('API searches are not allowed for this user role.')); + } + + try { + $redis = RedisTool::init(); + } catch (Exception $e) { + return null; // redis is not available, allow access + } + + $uuid = Configure::read('MISP.uuid') ?: 'no-uuid'; + $keyName = 'misp:' . $uuid . ':rate_limit:' . $user['id']; + $count = $redis->get($keyName); + + if ($count !== false && $count >= $rateLimit) { + return [ + 'exceeded' => true, + 'limit' => $rateLimit, + 'reset' => $redis->ttl($keyName), + 'remaining' => $rateLimit - $count, + ]; + } + + $newCount = $redis->incr($keyName); + if ($newCount === 1) { + $redis->expire($keyName, 900); + $reset = 900; + } else { + $reset = $redis->ttl($keyName); + } + + return [ + 'exceeded' => false, + 'limit' => $rateLimit, + 'reset' => $reset, + 'remaining' => $rateLimit - $newCount, + ]; } } diff --git a/app/Controller/Component/RestResponseComponent.php b/app/Controller/Component/RestResponseComponent.php index 585ee77c5..3f760f056 100644 --- a/app/Controller/Component/RestResponseComponent.php +++ b/app/Controller/Component/RestResponseComponent.php @@ -517,7 +517,7 @@ class RestResponseComponent extends Component if ($id) { $response['id'] = $id; } - return $this->__sendResponse($response, 403, $format); + return $this->prepareResponse($response, 403, $format); } /** @@ -562,7 +562,7 @@ class RestResponseComponent extends Component if ($id) { $response['id'] = $id; } - return $this->__sendResponse($response, 200, $format); + return $this->prepareResponse($response, 200, $format); } /** @@ -587,7 +587,7 @@ class RestResponseComponent extends Component * @return CakeResponse * @throws Exception */ - private function __sendResponse($response, $code, $format = false, $raw = false, $download = false, $headers = array()) + private function prepareResponse($response, $code, $format = false, $raw = false, $download = false, $headers = array()) { App::uses('TmpFileTool', 'Tools'); $format = !empty($format) ? strtolower($format) : 'json'; @@ -633,7 +633,7 @@ class RestResponseComponent extends Component } // If response is big array, encode items separately to save memory - if (is_array($response) && count($response) > 10000) { + if (is_array($response) && count($response) > 10000 && JsonTool::arrayIsList($response)) { $output = new TmpFileTool(); $output->write('['); @@ -775,7 +775,7 @@ class RestResponseComponent extends Component if (!empty($errors)) { $data['errors'] = $errors; } - return $this->__sendResponse($data, 200, $format, $raw, $download, $headers); + return $this->prepareResponse($data, 200, $format, $raw, $download, $headers); } /** @@ -807,7 +807,7 @@ class RestResponseComponent extends Component 'message' => $message, 'url' => $url ); - return $this->__sendResponse($message, $code, $format, $raw, false, $headers); + return $this->prepareResponse($message, $code, $format, $raw, false, $headers); } public function setHeader($header, $value) @@ -834,7 +834,7 @@ class RestResponseComponent extends Component } } $response['url'] = $this->__generateURL($actionArray, $controller, $params); - return $this->__sendResponse($response, 200, $format); + return $this->prepareResponse($response, 200, $format); } private function __setup() diff --git a/app/Controller/Component/RestSearchComponent.php b/app/Controller/Component/RestSearchComponent.php index 60467dc9f..99a67821f 100644 --- a/app/Controller/Component/RestSearchComponent.php +++ b/app/Controller/Component/RestSearchComponent.php @@ -152,6 +152,8 @@ class RestSearchComponent extends Component 'category', 'org', 'tags', + 'first_seen', + 'last_seen', 'from', 'to', 'last', @@ -184,6 +186,7 @@ class RestSearchComponent extends Component 'context', 'returnFormat', 'id', + 'uuid', 'type', 'from', 'to', @@ -191,7 +194,8 @@ class RestSearchComponent extends Component 'org_id', 'source', 'includeAttribute', - 'includeEvent' + 'includeEvent', + 'includeUuid', ], 'GalaxyCluster' => [ 'page', @@ -204,7 +208,7 @@ class RestSearchComponent extends Component 'distribution', 'org', 'orgc', - 'tag', + 'tag_name', 'custom', 'sgReferenceOnly', 'minimal', diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 573ae6612..7c87891fb 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -834,33 +834,29 @@ class EventsController extends AppController } if (empty($rules['limit'])) { - $events = array(); + $events = []; $i = 1; $rules['limit'] = 20000; while (true) { - $rules['page'] = $i; + $rules['page'] = $i++; $temp = $this->Event->find('all', $rules); $resultCount = count($temp); if ($resultCount !== 0) { - // this is faster and memory efficient than array_merge - foreach ($temp as $tempEvent) { - $events[] = $tempEvent; - } + array_push($events, ...$temp); } if ($resultCount < $rules['limit']) { break; } - $i++; } unset($temp); - $absolute_total = count($events); + $absoluteTotal = count($events); } else { $counting_rules = $rules; unset($counting_rules['limit']); unset($counting_rules['page']); - $absolute_total = $this->Event->find('count', $counting_rules); + $absoluteTotal = $this->Event->find('count', $counting_rules); - $events = $absolute_total === 0 ? [] : $this->Event->find('all', $rules); + $events = $absoluteTotal === 0 ? [] : $this->Event->find('all', $rules); } $isCsvResponse = $this->response->type() === 'text/csv'; @@ -979,7 +975,7 @@ class EventsController extends AppController $events = $export->eventIndex($events); } - return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, ['X-Result-Count' => $absolute_total]); + return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, ['X-Result-Count' => $absoluteTotal]); } private function __indexColumns() @@ -2383,7 +2379,7 @@ class EventsController extends AppController $results = $this->Event->addMISPExportFile($this->Auth->user(), $data, $isXml, $takeOwnership, $publish); } catch (Exception $e) { $this->log("Exception during processing MISP file import: {$e->getMessage()}"); - $this->Flash->error(__('Could not process MISP export file. %s.', $e->getMessage())); + $this->Flash->error(__('Could not process MISP export file. %s', $e->getMessage())); $this->redirect(['controller' => 'events', 'action' => 'add_misp_export']); } } @@ -3203,7 +3199,7 @@ class EventsController extends AppController $event = $this->Event->find('first', [ 'conditions' => Validation::uuid($id) ? ['Event.uuid' => $id] : ['Event.id' => $id], 'recursive' => -1, - 'fields' => ['id', 'info', 'publish_timestamp', 'orgc_id'], + 'fields' => ['id', 'info', 'publish_timestamp', 'orgc_id', 'user_id'], ]); if (empty($event)) { throw new NotFoundException(__('Invalid event.')); @@ -3222,6 +3218,16 @@ class EventsController extends AppController } } } + if ( + Configure::read('MISP.block_publishing_for_same_creator', false) && + $this->Auth->user()['id'] == $event['Event']['user_id'] + ) { + $message = __('Could not publish the event, the publishing user cannot be the same as the event creator as per this instance\'s configuration.'); + if (!$this->_isRest()) { + $this->Flash->error($message); + } + throw new MethodNotAllowedException($message); + } return $event; } @@ -3325,7 +3331,7 @@ class EventsController extends AppController $this->Flash->info(__('Warning, you are logged in as a site admin, any export that you generate will contain the FULL UNRESTRICTED data-set. If you would like to generate an export for your own organisation, please log in with a different user.')); } // Check if the background jobs are enabled - if not, fall back to old export page. - if (Configure::read('MISP.background_jobs') && !Configure::read('MISP.disable_cached_exports')) { + if (Configure::read('MISP.background_jobs') && !Configure::read('MISP.disable_cached_exports', true)) { $now = time(); // as a site admin we'll use the ADMIN identifier, not to overwrite the cached files of our own org with a file that includes too much data. @@ -3412,7 +3418,7 @@ class EventsController extends AppController public function downloadExport($type, $extra = null) { - if (Configure::read('MISP.disable_cached_exports')) { + if (Configure::read('MISP.disable_cached_exports', true)) { throw new MethodNotAllowedException(__('This feature is currently disabled')); } if ($this->_isSiteAdmin()) { diff --git a/app/Controller/JobsController.php b/app/Controller/JobsController.php index 6c0f238a5..630fd4b16 100644 --- a/app/Controller/JobsController.php +++ b/app/Controller/JobsController.php @@ -151,9 +151,12 @@ class JobsController extends AppController public function cache($type) { - if (Configure::read('MISP.disable_cached_exports')) { + if (Configure::read('MISP.disable_cached_exports', true)) { throw new MethodNotAllowedException('This feature is currently disabled'); } + if (!$this->request->is('post')) { + throw new MethodNotAllowedException('This endpoint only accept POST.'); + } if ($this->_isSiteAdmin()) { $target = 'All events.'; } else { diff --git a/app/Controller/OrganisationsController.php b/app/Controller/OrganisationsController.php index 45ea69bf3..0b400286e 100644 --- a/app/Controller/OrganisationsController.php +++ b/app/Controller/OrganisationsController.php @@ -481,6 +481,25 @@ class OrganisationsController extends AppController $extension = pathinfo($logo['name'], PATHINFO_EXTENSION); $filename = $orgId . '.' . ($extension === 'svg' ? 'svg' : 'png'); + if ($logo['size'] > 250*1024) { + $this->Flash->error(__('This organisation logo is too large, maximum file size allowed is 250kB.')); + return false; + } + + if ($extension !== 'svg' && $extension !== 'png') { + $this->Flash->error(__('Invalid file extension, Only PNG and SVG images are allowed.')); + return false; + } + + $imgMime = mime_content_type($logo['tmp_name']); + if ($extension === 'png' && !exif_imagetype($logo['tmp_name'])) { + $this->Flash->error(__('This is not a valid PNG image.')); + return false; + } else if ($extension === 'svg' && !($imgMime === 'image/svg+xml' || $imgMime === 'image/svg')) { + $this->Flash->error(__('This is not a valid SVG image.')); + return false; + } + if ($extension === 'svg' && !Configure::read('Security.enable_svg_logos')) { $this->Flash->error(__('Invalid file extension, SVG images are not allowed.')); return false; diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index 5f0dc0f9b..1d3c892d6 100644 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -1073,7 +1073,7 @@ class ServersController extends AppController ); $dumpResults = array(); $tempArray = array(); - foreach ($finalSettings as $k => $result) { + foreach ($finalSettings as $result) { if ($result['level'] == 3) { $issues['deprecated']++; } @@ -1105,18 +1105,19 @@ class ServersController extends AppController $diagnostic_errors = 0; App::uses('File', 'Utility'); App::uses('Folder', 'Utility'); + if ($tab === 'correlations') { $this->loadModel('Correlation'); $correlation_metrics = $this->Correlation->collectMetrics(); $this->set('correlation_metrics', $correlation_metrics); - } - if ($tab === 'files') { + } else if ($tab === 'files') { if (!empty(Configure::read('Security.disable_instance_file_uploads'))) { throw new MethodNotAllowedException(__('This functionality is disabled.')); } $files = $this->Server->grabFiles(); $this->set('files', $files); } + // Only run this check on the diagnostics tab if ($tab === 'diagnostics' || $tab === 'download' || $this->_isRest()) { $php_ini = php_ini_loaded_file(); @@ -1279,12 +1280,10 @@ class ServersController extends AppController $this->set('workerIssueCount', $workerIssueCount); $priorityErrorColours = array(0 => 'red', 1 => 'yellow', 2 => 'green'); $this->set('priorityErrorColours', $priorityErrorColours); - $this->set('phpversion', phpversion()); + $this->set('phpversion', PHP_VERSION); $this->set('phpmin', $this->phpmin); $this->set('phprec', $this->phprec); $this->set('phptoonew', $this->phptoonew); - $this->set('pythonmin', $this->pythonmin); - $this->set('pythonrec', $this->pythonrec); $this->set('title_for_layout', __('Diagnostics')); } @@ -1863,7 +1862,7 @@ class ServersController extends AppController } if (Configure::read('SimpleBackgroundJobs.enabled')) { - $this->Server->getBackgroundJobsTool()->purgeQueue($worker); + $this->Server->getBackgroundJobsTool()->clearQueue($worker); } else { // CakeResque $worker_array = array('cache', 'default', 'email', 'prio'); @@ -2183,7 +2182,7 @@ class ServersController extends AppController if ($this->_isRest()) { return $this->RestResponse->saveFailResponse('Servers', 'addFromJson', false, $this->Server->validationErrors, $this->response->type()); } else { - $this->Flash->error(__('Could not save the server. Error: %s', json_encode($this->Server->validationErrors, true))); + $this->Flash->error(__('Could not save the server. Error: %s', json_encode($this->Server->validationErrors))); $this->redirect(array('action' => 'index')); } } diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php index c2735ab6e..09bc73493 100644 --- a/app/Controller/UsersController.php +++ b/app/Controller/UsersController.php @@ -1244,8 +1244,6 @@ class UsersController extends AppController // login was successful, do everything that is needed such as logging and more: $this->_postlogin(); } else { - $dataSourceConfig = ConnectionManager::getDataSource('default')->config; - $dataSource = $dataSourceConfig['datasource']; // don't display authError before first login attempt if (str_replace("//", "/", $this->webroot . $this->Session->read('Auth.redirect')) == $this->webroot && $this->Session->read('Message.auth.message') == $this->Auth->authError) { $this->Session->delete('Message.auth'); @@ -1260,73 +1258,7 @@ class UsersController extends AppController } } - // - // Actions needed for the first access, when the database is not populated yet. - // - - // populate the DB with the first role (site admin) if it's empty - if (!$this->User->Role->hasAny()) { - $siteAdmin = array('Role' => array( - 'id' => 1, - 'name' => 'Site Admin', - 'permission' => 3, - 'perm_add' => 1, - 'perm_modify' => 1, - 'perm_modify_org' => 1, - 'perm_publish' => 1, - 'perm_sync' => 1, - 'perm_admin' => 1, - 'perm_audit' => 1, - 'perm_auth' => 1, - 'perm_site_admin' => 1, - 'perm_regexp_access' => 1, - 'perm_sharing_group' => 1, - 'perm_template' => 1, - 'perm_tagger' => 1, - )); - $this->User->Role->save($siteAdmin); - // PostgreSQL: update value of auto incremented serial primary key after setting the column by force - if ($dataSource === 'Database/Postgres') { - $sql = "SELECT setval('roles_id_seq', (SELECT MAX(id) FROM roles));"; - $this->User->Role->query($sql); - } - } - if (!$this->User->Organisation->hasAny(array('Organisation.local' => true))) { - $this->User->runUpdates(); - $date = date('Y-m-d H:i:s'); - $org = array('Organisation' => array( - 'id' => 1, - 'name' => !empty(Configure::read('MISP.org')) ? Configure::read('MISP.org') : 'ADMIN', - 'description' => 'Automatically generated admin organisation', - 'type' => 'ADMIN', - 'uuid' => CakeText::uuid(), - 'local' => 1, - 'date_created' => $date, - 'sector' => '', - 'nationality' => '' - )); - $this->User->Organisation->save($org); - // PostgreSQL: update value of auto incremented serial primary key after setting the column by force - if ($dataSource === 'Database/Postgres') { - $sql = "SELECT setval('organisations_id_seq', (SELECT MAX(id) FROM organisations));"; - $this->User->Organisation->query($sql); - } - $org_id = $this->User->Organisation->id; - } - // populate the DB with the first user if it's empty - if (!$this->User->hasAny()) { - if (!isset($org_id)) { - $hostOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.name' => Configure::read('MISP.org'), 'Organisation.local' => true), 'recursive' => -1)); - if (!empty($hostOrg)) { - $org_id = $hostOrg['Organisation']['id']; - } else { - $firstOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.local' => true), 'order' => 'Organisation.id ASC')); - $org_id = $firstOrg['Organisation']['id']; - } - } - $this->User->runUpdates(); - $this->User->createInitialUser($org_id); - } + $this->User->init(); } } diff --git a/app/Lib/Export/NidsExport.php b/app/Lib/Export/NidsExport.php index d4f8f2467..f450670ca 100644 --- a/app/Lib/Export/NidsExport.php +++ b/app/Lib/Export/NidsExport.php @@ -6,10 +6,7 @@ abstract class NidsExport public $classtype = 'trojan-activity'; - public $format = ""; // suricata (default), snort - - - public $checkWhitelist = true; + protected $format; // suricata (default), snort public $additional_params = array( 'contain' => array( @@ -17,36 +14,31 @@ abstract class NidsExport 'fields' => array('threat_level_id') ) ), - ); public function handler($data, $options = array()) { - $continue = empty($format); - $this->checkWhitelist = false; if ($options['scope'] === 'Attribute') { $this->export( array($data), - $options['user']['nids_sid'], - $options['returnFormat'], - $continue + $options['user']['nids_sid'] ); } else if ($options['scope'] === 'Event') { if (!empty($data['EventTag'])) { $data['Event']['EventTag'] = $data['EventTag']; } if (!empty($data['Attribute'])) { - $this->__convertFromEventFormat($data['Attribute'], $data, $options, $continue); + $this->convertFromEventFormat($data['Attribute'], $data, $options); } if (!empty($data['Object'])) { - $this->__convertFromEventFormatObject($data['Object'], $data, $options, $continue); + $this->convertFromEventFormatObject($data['Object'], $data, $options); } } return ''; } - private function __convertFromEventFormat($attributes, $event, $options = array(), $continue = false) { - + private function convertFromEventFormat($attributes, $event, $options = array()) + { $rearranged = array(); foreach ($attributes as $attribute) { $attributeTag = array(); @@ -62,15 +54,12 @@ abstract class NidsExport } $this->export( $rearranged, - $options['user']['nids_sid'], - $options['returnFormat'], - $continue + $options['user']['nids_sid'] ); return true; - } - private function __convertFromEventFormatObject($objects, $event, $options = array(), $continue = false) + private function convertFromEventFormatObject($objects, $event, $options = array()) { $rearranged = array(); foreach ($objects as $object) { @@ -93,20 +82,18 @@ abstract class NidsExport 'Event' => $event['Event'] ); } else { // In case no custom export exists for the object, the approach falls back to the attribute case - $this->__convertFromEventFormat($object['Attribute'], $event, $options, $continue); + $this->convertFromEventFormat($object['Attribute'], $event, $options); } } $this->export( $rearranged, - $options['user']['nids_sid'], - $options['returnFormat'], - $continue + $options['user']['nids_sid'] ); return true; } - public function header($options = array()) + public function header() { $this->explain(); return ''; @@ -122,7 +109,7 @@ abstract class NidsExport return ''; } - public function explain() + protected function explain() { $this->rules[] = '# MISP export of IDS rules - optimized for '.$this->format; $this->rules[] = '#'; @@ -136,21 +123,8 @@ abstract class NidsExport $this->rules[] = '# '; } - private $whitelist = null; - - - public function export($items, $startSid, $format="suricata", $continue = false) + protected function export($items, $startSid) { - $this->format = $format; - if ($this->checkWhitelist && !isset($this->Whitelist)) { - $this->Whitelist = ClassRegistry::init('Whitelist'); - $this->whitelist = $this->Whitelist->getBlockedValues(); - } - - // output a short explanation - if (!$continue) { - $this->explain(); - } // generate the rules foreach ($items as $item) { // retrieve all tags for this item to add them to the msg @@ -180,7 +154,6 @@ abstract class NidsExport $sid++; if (!empty($item['Attribute']['type'])) { // item is an 'Attribute' - switch ($item['Attribute']['type']) { // LATER nids - test all the snort attributes // LATER nids - add the tag keyword in the rules to capture network traffic @@ -195,7 +168,7 @@ abstract class NidsExport break; case 'email': $this->emailSrcRule($ruleFormat, $item['Attribute'], $sid); - $sid++; + $sid++; $this->emailDstRule($ruleFormat, $item['Attribute'], $sid); break; case 'email-src': @@ -228,17 +201,17 @@ abstract class NidsExport case 'ja3-fingerprint-md5': $this->ja3Rule($ruleFormat, $item['Attribute'], $sid); break; - case 'ja3s-fingerprint-md5': // Atribute type doesn't exists yet (2020-12-10) but ready when created. + case 'ja3s-fingerprint-md5': // Attribute type doesn't exists yet (2020-12-10) but ready when created. $this->ja3sRule($ruleFormat, $item['Attribute'], $sid); break; case 'snort': - $this->snortRule($ruleFormat, $item['Attribute'], $sid, $ruleFormatMsg, $ruleFormatReference); + $this->snortRule($item['Attribute'], $sid, $ruleFormatMsg, $ruleFormatReference); // no break default: break; } - } else if(!empty($item['Attribute']['name'])) { // Item is an 'Object' + } else if (!empty($item['Attribute']['name'])) { // Item is an 'Object' switch ($item['Attribute']['name']) { case 'network-connection': @@ -252,34 +225,30 @@ abstract class NidsExport } } - } - - return $this->rules; } - public function networkConnectionRule($ruleFormat, $object, &$sid) + protected function networkConnectionRule($ruleFormat, $object, &$sid) { - $attributes = NidsExport::getObjectAttributes($object); - if(!array_key_exists('layer4-protocol', $attributes)){ + if (!array_key_exists('layer4-protocol', $attributes)) { $attributes['layer4-protocol'] = 'ip'; // If layer-4 protocol is unknown, we roll-back to layer-3 ('ip') } - if(!array_key_exists('ip-src', $attributes)){ + if (!array_key_exists('ip-src', $attributes)) { $attributes['ip-src'] = '$HOME_NET'; // If ip-src is unknown, we roll-back to $HOME_NET } - if(!array_key_exists('ip-dst', $attributes)){ + if (!array_key_exists('ip-dst', $attributes)) { $attributes['ip-dst'] = '$HOME_NET'; // If ip-dst is unknown, we roll-back to $HOME_NET } - if(!array_key_exists('src-port', $attributes)){ + if (!array_key_exists('src-port', $attributes)) { $attributes['src-port'] = 'any'; // If src-port is unknown, we roll-back to 'any' } - if(!array_key_exists('dst-port', $attributes)){ + if (!array_key_exists('dst-port', $attributes)) { $attributes['dst-port'] = 'any'; // If dst-port is unknown, we roll-back to 'any' } - $this->rules[] = sprintf( + $this->rules[] = sprintf( $ruleFormat, false, $attributes['layer4-protocol'], // proto @@ -294,12 +263,10 @@ abstract class NidsExport $sid, // sid 1 // rev ); - } - - public function ddosRule($ruleFormat, $object, &$sid) + + protected function ddosRule($ruleFormat, $object, &$sid) { - $attributes = NidsExport::getObjectAttributes($object); if(!array_key_exists('protocol', $attributes)){ @@ -318,7 +285,7 @@ abstract class NidsExport $attributes['dst-port'] = 'any'; // If dst-port is unknown, we roll-back to 'any' } - $this->rules[] = sprintf( + $this->rules[] = sprintf( $ruleFormat, false, $attributes['protocol'], // proto @@ -333,12 +300,10 @@ abstract class NidsExport $sid, // sid 1 // rev ); - } - - public static function getObjectAttributes($object) + + protected static function getObjectAttributes($object) { - $attributes = array(); foreach ($object['Attribute'] as $attribute) { @@ -348,7 +313,7 @@ abstract class NidsExport return $attributes; } - public function domainIpRule($ruleFormat, $attribute, &$sid) + protected function domainIpRule($ruleFormat, $attribute, &$sid) { $values = explode('|', $attribute['value']); $attributeCopy = $attribute; @@ -361,7 +326,7 @@ abstract class NidsExport $this->ipSrcRule($ruleFormat, $attributeCopy, $sid); } - public function ipDstRule($ruleFormat, $attribute, &$sid) + protected function ipDstRule($ruleFormat, $attribute, &$sid) { $overruled = $this->checkWhitelist($attribute['value']); $ipport = NidsExport::getIpPort($attribute); @@ -382,7 +347,7 @@ abstract class NidsExport ); } - public function ipSrcRule($ruleFormat, $attribute, &$sid) + protected function ipSrcRule($ruleFormat, $attribute, &$sid) { $overruled = $this->checkWhitelist($attribute['value']); $ipport = NidsExport::getIpPort($attribute); @@ -403,7 +368,7 @@ abstract class NidsExport ); } - public function emailSrcRule($ruleFormat, $attribute, &$sid) + protected function emailSrcRule($ruleFormat, $attribute, &$sid) { $overruled = $this->checkWhitelist($attribute['value']); $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule @@ -425,7 +390,7 @@ abstract class NidsExport ); } - public function emailDstRule($ruleFormat, $attribute, &$sid) + protected function emailDstRule($ruleFormat, $attribute, &$sid) { $overruled = $this->checkWhitelist($attribute['value']); $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule @@ -447,7 +412,7 @@ abstract class NidsExport ); } - public function emailSubjectRule($ruleFormat, $attribute, &$sid) + protected function emailSubjectRule($ruleFormat, $attribute, &$sid) { // LATER nids - email-subject rule might not match because of line-wrapping $overruled = $this->checkWhitelist($attribute['value']); @@ -470,7 +435,7 @@ abstract class NidsExport ); } - public function emailAttachmentRule($ruleFormat, $attribute, &$sid) + protected function emailAttachmentRule($ruleFormat, $attribute, &$sid) { // LATER nids - email-attachment rule might not match because of line-wrapping $overruled = $this->checkWhitelist($attribute['value']); @@ -493,7 +458,7 @@ abstract class NidsExport ); } - public function hostnameRule($ruleFormat, $attribute, &$sid) + protected function hostnameRule($ruleFormat, $attribute, &$sid) { $overruled = $this->checkWhitelist($attribute['value']); $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule @@ -549,7 +514,7 @@ abstract class NidsExport ); } - public function domainRule($ruleFormat, $attribute, &$sid) + protected function domainRule($ruleFormat, $attribute, &$sid) { $overruled = $this->checkWhitelist($attribute['value']); $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule @@ -605,7 +570,7 @@ abstract class NidsExport ); } - public function urlRule($ruleFormat, $attribute, &$sid) + protected function urlRule($ruleFormat, $attribute, &$sid) { // TODO in hindsight, an url should not be excluded given a host or domain name. //$hostpart = parse_url($attribute['value'], PHP_URL_HOST); @@ -630,7 +595,7 @@ abstract class NidsExport ); } - public function userAgentRule($ruleFormat, $attribute, &$sid) + protected function userAgentRule($ruleFormat, $attribute, &$sid) { $overruled = $this->checkWhitelist($attribute['value']); $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule @@ -652,17 +617,17 @@ abstract class NidsExport ); } - public function ja3Rule($ruleFormat, $attribute, &$sid) + protected function ja3Rule($ruleFormat, $attribute, &$sid) { //Empty because Snort doesn't support JA3 Rules } - public function ja3sRule($ruleFormat, $attribute, &$sid) + protected function ja3sRule($ruleFormat, $attribute, &$sid) { //Empty because Snort doesn't support JA3S Rules } - public function snortRule($ruleFormat, $attribute, &$sid, $ruleFormatMsg, $ruleFormatReference) + protected function snortRule($attribute, &$sid, $ruleFormatMsg, $ruleFormatReference) { // LATER nids - test using lots of snort rules, some rules don't contain all the necessary to be a valid rule. @@ -678,46 +643,46 @@ abstract class NidsExport // tag - '/tag\s*:\s*.+?;/' $replaceCount = array(); $tmpRule = preg_replace('/sid\s*:\s*[0-9]+\s*;/', 'sid:' . $sid . ';', $tmpRule, -1, $replaceCount['sid']); - if (null == $tmpRule) { + if (null === $tmpRule) { return false; } // don't output the rule on error with the regex $tmpRule = preg_replace('/rev\s*:\s*[0-9]+\s*;/', 'rev:1;', $tmpRule, -1, $replaceCount['rev']); - if (null == $tmpRule) { + if (null === $tmpRule) { return false; } // don't output the rule on error with the regex $tmpRule = preg_replace('/classtype:[a-zA-Z_-]+;/', 'classtype:' . $this->classtype . ';', $tmpRule, -1, $replaceCount['classtype']); - if (null == $tmpRule) { + if (null === $tmpRule) { return false; } // don't output the rule on error with the regex $tmpRule = preg_replace('/msg\s*:\s*"(.*?)"\s*;/', sprintf($ruleFormatMsg, 'snort-rule | $1') . ';', $tmpRule, -1, $replaceCount['msg']); - if (null == $tmpRule) { + if (null === $tmpRule) { return false; } // don't output the rule on error with the regex $tmpRule = preg_replace('/reference\s*:\s*.+?;/', $ruleFormatReference . ';', $tmpRule, -1, $replaceCount['reference']); - if (null == $tmpRule) { + if (null === $tmpRule) { return false; } // don't output the rule on error with the regex $tmpRule = preg_replace('/reference\s*:\s*.+?;/', $ruleFormatReference . ';', $tmpRule, -1, $replaceCount['reference']); - if (null == $tmpRule) { + if (null === $tmpRule) { return false; } // don't output the rule on error with the regex // FIXME nids - implement priority overwriting // some values were not replaced, so we need to add them ourselves, and insert them in the rule $extraForRule = ""; - if (0 == $replaceCount['sid']) { + if (0 === $replaceCount['sid']) { $extraForRule .= 'sid:' . $sid . ';'; } - if (0 == $replaceCount['rev']) { + if (0 === $replaceCount['rev']) { $extraForRule .= 'rev:1;'; } - if (0 == $replaceCount['classtype']) { + if (0 === $replaceCount['classtype']) { $extraForRule .= 'classtype:' . $this->classtype . ';'; } - if (0 == $replaceCount['msg']) { - $extraForRule .= $tmpMessage . ';'; + if (0 === $replaceCount['msg']) { + $extraForRule .= $ruleFormatMsg . ';'; } - if (0 == $replaceCount['reference']) { + if (0 === $replaceCount['reference']) { $extraForRule .= $ruleFormatReference . ';'; } $tmpRule = preg_replace('/;\s*\)/', '; ' . $extraForRule . ')', $tmpRule); @@ -734,7 +699,7 @@ abstract class NidsExport * @param string $type the type of dns name - domain (default) or hostname * @return string raw snort compatible format of the dns name */ - public static function dnsNameToRawFormat($name, $type='domain') + protected static function dnsNameToRawFormat($name, $type='domain') { $rawName = ""; if ('hostname' == $type) { @@ -747,7 +712,7 @@ abstract class NidsExport // count the length of the part, and add |length| before $length = strlen($explodedName); if ($length > 255) { - log('WARNING: DNS name is too long for RFC: '.$name); + CakeLog::notice('WARNING: DNS name is too long for RFC: '.$name); } $hexLength = dechex($length); if (1 == strlen($hexLength)) { @@ -768,7 +733,7 @@ abstract class NidsExport * @param string $name dns name to be converted * @return string raw snort compatible format of the dns name */ - public static function dnsNameToMSDNSLogFormat($name) + protected static function dnsNameToMSDNSLogFormat($name) { $rawName = ""; // in MS DNS log format we can't use (0) to distinguish between hostname and domain (including subdomains) @@ -779,7 +744,7 @@ abstract class NidsExport // count the length of the part, and add |length| before $length = strlen($explodedName); if ($length > 255) { - log('WARNING: DNS name is too long for RFC: '.$name); + CakeLog::notice('WARNING: DNS name is too long for RFC: '.$name); } $hexLength = dechex($length); $rawName .= '(' . $hexLength . ')' . $explodedName; @@ -793,34 +758,32 @@ abstract class NidsExport /** * Replaces characters that are not allowed in a signature. * example: " is converted to |22| - * @param unknown_type $value + * @param string $value */ - public static function replaceIllegalChars($value) + protected static function replaceIllegalChars($value) { $replace_pairs = array( - '|' => '|7c|', // Needs to stay on top ! - '"' => '|22|', - ';' => '|3b|', - ':' => '|3a|', - '\\' => '|5c|', - '0x' => '|30 78|' - ); + '|' => '|7c|', // Needs to stay on top ! + '"' => '|22|', + ';' => '|3b|', + ':' => '|3a|', + '\\' => '|5c|', + '0x' => '|30 78|' + ); return strtr($value, $replace_pairs); } - public function checkWhitelist($value) + /** + * @deprecated + * @param $value + * @return false + */ + protected function checkWhitelist($value) { - if ($this->checkWhitelist && is_array($this->whitelist)) { - foreach ($this->whitelist as $wlitem) { - if (preg_match($wlitem, $value)) { - return true; - } - } - } return false; } - public static function getProtocolPort($protocol, $customPort) + protected static function getProtocolPort($protocol, $customPort) { if ($customPort == null) { switch ($protocol) { @@ -840,7 +803,7 @@ abstract class NidsExport } } - public static function getCustomIP($customIP) + protected static function getCustomIP($customIP) { if (filter_var($customIP, FILTER_VALIDATE_IP)) { return $customIP; @@ -853,7 +816,7 @@ abstract class NidsExport * @param array $attribute * @return array|string[] */ - public static function getIpPort($attribute) + protected static function getIpPort($attribute) { if (strpos($attribute['type'], 'port') !== false) { return explode('|', $attribute['value']); diff --git a/app/Lib/Export/NidsSnortExport.php b/app/Lib/Export/NidsSnortExport.php index d91ea7735..be1d2cf9f 100644 --- a/app/Lib/Export/NidsSnortExport.php +++ b/app/Lib/Export/NidsSnortExport.php @@ -4,11 +4,5 @@ App::uses('NidsExport', 'Export'); class NidsSnortExport extends NidsExport { - public function export($items, $startSid, $format = "suricata", $continue = false) - { - // set the specific format - $this->format = 'snort'; - // call the generic function - return parent::export($items, $startSid, $format, $continue); - } + protected $format = 'snort'; } diff --git a/app/Lib/Export/NidsSuricataExport.php b/app/Lib/Export/NidsSuricataExport.php index f1e61378b..ba25b2c9f 100644 --- a/app/Lib/Export/NidsSuricataExport.php +++ b/app/Lib/Export/NidsSuricataExport.php @@ -3,16 +3,10 @@ App::uses('NidsExport', 'Export'); class NidsSuricataExport extends NidsExport { - public function export($items, $startSid, $format = "suricata", $continue = false) - { - // set the specific format - $this->format = "suricata"; - // call the generic function - return parent::export($items, $startSid, $format, $continue); - } + protected $format = "suricata"; // below overwrite functions from NidsExport - public function hostnameRule($ruleFormat, $attribute, &$sid) + protected function hostnameRule($ruleFormat, $attribute, &$sid) { $overruled = $this->checkWhitelist($attribute['value']); $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule @@ -53,7 +47,7 @@ class NidsSuricataExport extends NidsExport ); } - public function domainRule($ruleFormat, $attribute, &$sid) + protected function domainRule($ruleFormat, $attribute, &$sid) { $overruled = $this->checkWhitelist($attribute['value']); $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule @@ -94,7 +88,7 @@ class NidsSuricataExport extends NidsExport ); } - public function urlRule($ruleFormat, $attribute, &$sid) + protected function urlRule($ruleFormat, $attribute, &$sid) { $createRule = true; $overruled = $this->checkWhitelist($attribute['value']); @@ -207,7 +201,7 @@ class NidsSuricataExport extends NidsExport } } - public function userAgentRule($ruleFormat, $attribute, &$sid) + protected function userAgentRule($ruleFormat, $attribute, &$sid) { $overruled = $this->checkWhitelist($attribute['value']); $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule @@ -230,7 +224,7 @@ class NidsSuricataExport extends NidsExport ); } - public function ja3Rule($ruleFormat, $attribute, &$sid) + protected function ja3Rule($ruleFormat, $attribute, &$sid) { $overruled = $this->checkWhitelist($attribute['value']); $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule @@ -253,7 +247,7 @@ class NidsSuricataExport extends NidsExport } // For Future use once JA3S Hash Attribute type is created - public function ja3sRule($ruleFormat, $attribute, &$sid) + protected function ja3sRule($ruleFormat, $attribute, &$sid) { $overruled = $this->checkWhitelist($attribute['value']); $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule diff --git a/app/Lib/Export/RPZExport.php b/app/Lib/Export/RPZExport.php index 594db8407..f46f8ec7f 100644 --- a/app/Lib/Export/RPZExport.php +++ b/app/Lib/Export/RPZExport.php @@ -2,107 +2,104 @@ class RPZExport { - private $__policies = array( - 'Local-Data' => array( - 'explanation' => 'returns the defined alternate location.', - 'action' => '$walled_garden', - 'setting_id' => 3, - ), - 'NXDOMAIN' => array( - 'explanation' => 'return NXDOMAIN (name does not exist) irrespective of actual result received.', - 'action' => '.', - 'setting_id' => 1, - ), - 'NODATA' => array( - 'explanation' => 'returns NODATA (name exists but no answers returned) irrespective of actual result received.', - 'action' => '*.', - 'setting_id' => 2, - ), - 'DROP' => array( - 'explanation' => 'timeout.', - 'action' => 'rpz-drop.', - 'setting_id' => 0, - ), - 'PASSTHRU' => array( - 'explanation' => 'lets queries through, but allows for logging the hits (useful for testing).', - 'action' => 'rpz-passthru.', - 'setting_id' => 4, - ), - 'TCP-only' => array( - 'explanation' => 'force the client to use TCP.', - 'action' => 'rpz-tcp-only.', - 'setting_id' => 5, - ), + const POLICIES = array( + 'Local-Data' => array( + 'explanation' => 'returns the defined alternate location.', + 'action' => '$walled_garden', + 'setting_id' => 3, + ), + 'NXDOMAIN' => array( + 'explanation' => 'return NXDOMAIN (name does not exist) irrespective of actual result received.', + 'action' => '.', + 'setting_id' => 1, + ), + 'NODATA' => array( + 'explanation' => 'returns NODATA (name exists but no answers returned) irrespective of actual result received.', + 'action' => '*.', + 'setting_id' => 2, + ), + 'DROP' => array( + 'explanation' => 'timeout.', + 'action' => 'rpz-drop.', + 'setting_id' => 0, + ), + 'PASSTHRU' => array( + 'explanation' => 'lets queries through, but allows for logging the hits (useful for testing).', + 'action' => 'rpz-passthru.', + 'setting_id' => 4, + ), + 'TCP-only' => array( + 'explanation' => 'force the client to use TCP.', + 'action' => 'rpz-tcp-only.', + 'setting_id' => 5, + ), ); - private $__items = array(); + private $items = array(); public $additional_params = array( 'flatten' => 1 ); - private $__rpzSettings = array(); - - private $__valid_policies = array('NXDOMAIN', 'NODATA', 'DROP', 'Local-Data', 'PASSTHRU', 'TCP-only'); + private $rpzSettings = array(); private $__server = null; - public $validTypes = array( + const VALID_TYPES = array( 'ip-src' => array( - 'value' => 'ip' + 'value' => 'ip' ), 'ip-dst' => array( - 'value' => 'ip' + 'value' => 'ip' ), 'domain' => array( - 'value' => 'domain' + 'value' => 'domain' ), 'domain|ip' => array( - 'value1' => 'domain', - 'value2' => 'ip' + 'value1' => 'domain', + 'value2' => 'ip' ), 'hostname' => array( - 'value' => 'hostname' + 'value' => 'hostname' ) ); public function handler($data, $options = array()) { if ($options['scope'] === 'Attribute') { - return $this->__attributeHandler($data, $options); + $this->attributeHandler($data); } else { - return $this->__eventHandler($data, $options); + $this->eventHandler($data); } + return ''; } - private function __eventHandler($event, $options = array()) { + private function eventHandler($event) + { foreach ($event['Attribute'] as $attribute) { - if (isset($this->validTypes[$attribute['type']])) { - if ($attribute['type'] == 'domain|ip') { + if (isset(self::VALID_TYPES[$attribute['type']])) { + if ($attribute['type'] === 'domain|ip') { $temp = explode('|', $attribute['value']); $attribute['value1'] = $temp[0]; $attribute['value2'] = $temp[1]; } - $this->__attributeHandler(array('Attribute' => $attribute, $options)); + $this->attributeHandler(array('Attribute' => $attribute)); } } - return ''; } - private function __attributeHandler($attribute, $options = array()) + private function attributeHandler($attribute) { if (isset($attribute['Attribute'])) { $attribute = $attribute['Attribute']; } - if (isset($this->validTypes[$attribute['type']])) { - foreach ($this->validTypes[$attribute['type']] as $field => $mapping) { - // get rid of the in_array check - if (empty($this->__items[$mapping]) || !isset($this->__items[$mapping][$attribute[$field]])) { - $this->__items[$mapping][$attribute[$field]] = true; + if (isset(self::VALID_TYPES[$attribute['type']])) { + foreach (self::VALID_TYPES[$attribute['type']] as $field => $mapping) { + if (!isset($this->items[$mapping][$attribute[$field]])) { + $this->items[$mapping][$attribute[$field]] = true; } } } - return ''; } public function header($options = array()) @@ -117,16 +114,16 @@ class RPZExport } } if (isset($options['filters'][$v])) { - $this->__rpzSettings[$v] = $options['filters'][$v]; + $this->rpzSettings[$v] = $options['filters'][$v]; } else { $tempSetting = Configure::read('Plugin.RPZ_' . $v); if (isset($tempSetting)) { - $this->__rpzSettings[$v] = Configure::read('Plugin.RPZ_' . $v); + $this->rpzSettings[$v] = $tempSetting; } else { if (empty($this->__server)) { $this->__server = ClassRegistry::init('Server'); } - $this->__rpzSettings[$v] = $this->__server->serverSettings['Plugin']['RPZ_' . $v]['value']; + $this->rpzSettings[$v] = $this->__server->serverSettings['Plugin']['RPZ_' . $v]['value']; } } } @@ -135,10 +132,7 @@ class RPZExport public function footer($options = array()) { - foreach ($this->__items as $k => $v) { - $this->__items[$k] = array_keys($this->__items[$k]); - } - return $this->export($this->__items, $this->__rpzSettings); + return $this->export($this->items, $this->rpzSettings); } public function separator() @@ -146,39 +140,32 @@ class RPZExport return ''; } - public function getPolicyById($id) + private function getPolicyById($id) { - foreach ($this->__policies as $k => $v) { - if ($id == $v['setting_id']) { + foreach (self::POLICIES as $k => $v) { + if ($id === $v['setting_id']) { return $k; } } + return null; } - public function getIdByPolicy($policy) + private function getIdByPolicy($policy) { - return $this->__policies[$policy]['setting_id']; + return self::POLICIES[$policy]['setting_id']; } - public function explain($type, $policy) + private function explain($type, $policy) { $explanations = array( 'ip' => '; The following list of IP addresses will ', 'domain' => '; The following domain names and all of their sub-domains will ', 'hostname' => '; The following hostnames will ' ); - $policy_explanations = array( - 'Local-Data' => 'returns the defined alternate location.', - 'NXDOMAIN' => 'return NXDOMAIN (name does not exist) irrespective of actual result received.', - 'NODATA' => 'returns NODATA (name exists but no answers returned) irrespective of actual result received.', - 'DROP' => 'timeout.', - 'PASSTHRU' => 'lets queries through, but allows for logging the hits (useful for testing).', - 'TCP-only' => 'force the client to use TCP.', - ); - return $explanations[$type] . $this->__policies[$policy]['explanation'] . PHP_EOL; + return $explanations[$type] . self::POLICIES[$policy]['explanation'] . PHP_EOL; } - public function buildHeader($rpzSettings) + private function buildHeader(array $rpzSettings) { $rpzSettings['serial'] = str_replace('$date', date('Ymd'), $rpzSettings['serial']); $rpzSettings['serial'] = str_replace('$time', time(), $rpzSettings['serial']); @@ -196,55 +183,55 @@ class RPZExport return $header; } - public function export($items, $rpzSettings) + private function export(array $items, array $rpzSettings) { $result = $this->buildHeader($rpzSettings); $policy = $this->getPolicyById($rpzSettings['policy']); - $action = $this->__policies[$policy]['action']; - if ($policy == 'Local-Data') { + $action = self::POLICIES[$policy]['action']; + if ($policy === 'Local-Data') { $action = str_replace('$walled_garden', $rpzSettings['walled_garden'], $action); } if (isset($items['ip'])) { $result .= $this->explain('ip', $policy); - foreach ($items['ip'] as $item) { - $result .= $this->__convertIP($item, $action); + foreach ($items['ip'] as $item => $foo) { + $result .= $this->convertIp($item, $action); } $result .= PHP_EOL; } if (isset($items['domain'])) { $result .= $this->explain('domain', $policy); - foreach ($items['domain'] as $item) { - $result .= $this->__convertdomain($item, $action); + foreach ($items['domain'] as $item => $foo) { + $result .= $this->convertDomain($item, $action); } $result .= PHP_EOL; } if (isset($items['hostname'])) { $result .= $this->explain('hostname', $policy); - foreach ($items['hostname'] as $item) { - $result .= $this->__converthostname($item, $action); + foreach ($items['hostname'] as $item => $foo) { + $result .= $this->convertHostname($item, $action); } $result .= PHP_EOL; } return $result; } - private function __convertdomain($input, $action) + private function convertDomain($input, $action) { return $input . ' CNAME ' . $action . PHP_EOL . '*.' . $input . ' CNAME ' . $action . PHP_EOL; } - private function __converthostname($input, $action) + private function convertHostname($input, $action) { return $input . ' CNAME ' . $action . PHP_EOL; } - private function __convertIP($input, $action) + private function convertIp($input, $action) { - $type = filter_var($input, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) ? 'ipv6' : 'ipv4'; - if ($type == 'ipv6') { + $isIpv6 = filter_var($input, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6); + if ($isIpv6) { $prefix = '128'; } else { $prefix = '32'; @@ -252,7 +239,8 @@ class RPZExport if (strpos($input, '/')) { list($input, $prefix) = explode('/', $input); } - return $prefix . '.' . $this->{'__' . $type}($input) . '.rpz-ip CNAME ' . $action . PHP_EOL; + $converted = $isIpv6 ? $this->__ipv6($input) : $this->__ipv4($input); + return $prefix . '.' . $converted . '.rpz-ip CNAME ' . $action . PHP_EOL; } private function __ipv6($input) diff --git a/app/Lib/Tools/ApcuCacheTool.php b/app/Lib/Tools/ApcuCacheTool.php new file mode 100644 index 000000000..625639eb9 --- /dev/null +++ b/app/Lib/Tools/ApcuCacheTool.php @@ -0,0 +1,198 @@ +prefix = $prefix; + } + + /** + * Fetches a value from the cache. + * + * @param string $key The unique key of this item in the cache. + * @param mixed $default Default value to return if the key does not exist. + * + * @return mixed The value of the item from the cache, or $default in case of cache miss. + * + * @throws \Psr\SimpleCache\InvalidArgumentException + * MUST be thrown if the $key string is not a legal value. + */ + public function get($key, $default = null) + { + $value = \apcu_fetch("$this->prefix:$key", $success); + if ($success) { + return $value; + } + return $default; + } + + /** + * Persists data in the cache, uniquely referenced by a key with an optional expiration TTL time. + * + * @param string $key The key of the item to store. + * @param mixed $value The value of the item to store, must be serializable. + * @param null|int|\DateInterval $ttl Optional. The TTL value of this item. If no value is sent and + * the driver supports TTL then the library may set a default value + * for it or let the driver take care of that. + * + * @return bool True on success and false on failure. + * + * @throws \Psr\SimpleCache\InvalidArgumentException + * MUST be thrown if the $key string is not a legal value. + */ + public function set($key, $value, $ttl = null) + { + return \apcu_store("$this->prefix:$key", $value, $this->tllToInt($ttl)); + } + + /** + * Delete an item from the cache by its unique key. + * + * @param string $key The unique cache key of the item to delete. + * + * @return bool True if the item was successfully removed. False if there was an error. + * + * @throws \Psr\SimpleCache\InvalidArgumentException + * MUST be thrown if the $key string is not a legal value. + */ + public function delete($key) + { + return \apcu_delete("$this->prefix:$key"); + } + + /** + * Wipes clean the entire cache's keys. + * + * @return bool True on success and false on failure. + */ + public function clear() + { + $iterator = new APCUIterator( + '/^' . preg_quote($this->prefix . ':', '/') . '/', + APC_ITER_NONE + ); + return \apcu_delete($iterator); + } + + /** + * Obtains multiple cache items by their unique keys. + * + * @param iterable $keys A list of keys that can obtained in a single operation. + * @param mixed $default Default value to return for keys that do not exist. + * + * @return iterable A list of key => value pairs. Cache keys that do not exist or are stale will have $default as value. + * + * @throws \Psr\SimpleCache\InvalidArgumentException + * MUST be thrown if $keys is neither an array nor a Traversable, + * or if any of the $keys are not a legal value. + */ + public function getMultiple($keys, $default = null) + { + $keysToFetch = $this->keysToFetch($keys); + $values = \apcu_fetch($keysToFetch); + foreach ($keysToFetch as $keyToFetch) { + if (!isset($values[$keyToFetch])) { + $values[$keyToFetch] = $default; + } + } + return $values; + } + + /** + * Persists a set of key => value pairs in the cache, with an optional TTL. + * + * @param iterable $values A list of key => value pairs for a multiple-set operation. + * @param null|int|\DateInterval $ttl Optional. The TTL value of this item. If no value is sent and + * the driver supports TTL then the library may set a default value + * for it or let the driver take care of that. + * + * @return bool True on success and false on failure. + * + * @throws \Psr\SimpleCache\InvalidArgumentException + * MUST be thrown if $values is neither an array nor a Traversable, + * or if any of the $values are not a legal value. + */ + public function setMultiple($values, $ttl = null) + { + $dataToSave = []; + foreach ($values as $key => $value) { + $dataToSave["$this->prefix:$key"] = $value; + } + return \apcu_store($dataToSave, null, $this->tllToInt($ttl)); + } + + /** + * Deletes multiple cache items in a single operation. + * + * @param iterable $keys A list of string-based keys to be deleted. + * + * @return bool True if the items were successfully removed. False if there was an error. + * + * @throws \Psr\SimpleCache\InvalidArgumentException + * MUST be thrown if $keys is neither an array nor a Traversable, + * or if any of the $keys are not a legal value. + */ + public function deleteMultiple($keys) + { + $keysToDelete = $this->keysToFetch($keys); + return \apcu_delete($keysToDelete); + } + + /** + * Determines whether an item is present in the cache. + * + * NOTE: It is recommended that has() is only to be used for cache warming type purposes + * and not to be used within your live applications operations for get/set, as this method + * is subject to a race condition where your has() will return true and immediately after, + * another script can remove it making the state of your app out of date. + * + * @param string $key The cache item key. + * + * @return bool + * + * @throws \Psr\SimpleCache\InvalidArgumentException + * MUST be thrown if the $key string is not a legal value. + */ + public function has($key) + { + return \apcu_exists("$this->prefix:$key"); + } + + /** + * @param iterable $keys + * @return array + */ + private function keysToFetch(iterable $keys): array + { + $keysToFetch = []; + foreach ($keys as $key) { + $keysToFetch[] = "$this->prefix:$key"; + } + return $keysToFetch; + } + + /** + * @param null|int|\DateInterval $ttl + * @return int + */ + private function tllToInt($ttl = null): int + { + if ($ttl === null) { + return 0; + } elseif (is_int($ttl)) { + return $ttl; + } elseif ($ttl instanceof \DateInterval) { + return $ttl->days * 86400 + $ttl->h * 3600 + $ttl->i * 60 + $ttl->s; + } else { + throw new \Psr\SimpleCache\InvalidArgumentException("Invalid ttl value '$ttl' provided."); + } + } +} \ No newline at end of file diff --git a/app/Lib/Tools/AttributeValidationTool.php b/app/Lib/Tools/AttributeValidationTool.php index 752fbe1e0..cc9c6e075 100644 --- a/app/Lib/Tools/AttributeValidationTool.php +++ b/app/Lib/Tools/AttributeValidationTool.php @@ -41,7 +41,7 @@ class AttributeValidationTool switch ($type) { case 'ip-src': case 'ip-dst': - return self::compressIpv6($value); + return self::normalizeIp($value); case 'md5': case 'sha1': case 'sha224': @@ -98,7 +98,7 @@ class AttributeValidationTool $parts[0] = $punyCode; } } - $parts[1] = self::compressIpv6($parts[1]); + $parts[1] = self::normalizeIp($parts[1]); return "$parts[0]|$parts[1]"; case 'filename|md5': case 'filename|sha1': @@ -175,7 +175,7 @@ class AttributeValidationTool } else { return $value; } - return self::compressIpv6($parts[0]) . '|' . $parts[1]; + return self::normalizeIp($parts[0]) . '|' . $parts[1]; case 'mac-address': case 'mac-eui-64': $value = str_replace(array('.', ':', '-', ' '), '', strtolower($value)); @@ -700,11 +700,30 @@ class AttributeValidationTool * @param string $value * @return string */ - private static function compressIpv6($value) + private static function normalizeIp($value) { + // If IP is a CIDR + if (strpos($value, '/')) { + list($ip, $range) = explode('/', $value, 2); + + // Compress IPv6 + if (strpos($ip, ':') && $converted = inet_pton($ip)) { + $ip = inet_ntop($converted); + } + + // If IP is in CIDR format, but the network is 32 for IPv4 or 128 for IPv6, normalize to non CIDR type + if (($range === '32' && strpos($value, '.')) || ($range === '128' && strpos($value, ':'))) { + return $ip; + } + + return "$ip/$range"; + } + + // Compress IPv6 if (strpos($value, ':') && $converted = inet_pton($value)) { return inet_ntop($converted); } + return $value; } diff --git a/app/Lib/Tools/BackgroundJobs/BackgroundJob.php b/app/Lib/Tools/BackgroundJobs/BackgroundJob.php index b7a45d86d..57d2b5fc6 100644 --- a/app/Lib/Tools/BackgroundJobs/BackgroundJob.php +++ b/app/Lib/Tools/BackgroundJobs/BackgroundJob.php @@ -66,8 +66,9 @@ class BackgroundJob implements JsonSerializable /** * Run the job command + * @param callable|null $runningCallback */ - public function run(): void + public function run(callable $runningCallback = null): void { $descriptorSpec = [ 1 => ["pipe", "w"], // stdout @@ -88,7 +89,7 @@ class BackgroundJob implements JsonSerializable ['BACKGROUND_JOB_ID' => $this->id] ); - $this->pool($process, $pipes); + $this->pool($process, $pipes, $runningCallback); if ($this->returnCode === 0 && empty($stderr)) { $this->setStatus(BackgroundJob::STATUS_COMPLETED); @@ -98,7 +99,13 @@ class BackgroundJob implements JsonSerializable } } - private function pool($process, array $pipes) + /** + * @param resource $process + * @param array $pipes + * @param callable|null $runningCallback + * @return void + */ + private function pool($process, array $pipes, callable $runningCallback = null) { stream_set_blocking($pipes[1], false); stream_set_blocking($pipes[2], false); @@ -106,6 +113,14 @@ class BackgroundJob implements JsonSerializable $this->output = ''; $this->error = ''; + if ($runningCallback) { + $status = proc_get_status($process); + if ($status === false) { + throw new RuntimeException("Could not get process status"); + } + $runningCallback($status); + } + while (true) { $read = [$pipes[1], $pipes[2]]; $write = null; @@ -118,6 +133,12 @@ class BackgroundJob implements JsonSerializable $this->error .= stream_get_contents($pipes[2]); } $status = proc_get_status($process); + if ($status === false) { + throw new RuntimeException("Could not get process status"); + } + if ($runningCallback) { + $runningCallback($status); + } if (!$status['running']) { // Just in case read rest data from stream $this->output .= stream_get_contents($pipes[1]); @@ -153,6 +174,9 @@ class BackgroundJob implements JsonSerializable return ['id', 'command', 'args', 'createdAt', 'updatedAt', 'status', 'output', 'error', 'metadata']; } + /** + * @return string Background job ID in UUID format + */ public function id(): string { return $this->id; diff --git a/app/Lib/Tools/BackgroundJobs/Worker.php b/app/Lib/Tools/BackgroundJobs/Worker.php index 90a5c27f9..479dfcfdc 100644 --- a/app/Lib/Tools/BackgroundJobs/Worker.php +++ b/app/Lib/Tools/BackgroundJobs/Worker.php @@ -65,7 +65,7 @@ class Worker implements JsonSerializable ]; } - public function pid(): ?int + public function pid(): int { return $this->pid; } diff --git a/app/Lib/Tools/BackgroundJobsTool.php b/app/Lib/Tools/BackgroundJobsTool.php index 37e53883e..857baaac8 100644 --- a/app/Lib/Tools/BackgroundJobsTool.php +++ b/app/Lib/Tools/BackgroundJobsTool.php @@ -91,7 +91,8 @@ class BackgroundJobsTool ]; const JOB_STATUS_PREFIX = 'job_status', - DATA_CONTENT_PREFIX = 'data_content'; + DATA_CONTENT_PREFIX = 'data_content', + RUNNING_JOB_PREFIX = 'running'; /** @var array */ private $settings; @@ -277,6 +278,54 @@ class BackgroundJobsTool return null; } + /** + * @param Worker $worker + * @param BackgroundJob $job + * @param int|null $pid + * @return void + * @throws RedisException + */ + public function markAsRunning(Worker $worker, BackgroundJob $job, $pid = null) + { + $key = self::RUNNING_JOB_PREFIX . ':' . $worker->queue() . ':' . $job->id(); + $this->RedisConnection->setex($key, 60, [ + 'worker_pid' => $worker->pid(), + 'process_pid' => $pid, + ]); + } + + /** + * @param Worker $worker + * @param BackgroundJob $job + * @return void + * @throws RedisException + */ + public function removeFromRunning(Worker $worker, BackgroundJob $job) + { + $key = self::RUNNING_JOB_PREFIX . ':' . $worker->queue() . ':' . $job->id(); + $this->RedisConnection->del($key); + } + + /** + * Return current running jobs + * @return array + * @throws RedisException + */ + public function runningJobs(): array + { + $pattern = $this->RedisConnection->_prefix(self::RUNNING_JOB_PREFIX . ':*'); + $keys = RedisTool::keysByPattern($this->RedisConnection, $pattern); + + $jobIds = []; + foreach ($keys as $key) { + $parts = explode(':', $key); + $queue = $parts[2]; + $jobId = $parts[3]; + $jobIds[$queue][$jobId] = $this->RedisConnection->get(self::RUNNING_JOB_PREFIX . ":$queue:$jobId"); + } + return $jobIds; + } + /** * Get the job status. * @@ -500,19 +549,6 @@ class BackgroundJobsTool $this->getSupervisor()->startProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart); } - /** - * Purge queue - * - * @param string $queue - * @return void - */ - public function purgeQueue(string $queue) - { - $this->validateQueue($queue); - - $this->RedisConnection->del($queue); - } - /** * Return Background Jobs status * @@ -728,8 +764,7 @@ class BackgroundJobsTool * * @param integer $pid * @return \Supervisor\Process - * - * @throws NotFoundException + * @throws NotFoundException|Exception */ private function getProcessByPid(int $pid): \Supervisor\Process { diff --git a/app/Lib/Tools/BetterCakeEventManager.php b/app/Lib/Tools/BetterCakeEventManager.php index 9c0f41baf..fa6903f6e 100644 --- a/app/Lib/Tools/BetterCakeEventManager.php +++ b/app/Lib/Tools/BetterCakeEventManager.php @@ -52,10 +52,10 @@ class BetterCakeEventManager extends CakeEventManager $result = []; foreach ($priorities as $priority) { if (isset($globalListeners[$priority])) { - $result = array_merge($result, $globalListeners[$priority]); + array_push($result, ...$globalListeners[$priority]); } if (isset($localListeners[$priority])) { - $result = array_merge($result, $localListeners[$priority]); + array_push($result, ...$localListeners[$priority]); } } return $result; diff --git a/app/Lib/Tools/BetterSecurity.php b/app/Lib/Tools/BetterSecurity.php index 0e98f108c..302d63980 100644 --- a/app/Lib/Tools/BetterSecurity.php +++ b/app/Lib/Tools/BetterSecurity.php @@ -7,8 +7,8 @@ class BetterSecurity /** * @param string $plain - * @param string $key - * @return string + * @param string $key Encryption key + * @return string Cipher text with IV and tag * @throws Exception */ public static function encrypt($plain, $key) @@ -33,17 +33,17 @@ class BetterSecurity } /** - * @param string $cipher - * @param string $key + * @param string $cipherText Cipher text with IV and tag + * @param string $key Decryption key * @return string * @throws Exception */ - public static function decrypt($cipher, $key) + public static function decrypt($cipherText, $key) { if (strlen($key) < 32) { throw new Exception('Invalid key, key must be at least 256 bits (32 bytes) long.'); } - if (empty($cipher)) { + if (empty($cipherText)) { throw new Exception('The data to decrypt cannot be empty.'); } @@ -52,12 +52,18 @@ class BetterSecurity $ivSize = openssl_cipher_iv_length(self::METHOD); - // Split out hmac for comparison - $iv = substr($cipher, 0, $ivSize); - $tag = substr($cipher, $ivSize, self::TAG_SIZE); - $cipher = substr($cipher, $ivSize + self::TAG_SIZE); + if (strlen($cipherText) < $ivSize + self::TAG_SIZE) { + $length = strlen($cipherText); + $minLength = $ivSize + self::TAG_SIZE; + throw new Exception("Provided cipher text is too short, $length bytes provided, expected at least $minLength bytes."); + } - $decrypted = openssl_decrypt($cipher, self::METHOD, $key, true, $iv, $tag); + // Split out hmac for comparison + $iv = substr($cipherText, 0, $ivSize); + $tag = substr($cipherText, $ivSize, self::TAG_SIZE); + $cipherText = substr($cipherText, $ivSize + self::TAG_SIZE); + + $decrypted = openssl_decrypt($cipherText, self::METHOD, $key, OPENSSL_RAW_DATA, $iv, $tag); if ($decrypted === false) { throw new Exception('Could not decrypt. Maybe invalid encryption key?'); } diff --git a/app/Lib/Tools/ComplexTypeTool.php b/app/Lib/Tools/ComplexTypeTool.php index e9e4e123e..25ddbdfea 100644 --- a/app/Lib/Tools/ComplexTypeTool.php +++ b/app/Lib/Tools/ComplexTypeTool.php @@ -308,14 +308,13 @@ class ComplexTypeTool */ private function parseFreetext($input) { - $input = str_replace("\xc2\xa0", ' ', $input); // non breaking space to normal space - $input = preg_replace('/\p{C}+/u', ' ', $input); - $iocArray = preg_split("/\r\n|\n|\r|\s|\s+|,|\<|\>|;/", $input); + // convert non breaking space to normal space and all unicode chars from "other" category + $input = preg_replace("/\p{C}+|\xc2\xa0/u", ' ', $input); + $iocArray = preg_split("/\r\n|\n|\r|\s|\s+|,|<|>|;/", $input); preg_match_all('/\"([^\"]+)\"/', $input, $matches); - foreach ($matches[1] as $match) { - $iocArray[] = $match; - } + array_push($iocArray, ...$matches[1]); + return $iocArray; } diff --git a/app/Lib/Tools/CurlClient.php b/app/Lib/Tools/CurlClient.php new file mode 100644 index 000000000..bf8966af5 --- /dev/null +++ b/app/Lib/Tools/CurlClient.php @@ -0,0 +1,377 @@ +timeout = $params['timeout']; + } + if (isset($params['ssl_cafile'])) { + $this->caFile = $params['ssl_cafile']; + } + if (isset($params['ssl_local_cert'])) { + $this->localCert = $params['ssl_local_cert']; + } + if (isset($params['compress'])) { + $this->compress = $params['compress']; + } + if (isset($params['ssl_crypto_method'])) { + $this->cryptoMethod = $this->convertCryptoMethod($params['ssl_crypto_method']); + } + if (isset($params['ssl_allow_self_signed'])) { + $this->allowSelfSigned = $params['ssl_allow_self_signed']; + } + if (isset($params['ssl_verify_peer'])) { + $this->verifyPeer = $params['ssl_verify_peer']; + } + $this->defaultOptions = $this->generateDefaultOptions(); + } + + /** + * @param string $uri + * @param array $query + * @param array $request + * @return HttpSocketResponseExtended + */ + public function head($uri = null, $query = [], $request = []) + { + return $this->internalRequest('HEAD', $uri, $query, $request); + } + + /** + * @param string $uri + * @param array $query + * @param array $request + * @return HttpSocketResponseExtended + */ + public function get($uri = null, $query = [], $request = []) + { + return $this->internalRequest('GET', $uri, $query, $request); + } + + /** + * @param string $uri + * @param array $data + * @param array $request + * @return HttpSocketResponseExtended + */ + public function post($uri = null, $data = [], $request = []) + { + return $this->internalRequest('POST', $uri, $data, $request); + } + + /** + * @param string $uri + * @param array$data + * @param $request + * @return HttpSocketResponseExtended + */ + public function put($uri = null, $data = [], $request = []) + { + return $this->internalRequest('PUT', $uri, $data, $request); + } + + /** + * @param string $uri + * @param array $data + * @param array $request + * @return HttpSocketResponseExtended + */ + public function patch($uri = null, $data = [], $request = []) + { + return $this->internalRequest('PATCH', $uri, $data, $request); + } + + /** + * @param string $uri + * @param array $data + * @param array $request + * @return HttpSocketResponseExtended + */ + public function delete($uri = null, $data = array(), $request = array()) + { + return $this->internalRequest('DELETE', $uri, $data, $request); + } + + public function url($url = null, $uriTemplate = null) + { + throw new Exception('Not implemented'); + } + + public function request($request = array()) + { + throw new Exception('Not implemented'); + } + + public function setContentResource($resource) + { + throw new Exception('Not implemented'); + } + + public function getMetaData() + { + return null; // not supported by curl extension + } + + /** + * @param string $host + * @param int $port + * @param string $method + * @param string $user + * @param string $pass + * @return void + */ + public function configProxy($host, $port = 3128, $method = null, $user = null, $pass = null) + { + if (empty($host)) { + $this->proxy = []; + return; + } + if (is_array($host)) { + $this->proxy = $host + ['host' => null]; + return; + } + $this->proxy = compact('host', 'port', 'method', 'user', 'pass'); + } + + /** + * @param string $method + * @param string $url + * @param array|string $query + * @param array $request + * @return HttpSocketResponseExtended + */ + private function internalRequest($method, $url, $query, $request) + { + if (empty($url)) { + throw new InvalidArgumentException("No URL provided."); + } + + if (!$this->ch) { + // Share handle between requests to allow keep connection alive between requests + $this->ch = curl_init(); + if (!$this->ch) { + throw new \RuntimeException("Could not initialize curl"); + } + } else { + // Reset options, so we can do another request + curl_reset($this->ch); + } + + if (($method === 'GET' || $method === 'HEAD') && !empty($query)) { + $url .= '?' . http_build_query($query, '', '&', PHP_QUERY_RFC3986); + } + + $options = $this->defaultOptions; + $options[CURLOPT_URL] = $url; + $options[CURLOPT_CUSTOMREQUEST] = $method; + + if (($method === 'POST' || $method === 'DELETE' || $method === 'PUT' || $method === 'PATCH') && !empty($query)) { + $options[CURLOPT_POSTFIELDS] = $query; + } + + if (!empty($request['header'])) { + $headers = []; + foreach ($request['header'] as $key => $value) { + if (is_array($value)) { + $value = implode(', ', $value); + } + $headers[] = "$key: $value"; + } + $options[CURLOPT_HTTPHEADER] = $headers; + } + + // Parse response headers + $responseHeaders = []; + $options[CURLOPT_HEADERFUNCTION] = function ($curl, $header) use (&$responseHeaders){ + $len = strlen($header); + $header = explode(':', $header, 2); + if (count($header) < 2) { // ignore invalid headers + return $len; + } + $key = strtolower(trim($header[0])); + $value = trim($header[1]); + + if (isset($responseHeaders[$key])) { + $responseHeaders[$key] = array_merge((array)$responseHeaders[$key], [$value]); + } else { + $responseHeaders[$key] = $value; + } + return $len; + }; + + if (!curl_setopt_array($this->ch, $options)) { + throw new \RuntimeException('curl error: Could not set options'); + } + + // Download the given URL, and return output + $output = curl_exec($this->ch); + + if ($output === false) { + $errorCode = curl_errno($this->ch); + $errorMessage = curl_error($this->ch); + if (!empty($errorMessage)) { + $errorMessage = ": $errorMessage"; + } + throw new SocketException("curl error $errorCode '" . curl_strerror($errorCode) . "'" . $errorMessage); + } + + $code = curl_getinfo($this->ch, CURLINFO_HTTP_CODE); + return $this->constructResponse($output, $responseHeaders, $code); + } + + public function disconnect() + { + if ($this->ch) { + curl_close($this->ch); + $this->ch = null; + } + } + + /** + * @param string $body + * @param array $headers + * @param int $code + * @return HttpSocketResponseExtended + */ + private function constructResponse($body, array $headers, $code) + { + if (isset($responseHeaders['content-encoding']) && $responseHeaders['content-encoding'] === 'zstd') { + if (!function_exists('zstd_uncompress')) { + throw new SocketException('Response is zstd encoded, but PHP do not support zstd decoding.'); + } + $body = zstd_uncompress($body); + if ($body === false) { + throw new SocketException('Could not decode zstd encoded response.'); + } + } + + $response = new HttpSocketResponseExtended(); + $response->code = $code; + $response->body = $body; + $response->headers = $headers; + return $response; + } + + /** + * @param int $cryptoMethod + * @return int + */ + private function convertCryptoMethod($cryptoMethod) + { + switch ($cryptoMethod) { + case STREAM_CRYPTO_METHOD_TLSv1_0_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_1_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT: + return CURL_SSLVERSION_TLSv1; + case STREAM_CRYPTO_METHOD_TLSv1_1_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT: + return CURL_SSLVERSION_TLSv1_1; + case STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT: + return CURL_SSLVERSION_TLSv1_2; + case STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT: + return CURL_SSLVERSION_TLSv1_3; + default: + throw new InvalidArgumentException("Unsupported crypto method value $cryptoMethod"); + } + } + + /** + * @return array + */ + private function generateDefaultOptions() + { + $options = [ + CURLOPT_FOLLOWLOCATION => true, // Allows to follow redirect + CURLOPT_MAXREDIRS => 10, + CURLOPT_RETURNTRANSFER => true, // Should cURL return or print out the data? (true = return, false = print) + CURLOPT_HEADER => false, // Include header in result? + CURLOPT_TIMEOUT => $this->timeout, // Timeout in seconds + CURLOPT_PROTOCOLS => CURLPROTO_HTTPS | CURLPROTO_HTTP, // be sure that only HTTP and HTTPS protocols are enabled, + ]; + + if ($this->caFile) { + $options[CURLOPT_CAINFO] = $this->caFile; + } + + if ($this->localCert) { + $options[CURLOPT_SSLCERT] = $this->localCert; + } + + if ($this->cryptoMethod) { + $options[CURLOPT_SSLVERSION] = $this->cryptoMethod; + } + + if ($this->compress) { + $options[CURLOPT_ACCEPT_ENCODING] = $this->supportedEncodings(); + } + + if ($this->allowSelfSigned) { + $options[CURLOPT_SSL_VERIFYPEER] = $this->verifyPeer; + $options[CURLOPT_SSL_VERIFYHOST] = 0; + } + + if (!empty($this->proxy)) { + $options[CURLOPT_PROXY] = "{$this->proxy['host']}:{$this->proxy['port']}"; + if (!empty($this->proxy['method']) && isset($this->proxy['user'], $this->proxy['pass'])) { + $options[CURLOPT_PROXYUSERPWD] = "{$this->proxy['user']}:{$this->proxy['pass']}"; + } + } + + return $options; + } + + /** + * @return string + */ + private function supportedEncodings() + { + $encodings = []; + // zstd is not supported by curl itself, but add support if PHP zstd extension is installed + if (function_exists('zstd_uncompress')) { + $encodings[] = 'zstd'; + } + // brotli and gzip is supported by curl itself if it is compiled with these features + $info = curl_version(); + if (defined('CURL_VERSION_BROTLI') && $info['features'] & CURL_VERSION_BROTLI) { + $encodings[] = 'br'; + } + if ($info['features'] & CURL_VERSION_LIBZ) { + $encodings[] = 'gzip, deflate'; + } + return implode(', ', $encodings); + } +} \ No newline at end of file diff --git a/app/Lib/Tools/GitTool.php b/app/Lib/Tools/GitTool.php index f9a77f5c8..ff3988151 100644 --- a/app/Lib/Tools/GitTool.php +++ b/app/Lib/Tools/GitTool.php @@ -12,11 +12,7 @@ class GitTool public static function getLatestTags(HttpSocketExtended $HttpSocket) { $url = 'https://api.github.com/repos/MISP/MISP/tags?per_page=10'; - $response = $HttpSocket->get($url); - if (!$response->isOk()) { - throw new HttpSocketHttpException($response, $url); - } - return $response->json(); + return self::gitHubRequest($HttpSocket, $url); } /** @@ -28,11 +24,7 @@ class GitTool public static function getLatestCommit(HttpSocketExtended $HttpSocket) { $url = 'https://api.github.com/repos/MISP/MISP/commits?per_page=1'; - $response = $HttpSocket->get($url); - if (!$response->isOk()) { - throw new HttpSocketHttpException($response, $url); - } - $data = $response->json(); + $data = self::gitHubRequest($HttpSocket, $url); if (!isset($data[0]['sha'])) { throw new Exception("Response do not contains requested data."); } @@ -40,20 +32,49 @@ class GitTool } /** + * @param HttpSocketExtended $HttpSocket + * @param string $url + * @return array + * @throws HttpSocketHttpException + * @throws HttpSocketJsonException + */ + private static function gitHubRequest(HttpSocketExtended $HttpSocket, $url) + { + $response = $HttpSocket->get($url, [], ['header' => ['User-Agent' => 'MISP']]); + if (!$response->isOk()) { + throw new HttpSocketHttpException($response, $url); + } + return $response->json(); + } + + /** + * Returns current SHA1 hash of current commit * `git rev-parse HEAD` + * @param string $repoPath * @return string * @throws Exception */ - public static function currentCommit() + public static function currentCommit($repoPath) { - $head = rtrim(FileAccessTool::readFromFile(ROOT . '/.git/HEAD')); + if (is_file($repoPath . '/.git')) { + $fileContent = FileAccessTool::readFromFile($repoPath . '/.git'); + if (substr($fileContent, 0, 8) === 'gitdir: ') { + $gitDir = $repoPath . '/' . trim(substr($fileContent, 8)) . '/'; + } else { + throw new Exception("$repoPath/.git is file, but contains non expected content $fileContent"); + } + } else { + $gitDir = $repoPath . '/.git/'; + } + + $head = rtrim(FileAccessTool::readFromFile($gitDir . 'HEAD')); if (substr($head, 0, 5) === 'ref: ') { $path = substr($head, 5); - return rtrim(FileAccessTool::readFromFile(ROOT . '/.git/' . $path)); + return rtrim(FileAccessTool::readFromFile($gitDir . $path)); } else if (strlen($head) === 40) { return $head; } else { - throw new Exception("Invalid head $head"); + throw new Exception("Invalid head '$head' in $gitDir/HEAD"); } } @@ -94,30 +115,18 @@ class GitTool return $output; } - /** - * @param string $submodule Path to Git repo - * @return string|null - */ - public static function submoduleCurrentCommit($submodule) - { - try { - $commit = ProcessTool::execute(['git', 'rev-parse', 'HEAD'], $submodule); - } catch (ProcessException $e) { - return null; - } - return rtrim($commit); - } - /** * @param string $commit * @param string|null $submodule Path to Git repo * @return int|null + * @throws Exception */ public static function commitTimestamp($commit, $submodule = null) { try { $timestamp = ProcessTool::execute(['git', 'show', '-s', '--pretty=format:%ct', $commit], $submodule); } catch (ProcessException $e) { + CakeLog::notice("Could not get Git commit timestamp for $submodule: {$e->getMessage()}"); return null; } return (int)rtrim($timestamp); diff --git a/app/Lib/Tools/HttpSocketExtended.php b/app/Lib/Tools/HttpSocketExtended.php index 7213fc5fa..9ca042c35 100644 --- a/app/Lib/Tools/HttpSocketExtended.php +++ b/app/Lib/Tools/HttpSocketExtended.php @@ -18,10 +18,15 @@ class HttpSocketHttpException extends Exception { $this->response = $response; $this->url = $url; + $message = "Remote server returns HTTP error code $response->code"; if ($url) { $message .= " for URL $url"; } + if ($response->body) { + $message .= ': ' . substr($response->body, 0, 100); + } + parent::__construct($message, (int)$response->code); } diff --git a/app/Lib/Tools/JSONConverterTool.php b/app/Lib/Tools/JSONConverterTool.php index 7ede8dd25..04666aee1 100644 --- a/app/Lib/Tools/JSONConverterTool.php +++ b/app/Lib/Tools/JSONConverterTool.php @@ -153,7 +153,7 @@ class JSONConverterTool return; } yield '{"Event":{'; - $firstKey = key($event['Event']); + $firstKey = array_key_first($event['Event']); foreach ($event['Event'] as $key => $value) { if ($key === 'Attribute' || $key === 'Object') { // Encode every object or attribute separately yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":["; diff --git a/app/Lib/Tools/JsonTool.php b/app/Lib/Tools/JsonTool.php index 0760dcade..c23abc7e7 100644 --- a/app/Lib/Tools/JsonTool.php +++ b/app/Lib/Tools/JsonTool.php @@ -9,10 +9,7 @@ class JsonTool */ public static function encode($value, $prettyPrint = false) { - $flags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE; - if (defined('JSON_THROW_ON_ERROR')) { - $flags |= JSON_THROW_ON_ERROR; // Throw exception on error if supported - } + $flags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR; if ($prettyPrint) { $flags |= JSON_PRETTY_PRINT; } @@ -34,16 +31,8 @@ class JsonTool } catch (SimdJsonException $e) { throw new JsonException($e->getMessage(), $e->getCode(), $e); } - } elseif (defined('JSON_THROW_ON_ERROR')) { - // JSON_THROW_ON_ERROR is supported since PHP 7.3 - return json_decode($value, true, 512, JSON_THROW_ON_ERROR); - } else { - $decoded = json_decode($value, true); - if ($decoded === null) { - throw new UnexpectedValueException('Could not parse JSON: ' . json_last_error_msg(), json_last_error()); - } - return $decoded; } + return json_decode($value, true, 512, JSON_THROW_ON_ERROR); } /** @@ -78,4 +67,39 @@ class JsonTool return false; } } + + /** + * @see https://www.php.net/manual/en/function.array-is-list.php + * @param array $array + * @return bool + */ + public static function arrayIsList(array $array) + { + if (function_exists('array_is_list')) { + return array_is_list($array); + } + + $i = -1; + foreach ($array as $k => $v) { + ++$i; + if ($k !== $i) { + return false; + } + } + return true; + } + + /** + * JSON supports just unicode strings. This helper method converts non unicode chars to Unicode Replacement Character U+FFFD (UTF-8) + * @param string $string + * @return string + */ + public static function escapeNonUnicode($string) + { + if (mb_check_encoding($string, 'UTF-8')) { + return $string; // string is valid unicode + } + + return htmlspecialchars_decode(htmlspecialchars($string, ENT_SUBSTITUTE, 'UTF-8')); + } } diff --git a/app/Lib/Tools/ProcessTool.php b/app/Lib/Tools/ProcessTool.php index 26d48cc47..b0003938b 100644 --- a/app/Lib/Tools/ProcessTool.php +++ b/app/Lib/Tools/ProcessTool.php @@ -8,14 +8,14 @@ class ProcessException extends Exception private $stdout; /** - * @param string|array $command + * @param array $command * @param int $returnCode * @param string $stderr * @param string $stdout */ - public function __construct($command, $returnCode, $stderr, $stdout) + public function __construct(array $command, $returnCode, $stderr, $stdout) { - $commandForException = is_array($command) ? implode(' ', $command) : $command; + $commandForException = implode(' ', $command); $message = "Command '$commandForException' finished with error code $returnCode.\nSTDERR: '$stderr'\nSTDOUT: '$stdout'"; $this->stderr = $stderr; $this->stdout = $stdout; @@ -56,11 +56,6 @@ class ProcessTool self::logMessage('Running command ' . implode(' ', $command)); } - // PHP older than 7.4 do not support proc_open with array, so we need to convert values to string manually - if (PHP_VERSION_ID < 70400) { - $command = array_map('escapeshellarg', $command); - $command = implode(' ', $command); - } $process = proc_open($command, $descriptorSpec, $pipes, $cwd); if (!$process) { $commandForException = self::commandFormat($command); @@ -136,8 +131,8 @@ class ProcessTool * @param array|string $command * @return string */ - private static function commandFormat($command) + private static function commandFormat(array $command) { - return is_array($command) ? implode(' ', $command) : $command; + return implode(' ', $command); } } diff --git a/app/Lib/Tools/PubSubTool.php b/app/Lib/Tools/PubSubTool.php index 6a17a020a..b03cfb547 100644 --- a/app/Lib/Tools/PubSubTool.php +++ b/app/Lib/Tools/PubSubTool.php @@ -178,8 +178,12 @@ class PubSubTool public function killService() { + $settings = $this->getSetSettings(); + if ($settings['supervisor_managed']) { + throw new RuntimeException('ZeroMQ server is managed by supervisor, it is not possible to restart it.'); + } + if ($this->checkIfRunning()) { - $settings = $this->getSetSettings(); $redis = $this->createRedisConnection($settings); $redis->rPush('command', 'kill'); sleep(1); @@ -213,12 +217,16 @@ class PubSubTool public function restartServer() { + $settings = $this->getSetSettings(); + if ($settings['supervisor_managed']) { + throw new RuntimeException('ZeroMQ server is managed by supervisor, it is not possible to restart it.'); + } + if (!$this->checkIfRunning()) { if (!$this->killService()) { return 'Could not kill the previous instance of the ZeroMQ script.'; } } - $settings = $this->getSetSettings(); $this->setupPubServer($settings); if ($this->checkIfRunning() === false) { return 'Failed starting the ZeroMQ script.'; @@ -226,12 +234,22 @@ class PubSubTool return true; } + public function createConfigFile() + { + $settings = $this->getSetSettings(); + $this->saveSettingToFile($settings); + } + /** * @param array $settings * @throws Exception */ private function setupPubServer(array $settings) { + if ($settings['supervisor_managed']) { + return; // server is managed by supervisor, we don't need to check if is running or start it when not + } + if ($this->checkIfRunning() === false) { if ($this->checkIfRunning(self::OLD_PID_LOCATION)) { // Old version is running, kill it and start again new one. @@ -250,6 +268,7 @@ class PubSubTool * @param string|array $data * @return bool * @throws JsonException + * @throws RedisException */ private function pushToRedis($ns, $data) { @@ -295,9 +314,12 @@ class PubSubTool FileAccessTool::writeToFile($settingFilePath, JsonTool::encode($settings)); } + /** + * @return array + */ private function getSetSettings() { - $settings = array( + $settings = [ 'redis_host' => 'localhost', 'redis_port' => 6379, 'redis_password' => '', @@ -307,7 +329,8 @@ class PubSubTool 'port' => '50000', 'username' => null, 'password' => null, - ); + 'supervisor_managed' => false, + ]; $pluginConfig = Configure::read('Plugin'); foreach ($settings as $key => $setting) { diff --git a/app/Lib/Tools/RedisTool.php b/app/Lib/Tools/RedisTool.php index 8a68dfe20..6d330310b 100644 --- a/app/Lib/Tools/RedisTool.php +++ b/app/Lib/Tools/RedisTool.php @@ -57,25 +57,34 @@ class RedisTool /** * @param Redis $redis * @param string|array $pattern - * @return int|Redis Number of deleted keys or instance of Redis if used in MULTI mode + * @return Generator * @throws RedisException */ - public static function deleteKeysByPattern(Redis $redis, $pattern) + public static function keysByPattern(Redis $redis, $pattern) { if (is_string($pattern)) { $pattern = [$pattern]; } - $allKeys = []; foreach ($pattern as $p) { $iterator = null; while (false !== ($keys = $redis->scan($iterator, $p, 1000))) { foreach ($keys as $key) { - $allKeys[] = $key; + yield $key; } } } + } + /** + * @param Redis $redis + * @param string|array $pattern + * @return int|Redis Number of deleted keys or instance of Redis if used in MULTI mode + * @throws RedisException + */ + public static function deleteKeysByPattern(Redis $redis, $pattern) + { + $allKeys = iterator_to_array(self::keysByPattern($redis, $pattern)); if (empty($allKeys)) { return 0; } @@ -153,11 +162,7 @@ class RedisTool return false; } - if (self::$serializer === null) { - self::$serializer = Configure::read('MISP.redis_serializer') ?: false; - } - - if (self::$serializer === 'igbinary') { + if ($string[0] === "\x00") { return igbinary_unserialize($string); } else { return JsonTool::decode($string); diff --git a/app/Lib/Tools/ServerSyncTool.php b/app/Lib/Tools/ServerSyncTool.php index 8f9c1c014..b22ad08b4 100644 --- a/app/Lib/Tools/ServerSyncTool.php +++ b/app/Lib/Tools/ServerSyncTool.php @@ -355,6 +355,14 @@ class ServerSyncTool return $this->server['Server']['id']; } + /** + * @return string + */ + public function serverName() + { + return $this->server['Server']['name']; + } + /** * @return array */ diff --git a/app/Lib/Tools/SyncTool.php b/app/Lib/Tools/SyncTool.php index 157123d21..df14851ea 100644 --- a/app/Lib/Tools/SyncTool.php +++ b/app/Lib/Tools/SyncTool.php @@ -1,5 +1,4 @@ configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']); diff --git a/app/Model/AccessLog.php b/app/Model/AccessLog.php index f1f70b453..3c27ac3b7 100644 --- a/app/Model/AccessLog.php +++ b/app/Model/AccessLog.php @@ -71,12 +71,6 @@ class AccessLog extends AppModel { $accessLog = &$this->data['AccessLog']; - $this->externalLog($accessLog); - - if (Configure::read('MISP.log_paranoid_skip_db')) { - return; - } - // Truncate foreach (['request_id', 'user_agent', 'url'] as $field) { if (isset($accessLog[$field]) && strlen($accessLog[$field]) > 255) { @@ -202,7 +196,7 @@ class AccessLog extends AppModel if ($includeSqlQueries && !empty($sqlLog['log'])) { foreach ($sqlLog['log'] as &$log) { - $log['query'] = $this->escapeNonUnicode($log['query']); + $log['query'] = JsonTool::escapeNonUnicode($log['query']); unset($log['affected']); // affected is the same as numRows unset($log['params']); // no need to save for your use case } @@ -214,6 +208,12 @@ class AccessLog extends AppModel $data['query_count'] = $queryCount; $data['duration'] = (int)((microtime(true) - $requestTime->format('U.u')) * 1000); // in milliseconds + $this->externalLog($data); + + if (Configure::read('MISP.log_paranoid_skip_db')) { + return true; // do not save access log to database + } + try { return $this->save($data, ['atomic' => false]); } catch (Exception $e) { @@ -226,7 +226,7 @@ class AccessLog extends AppModel * @param array $data * @return void */ - public function externalLog(array $data) + private function externalLog(array $data) { if ($this->pubToZmq('audit')) { $this->getPubSubTool()->publish($data, 'audit', 'log'); @@ -310,36 +310,4 @@ class AccessLog extends AppModel } return $data; } - - /** - * @param $string - * @return string - */ - private function escapeNonUnicode($string) - { - if (json_encode($string, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_LINE_TERMINATORS) !== false) { - return $string; // string is valid unicode - } - - if (function_exists('mb_str_split')) { - $result = mb_str_split($string); - } else { - $result = []; - $length = mb_strlen($string); - for ($i = 0; $i < $length; $i++) { - $result[] = mb_substr($string, $i, 1); - } - } - - $string = ''; - foreach ($result as $char) { - if (strlen($char) === 1 && !preg_match('/[[:print:]]/', $char)) { - $string .= '\x' . bin2hex($char); - } else { - $string .= $char; - } - } - - return $string; - } } \ No newline at end of file diff --git a/app/Model/AdminSetting.php b/app/Model/AdminSetting.php index 52179e29f..81c239e24 100644 --- a/app/Model/AdminSetting.php +++ b/app/Model/AdminSetting.php @@ -90,6 +90,7 @@ class AdminSetting extends AppModel $time = time(); $this->__deleteScriptTmpFiles($time); $this->__deleteTaxiiTmpFiles($time); + $this->__deleteCachedExportFiles($time); } private function __deleteScriptTmpFiles($time) { @@ -107,6 +108,29 @@ class AdminSetting extends AppModel } } + private function __deleteCachedExportFiles($time) { + $cache_path = APP . 'tmp/cached_exports'; + $cache_dir = new Folder($cache_path); + $cache_data = $cache_dir->read(false, false); + if (!empty($cache_data[0])) { + foreach ($cache_data[0] as $cache_export_dir) { + $tmp_dir = new Folder($cache_path . '/' . $cache_export_dir); + $cache_export_dir_contents = $tmp_dir->read(false, false); + if (!empty(count($cache_export_dir_contents[1]))) { + $files_count = count($cache_export_dir_contents[1]); + $files_removed = 0; + foreach ($cache_export_dir_contents[1] as $tmp_file) { + $tmp_file = new File($cache_path . '/' . $cache_export_dir . '/' . $tmp_file); + if ($time > $tmp_file->lastChange() + 3600) { + $tmp_file->delete(); + $files_removed += 1; + } + } + } + } + } + } + private function __deleteTaxiiTmpFiles($time) { $taxii_path = APP . 'files/scripts/tmp/Taxii'; $taxii_dir = new Folder($taxii_path); @@ -114,13 +138,13 @@ class AdminSetting extends AppModel if (!empty($taxii_contents[0])) { foreach ($taxii_contents[0] as $taxii_temp_dir) { if (preg_match('/^[a-zA-Z0-9]{12}$/', $taxii_temp_dir)) { - $tmp_dir = new Folder($taxii_path . $taxii_temp_dir); + $tmp_dir = new Folder($taxii_path . '/' .$taxii_temp_dir); $taxii_temp_dir_contents = $tmp_dir->read(false, false); if (!empty(count($taxii_temp_dir_contents[1]))) { $files_count = count($taxii_temp_dir_contents[1]); $files_removed = 0; foreach ($taxii_temp_dir_contents[1] as $tmp_file) { - $tmp_file = new File($taxii_path . $taxii_temp_dir . '/' . $tmp_file); + $tmp_file = new File($taxii_path . '/' . $taxii_temp_dir . '/' . $tmp_file); if ($time > $tmp_file->lastChange() + 3600) { $tmp_file->delete(); $files_removed += 1; diff --git a/app/Model/Allowedlist.php b/app/Model/Allowedlist.php index aea47d01a..d55fee63e 100644 --- a/app/Model/Allowedlist.php +++ b/app/Model/Allowedlist.php @@ -89,6 +89,9 @@ class Allowedlist extends AppModel if ($isAttributeArray) { // loop through each attribute and unset the ones that are allowedlisted foreach ($data as $k => $attribute) { + if (empty($attribute['Attribute'])) { + $attribute = ['Attribute' => $attribute]; + } // loop through each allowedlist item and run a preg match against the attribute value. If it matches, unset the attribute foreach ($allowedlists as $wlitem) { if (preg_match($wlitem, $attribute['Attribute']['value'])) { diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 2f2bae103..0d6b4f53f 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -86,7 +86,7 @@ class AppModel extends Model 99 => false, 100 => false, 101 => false, 102 => false, 103 => false, 104 => false, 105 => false, 106 => false, 107 => false, 108 => false, 109 => false, 110 => false, 111 => false, 112 => false, 113 => true, 114 => false, 115 => false, 116 => false, - 117 => false, 118 => false + 117 => false, 118 => false, 119 => false, ); const ADVANCED_UPDATES_DESCRIPTION = array( @@ -2006,6 +2006,9 @@ class AppModel extends Model case 118: $sqlArray[] = "ALTER TABLE `event_reports` MODIFY `content` mediumtext;"; break; + case 119: + $sqlArray[] = "ALTER TABLE `access_logs` MODIFY `action` varchar(191) NOT NULL"; + break; case 'fixNonEmptySharingGroupID': $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; @@ -2374,9 +2377,9 @@ class AppModel extends Model } // alternative to the build in notempty/notblank validation functions, compatible with cakephp <= 2.6 and cakephp and cakephp >= 2.7 - public function valueNotEmpty($value) + public function valueNotEmpty(array $value) { - $field = array_keys($value)[0]; + $field = array_key_first($value); $value = trim($value[$field]); if (!empty($value)) { return true; @@ -2384,27 +2387,27 @@ class AppModel extends Model return ucfirst($field) . ' cannot be empty.'; } - public function valueIsJson($value) + public function valueIsJson(array $value) { - $value = array_values($value)[0]; + $value = current($value); if (!JsonTool::isValid($value)) { return __('Invalid JSON.'); } return true; } - public function valueIsID($value) + public function valueIsID(array $value) { - $field = array_keys($value)[0]; + $field = array_key_first($value); if (!is_numeric($value[$field]) || $value[$field] < 0) { return 'Invalid ' . ucfirst($field) . ' ID'; } return true; } - public function stringNotEmpty($value) + public function stringNotEmpty(array $value) { - $field = array_keys($value)[0]; + $field = array_key_first($value); $value = trim($value[$field]); if (!isset($value) || ($value == false && $value !== "0")) { return ucfirst($field) . ' cannot be empty.'; @@ -3267,14 +3270,13 @@ class AppModel extends Model * Returns MISP version from VERSION.json file as array with major, minor and hotfix keys. * * @return array - * @throws JsonException + * @throws Exception */ public function checkMISPVersion() { static $versionArray; if ($versionArray === null) { - $content = FileAccessTool::readFromFile(ROOT . DS . 'VERSION.json'); - $versionArray = JsonTool::decode($content); + $versionArray = FileAccessTool::readJsonFromFile(ROOT . DS . 'VERSION.json', true); } return $versionArray; } @@ -3290,7 +3292,7 @@ class AppModel extends Model if ($commit === null) { App::uses('GitTool', 'Tools'); try { - $commit = GitTool::currentCommit(); + $commit = GitTool::currentCommit(ROOT); } catch (Exception $e) { $this->logException('Could not get current git commit', $e, LOG_NOTICE); $commit = false; @@ -3714,7 +3716,7 @@ class AppModel extends Model if (!$isRule) { $args = func_get_args(); $fields = $args[1]; - $or = isset($args[2]) ? $args[2] : true; + $or = $args[2] ?? true; } } if (!is_array($fields)) { @@ -3859,8 +3861,7 @@ class AppModel extends Model protected function isMysql() { $dataSource = ConnectionManager::getDataSource('default'); - $dataSourceName = $dataSource->config['datasource']; - return $dataSourceName === 'Database/Mysql' || $dataSourceName === 'Database/MysqlObserver' || $dataSourceName === 'Database/MysqlExtended' || $dataSource instanceof Mysql; + return $dataSource instanceof Mysql; } /** @@ -3996,21 +3997,21 @@ class AppModel extends Model "); } - public function findOrder($order, $order_model, $valid_order_fields) + public function findOrder($order, $orderModel, $validOrderFields) { if (!is_array($order)) { - $order_rules = explode(' ', strtolower($order)); - $order_field = explode('.', $order_rules[0]); - $order_field = end($order_field); - if (in_array($order_field, $valid_order_fields)) { + $orderRules = explode(' ', strtolower($order)); + $orderField = explode('.', $orderRules[0]); + $orderField = end($orderField); + if (in_array($orderField, $validOrderFields, true)) { $direction = 'asc'; - if (!empty($order_rules[1]) && trim($order_rules[1]) === 'desc') { + if (!empty($orderRules[1]) && trim($orderRules[1]) === 'desc') { $direction = 'desc'; } } else { return null; } - return $order_model . '.' . $order_field . ' ' . $direction; + return $orderModel . '.' . $orderField . ' ' . $direction; } return null; } @@ -4020,6 +4021,12 @@ class AppModel extends Model */ public function _remoteIp() { + static $remoteIp; + + if ($remoteIp) { + return $remoteIp; + } + $clientIpHeader = Configure::read('MISP.log_client_ip_header'); if ($clientIpHeader && isset($_SERVER[$clientIpHeader])) { $headerValue = $_SERVER[$clientIpHeader]; @@ -4027,9 +4034,12 @@ class AppModel extends Model if (($commaPos = strpos($headerValue, ',')) !== false) { $headerValue = substr($headerValue, 0, $commaPos); } - return trim($headerValue); + $remoteIp = trim($headerValue); + } else { + $remoteIp = $_SERVER['REMOTE_ADDR'] ?? null; } - return $_SERVER['REMOTE_ADDR'] ?? null; + + return $remoteIp; } public function find($type = 'first', $query = array()) @@ -4062,7 +4072,7 @@ class AppModel extends Model return false; } - public function checkParam($param) + private function checkParam($param) { return preg_match('/^[\w\_\-\. ]+$/', $param); } diff --git a/app/Model/AttachmentScan.php b/app/Model/AttachmentScan.php index da9642adc..2d6f0b837 100644 --- a/app/Model/AttachmentScan.php +++ b/app/Model/AttachmentScan.php @@ -189,6 +189,7 @@ class AttachmentScan extends AppModel /** @var Job $job */ $job = ClassRegistry::init('Job'); if ($jobId && !$job->exists($jobId)) { + $this->log("Job with ID $jobId not found in database", LOG_NOTICE); $jobId = null; } @@ -252,12 +253,12 @@ class AttachmentScan extends AppModel $infected = $this->scanAttachment($type, $attribute[$type], $moduleInfo); if ($infected === true) { $virusFound++; + $scanned++; + } else if ($infected === false) { + $scanned++; } - $scanned++; - } catch (NotFoundException $e) { - // skip if file doesn't exists } catch (Exception $e) { - $this->logException("Could not scan attachment for $type {$attribute['Attribute']['id']}", $e); + $this->logException("Could not scan attachment for $type {$attribute['Attribute']['id']}", $e, LOG_WARNING); $fails++; } @@ -297,14 +298,14 @@ class AttachmentScan extends AppModel $job = ClassRegistry::init('Job'); $jobId = $job->createJob( 'SYSTEM', - Job::WORKER_DEFAULT, + Job::WORKER_PRIO, 'virus_scan', ($type === self::TYPE_ATTRIBUTE ? 'Attribute: ' : 'Shadow attribute: ') . $attribute['id'], 'Scanning...' ); $this->getBackgroundJobsTool()->enqueue( - BackgroundJobsTool::DEFAULT_QUEUE, + BackgroundJobsTool::PRIO_QUEUE, BackgroundJobsTool::CMD_ADMIN, [ 'scanAttachment', @@ -319,10 +320,12 @@ class AttachmentScan extends AppModel } /** + * Return true if attachment is infected, null if attachment was not scanned and false if attachment is OK + * * @param string $type * @param array $attribute * @param array $moduleInfo - * @return bool|null Return true if attachment is infected. + * @return bool|null * @throws Exception */ private function scanAttachment($type, array $attribute, array $moduleInfo) @@ -351,11 +354,10 @@ class AttachmentScan extends AppModel return false; // empty file is automatically considered as not infected } - - /* if ($file->size() > 50 * 1024 * 1024) { - $this->log("File '$file->path' is bigger than 50 MB, will be not scanned.", LOG_NOTICE); - return false; - }*/ + if ($fileSize > 25 * 1024 * 1024) { + $this->log("File '$file->path' is bigger than 25 MB, will be not scanned.", LOG_NOTICE); + return null; + } $fileContent = $file->read(); if ($fileContent === false) { diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index feb58d529..2d82063d1 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -434,7 +434,7 @@ class Attribute extends AppModel public function afterSave($created, $options = array()) { // Passing event in `parentEvent` field will speed up correlation - $passedEvent = isset($options['parentEvent']) ? $options['parentEvent'] : false; + $passedEvent = $options['parentEvent'] ?? false; $attribute = $this->data['Attribute']; @@ -545,6 +545,28 @@ class Attribute extends AppModel return $result; } + /** + * This method is called after all data are successfully saved into database + * @return void + * @throws Exception + */ + private function afterDatabaseSave(array $data) + { + $attribute = $data['Attribute']; + if (isset($attribute['type']) && $this->typeIsAttachment($attribute['type'])) { + $this->loadAttachmentScan()->backgroundScan(AttachmentScan::TYPE_ATTRIBUTE, $attribute); + } + } + + public function save($data = null, $validate = true, $fieldList = array()) + { + $result = parent::save($data, $validate, $fieldList); + if ($result) { + $this->afterDatabaseSave($result); + } + return $result; + } + public function beforeDelete($cascade = true) { // delete attachments from the disk @@ -786,7 +808,7 @@ class Attribute extends AppModel // check whether the variable is null or datetime public function datetimeOrNull($fields) { - $seen = array_values($fields)[0]; + $seen = current($fields); if ($seen === null) { return true; } @@ -881,7 +903,6 @@ class Attribute extends AppModel } $result = $this->loadAttachmentTool()->save($attribute['event_id'], $attribute['id'], $attribute['data']); if ($result) { - $this->loadAttachmentScan()->backgroundScan(AttachmentScan::TYPE_ATTRIBUTE, $attribute); // Clean thumbnail cache if ($this->isImage($attribute) && Configure::read('MISP.thumbnail_in_redis')) { $redis = RedisTool::init(); @@ -1224,38 +1245,96 @@ class Attribute extends AppModel $this->Correlation->purgeCorrelations($eventId); } - public function reportValidationIssuesAttributes($eventId) + /** + * This method is useful if you want to iterate all attributes sorted by ID + * @param array $conditions + * @param array $fields + * @param bool|string $callbacks + * @return Generator|void + */ + public function fetchAttributesInChunks(array $conditions = [], array $fields = [], $callbacks = true) + { + $query = [ + 'recursive' => -1, + 'conditions' => $conditions, + 'limit' => 500, + 'order' => ['Attribute.id'], + 'fields' => $fields, + 'callbacks' => $callbacks, + ]; + + while (true) { + $attributes = $this->find('all', $query); + foreach ($attributes as $attribute) { + yield $attribute; + } + $count = count($attributes); + if ($count < 500) { + return; + } + $lastAttribute = $attributes[$count - 1]; + $query['conditions']['Attribute.id >'] = $lastAttribute['Attribute']['id']; + } + } + + /** + * @param int|null $eventId + * @return Generator + */ + public function reportValidationIssuesAttributes($eventId = null) { $conditions = array(); if ($eventId && is_numeric($eventId)) { $conditions = array('event_id' => $eventId); } - $attributeIds = $this->find('column', array( - 'fields' => array('id'), - 'conditions' => $conditions - )); - $chunks = array_chunk($attributeIds, 500); + $attributes = $this->fetchAttributesInChunks($conditions); - $result = array(); - foreach ($chunks as $chunk) { - $attributes = $this->find('all', array('recursive' => -1, 'conditions' => array('id' => $chunk))); - foreach ($attributes as $attribute) { - $this->set($attribute); - if (!$this->validates()) { - $resultErrors = array(); - foreach ($this->validationErrors as $field => $error) { - $resultErrors[$field] = array('value' => $attribute['Attribute'][$field], 'error' => $error[0]); - } - $result[] = [ - 'id' => $attribute['Attribute']['id'], - 'error' => $resultErrors, - 'details' => 'Event ID: [' . $attribute['Attribute']['event_id'] . "] - Category: [" . $attribute['Attribute']['category'] . "] - Type: [" . $attribute['Attribute']['type'] . "] - Value: [" . $attribute['Attribute']['value'] . ']', - ]; + foreach ($attributes as $attribute) { + $this->set($attribute); + if (!$this->validates()) { + $resultErrors = []; + foreach ($this->validationErrors as $field => $error) { + $resultErrors[$field] = ['value' => $attribute['Attribute'][$field], 'error' => $error[0]]; } + yield [ + 'id' => $attribute['Attribute']['id'], + 'error' => $resultErrors, + 'details' => 'Event ID: [' . $attribute['Attribute']['event_id'] . "] - Category: [" . $attribute['Attribute']['category'] . "] - Type: [" . $attribute['Attribute']['type'] . "] - Value: [" . $attribute['Attribute']['value'] . ']', + ]; + } + } + } + + /** + * @param bool $dryRun If true, no changes will be made to + * @return Generator + * @throws Exception + */ + public function normalizeIpAddress($dryRun = false) + { + $attributes = $this->fetchAttributesInChunks([ + 'Attribute.type' => ['ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'domain|ip'], + ]); + + foreach ($attributes as $attribute) { + $value = $attribute['Attribute']['value']; + $normalizedValue = AttributeValidationTool::modifyBeforeValidation($attribute['Attribute']['type'], $value); + if ($value !== $normalizedValue) { + if (!$dryRun) { + $attribute['Attribute']['value'] = $normalizedValue; + $this->save($attribute, true, ['value1', 'value2']); + } + + yield [ + 'id' => (int) $attribute['Attribute']['id'], + 'event_id' => (int) $attribute['Attribute']['event_id'], + 'type' => $attribute['Attribute']['type'], + 'value' => $value, + 'normalized_value' => $normalizedValue, + ]; } } - return $result; } /** @@ -1610,6 +1689,7 @@ class Attribute extends AppModel * @param array $user * @param array $options * @param int|false $result_count If false, count is not fetched + * @param bool $real_count * @return array * @throws Exception */ @@ -3096,8 +3176,7 @@ class Attribute extends AppModel $exportTool->additional_params ); } - ClassRegistry::init('ConnectionManager'); - $db = ConnectionManager::getDataSource('default'); + $tmpfile = new TmpFileTool(); $tmpfile->write($exportTool->header($exportToolParams)); $loop = false; @@ -3673,7 +3752,7 @@ class Attribute extends AppModel ); } - private function findAttributeByValue($attribute) + private function findAttributeByValue(array $attribute) { $type = $attribute['type']; $conditions = [ diff --git a/app/Model/AuditLog.php b/app/Model/AuditLog.php index 9b0718ae9..36fd85ab2 100644 --- a/app/Model/AuditLog.php +++ b/app/Model/AuditLog.php @@ -10,6 +10,7 @@ class AuditLog extends AppModel { const BROTLI_HEADER = "\xce\xb2\xcf\x81"; const COMPRESS_MIN_LENGTH = 256; + const CHANGE_MAX_SIZE = 64 * 1024; // MySQL type blob const ACTION_ADD = 'add', ACTION_EDIT = 'edit', @@ -235,6 +236,10 @@ class AuditLog extends AppModel if (isset($auditLog['change'])) { $auditLog['change'] = $this->encodeChange($auditLog['change']); + if (strlen($auditLog['change']) > self::CHANGE_MAX_SIZE) { + // Change is too big to save in database, skipping + $auditLog['change'] = null; + } } } diff --git a/app/Model/AuthKey.php b/app/Model/AuthKey.php index 7f3f4c1c8..1afce723a 100644 --- a/app/Model/AuthKey.php +++ b/app/Model/AuthKey.php @@ -207,7 +207,7 @@ class AuthKey extends AppModel */ private function updateUniqueIp(array $authkey) { - if (Configure::read("MISP.disable_seen_ips_authkeys")) { + if (PHP_SAPI === 'cli' || Configure::read("MISP.disable_seen_ips_authkeys")) { return; } diff --git a/app/Model/Correlation.php b/app/Model/Correlation.php index 812372514..ab43f8606 100644 --- a/app/Model/Correlation.php +++ b/app/Model/Correlation.php @@ -153,7 +153,7 @@ class Correlation extends AppModel if (!empty($eventIds)) { $eventCount = count($eventIds); foreach ($eventIds as $j => $currentEventId) { - $attributeCount += $this->__iteratedCorrelation( + $attributeCount += $this->iteratedCorrelation( $jobId, $full, $attributeId, @@ -179,7 +179,7 @@ class Correlation extends AppModel * @return int * @throws Exception */ - private function __iteratedCorrelation( + private function iteratedCorrelation( $jobId = false, $full = false, $attributeId = null, @@ -215,30 +215,14 @@ class Correlation extends AppModel if ($attributeId) { $attributeConditions['Attribute.id'] = $attributeId; } - $query = [ - 'recursive' => -1, - 'conditions' => $attributeConditions, - // fetch just necessary fields to save memory - 'fields' => $this->getFieldRules(), - 'order' => 'Attribute.id', - 'limit' => 5000, - 'callbacks' => false, // memory leak fix - ]; + + $attributes = $this->Attribute->fetchAttributesInChunks($attributeConditions, $this->getFieldRules(), false); + $attributeCount = 0; - do { - $attributes = $this->Attribute->find('all', $query); - foreach ($attributes as $attribute) { - $this->afterSaveCorrelation($attribute['Attribute'], $full, $event); - } - $fetchedAttributes = count($attributes); - unset($attributes); - $attributeCount += $fetchedAttributes; - if ($fetchedAttributes === 5000) { // maximum number of attributes fetched, continue in next loop - $query['conditions']['Attribute.id >'] = $attribute['Attribute']['id']; - } else { - break; - } - } while (true); + foreach ($attributes as $attribute) { + $this->afterSaveCorrelation($attribute['Attribute'], $full, $event); + ++$attributeCount; + } // Generating correlations can take long time, so clear caches after each event to refresh them $this->cidrListCache = null; diff --git a/app/Model/Datasource/Database/MysqlExtended.php b/app/Model/Datasource/Database/MysqlExtended.php index df20c4281..4504a3c9f 100644 --- a/app/Model/Datasource/Database/MysqlExtended.php +++ b/app/Model/Datasource/Database/MysqlExtended.php @@ -7,6 +7,14 @@ App::uses('Mysql', 'Model/Datasource/Database'); */ class MysqlExtended extends Mysql { + const PDO_MAP = [ + 'integer' => PDO::PARAM_INT, + 'float' => PDO::PARAM_STR, + 'boolean' => PDO::PARAM_BOOL, + 'string' => PDO::PARAM_STR, + 'text' => PDO::PARAM_STR + ]; + /** * Output MD5 as binary, that is faster and uses less memory * @param string $value @@ -157,15 +165,9 @@ class MysqlExtended extends Mysql public function insertMulti($table, $fields, $values) { $table = $this->fullTableName($table); - $holder = implode(',', array_fill(0, count($fields), '?')); + $holder = substr(str_repeat('?,', count($fields)), 0, -1); $fields = implode(',', array_map([$this, 'name'], $fields)); - $pdoMap = [ - 'integer' => PDO::PARAM_INT, - 'float' => PDO::PARAM_STR, - 'boolean' => PDO::PARAM_BOOL, - 'string' => PDO::PARAM_STR, - 'text' => PDO::PARAM_STR - ]; + $columnMap = []; foreach ($values[key($values)] as $key => $val) { if (is_int($val)) { @@ -174,21 +176,21 @@ class MysqlExtended extends Mysql $columnMap[$key] = PDO::PARAM_BOOL; } else { $type = $this->introspectType($val); - $columnMap[$key] = $pdoMap[$type]; + $columnMap[$key] = self::PDO_MAP[$type]; } } $sql = "INSERT INTO $table ($fields) VALUES "; - $sql .= implode(',', array_fill(0, count($values), "($holder)")); + $sql .= substr(str_repeat("($holder),", count($values)), 0, -1); $statement = $this->_connection->prepare($sql); $valuesList = array(); - $i = 1; + $i = 0; foreach ($values as $value) { foreach ($value as $col => $val) { if ($this->fullDebug) { $valuesList[] = $val; } - $statement->bindValue($i++, $val, $columnMap[$col]); + $statement->bindValue(++$i, $val, $columnMap[$col]); } } $result = $statement->execute(); diff --git a/app/Model/Event.php b/app/Model/Event.php index d6b9ba4c7..75c52b4fc 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -3743,7 +3743,10 @@ class Event extends AppModel unset($this->Attribute->validate['value']['uniqueValue']); // unset this - we are saving a new event, there are no values to compare against and event_id is not set in the attributes } unset($data['Event']['id']); - if (isset($data['Event']['published']) && $data['Event']['published'] && $user['Role']['perm_publish'] == 0) { + if ( + (Configure::read('MISP.block_publishing_for_same_creator', false) && !$user['Role']['perm_sync']) || + (isset($data['Event']['published']) && $data['Event']['published'] && $user['Role']['perm_publish'] == 0) + ) { $data['Event']['published'] = 0; } if (isset($data['Event']['uuid'])) { @@ -4059,7 +4062,10 @@ class Event extends AppModel } else { return array('error' => 'Event could not be saved: Could not find the local event.'); } - if (!empty($data['Event']['published']) && !$user['Role']['perm_publish']) { + if ( + (Configure::read('MISP.block_publishing_for_same_creator', false) && !$user['Role']['perm_sync'] && $user['id'] == $existingEvent['Event']['user_id']) || + (!empty($data['Event']['published']) && !$user['Role']['perm_publish']) + ) { $data['Event']['published'] = 0; } if (!isset($data['Event']['published'])) { @@ -4190,7 +4196,7 @@ class Event extends AppModel if ((true != Configure::read('MISP.disablerestalert')) && (empty($server) || empty($server['Server']['publish_without_email']))) { $this->sendAlertEmailRouter($id, $user, $existingEvent['Event']['publish_timestamp']); } - $this->publish($existingEvent['Event']['id']); + $this->publish($existingEvent['Event']['id'], $passAlong); } if ($jobId) { $eventLock->deleteBackgroundJobLock($data['Event']['id'], $jobId); @@ -5952,7 +5958,9 @@ class Event extends AppModel $this->add_original_file($decoded['original'], $originalFile, $created_id, $stixVersion); } if ($publish && $user['Role']['perm_publish']) { - $this->publish($created_id); + if (!Configure::read('MISP.block_publishing_for_same_creator', false) || $user['Role']['perm_sync']) { + $this->publish($created_id); + } } return $created_id; } else if (is_numeric($result)) { diff --git a/app/Model/Feed.php b/app/Model/Feed.php index 185b9253b..ef448a536 100644 --- a/app/Model/Feed.php +++ b/app/Model/Feed.php @@ -2062,6 +2062,7 @@ class Feed extends AppModel $contentType = $response->getHeader('content-type'); if ($contentType === 'application/zip') { $zipFilePath = FileAccessTool::writeToTempFile($response->body); + unset($response->body); // cleanup variable to reduce memory usage try { $response->body = $this->unzipFirstFile($zipFilePath); @@ -2198,7 +2199,7 @@ class Feed extends AppModel ZipArchive::ER_READ => 'read error', ZipArchive::ER_SEEK => 'seek error', ]; - $message = isset($errorCodes[$result]) ? $errorCodes[$result] : 'error ' . $result; + $message = $errorCodes[$result] ?? 'error ' . $result; throw new Exception("Remote server returns ZIP file, that cannot be open ($message)"); } diff --git a/app/Model/FuzzyCorrelateSsdeep.php b/app/Model/FuzzyCorrelateSsdeep.php index 236ef9d93..d3d453872 100644 --- a/app/Model/FuzzyCorrelateSsdeep.php +++ b/app/Model/FuzzyCorrelateSsdeep.php @@ -9,38 +9,32 @@ class FuzzyCorrelateSsdeep extends AppModel public function ssdeep_prepare($hash) { - list($block_size, $hash) = explode(':', $hash, 2); - - $chars = array(); - for ($i = 0; $i < strlen($hash); $i++) { - if (!in_array($hash[$i], $chars, true)) { - $chars[] = $hash[$i]; - } - } + list($blockSize, $hash) = explode(':', $hash, 2); + + $uniqueChars = array_unique(str_split($hash), SORT_REGULAR); + $search = true; while ($search) { $search = false; - foreach ($chars as $c) { + foreach ($uniqueChars as $c) { if (strpos($hash, $c . $c . $c . $c)) { $hash = str_replace($c . $c . $c . $c, $c . $c . $c, $hash); $search = true; } } } - $hash = explode(':', $hash); - $block_data = $hash[0]; - $double_block_data = $hash[1]; - // (struct.unpack("get_all_7_char_chunks($block_data), - $this->get_all_7_char_chunks($double_block_data) - ); - return $result; + $hash = explode(':', $hash); + list($block_data, $double_block_data) = $hash; + + return [ + $blockSize, + $this->getAll7CharChunks($block_data), + $this->getAll7CharChunks($double_block_data) + ]; } - public function get_all_7_char_chunks($hash) + private function getAll7CharChunks($hash) { $results = array(); for ($i = 0; $i < strlen($hash) - 6; $i++) { @@ -56,16 +50,22 @@ class FuzzyCorrelateSsdeep extends AppModel return $results; } + /** + * @param string $hash + * @param int $attributeId + * @return array + */ public function query_ssdeep_chunks($hash, $attributeId) { $chunks = $this->ssdeep_prepare($hash); + $bothPartChunks = array_merge($chunks[1], $chunks[2]); // Original algo from article https://www.virusbulletin.com/virusbulletin/2015/11/optimizing-ssdeep-use-scale // also propose to insert chunk size to database, but current database schema doesn't contain that column. // This optimisation can be add in future versions. $result = $this->find('column', array( 'conditions' => array( - 'FuzzyCorrelateSsdeep.chunk' => array_merge($chunks[1], $chunks[2]), + 'FuzzyCorrelateSsdeep.chunk' => $bothPartChunks, ), 'fields' => array('FuzzyCorrelateSsdeep.attribute_id'), 'unique' => true, @@ -73,15 +73,11 @@ class FuzzyCorrelateSsdeep extends AppModel $toSave = []; $attributeId = (int) $attributeId; - foreach (array(1, 2) as $type) { - foreach ($chunks[$type] as $chunk) { - $toSave[] = [$attributeId, $chunk]; - } - } - if (!empty($toSave)) { - $db = $this->getDataSource(); - $db->insertMulti($this->table, ['attribute_id', 'chunk'], $toSave); + foreach ($bothPartChunks as $chunk) { + $toSave[] = [$attributeId, $chunk]; } + $db = $this->getDataSource(); + $db->insertMulti($this->table, ['attribute_id', 'chunk'], $toSave); return $result; } diff --git a/app/Model/Galaxy.php b/app/Model/Galaxy.php index 906b506f6..bb8138f7d 100644 --- a/app/Model/Galaxy.php +++ b/app/Model/Galaxy.php @@ -264,7 +264,7 @@ class Galaxy extends AppModel $fields = array('galaxy_cluster_id', 'key', 'value'); $db->insertMulti('galaxy_elements', $fields, $elements); } - $allRelations = array_merge($allRelations, $relations); + array_push($allRelations, ...$relations); } // Save relation as last part when all clusters are created if (!empty($allRelations)) { @@ -287,24 +287,42 @@ class Galaxy extends AppModel if (empty($galaxy['uuid'])) { return false; } - $existingGalaxy = $this->find('first', array( + + $existingGalaxy = $this->find('first', [ 'recursive' => -1, - 'conditions' => array('Galaxy.uuid' => $galaxy['uuid']) - )); - if (empty($existingGalaxy)) { - if ($user['Role']['perm_site_admin'] || $user['Role']['perm_galaxy_editor']) { - $this->create(); - unset($galaxy['id']); - $this->save($galaxy); - $existingGalaxy = $this->find('first', array( - 'recursive' => -1, - 'conditions' => array('Galaxy.id' => $this->id) - )); - } else { - return false; + 'conditions' => ['Galaxy.uuid' => $galaxy['uuid']], + ]); + + unset($galaxy['id']); + if (!empty($existingGalaxy)) { + // check if provided galaxy has the same fields as galaxy that are saved in database + $fieldsToSave = []; + foreach (array_keys(array_intersect_key($existingGalaxy, $galaxy)) as $key) { + if ($existingGalaxy['Galaxy'][$key] != $galaxy[$key]) { + $fieldsToSave[$key] = $galaxy[$key]; + } } + } else { + $fieldsToSave = $galaxy; } - return $existingGalaxy; + + if (empty($fieldsToSave) && !empty($existingGalaxy)) { + return $existingGalaxy; // galaxy already exists and galaxy fields are the same + } + + if (!$user['Role']['perm_site_admin'] && !$user['Role']['perm_galaxy_editor']) { + return false; // user has no permission to modify galaxy + } + + if (empty($existingGalaxy)) { + $this->create(); + } + + $this->save($fieldsToSave); + return $this->find('first', [ + 'recursive' => -1, + 'conditions' => ['Galaxy.id' => $this->id], + ]); } /** diff --git a/app/Model/GalaxyElement.php b/app/Model/GalaxyElement.php index b648ce605..bb9289532 100644 --- a/app/Model/GalaxyElement.php +++ b/app/Model/GalaxyElement.php @@ -48,37 +48,6 @@ class GalaxyElement extends AppModel $this->saveMany($tempElements); } - public function update($galaxy_id, $oldClusters, $newClusters) - { - $elementsToSave = array(); - // Since we are dealing with flat files as the end all be all content, we are safe to just drop all of the old clusters and recreate them. - foreach ($oldClusters as $oldCluster) { - $this->deleteAll(array('GalaxyElement.galaxy_cluster_id' => $oldCluster['GalaxyCluster']['id'])); - } - foreach ($newClusters as $newCluster) { - $tempCluster = array(); - foreach ($newCluster as $key => $value) { - // Don't store the reserved fields as elements - if ($key == 'description' || $key == 'value') { - continue; - } - if (is_array($value)) { - foreach ($value as $arrayElement) { - $tempCluster[] = array('key' => $key, 'value' => $arrayElement); - } - } else { - $tempCluster[] = array('key' => $key, 'value' => $value); - } - } - - foreach ($tempCluster as $key => $value) { - $tempCluster[$key]['galaxy_cluster_id'] = $oldCluster['GalaxyCluster']['id']; - } - $elementsToSave = array_merge($elementsToSave, $tempCluster); - } - $this->saveMany($elementsToSave); - } - public function captureElements($user, $elements, $clusterId) { $tempElements = array(); diff --git a/app/Model/MispObject.php b/app/Model/MispObject.php index 6f9600aec..172a5ae21 100644 --- a/app/Model/MispObject.php +++ b/app/Model/MispObject.php @@ -153,6 +153,8 @@ class MispObject extends AppModel 'object_name' => array('function' => 'set_filter_object_name'), 'object_template_uuid' => array('function' => 'set_filter_object_template_uuid'), 'object_template_version' => array('function' => 'set_filter_object_template_version'), + 'first_seen' => array('function' => 'set_filter_seen'), + 'last_seen' => array('function' => 'set_filter_seen'), 'deleted' => array('function' => 'set_filter_deleted') ), 'Event' => array( @@ -181,8 +183,8 @@ class MispObject extends AppModel 'deleted' => array('function' => 'set_filter_deleted'), 'timestamp' => array('function' => 'set_filter_timestamp'), 'attribute_timestamp' => array('function' => 'set_filter_timestamp'), - 'first_seen' => array('function' => 'set_filter_seen'), - 'last_seen' => array('function' => 'set_filter_seen'), + //'first_seen' => array('function' => 'set_filter_seen'), + //'last_seen' => array('function' => 'set_filter_seen'), 'to_ids' => array('function' => 'set_filter_to_ids'), 'comment' => array('function' => 'set_filter_comment') ) @@ -1678,7 +1680,9 @@ class MispObject extends AppModel $results = $this->Sightingdb->attachToObjects($results, $user); } $params['page'] += 1; - $results = $this->Allowedlist->removeAllowedlistedFromArray($results, true); + foreach ($results as $k => $result) { + $results[$k]['Attribute'] = $this->Allowedlist->removeAllowedlistedFromArray($result['Attribute'], true); + } $results = array_values($results); $i = 0; foreach ($results as $object) { diff --git a/app/Model/Module.php b/app/Model/Module.php index cb99a10fd..337861efb 100644 --- a/app/Model/Module.php +++ b/app/Model/Module.php @@ -50,6 +50,8 @@ class Module extends AppModel ) ); + private $httpSocket = []; + public function validateIPField($value) { if (!filter_var($value, FILTER_VALIDATE_IP) === false) { @@ -309,16 +311,9 @@ class Module extends AppModel if (!$serverUrl) { throw new Exception("Module type $moduleFamily is not enabled."); } - App::uses('HttpSocketExtended', 'Tools'); - $httpSocketSetting = ['timeout' => $timeout]; - $sslSettings = array('ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_verify_peer', 'ssl_cafile'); - foreach ($sslSettings as $sslSetting) { - $value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting); - if ($value && $value !== '') { - $httpSocketSetting[$sslSetting] = $value; - } - } - $httpSocket = new HttpSocketExtended($httpSocketSetting); + + $httpSocket = $this->initHttpSocket($moduleFamily, $timeout); + $request = []; if ($moduleFamily === 'Cortex') { if (!empty(Configure::read('Plugin.' . $moduleFamily . '_authkey'))) { @@ -422,4 +417,37 @@ class Module extends AppModel return false; } + + /** + * @param string $moduleFamily + * @param int $timeout + * @return HttpSocketExtended|CurlClient + */ + private function initHttpSocket($moduleFamily, $timeout) + { + $unique = "$moduleFamily:$timeout"; + + if (isset($this->httpSocket[$unique])) { + return $this->httpSocket[$unique]; + } + + $httpSocketSetting = ['timeout' => $timeout]; + $sslSettings = ['ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_cafile']; + foreach ($sslSettings as $sslSetting) { + $value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting); + if ($value && $value !== '') { + $httpSocketSetting[$sslSetting] = $value; + } + } + + if (function_exists('curl_init')) { + App::uses('CurlClient', 'Tools'); + $httpSocket = new CurlClient($httpSocketSetting); + } else { + App::uses('HttpSocketExtended', 'Tools'); + $httpSocket = new HttpSocketExtended($httpSocketSetting); + } + + return $this->httpSocket[$unique] = $httpSocket; + } } diff --git a/app/Model/Organisation.php b/app/Model/Organisation.php index 730944b19..c03e3066a 100644 --- a/app/Model/Organisation.php +++ b/app/Model/Organisation.php @@ -76,14 +76,28 @@ class Organisation extends AppModel ); const ORGANISATION_ASSOCIATIONS = array( + 'AccessLog' => array('table' => 'access_logs', 'fields' => array('org_id')), + 'AuditLog' => array('table' => 'audit_logs', 'fields' => array('org_id')), 'Correlation' => array('table' => 'correlations', 'fields' => array('org_id')), + 'Cerebrate' => array('table' => 'cerebrates', 'fields' => array('org_id')), + 'Dashboard' => array('table' => 'dashboards', 'fields' => array('restrict_to_org_id')), 'Event' => array('table' => 'events', 'fields' => array('org_id', 'orgc_id')), + 'EventGraph' => array('table' => 'event_graph', 'fields' => array('org_id')), + 'Feed' => array('table' => 'feeds', 'fields' => array('orgc_id')), + 'GalaxyCluster' => array('table' => 'galaxy_clusters', 'fields' => array('org_id', 'orgc_id')), + 'ObjectTemplate' => array('table' => 'object_templates', 'fields' => array('org_id')), 'Job' => array('table' => 'jobs', 'fields' => array('org_id')), + 'RestClientHistory' => array('table' => 'rest_client_histories', 'fields' => array('org_id')), 'Server' => array('table' => 'servers', 'fields' => array('org_id', 'remote_org_id')), 'ShadowAttribute' => array('table' => 'shadow_attributes', 'fields' => array('org_id', 'event_org_id')), 'SharingGroup' => array('table' => 'sharing_groups', 'fields' => array('org_id')), 'SharingGroupOrg' => array('table' => 'sharing_group_orgs', 'fields' => array('org_id')), + 'SharingGroupBlueprint' => array('table' => 'sharing_group_blueprints', 'fields' => array('org_id')), + 'Sighting' => array('table' => 'sightings', 'fields' => array('org_id')), + 'SightingdbOrg' => array('table' => 'sightingdb_orgs', 'fields' => array('org_id')), 'Thread' => array('table' => 'threads', 'fields' => array('org_id')), + 'Tag' => array('table' => 'tags', 'fields' => array('org_id')), + 'TagCollection' => array('table' => 'tag_collections', 'fields' => array('org_id')), 'User' => array('table' => 'users', 'fields' => array('org_id')) ); @@ -287,6 +301,9 @@ class Organisation extends AppModel public function orgMerge($id, $request, $user) { $currentOrg = $this->find('first', array('recursive' => -1, 'conditions' => array('Organisation.id' => $id))); + if (isset($currentOrg['Organisation']['restricted_to_domain'])) { + $currentOrg['Organisation']['restricted_to_domain'] = json_encode($currentOrg['Organisation']['restricted_to_domain']); + } $currentOrgUserCount = $this->User->find('count', array( 'conditions' => array('User.org_id' => $id) )); diff --git a/app/Model/Server.php b/app/Model/Server.php index 0ead31d16..14a3f4ab6 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -472,7 +472,21 @@ class Server extends AppModel return false; } - private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, Event $eventModel, $server, $user, $jobId, $force = false, $headers = false, $body = false) + /** + * @param array $event + * @param int|string $eventId + * @param array $successes + * @param array $fails + * @param Event $eventModel + * @param array $server + * @param array $user + * @param int $jobId + * @param bool $force + * @param HttpSocketResponseExtended $response + * @return false|void + * @throws Exception + */ + private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, Event $eventModel, $server, $user, $jobId, $force = false, $response) { // check if the event already exist (using the uuid) $existingEvent = $eventModel->find('first', [ @@ -485,7 +499,7 @@ class Server extends AppModel if (!$existingEvent) { // add data for newly imported events if (isset($event['Event']['protected']) && $event['Event']['protected']) { - if (!$eventModel->CryptographicKey->validateProtectedEvent($body, $user, $headers['x-pgp-signature'], $event)) { + if (!$eventModel->CryptographicKey->validateProtectedEvent($response->body, $user, $response->getHeader('x-pgp-signature'), $event)) { $fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.'); return false; } @@ -505,7 +519,7 @@ class Server extends AppModel $fails[$eventId] = __('Blocked an edit to an event that was created locally. This can happen if a synchronised event that was created on this instance was modified by an administrator on the remote side.'); } else { if ($existingEvent['Event']['protected']) { - if (!$eventModel->CryptographicKey->validateProtectedEvent($body, $user, $headers['x-pgp-signature'], $existingEvent)) { + if (!$eventModel->CryptographicKey->validateProtectedEvent($response->body, $user, $response->getHeader('x-pgp-signature'), $existingEvent)) { $fails[$eventId] = __('Event failed the validation checks. The remote instance claims that the event can be signed with a valid key which is sus.'); } } @@ -549,12 +563,10 @@ class Server extends AppModel $params['excludeLocalTags'] = 1; } try { - $event = $serverSync->fetchEvent($eventId, $params); - $headers = $event->headers; - $body = $event->body; - $event = $event->json(); + $response = $serverSync->fetchEvent($eventId, $params); + $event = $response->json(); } catch (Exception $e) { - $this->logException("Failed downloading the event $eventId from remote server {$serverSync->serverId()}", $e); + $this->logException("Failed to download the event $eventId from remote server {$serverSync->serverId()} '{$serverSync->serverName()}'", $e); $fails[$eventId] = __('failed downloading the event'); return false; } @@ -568,7 +580,7 @@ class Server extends AppModel } return false; } - $this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $headers, $body); + $this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $response); return true; } @@ -2359,23 +2371,21 @@ class Server extends AppModel return $setting; } - public function serverSettingsEditValue(array $user, array $setting, $value, $forceSave = false) + /** + * @param array|string $user + * @param array $setting + * @param mixed $value + * @param bool $forceSave + * @return mixed|string|true|null + * @throws Exception + */ + public function serverSettingsEditValue($user, array $setting, $value, $forceSave = false) { if (isset($setting['beforeHook'])) { - $beforeResult = call_user_func_array(array($this, $setting['beforeHook']), array($setting['name'], $value)); + $beforeResult = $this->{$setting['beforeHook']}($setting['name'], $value); if ($beforeResult !== true) { - $this->Log = ClassRegistry::init('Log'); - $this->Log->create(); - $this->Log->saveOrFailSilently(array( - 'org' => $user['Organisation']['name'], - 'model' => 'Server', - 'model_id' => 0, - 'email' => $user['email'], - 'action' => 'serverSettingsEdit', - 'user_id' => $user['id'], - 'title' => 'Server setting issue', - 'change' => 'There was an issue witch changing ' . $setting['name'] . ' to ' . $value . '. The error message returned is: ' . $beforeResult . 'No changes were made.', - )); + $change = 'There was an issue witch changing ' . $setting['name'] . ' to ' . $value . '. The error message returned is: ' . $beforeResult . 'No changes were made.'; + $this->loadLog()->createLogEntry($user, 'serverSettingsEdit', 'Server', 0, 'Server setting issue', $change); return $beforeResult; } } @@ -2384,7 +2394,7 @@ class Server extends AppModel if ($setting['type'] === 'boolean') { $value = (bool)$value; } else if ($setting['type'] === 'numeric') { - $value = (int)($value); + $value = (int)$value; } if (isset($setting['test'])) { if ($setting['test'] instanceof Closure) { @@ -2425,7 +2435,7 @@ class Server extends AppModel if ($setting['afterHook'] instanceof Closure) { $afterResult = $setting['afterHook']($setting['name'], $value, $oldValue); } else { - $afterResult = call_user_func_array(array($this, $setting['afterHook']), array($setting['name'], $value, $oldValue)); + $afterResult = $this->{$setting['afterHook']}($setting['name'], $value, $oldValue); } if ($afterResult !== true) { $change = 'There was an issue after setting a new setting. The error message returned is: ' . $afterResult; @@ -2434,9 +2444,8 @@ class Server extends AppModel } } return true; - } else { - return __('Something went wrong. MISP tried to save a malformed config file. Setting change reverted.'); } + return __('Something went wrong. MISP tried to save a malformed config file or you dont have permission to write to config file. Setting change reverted.'); } /** @@ -2539,10 +2548,10 @@ class Server extends AppModel 'name' => __('Organisation logos'), 'description' => __('The logo used by an organisation on the event index, event view, discussions, proposals, etc. Make sure that the filename is in the org.png format, where org is the case-sensitive organisation name.'), 'expected' => array(), - 'valid_format' => __('48x48 pixel .png files'), + 'valid_format' => __('48x48 pixel .png files or .svg file'), 'path' => APP . 'webroot' . DS . 'img' . DS . 'orgs', - 'regex' => '.*\.(png|PNG)$', - 'regex_error' => __('Filename must be in the following format: *.png'), + 'regex' => '.*\.(png|svg)$', + 'regex_error' => __('Filename must be in the following format: *.png or *.svg'), 'files' => array(), ), 'img' => array( @@ -2578,6 +2587,7 @@ class Server extends AppModel 'read' => $f->isReadable(), 'write' => $f->isWritable(), 'execute' => $f->isExecutable(), + 'link' => $f->isLink(), ]; } } @@ -4155,12 +4165,13 @@ class Server extends AppModel private function checkRemoteVersion($HttpSocket) { try { - $json_decoded_tags = GitTool::getLatestTags($HttpSocket); + $tags = GitTool::getLatestTags($HttpSocket); } catch (Exception $e) { + $this->logException('Could not retrieve latest tags from GitHub', $e, LOG_NOTICE); return false; } // find the latest version tag in the v[major].[minor].[hotfix] format - foreach ($json_decoded_tags as $tag) { + foreach ($tags as $tag) { if (preg_match('/^v[0-9]+\.[0-9]+\.[0-9]+$/', $tag['name'])) { return $this->checkVersion($tag['name']); } @@ -4182,7 +4193,7 @@ class Server extends AppModel try { $latestCommit = GitTool::getLatestCommit($HttpSocket); } catch (Exception $e) { - $latestCommit = false; + $this->logException('Could not retrieve version from GitHub', $e, LOG_NOTICE); } } @@ -4202,6 +4213,7 @@ class Server extends AppModel try { return GitTool::currentBranch(); } catch (Exception $e) { + $this->logException('Could not retrieve current Git branch', $e, LOG_NOTICE); return false; } } @@ -4252,38 +4264,38 @@ class Server extends AppModel 'app/files/scripts/misp-opendata', 'app/files/scripts/python-maec', 'app/files/scripts/python-stix', - ); return in_array($submodule, $accepted_submodules_names, true); } /** - * @param string $submodule_name - * @param string $superproject_submodule_commit_id + * @param string $submoduleName + * @param string $superprojectSubmoduleCommitId * @return array + * @throws Exception */ - private function getSubmoduleGitStatus($submodule_name, $superproject_submodule_commit_id) + private function getSubmoduleGitStatus($submoduleName, $superprojectSubmoduleCommitId) { - $path = APP . '../' . $submodule_name; - $submodule_name = (strpos($submodule_name, '/') >= 0 ? explode('/', $submodule_name) : $submodule_name); - $submodule_name = end($submodule_name); + $path = APP . '../' . $submoduleName; + $submoduleName = (strpos($submoduleName, '/') >= 0 ? explode('/', $submoduleName) : $submoduleName); + $submoduleName = end($submoduleName); - $submoduleCurrentCommitId = GitTool::submoduleCurrentCommit($path); + $submoduleCurrentCommitId = GitTool::currentCommit($path); $currentTimestamp = GitTool::commitTimestamp($submoduleCurrentCommitId, $path); - if ($submoduleCurrentCommitId !== $superproject_submodule_commit_id) { - $remoteTimestamp = GitTool::commitTimestamp($superproject_submodule_commit_id, $path); + if ($submoduleCurrentCommitId !== $superprojectSubmoduleCommitId) { + $remoteTimestamp = GitTool::commitTimestamp($superprojectSubmoduleCommitId, $path); } else { $remoteTimestamp = $currentTimestamp; } $status = array( - 'moduleName' => $submodule_name, + 'moduleName' => $submoduleName, 'current' => $submoduleCurrentCommitId, 'currentTimestamp' => $currentTimestamp, - 'remote' => $superproject_submodule_commit_id, + 'remote' => $superprojectSubmoduleCommitId, 'remoteTimestamp' => $remoteTimestamp, - 'upToDate' => '', + 'upToDate' => 'error', 'isReadable' => is_readable($path) && is_readable($path . '/.git'), ); @@ -4295,15 +4307,11 @@ class Server extends AppModel } else { $status['upToDate'] = 'younger'; } - } else { - $status['upToDate'] = 'error'; } if ($status['isReadable'] && !empty($status['remoteTimestamp']) && !empty($status['currentTimestamp'])) { - $date1 = new DateTime(); - $date1->setTimestamp($status['remoteTimestamp']); - $date2 = new DateTime(); - $date2->setTimestamp($status['currentTimestamp']); + $date1 = new DateTime("@{$status['remoteTimestamp']}"); + $date2 = new DateTime("@{$status['currentTimestamp']}"); $status['timeDiff'] = $date1->diff($date2); } else { $status['upToDate'] = 'error'; @@ -4793,11 +4801,11 @@ class Server extends AppModel $results = [ __('User') => $user['User']['email'], - __('Role name') => isset($user['Role']['name']) ? $user['Role']['name'] : __('Unknown, outdated instance'), + __('Role name') => $user['Role']['name'] ?? __('Unknown, outdated instance'), __('Sync flag') => isset($user['Role']['perm_sync']) ? ($user['Role']['perm_sync'] ? __('Yes') : __('No')) : __('Unknown, outdated instance'), ]; - if (isset($response->headers['X-Auth-Key-Expiration'])) { - $date = new DateTime($response->headers['X-Auth-Key-Expiration']); + if ($response->getHeader('X-Auth-Key-Expiration')) { + $date = new DateTime($response->getHeader('X-Auth-Key-Expiration')); $results[__('Auth key expiration')] = $date->format('Y-m-d H:i:s'); } return $results; @@ -4935,6 +4943,28 @@ class Server extends AppModel return $this->saveMany($toSave, ['validate' => false, 'fields' => ['authkey']]); } + /** + * @param string $encryptionKey + * @return bool + * @throws Exception + */ + public function isEncryptionKeyValid($encryptionKey) + { + $servers = $this->find('list', [ + 'fields' => ['Server.id', 'Server.authkey'], + ]); + foreach ($servers as $id => $authkey) { + if (EncryptedValue::isEncrypted($authkey)) { + try { + BetterSecurity::decrypt(substr($authkey, 2), $encryptionKey); + } catch (Exception $e) { + throw new Exception("Could not decrypt auth key for server #$id", 0, $e); + } + } + } + return true; + } + /** * Return all Attribute and Object types */ @@ -5143,9 +5173,9 @@ class Server extends AppModel 'type' => 'string', ), 'disable_cached_exports' => array( - 'level' => 1, - 'description' => __('Cached exports can take up a considerable amount of space and can be disabled instance wide using this setting. Disabling the cached exports is not recommended as it\'s a valuable feature, however, if your server is having free space issues it might make sense to take this step.'), - 'value' => false, + 'level' => 2, + 'description' => __('Cached exports can take up a considerable amount of space and can be disabled instance wide using this setting. Even tough the feature is deprecated and will be removed in the future, you can still decide to enable it.'), + 'value' => true, 'null' => true, 'test' => 'testDisableCache', 'type' => 'boolean', @@ -6143,6 +6173,14 @@ class Server extends AppModel 'type' => 'boolean', 'null' => true, ], + 'block_publishing_for_same_creator' => [ + 'level' => self::SETTING_OPTIONAL, + 'description' => __('Enabling this setting will make MISP block event publishing in the case of the publisher being the same user as the event creator.'), + 'value' => false, + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true, + ], 'self_update' => [ 'level' => self::SETTING_CRITICAL, 'description' => __('Enable the GUI button for MISP self-update on the Diagnostics page.'), diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php index d19148bf8..7384869b9 100644 --- a/app/Model/Sighting.php +++ b/app/Model/Sighting.php @@ -1017,16 +1017,16 @@ class Sighting extends AppModel * @return TmpFileTool * @throws Exception */ - public function restSearch(array $user, $returnFormat, $filters) + public function restSearch(array $user, $returnFormat, array $filters) { $allowedContext = array('event', 'attribute'); // validate context if (isset($filters['context']) && !in_array($filters['context'], $allowedContext, true)) { - throw new MethodNotAllowedException(__('Invalid context %s.', $filters['context'])); + throw new BadRequestException(__('Invalid context %s.', $filters['context'])); } // ensure that an id or uuid is provided if context is set if (!empty($filters['context']) && !(isset($filters['id']) || isset($filters['uuid'])) ) { - throw new MethodNotAllowedException(__('An ID or UUID must be provided if the context is set.')); + throw new BadRequestException(__('An ID or UUID must be provided if the context is set.')); } if (!isset($this->validFormats[$returnFormat][1])) { @@ -1396,7 +1396,7 @@ class Sighting extends AppModel try { $sightings = $serverSync->fetchSightingsForEvents($chunk); } catch (Exception $e) { - $this->logException("Failed downloading the sightings from {$serverSync->server()['Server']['name']}.", $e); + $this->logException("Failed to download sightings from {$serverSync->server()['Server']['name']}.", $e); continue; } diff --git a/app/Model/SystemSetting.php b/app/Model/SystemSetting.php index d30eefcd4..875a57c75 100644 --- a/app/Model/SystemSetting.php +++ b/app/Model/SystemSetting.php @@ -46,7 +46,7 @@ class SystemSetting extends AppModel { /** @var self $systemSetting */ $systemSetting = ClassRegistry::init('SystemSetting'); - if (!$systemSetting->databaseExists()) { + if (!$systemSetting->tableExists()) { return; } $settings = $systemSetting->getSettings(); @@ -58,7 +58,7 @@ class SystemSetting extends AppModel } } - public function databaseExists() + private function tableExists() { $tables = ConnectionManager::getDataSource($this->useDbConfig)->listSources(); return in_array('system_settings', $tables, true); @@ -154,6 +154,32 @@ class SystemSetting extends AppModel return $this->saveMany($toSave); } + /** + * Check if provided encryption key is valid for all encrypted settings + * @param string $encryptionKey + * @return bool + * @throws Exception + */ + public function isEncryptionKeyValid($encryptionKey) + { + $settings = $this->find('list', [ + 'fields' => ['SystemSetting.setting', 'SystemSetting.value'], + ]); + foreach ($settings as $setting => $value) { + if (!self::isSensitive($setting)) { + continue; + } + if (EncryptedValue::isEncrypted($value)) { + try { + BetterSecurity::decrypt(substr($value, 2), $encryptionKey); + } catch (Exception $e) { + throw new Exception("Could not decrypt `$setting` setting.", 0, $e); + } + } + } + return true; + } + /** * Sensitive setting are passwords or api keys. * @param string $setting Setting name diff --git a/app/Model/User.php b/app/Model/User.php index 1965ec560..b0d1d0358 100644 --- a/app/Model/User.php +++ b/app/Model/User.php @@ -659,21 +659,18 @@ class User extends AppModel public function getUserById($id) { if (empty($id)) { - throw new NotFoundException('Invalid user ID.'); + throw new InvalidArgumentException('Invalid user ID.'); } - return $this->find( - 'first', - array( - 'conditions' => array('User.id' => $id), - 'recursive' => -1, - 'contain' => array( - 'Organisation', - 'Role', - 'Server', - 'UserSetting', - ) - ) - ); + return $this->find('first', [ + 'conditions' => ['User.id' => $id], + 'recursive' => -1, + 'contain' => [ + 'Organisation', + 'Role', + 'Server', + 'UserSetting', + ] + ]); } /** @@ -740,7 +737,7 @@ class User extends AppModel ], ]); if (empty($user)) { - return $user; + return null; } return $this->rearrangeToAuthForm($user); } @@ -861,6 +858,10 @@ class User extends AppModel return true; } + if (!isset($user['User'])) { + throw new InvalidArgumentException("Invalid user model provided."); + } + if ($user['User']['disabled'] || !$this->checkIfUserIsValid($user['User'])) { return true; } @@ -937,6 +938,11 @@ class User extends AppModel */ public function describeAuthFields() { + static $fields; // generate array just once + if ($fields) { + return $fields; + } + $fields = $this->schema(); // Do not include keys, because they are big and usually not necessary unset($fields['gpgkey']); @@ -1105,13 +1111,18 @@ class User extends AppModel return $hashed; } - public function createInitialUser($org_id) + /** + * @param int $orgId + * @return string User auth key + * @throws Exception + */ + public function createInitialUser($orgId) { $authKey = $this->generateAuthKey(); $admin = array('User' => array( 'id' => 1, 'email' => 'admin@admin.test', - 'org_id' => $org_id, + 'org_id' => $orgId, 'password' => 'admin', 'confirm_password' => 'admin', 'authkey' => $authKey, @@ -1123,7 +1134,6 @@ class User extends AppModel $this->validator()->remove('password'); // password is too simple, remove validation $this->save($admin); if (!empty(Configure::read("Security.advanced_authkeys"))) { - $this->AuthKey = ClassRegistry::init('AuthKey'); $newKey = [ 'authkey' => $authKey, 'user_id' => 1, @@ -2068,12 +2078,10 @@ class User extends AppModel return false; } - $cutoff = $redis->get('misp:session_destroy:' . $id); - $allcutoff = $redis->get('misp:session_destroy:all'); + list($cutoff, $allcutoff) = $redis->mGet(['misp:session_destroy:' . $id, 'misp:session_destroy:all']); if ( empty($cutoff) || ( - !empty($cutoff) && !empty($allcutoff) && $allcutoff < $cutoff ) @@ -2156,7 +2164,7 @@ class User extends AppModel if (!ctype_alnum($token)) { return false; } - $redis = $this->setupRedis(); + $redis = RedisTool::init(); $userId = $redis->get('misp:forgot:' . $token); if (empty($userId)) { return false; @@ -2167,8 +2175,78 @@ class User extends AppModel public function purgeForgetToken($token) { - $redis = $this->setupRedis(); - $userId = $redis->del('misp:forgot:' . $token); + $redis = RedisTool::init(); + $redis->del('misp:forgot:' . $token); return true; } + + /** + * Create default Role, Organisation and User + * @return string|null Created user auth key + * @throws Exception + */ + public function init() + { + if (!$this->Role->hasAny()) { + $siteAdmin = ['Role' => [ + 'id' => 1, + 'name' => 'Site Admin', + 'permission' => 3, + 'perm_add' => 1, + 'perm_modify' => 1, + 'perm_modify_org' => 1, + 'perm_publish' => 1, + 'perm_sync' => 1, + 'perm_admin' => 1, + 'perm_audit' => 1, + 'perm_auth' => 1, + 'perm_site_admin' => 1, + 'perm_regexp_access' => 1, + 'perm_sharing_group' => 1, + 'perm_template' => 1, + 'perm_tagger' => 1, + ]]; + $this->Role->save($siteAdmin); + // PostgreSQL: update value of auto incremented serial primary key after setting the column by force + if (!$this->isMysql()) { + $sql = "SELECT setval('roles_id_seq', (SELECT MAX(id) FROM roles));"; + $this->Role->query($sql); + } + } + + if (!$this->Organisation->hasAny(['Organisation.local' => true])) { + $this->runUpdates(); + $org = ['Organisation' => [ + 'id' => 1, + 'name' => !empty(Configure::read('MISP.org')) ? Configure::read('MISP.org') : 'ADMIN', + 'description' => 'Automatically generated admin organisation', + 'type' => 'ADMIN', + 'date_created' => date('Y-m-d H:i:s'), + 'local' => 1, + ]]; + $this->Organisation->save($org); + // PostgreSQL: update value of auto incremented serial primary key after setting the column by force + if (!$this->isMysql()) { + $sql = "SELECT setval('organisations_id_seq', (SELECT MAX(id) FROM organisations));"; + $this->Organisation->query($sql); + } + $orgId = $this->Organisation->id; + } + + if (!$this->hasAny()) { + if (!isset($orgId)) { + $hostOrg = $this->Organisation->find('first', array('conditions' => array('Organisation.name' => Configure::read('MISP.org'), 'Organisation.local' => true), 'recursive' => -1)); + if (!empty($hostOrg)) { + $orgId = $hostOrg['Organisation']['id']; + } else { + $firstOrg = $this->Organisation->find('first', array('conditions' => array('Organisation.local' => true), 'order' => 'Organisation.id ASC')); + $orgId = $firstOrg['Organisation']['id']; + } + } + $this->runUpdates(); + return $this->createInitialUser($orgId); + } + + return null; + } } diff --git a/app/Model/UserLoginProfile.php b/app/Model/UserLoginProfile.php index 27ca1081a..1d7e7ee33 100644 --- a/app/Model/UserLoginProfile.php +++ b/app/Model/UserLoginProfile.php @@ -36,22 +36,55 @@ class UserLoginProfile extends AppModel ]; const BROWSER_CACHE_DIR = APP . DS . 'tmp' . DS . 'browscap'; - const BROWSER_INI_FILE = APP . DS . 'files' . DS . 'browscap'. DS . 'browscap.ini'; // Browscap file managed by MISP - https://browscap.org/stream?q=Lite_PHP_BrowsCapINI + const BROWSER_INI_FILE = APP . DS . 'files' . DS . 'browscap'. DS . 'browscap.ini.gz'; // Browscap file managed by MISP - https://browscap.org/stream?q=Lite_PHP_BrowsCapINI const GEOIP_DB_FILE = APP . DS . 'files' . DS . 'geo-open' . DS . 'GeoOpen-Country.mmdb'; // GeoIP file managed by MISP - https://data.public.lu/en/datasets/geo-open-ip-address-geolocation-per-country-in-mmdb-format/ private $userProfile; private $knownUserProfiles = []; - private function _buildBrowscapCache() + private function browscapGetBrowser() { - $this->log("Browscap - building new cache from browscap.ini file.", LOG_INFO); - $fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR); - $cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache); - $logger = new \Monolog\Logger('name'); - $bc = new \BrowscapPHP\BrowscapUpdater($cache, $logger); - $bc->convertFile(UserLoginProfile::BROWSER_INI_FILE); + + if (function_exists('apcu_fetch')) { + App::uses('ApcuCacheTool', 'Tools'); + $cache = new ApcuCacheTool('misp:browscap'); + } else { + $fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR); + $cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache); + } + + try { + $bc = new \BrowscapPHP\Browscap($cache, $logger); + return $bc->getBrowser(); + } catch (\BrowscapPHP\Exception $e) { + $this->log("Browscap - building new cache from browscap.ini file.", LOG_INFO); + $bcUpdater = new \BrowscapPHP\BrowscapUpdater($cache, $logger); + $bcUpdater->convertString(FileAccessTool::readCompressedFile(UserLoginProfile::BROWSER_INI_FILE)); + } + + $bc = new \BrowscapPHP\Browscap($cache, $logger); + return $bc->getBrowser(); + } + + /** + * @param string $ip + * @return string|null + */ + public function countryByIp($ip) + { + if (class_exists('GeoIp2\Database\Reader')) { + $geoDbReader = new GeoIp2\Database\Reader(UserLoginProfile::GEOIP_DB_FILE); + try { + $record = $geoDbReader->country($ip); + return $record->country->isoCode; + } catch (InvalidArgumentException $e) { + $this->logException("Could not get country code for IP address", $e, LOG_NOTICE); + return null; + } + } + return null; } public function beforeSave($options = []) @@ -76,16 +109,7 @@ class UserLoginProfile extends AppModel if (!$this->userProfile) { // below uses https://github.com/browscap/browscap-php if (class_exists('\BrowscapPHP\Browscap')) { - try { - $fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR); - $cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache); - $logger = new \Monolog\Logger('name'); - $bc = new \BrowscapPHP\Browscap($cache, $logger); - $browser = $bc->getBrowser(); - } catch (\BrowscapPHP\Exception $e) { - $this->_buildBrowscapCache(); - return $this->_getUserProfile(); - } + $browser = $this->browscapGetBrowser(); } else { // a primitive OS & browser extraction capability $ua = $_SERVER['HTTP_USER_AGENT'] ?? null; @@ -100,18 +124,7 @@ class UserLoginProfile extends AppModel $browser->browser = "browser"; } $ip = $this->_remoteIp(); - if (class_exists('GeoIp2\Database\Reader')) { - try { - $geoDbReader = new GeoIp2\Database\Reader(UserLoginProfile::GEOIP_DB_FILE); - $record = $geoDbReader->country($ip); - $country = $record->country->isoCode; - } catch (InvalidArgumentException $e) { - $this->logException("Could not get country code for IP address", $e); - $country = 'None'; - } - } else { - $country = 'None'; - } + $country = $this->countryByIp($ip) ?? 'None'; $this->userProfile = [ 'user_agent' => $_SERVER['HTTP_USER_AGENT'] ?? null, 'ip' => $ip, @@ -247,13 +260,13 @@ class UserLoginProfile extends AppModel public function emailNewLogin(array $user) { if (!Configure::read('MISP.disable_emailing')) { - $date_time = date('c'); - + $user = $this->User->getUserById($user['id']); // fetch in database format + $datetime = date('c'); // ISO 8601 date $body = new SendEmailTemplate('userloginprofile_newlogin'); $body->set('userLoginProfile', $this->User->UserLoginProfile->_getUserProfile()); $body->set('baseurl', Configure::read('MISP.baseurl')); $body->set('misp_org', Configure::read('MISP.org')); - $body->set('date_time', $date_time); + $body->set('date_time', $datetime); // Fetch user that contains also PGP or S/MIME keys for e-mail encryption $this->User->sendEmail($user, $body, false, "[" . Configure::read('MISP.org') . " MISP] New sign in."); } diff --git a/app/Model/Warninglist.php b/app/Model/Warninglist.php index b4313ddc2..bc3e649d4 100644 --- a/app/Model/Warninglist.php +++ b/app/Model/Warninglist.php @@ -390,8 +390,7 @@ class Warninglist extends AppModel $warninglistId = (int)$this->id; $result = true; - $keys = array_keys($list['list']); - if ($keys === array_keys($keys)) { + if (JsonTool::arrayIsList($list['list'])) { foreach (array_chunk($list['list'], 1000) as $chunk) { $valuesToInsert = []; foreach ($chunk as $value) { diff --git a/app/Plugin/BinaryFileCache/Engine/BinaryFileEngine.php b/app/Plugin/BinaryFileCache/Engine/BinaryFileEngine.php new file mode 100644 index 000000000..c39173561 --- /dev/null +++ b/app/Plugin/BinaryFileCache/Engine/BinaryFileEngine.php @@ -0,0 +1,276 @@ + 'BinaryFile', + 'path' => CACHE, + 'prefix' => 'cake_', + 'serialize' => true, + 'mask' => 0660, + ]; + CacheEngine::init($settings); + + $this->useIgbinary = function_exists('igbinary_serialize'); + if (substr($this->settings['path'], -1) !== DS) { + $this->settings['path'] .= DS; + } + if (!empty($this->_groupPrefix)) { + $this->_groupPrefix = str_replace('_', DS, $this->_groupPrefix); + } + return $this->_active(); + } + + /** + * @param string $key + * @param mixed $data + * @param int $duration + * @return bool + */ + public function write($key, $data, $duration) + { + if (!$this->_init) { + return false; + } + + $fileInfo = $this->cacheFilePath($key); + $resource = $this->createFile($fileInfo); + if (!$resource) { + return false; + } + + if (!empty($this->settings['serialize'])) { + if ($this->useIgbinary) { + $data = igbinary_serialize($data); + if ($data === null) { + return false; + } + } else { + $data = serialize($data); + } + } + + $expires = pack("q", time() + $duration); + + flock($resource, LOCK_EX); + + ftruncate($resource, 0); + + $result = fwrite($resource, $expires); + if ($result !== self::BINARY_CACHE_TIME_LENGTH) { + $this->handleWriteError($fileInfo); + fclose($resource); + return false; + } + + $result = fwrite($resource, $data); + if ($result !== strlen($data)) { + $this->handleWriteError($fileInfo); + fclose($resource); + return false; + } + + fclose($resource); + + return true; + } + + /** + * @param string $key + * @return false|mixed|string + */ + public function read($key) + { + if (!$this->_init) { + return false; + } + + $fileInfo = $this->cacheFilePath($key); + + $exists = file_exists($fileInfo->getPathname()); + if (!$exists) { + return false; + } + + $resource = $this->openFile($fileInfo); + if (!$resource) { + return false; + } + + $time = time(); + + flock($resource, LOCK_SH); + + $cacheTimeBinary = fread($resource, self::BINARY_CACHE_TIME_LENGTH); + if (!$cacheTimeBinary) { + fclose($resource); + return false; + } + + $cacheTime = $this->unpackCacheTime($cacheTimeBinary); + if ($cacheTime < $time || ($time + $this->settings['duration']) < $cacheTime) { + fclose($resource); + return false; // already expired + } + + $data = stream_get_contents($resource, null, self::BINARY_CACHE_TIME_LENGTH); + fclose($resource); + + if (!empty($this->settings['serialize'])) { + if ($this->useIgbinary) { + $data = igbinary_unserialize($data); + } else { + $data = unserialize($data); + } + } + + return $data; + } + + /** + * @param string $path + * @param int $now + * @param int $threshold + * @return void + */ + protected function _clearDirectory($path, $now, $threshold) + { + $prefixLength = strlen($this->settings['prefix']); + + if (!is_dir($path)) { + return; + } + + $dir = dir($path); + if ($dir === false) { + return; + } + + while (($entry = $dir->read()) !== false) { + if (substr($entry, 0, $prefixLength) !== $this->settings['prefix']) { + continue; + } + + try { + $file = new SplFileObject($path . $entry, 'rb'); + } catch (Exception $e) { + continue; + } + + if ($threshold) { + $mtime = $file->getMTime(); + if ($mtime > $threshold) { + continue; + } + $expires = $this->unpackCacheTime($file->fread(self::BINARY_CACHE_TIME_LENGTH)); + if ($expires > $now) { + continue; + } + } + if ($file->isFile()) { + $filePath = $file->getRealPath(); + $file = null; + @unlink($filePath); + } + } + } + + /** + * @param SplFileInfo $fileInfo + * @return false|resource + */ + private function createFile(SplFileInfo $fileInfo) + { + $exists = file_exists($fileInfo->getPathname()); + if (!$exists) { + $resource = $this->openFile($fileInfo, 'cb'); + if ($resource && !chmod($fileInfo->getPathname(), (int)$this->settings['mask'])) { + trigger_error(__d( + 'cake_dev', 'Could not apply permission mask "%s" on cache file "%s"', + [$fileInfo->getPathname(), $this->settings['mask']]), E_USER_WARNING); + } + return $resource; + } + + return $this->openFile($fileInfo, 'cb'); + } + + /** + * @param SplFileInfo $fileInfo + * @param string $mode + * @return false|resource + */ + private function openFile(SplFileInfo $fileInfo, $mode = 'rb') + { + $resource = fopen($fileInfo->getPathname(), $mode); + if (!$resource) { + trigger_error(__d( + 'cake_dev', 'Could not open file %s', + array($fileInfo->getPathname())), E_USER_WARNING); + } + return $resource; + } + + /** + * @param string $key + * @return SplFileInfo + */ + private function cacheFilePath(string $key): SplFileInfo + { + $groups = null; + if (!empty($this->_groupPrefix)) { + $groups = vsprintf($this->_groupPrefix, $this->groups()); + } + $dir = $this->settings['path'] . $groups; + + if (!is_dir($dir)) { + mkdir($dir, 0775, true); + } + + $suffix = '.bin'; + if ($this->settings['serialize'] && $this->useIgbinary) { + $suffix = '.igbin'; + } + + return new SplFileInfo($dir . $key . $suffix); + } + + /** + * @param SplFileInfo $fileInfo + * @return void + */ + private function handleWriteError(SplFileInfo $fileInfo) + { + unlink($fileInfo->getPathname()); // delete file in case file was just partially written + trigger_error(__d( + 'cake_dev', 'Could not write to file %s', + array($fileInfo->getPathname())), E_USER_WARNING); + } + + /** + * @param string $cacheTimeBinary + * @return int + */ + private function unpackCacheTime($cacheTimeBinary) + { + if ($cacheTimeBinary === false || strlen($cacheTimeBinary) !== self::BINARY_CACHE_TIME_LENGTH) { + throw new InvalidArgumentException("Invalid cache time in binary format provided '$cacheTimeBinary'"); + } + return unpack("q", $cacheTimeBinary)[1]; + } +} \ No newline at end of file diff --git a/app/Plugin/EcsLog/Lib/Log/Engine/EcsLog.php b/app/Plugin/EcsLog/Lib/Log/Engine/EcsLog.php index 6cb3c8d4b..4e08b1eb3 100644 --- a/app/Plugin/EcsLog/Lib/Log/Engine/EcsLog.php +++ b/app/Plugin/EcsLog/Lib/Log/Engine/EcsLog.php @@ -57,7 +57,7 @@ class EcsLog implements CakeLogInterface 'log' => [ 'level' => $type, ], - 'message' => $message, + 'message' => JsonTool::escapeNonUnicode($message), ]; static::writeMessage($message); diff --git a/app/Plugin/OidcAuth/Lib/Oidc.php b/app/Plugin/OidcAuth/Lib/Oidc.php index 67f712088..d8d4dd210 100644 --- a/app/Plugin/OidcAuth/Lib/Oidc.php +++ b/app/Plugin/OidcAuth/Lib/Oidc.php @@ -28,7 +28,6 @@ class Oidc $claims = $oidc->getVerifiedClaims(); $mispUsername = $claims->email ?? $oidc->requestUserInfo('email'); - if (empty($mispUsername)) { $sub = $claims->sub ?? 'UNKNOWN'; throw new Exception("OIDC user $sub doesn't have email address, that is required by MISP."); @@ -66,13 +65,13 @@ class Oidc $roleProperty = $this->getConfig('roles_property', 'roles'); $roles = $claims->{$roleProperty} ?? $oidc->requestUserInfo($roleProperty); if ($roles === null) { - $this->log($mispUsername, "Role property `$roleProperty` is missing in claims.", LOG_WARNING); + $this->log($mispUsername, "Role property `$roleProperty` is missing in claims, access prohibited.", LOG_WARNING); return false; } $roleId = $this->getUserRole($roles, $mispUsername); if ($roleId === null) { - $this->log($mispUsername, 'No role was assigned.'); + $this->log($mispUsername, 'No role was assigned, access prohibited.', LOG_WARNING); if ($user) { $this->block($user); } diff --git a/app/Test/AttributeValidationToolTest.php b/app/Test/AttributeValidationToolTest.php index b1781220d..b08a13fda 100644 --- a/app/Test/AttributeValidationToolTest.php +++ b/app/Test/AttributeValidationToolTest.php @@ -124,6 +124,16 @@ class AttributeValidationToolTest extends TestCase ]); } + public function testRemoveCidrFromIp(): void + { + $this->assertEquals('127.0.0.1', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1/32')); + $this->assertEquals('127.0.0.1/31', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1/31')); + $this->assertEquals('example.com|1234:fd2:5621:1:89::4500', AttributeValidationTool::modifyBeforeValidation('domain|ip', 'example.com|1234:0fd2:5621:0001:0089:0000:0000:4500/128')); + $this->assertEquals('1234:fd2:5621:1:89::4500|80', AttributeValidationTool::modifyBeforeValidation('ip-src|port', '1234:0fd2:5621:0001:0089:0000:0000:4500/128|80')); + $this->assertEquals('1234:fd2:5621:1:89::4500/127|80', AttributeValidationTool::modifyBeforeValidation('ip-src|port', '1234:0fd2:5621:0001:0089:0000:0000:4500/127|80')); + $this->assertEquals('127.0.0.1', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1')); + } + public function testCompressIpv6(): void { $this->assertEquals('1234:fd2:5621:1:89::4500', AttributeValidationTool::modifyBeforeValidation('ip-src', '1234:0fd2:5621:0001:0089:0000:0000:4500')); diff --git a/app/Test/ComplexTypeToolTest.php b/app/Test/ComplexTypeToolTest.php index 54acead69..16d4a6273 100644 --- a/app/Test/ComplexTypeToolTest.php +++ b/app/Test/ComplexTypeToolTest.php @@ -527,10 +527,31 @@ EOT; public function testCheckFreeTextNonBreakableSpace(): void { $complexTypeTool = new ComplexTypeTool(); + $results = $complexTypeTool->checkFreeText("127.0.0.1\xc2\xa0127.0.0.2"); $this->assertCount(2, $results); $this->assertEquals('127.0.0.1', $results[0]['value']); $this->assertEquals('ip-dst', $results[0]['default_type']); + + $results = $complexTypeTool->checkFreeText("127.0.0.1\xc2\xa0\xc2\xa0127.0.0.2"); + $this->assertCount(2, $results); + $this->assertEquals('127.0.0.1', $results[0]['value']); + $this->assertEquals('ip-dst', $results[0]['default_type']); + } + + public function testCheckFreeTextControlCharToSpace(): void + { + $complexTypeTool = new ComplexTypeTool(); + + $results = $complexTypeTool->checkFreeText("127.0.0.1\x1d127.0.0.2"); + $this->assertCount(2, $results); + $this->assertEquals('127.0.0.1', $results[0]['value']); + $this->assertEquals('ip-dst', $results[0]['default_type']); + + $results = $complexTypeTool->checkFreeText("127.0.0.1\x1d\x1d127.0.0.2"); + $this->assertCount(2, $results); + $this->assertEquals('127.0.0.1', $results[0]['value']); + $this->assertEquals('ip-dst', $results[0]['default_type']); } public function testCheckFreeTextQuoted(): void diff --git a/app/Test/JSONConverterToolTest.php b/app/Test/JSONConverterToolTest.php index 5e7758148..adc47fb4a 100644 --- a/app/Test/JSONConverterToolTest.php +++ b/app/Test/JSONConverterToolTest.php @@ -1,4 +1,5 @@ assertEquals($jsonNormalWithoutSpaces, $jsonStreamWithoutSpaces); - if (defined('JSON_THROW_ON_ERROR')) { - json_decode($json, true, 512, JSON_THROW_ON_ERROR); - $this->assertTrue(true); - } else { - $this->assertNotNull(json_decode($json)); - } + $this->assertTrue(JsonTool::isValid($json)); } } diff --git a/app/View/AuthKeys/view.ctp b/app/View/AuthKeys/view.ctp index 2dbac015a..5bd195ff0 100644 --- a/app/View/AuthKeys/view.ctp +++ b/app/View/AuthKeys/view.ctp @@ -3,12 +3,12 @@ $keyUsageCsv = null; if (isset($keyUsage)) { $todayString = date('Y-m-d'); $today = strtotime($todayString); - $startDate = key($keyUsage); // oldest date for sparkline + $startDate = array_key_first($keyUsage); // oldest date for sparkline $startDate = strtotime($startDate) - (3600 * 24 * 3); $keyUsageCsv = 'Date,Close\n'; for ($date = $startDate; $date <= $today; $date += (3600 * 24)) { $dateAsString = date('Y-m-d', $date); - $keyUsageCsv .= $dateAsString . ',' . (isset($keyUsage[$dateAsString]) ? $keyUsage[$dateAsString] : 0) . '\n'; + $keyUsageCsv .= $dateAsString . ',' . ($keyUsage[$dateAsString] ?? '0') . '\n'; } } else { $lastUsed = null; diff --git a/app/View/Elements/global_menu.ctp b/app/View/Elements/global_menu.ctp index 4a8e50968..5cdfae0c1 100755 --- a/app/View/Elements/global_menu.ctp +++ b/app/View/Elements/global_menu.ctp @@ -277,11 +277,6 @@ 'url' => $baseurl . '/servers/createSync', 'requirement' => $isAclSync && !$isSiteAdmin ), - array( - 'text' => __('Import Server Settings'), - 'url' => $baseurl . '/servers/import', - 'requirement' => $this->Acl->canAccess('servers', 'import'), - ), array( 'text' => __('Remote Servers'), 'url' => $baseurl . '/servers/index', @@ -292,11 +287,6 @@ 'url' => $baseurl . '/feeds/index', 'requirement' => $this->Acl->canAccess('feeds', 'index'), ), - array( - 'text' => __('Search Feed Caches'), - 'url' => $baseurl . '/feeds/searchCaches', - 'requirement' => $this->Acl->canAccess('feeds', 'searchCaches'), - ), array( 'text' => __('SightingDB'), 'url' => $baseurl . '/sightingdb/index', @@ -313,7 +303,7 @@ 'requirement' => $this->Acl->canAccess('cerebrates', 'index'), ), array( - 'text' => __('List Taxii Servers'), + 'text' => __('TAXII Servers'), 'url' => $baseurl . '/TaxiiServers/index', 'requirement' => $this->Acl->canAccess('taxiiServers', 'index'), ), diff --git a/app/View/Elements/healthElements/files.ctp b/app/View/Elements/healthElements/files.ctp index 0221d8124..2d13b4767 100644 --- a/app/View/Elements/healthElements/files.ctp +++ b/app/View/Elements/healthElements/files.ctp @@ -1,3 +1,10 @@ +

- :
- :
- :
+ :
+ :
+ :
- :
+ :
    $expectedValue): $colour = 'red'; @@ -24,7 +31,7 @@ endif; ?>
- +
@@ -35,19 +42,10 @@ 1) { - $f['filesize'] = $f['filesize'] / 1024; - $sizeUnit = "KB"; - if (($f['filesize'] / 1024) > 1) { - $f['filesize'] = $f['filesize'] / 1024; - $sizeUnit = "MB"; - } - $f['filesize'] = round($f['filesize'], 1); - } ?> @@ -55,7 +53,7 @@ $ev): - if ($f['filename'] == $ev) echo h($ek) . "
"; + if ($f['filename'] == $ev) echo h($ek) . "
"; endforeach; else: echo __('N/A'); @@ -63,7 +61,7 @@ ?>
- + @@ -93,5 +91,4 @@ echo $this->Form->end(); endforeach; ?> - diff --git a/app/View/Events/export.ctp b/app/View/Events/export.ctp index 60e29ed79..77e3f09c9 100755 --- a/app/View/Events/export.ctp +++ b/app/View/Events/export.ctp @@ -3,6 +3,10 @@


+ +

+ +