diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..f2da0572a --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,67 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ 2.4 ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ 2.4 ] + schedule: + - cron: '31 16 * * 1' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + language: [ 'javascript', 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7e5fb90a9..27f480b65 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest] + os: [ubuntu-20.04] php: ['7.2', '7.3', '7.4'] # Steps represent a sequence of tasks that will be executed as part of the job @@ -66,9 +66,14 @@ jobs: run: | git submodule update --init --recursive sudo apt-get -y update - sudo apt-get -y install python3 python3-venv virtualenv python3-pip python3-nose python3-redis python3-lxml apache2 curl libapache2-mod-php libfuzzy-dev libemail-address-perl libemail-outlook-message-perl - sudo pip3 install --upgrade pip setuptools requests pyzmq poetry + if [[ $php_version == "7.4" ]]; then + # Repo is missing for unknown reason + LC_ALL=C.UTF-8 sudo apt-add-repository ppa:ondrej/php -y + fi + sudo apt-get -y install curl python3 python3-zmq python3-requests python3-pip python3-nose python3-redis python3-lxml apache2 libapache2-mod-php$php_version libfuzzy-dev + sudo pip3 install virtualenv # virtualenv must be instaled from pip and not from ubuntu pacckages sudo pip3 install --upgrade -r requirements.txt + curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - sudo chown $USER:www-data $HOME/.composer pushd app sudo -H -u $USER php composer.phar install --no-progress @@ -185,7 +190,7 @@ jobs: sudo chmod -R 777 ./tests # Start workers # Dirty install python stuff - virtualenv -p python3 ./venv + python3 -m virtualenv -p python3 ./venv sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python"' . ./venv/bin/activate pushd cti-python-stix2 @@ -209,9 +214,11 @@ jobs: - name: Run tests run: | + source $HOME/.poetry/env # enable poetry binary ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/ ./app/Vendor/bin/phpunit app/Test/ComplexTypeToolTest.php ./app/Vendor/bin/phpunit app/Test/JSONConverterToolTest.php + ./app/Vendor/bin/phpunit app/Test/CidrToolTest.php # Ensure the perms of config files sudo chown -R $USER:www-data `pwd`/app/Config @@ -220,14 +227,15 @@ jobs: sudo chown -R $USER:www-data `pwd`/app/Config sudo chmod -R 777 `pwd`/app/Config - python3 tests/testlive_security.py - pushd tests ./curl_tests_GH.sh $AUTH $HOST popd + pushd PyMISP poetry install -E fileobjects -E openioc -E virustotal -E docs -E pdfexport -E email poetry run python tests/testlive_comprehensive.py + poetry add lxml + poetry run python ../tests/testlive_security.py -v poetry run python tests/test_mispevent.py popd cp PyMISP/tests/keys.py PyMISP/examples/events/ diff --git a/INSTALL/INSTALL.sh b/INSTALL/INSTALL.sh index f6471e6a3..5c5eca35a 100755 --- a/INSTALL/INSTALL.sh +++ b/INSTALL/INSTALL.sh @@ -620,7 +620,7 @@ preInstall () { DBPASSWORD_MISP=$(cat database.php |grep -v // |grep -e password |tr -d \' |tr -d \ |tr -d , |tr -d \> |cut -f 2 -d=) DBUSER_MISP=$(cat database.php |grep -v // |grep -e login |tr -d \' |tr -d \ |tr -d , |tr -d \> |cut -f 2 -d=) DBNAME=$(cat database.php |grep -v // |grep -e database |tr -d \' |tr -d \ |tr -d , |tr -d \> |cut -f 2 -d=) - AUTH_KEY=$(mysql --disable-column-names -B -u $DBUSER_MISP -p"$DBPASSWORD_MISP" $DBNAME -e 'SELECT authkey FROM users WHERE role_id=1 LIMIT 1') + AUTH_KEY=$(mysql -h $DBHOST --disable-column-names -B -u $DBUSER_MISP -p"$DBPASSWORD_MISP" $DBNAME -e 'SELECT authkey FROM users WHERE role_id=1 LIMIT 1') # Check if db exists [[ -d "/var/lib/mysql/$DBNAME" ]] && MISP_DB_DIR_EXISTS=1 && echo "/var/lib/mysql/$DBNAME exists" @@ -675,6 +675,7 @@ kaliSpaceSaver () { echo "${RED}Not implement${NC}" } +# FIXME: Kali now uses kali/kali instead of root/toor # Because Kali is l33t we make sure we DO NOT run as root kaliOnTheR0ckz () { totalRoot=$(df -k | grep /$ |awk '{ print $2 }') @@ -776,6 +777,7 @@ installRNG () { kaliUpgrade () { debug "Running various Kali upgrade tasks" checkAptLock + sudo DEBIAN_FRONTEND=noninteractive apt update sudo DEBIAN_FRONTEND=noninteractive apt install --only-upgrade bash libc6 -y sudo DEBIAN_FRONTEND=noninteractive apt autoremove -y } @@ -1071,8 +1073,8 @@ nuke () { sleep 10 sudo rm -rvf /usr/local/src/{misp-modules,viper,mail_to_misp,LIEF,faup} sudo rm -rvf /var/www/MISP - sudo mysqladmin drop misp - sudo mysql -e "DROP USER misp@localhost" + sudo mysqladmin -h $DBHOST drop misp + sudo mysql -h $DBHOST -e "DROP USER misp@localhost" } # Final function to let the user know what happened @@ -1197,8 +1199,9 @@ installDepsPhp74 () { libapache2-mod-php \ php php-cli \ php-dev \ - php-json php-xml php-mysql php-opcache php-readline php-mbstring php-zip \ + php-json php-xml php-mysql php7.4-opcache php-readline php-mbstring php-zip \ php-redis php-gnupg \ + php-intl php-bcmath \ php-gd for key in upload_max_filesize post_max_size max_execution_time max_input_time memory_limit @@ -1250,6 +1253,7 @@ installDepsPhp72 () { php-dev \ php-json php-xml php-mysql php7.2-opcache php-readline php-mbstring php-zip \ php-redis php-gnupg \ + php-intl php-bcmath \ php-gd for key in upload_max_filesize post_max_size max_execution_time max_input_time memory_limit @@ -1292,25 +1296,25 @@ prepareDB () { debug "Setting up database" # Kill the anonymous users - sudo mysql -e "DROP USER IF EXISTS ''@'localhost'" + sudo mysql -h $DBHOST -e "DROP USER IF EXISTS ''@'localhost'" # Because our hostname varies we'll use some Bash magic here. - sudo mysql -e "DROP USER IF EXISTS ''@'$(hostname)'" + sudo mysql -h $DBHOST -e "DROP USER IF EXISTS ''@'$(hostname)'" # Kill off the demo database - sudo mysql -e "DROP DATABASE IF EXISTS test" + sudo mysql -h $DBHOST -e "DROP DATABASE IF EXISTS test" # No root remote logins - sudo mysql -e "DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1')" + sudo mysql -h $DBHOST -e "DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1')" # Make sure that NOBODY can access the server without a password - sudo mysqladmin -u "${DBUSER_ADMIN}" password "${DBPASSWORD_ADMIN}" + sudo mysqladmin -h $DBHOST -u "${DBUSER_ADMIN}" password "${DBPASSWORD_ADMIN}" # Make our changes take effect - sudo mysql -e "FLUSH PRIVILEGES" + sudo mysql -h $DBHOST -e "FLUSH PRIVILEGES" - sudo mysql -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "CREATE DATABASE ${DBNAME};" - sudo mysql -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "CREATE USER '${DBUSER_MISP}'@'localhost' IDENTIFIED BY '${DBPASSWORD_MISP}';" - sudo mysql -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "GRANT USAGE ON *.* to '${DBUSER_MISP}'@'localhost';" - sudo mysql -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "GRANT ALL PRIVILEGES on ${DBNAME}.* to '${DBUSER_MISP}'@'localhost';" - sudo mysql -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "FLUSH PRIVILEGES;" + sudo mysql -h $DBHOST -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "CREATE DATABASE ${DBNAME};" + sudo mysql -h $DBHOST -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "CREATE USER '${DBUSER_MISP}'@'localhost' IDENTIFIED BY '${DBPASSWORD_MISP}';" + sudo mysql -h $DBHOST -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "GRANT USAGE ON *.* to '${DBUSER_MISP}'@'localhost';" + sudo mysql -h $DBHOST -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "GRANT ALL PRIVILEGES on ${DBNAME}.* to '${DBUSER_MISP}'@'localhost';" + sudo mysql -h $DBHOST -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "FLUSH PRIVILEGES;" # Import the empty MISP database from MYSQL.sql - ${SUDO_WWW} cat ${PATH_TO_MISP}/INSTALL/MYSQL.sql | mysql -u "${DBUSER_MISP}" -p"${DBPASSWORD_MISP}" ${DBNAME} + ${SUDO_WWW} cat ${PATH_TO_MISP}/INSTALL/MYSQL.sql | mysql -h $DBHOST -u "${DBUSER_MISP}" -p"${DBPASSWORD_MISP}" ${DBNAME} fi } @@ -1372,7 +1376,7 @@ installCore () { for dependency in CybOXProject/python-cybox STIXProject/python-stix MAECProject/python-maec CybOXProject/mixbox; do false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git clone https://github.com/${dependency}.git ${PATH_TO_MISP_SCRIPTS}/${dependency##*/}; done ${SUDO_WWW} git -C ${PATH_TO_MISP_SCRIPTS}/${dependency##*/} config core.filemode false - ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP_SCRIPTS}/${dependency##/*} + ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP_SCRIPTS}/${dependency##*/} done debug "Install python-stix2" @@ -1384,8 +1388,8 @@ installCore () { # FIXME: Remove libfaup etc once the egg has the library baked-in sudo apt-get install cmake libcaca-dev liblua5.3-dev -y cd /tmp - false; while [[ $? -ne 0 ]]; do [[ ! -d "faup" ]] && ${SUDO_CMD} git clone git://github.com/stricaud/faup.git faup; done - false; while [[ $? -ne 0 ]]; do [[ ! -d "gtcaca" ]] && ${SUDO_CMD} git clone git://github.com/stricaud/gtcaca.git gtcaca; done + false; while [[ $? -ne 0 ]]; do [[ ! -d "faup" ]] && ${SUDO_CMD} git clone https://github.com/stricaud/faup.git faup; done + false; while [[ $? -ne 0 ]]; do [[ ! -d "gtcaca" ]] && ${SUDO_CMD} git clone https://github.com/stricaud/gtcaca.git gtcaca; done sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca cd gtcaca ${SUDO_CMD} mkdir -p build @@ -1421,7 +1425,7 @@ installCore () { ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U setuptools pip lief zmq redis python-magic plyara for dependency in CybOXProject/python-cybox STIXProject/python-stix MAECProject/python-maec CybOXProject/mixbox; do false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git -C ${PATH_TO_MISP_SCRIPTS}/${dependency##*/} pull; done - ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP_SCRIPTS}/${dependency##/*} + ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP_SCRIPTS}/${dependency##*/} done ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP}/cti-python-stix2 @@ -1557,6 +1561,7 @@ coreCAKE () { # Various plugin sightings settings $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Sightings_policy" 0 $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Sightings_anonymise" false + $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Sightings_anonymise_as" 1 $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Sightings_range" 365 $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Sightings_sighting_db_enable" false @@ -1592,11 +1597,14 @@ coreCAKE () { $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.default_event_threat_level" 4 $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.newUserText" "Dear new MISP user,\\n\\nWe would hereby like to welcome you to the \$org MISP community.\\n\\n Use the credentials below to log into MISP at \$misp, where you will be prompted to manually change your password to something of your own choice.\\n\\nUsername: \$username\\nPassword: \$password\\n\\nIf you have any questions, don't hesitate to contact us at: \$contact.\\n\\nBest regards,\\nYour \$org MISP support team" $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.passwordResetText" "Dear MISP user,\\n\\nA password reset has been triggered for your account. Use the below provided temporary password to log into MISP at \$misp, where you will be prompted to manually change your password to something of your own choice.\\n\\nUsername: \$username\\nYour temporary password: \$password\\n\\nIf you have any questions, don't hesitate to contact us at: \$contact.\\n\\nBest regards,\\nYour \$org MISP support team" - $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.enableEventBlacklisting" true - $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.enableOrgBlacklisting" true + $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.enableEventBlocklisting" true + $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.enableOrgBlocklisting" true $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.log_client_ip" false $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.log_auth" false $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.disableUserSelfManagement" false + $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.disable_user_login_change" false + $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.disable_user_password_change" false + $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.disable_user_add" false $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.block_event_alert" false $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.block_event_alert_tag" "no-alerts=\"true\"" $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.block_old_event_alert" false @@ -1617,6 +1625,10 @@ coreCAKE () { $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.event_view_filter_fields" "id, uuid, value, comment, type, category, Tag.name" # Force defaults to make MISP Server Settings less GREEN + $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "debug" 0 + $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Security.auth_enforced" false + $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Security.rest_client_baseurl" "" + $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Security.advanced_authkeys" false $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Security.password_policy_length" 12 $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Security.password_policy_complexity" '/^((?=.*\d)|(?=.*\W+))(?![\n])(?=.*[A-Z])(?=.*[a-z]).*$|.{16,}/' $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Security.self_registration_message" "If you would like to send us a registration request, please fill out the form below. Make sure you fill out as much information as possible in order to ease the task of the administrators." @@ -1632,7 +1644,7 @@ coreCAKE () { updateGOWNT () { # AUTH_KEY Place holder in case we need to **curl** somehing in the future # - $SUDO_WWW $RUN_MYSQL -- mysql -u $DBUSER_MISP -p$DBPASSWORD_MISP misp -e "SELECT authkey FROM users;" | tail -1 > /tmp/auth.key + $SUDO_WWW $RUN_MYSQL -- mysql -h $DBHOST -u $DBUSER_MISP -p$DBPASSWORD_MISP misp -e "SELECT authkey FROM users;" | tail -1 > /tmp/auth.key AUTH_KEY=$(cat /tmp/auth.key) rm /tmp/auth.key @@ -1731,8 +1743,8 @@ mispmodules () { fi # Install faup/gtcaca - [[ ! -d "faup" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone git://github.com/stricaud/faup.git faup; done - [[ ! -d "gtcaca" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca; done + [[ ! -d "faup" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone https://github.com/stricaud/faup.git faup; done + [[ ! -d "gtcaca" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone https://github.com/stricaud/gtcaca.git gtcaca; done sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca # Install gtcaca cd gtcaca @@ -1740,7 +1752,7 @@ mispmodules () { cd build $SUDO_CMD cmake .. && $SUDO_CMD make sudo make install - cd /usr/loca/src/faup + cd /usr/local/src/faup # Install faup $SUDO_CMD mkdir -p build cd build @@ -1757,6 +1769,7 @@ mispmodules () { $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I -r REQUIREMENTS sudo chgrp staff . $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I . + $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install censys pyfaup # Start misp-modules as a service sudo cp /usr/local/src/misp-modules/etc/systemd/system/misp-modules.service /etc/systemd/system/ @@ -1769,6 +1782,7 @@ mispmodules () { # Enable Enrichment, set better timeouts $SUDO_WWW $CAKE Admin setSetting "Plugin.Enrichment_services_enable" true $SUDO_WWW $CAKE Admin setSetting "Plugin.Enrichment_hover_enable" true + $SUDO_WWW $CAKE Admin setSetting "Plugin.Enrichment_hover_popover_only" false $SUDO_WWW $CAKE Admin setSetting "Plugin.Enrichment_timeout" 300 $SUDO_WWW $CAKE Admin setSetting "Plugin.Enrichment_hover_timeout" 150 # TODO:"Investigate why the next one fails" @@ -1903,8 +1917,8 @@ mail2misp () { sudo apt-get install cmake libcaca-dev liblua5.3-dev -y false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone https://github.com/MISP/mail_to_misp.git; done ## TODO: The below fails miserably (obviously) if faup/gtcac dirs exist, let's just make the dangerous assumption (for the sake of the installer, that they exist) - ##[[ ! -d "faup" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone git://github.com/stricaud/faup.git faup; done - ##[[ ! -d "gtcaca" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca; done + ##[[ ! -d "faup" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone https://github.com/stricaud/faup.git faup; done + ##[[ ! -d "gtcaca" ]] && false; while [[ $? -ne 0 ]]; do $SUDO_CMD git clone https://github.com/stricaud/gtcaca.git gtcaca; done sudo chown -R ${MISP_USER}:${MISP_USER} faup mail_to_misp gtcaca cd gtcaca $SUDO_CMD mkdir -p build @@ -2172,8 +2186,8 @@ installCoreRHEL () { # BROKEN: This needs to be tested on RHEL/CentOS ##sudo apt-get install cmake libcaca-dev liblua5.3-dev -y cd /tmp - [[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup - [[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca + [[ ! -d "faup" ]] && $SUDO_CMD git clone https://github.com/stricaud/faup.git faup + [[ ! -d "gtcaca" ]] && $SUDO_CMD git clone https://github.com/stricaud/gtcaca.git gtcaca sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca cd gtcaca $SUDO_CMD mkdir -p build @@ -2296,12 +2310,12 @@ EOF sudo systemctl restart rh-mariadb102-mariadb - scl enable rh-mariadb102 "mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e 'CREATE DATABASE $DBNAME;'" - scl enable rh-mariadb102 "mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e \"GRANT USAGE on *.* to $DBUSER_MISP@localhost IDENTIFIED by '$DBPASSWORD_MISP';\"" - scl enable rh-mariadb102 "mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e \"GRANT ALL PRIVILEGES on $DBNAME.* to '$DBUSER_MISP'@'localhost';\"" - scl enable rh-mariadb102 "mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e 'FLUSH PRIVILEGES;'" + scl enable rh-mariadb102 "mysql -h $DBHOST -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e 'CREATE DATABASE $DBNAME;'" + scl enable rh-mariadb102 "mysql -h $DBHOST -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e \"GRANT USAGE on *.* to $DBUSER_MISP@localhost IDENTIFIED by '$DBPASSWORD_MISP';\"" + scl enable rh-mariadb102 "mysql -h $DBHOST -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e \"GRANT ALL PRIVILEGES on $DBNAME.* to '$DBUSER_MISP'@'localhost';\"" + scl enable rh-mariadb102 "mysql -h $DBHOST -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e 'FLUSH PRIVILEGES;'" - $SUDO_WWW cat $PATH_TO_MISP/INSTALL/MYSQL.sql | sudo scl enable rh-mariadb102 "mysql -u $DBUSER_MISP -p$DBPASSWORD_MISP $DBNAME" + $SUDO_WWW cat $PATH_TO_MISP/INSTALL/MYSQL.sql | sudo scl enable rh-mariadb102 "mysql -h $DBHOST -u $DBUSER_MISP -p$DBPASSWORD_MISP $DBNAME" } apacheConfig_RHEL () { @@ -2529,6 +2543,7 @@ mispmodulesRHEL () { # pip install $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U -I -r REQUIREMENTS $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U . + $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install pyfaup censys sudo yum install rubygem-rouge rubygem-asciidoctor zbar-devel opencv-devel -y echo "[Unit] @@ -2770,10 +2785,10 @@ installSupported () { # Install PHP 7.2 Dependencies - functionLocation('INSTALL.ubuntu1804.md') [[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp72 elif [[ "$PHP_VER" == 7.3 ]]; then - # Install PHP 7.4 Dependencies - functionLocation('INSTALL.ubuntu2004.md') + # Install PHP 7.3 Dependencies - functionLocation('generic/supportFunctions.md') [[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp73 elif [[ "$PHP_VER" == 7.4 ]]; then - # Install PHP 7.3 Dependencies - functionLocation('generic/supportFunctions.md') + # Install PHP 7.4 Dependencies - functionLocation('INSTALL.ubuntu2004.md') [[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp74 elif [[ "$PHP_VER" == 7.0 ]]; then # Install PHP 7.0 Dependencies - functionLocation('generic/supportFunctions.md') @@ -2875,7 +2890,7 @@ installSupported () { # Main Kali Install function installMISPonKali () { - # Kali might have a bug on installs where libc6 is not up to date, this forces bash and libc to update - functionLocation('') + # Kali might have a bug on installs where libc6 is not up to date, this forces bash and libc to update - functionLocation('generic/supportFunctions.md') kaliUpgrade # Set locale if not set - functionLocation('generic/supportFunctions.md') @@ -2884,8 +2899,8 @@ installMISPonKali () { # Set Base URL - functionLocation('generic/supportFunctions.md') setBaseURL - # Install PHP 7.3 Dependencies - functionLocation('generic/supportFunctions.md') - installDepsPhp73 + # Install PHP 7.4 Dependencies - functionLocation('INSTALL.ubuntu2004.md') + installDepsPhp74 # Set custom Kali only variables and tweaks space @@ -2903,12 +2918,12 @@ installMISPonKali () { installCoreDeps debug "Enabling redis and gnupg modules" - sudo phpenmod -v 7.3 redis - sudo phpenmod -v 7.3 gnupg + sudo phpenmod -v 7.4 redis + sudo phpenmod -v 7.4 gnupg debug "Apache2 ops: dismod: status - dissite: 000-default enmod: ssl rewrite headers php7.3 ensite: default-ssl" sudo a2dismod status - sudo a2enmod ssl rewrite headers php7.3 + sudo a2enmod ssl rewrite headers php7.4 sudo a2dissite 000-default sudo a2ensite default-ssl @@ -3011,26 +3026,18 @@ installMISPonKali () { debug "Setting up database" if [[ ! -e /var/lib/mysql/misp/users.ibd ]]; then - echo " - set timeout 10 - spawn sudo mysql_secure_installation - expect \"Enter current password for root (enter for none):\" - send -- \"\r\" - expect \"Set root password?\" - send -- \"y\r\" - expect \"New password:\" - send -- \"${DBPASSWORD_ADMIN}\r\" - expect \"Re-enter new password:\" - send -- \"${DBPASSWORD_ADMIN}\r\" - expect \"Remove anonymous users?\" - send -- \"y\r\" - expect \"Disallow root login remotely?\" - send -- \"y\r\" - expect \"Remove test database and access to it?\" - send -- \"y\r\" - expect \"Reload privilege tables now?\" - send -- \"y\r\" - expect eof" | expect -f - + # Kill the anonymous users + sudo mysql -h $DBHOST -e "DROP USER IF EXISTS ''@'localhost'" + # Because our hostname varies we'll use some Bash magic here. + sudo mysql -h $DBHOST -e "DROP USER IF EXISTS ''@'$(hostname)'" + # Kill off the demo database + sudo mysql -h $DBHOST -e "DROP DATABASE IF EXISTS test" + # No root remote logins + sudo mysql -h $DBHOST -e "DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1')" + # Make sure that NOBODY can access the server without a password + sudo mysqladmin -h $DBHOST -u "${DBUSER_ADMIN}" password "${DBPASSWORD_ADMIN}" + # Make our changes take effect + sudo mysql -h $DBHOST -e "FLUSH PRIVILEGES" sudo mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "CREATE DATABASE $DBNAME;" sudo mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "GRANT USAGE ON *.* TO $DBUSER_MISP@localhost IDENTIFIED BY '$DBPASSWORD_MISP';" @@ -3281,9 +3288,6 @@ x86_64-debian-stretch x86_64-debian-buster x86_64-ubuntu-bionic x86_64-ubuntu-focal -x86_64-kali-2020.1 -x86_64-kali-2020.2 -x86_64-kali-2020.3 x86_64-kali-2020.4 armv6l-raspbian-stretch armv7l-raspbian-stretch diff --git a/INSTALL/INSTALL.sh.sfv b/INSTALL/INSTALL.sh.sfv index 86cb07058..632ff3b5b 100644 --- a/INSTALL/INSTALL.sh.sfv +++ b/INSTALL/INSTALL.sh.sfv @@ -1,5 +1,5 @@ -; Generated by RHash v1.3.9 on 2020-12-01 at 09:56.39 +; Generated by RHash v1.3.9 on 2021-02-03 at 15:13.46 ; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/ ; -; 136271 09:56.39 2020-12-01 INSTALL.sh -INSTALL.sh 3EBA91CFD1E3B5F87417A2943BBACD91EBE15A3C FE4C15F83C1BC84CF2099DD4F71382B0726EF620EA6740C8587FF6797067281B 37E30E0D3C58745F0E4F82AA7770D69C30750B809DDA210AA371E6557C07004972DFBE44775B31192171E56DA0641292 376336E61DF47AFA4DC701ED58658092580A7A2F501DD63994066C919B3E90E351631A33761887373529D88BC37029CE135942D15BE3BBB07D004A47D61A6C4C +; 137499 15:13.46 2021-02-03 INSTALL.sh +INSTALL.sh 5645164D7C2701EC0E0FF7D33CD8263D41B27947 803115D518C0EF187B041B942B0298CEAF329F4ADD07DB405B508D6E7ABB5D45 6B8019972E761EDC6E8CD1335E8B280376315DD762F6C98AE36C149D7960A0FC5D3C6E3F228DCB2949BDCB747942B557 ABE5D6541D23895E863BAB8BE306E54058897823EA703E6A8A225097ED8CA24F365441E8E98851576E5289B34D81C351C1A1A2520A0865BFE0D37AD77D018282 diff --git a/INSTALL/INSTALL.sh.sha1 b/INSTALL/INSTALL.sh.sha1 index ec12ebe75..00583f353 100644 --- a/INSTALL/INSTALL.sh.sha1 +++ b/INSTALL/INSTALL.sh.sha1 @@ -1 +1 @@ -3eba91cfd1e3b5f87417a2943bbacd91ebe15a3c INSTALL.sh +5645164d7c2701ec0e0ff7d33cd8263d41b27947 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha256 b/INSTALL/INSTALL.sh.sha256 index 790f62fe5..0a6813d13 100644 --- a/INSTALL/INSTALL.sh.sha256 +++ b/INSTALL/INSTALL.sh.sha256 @@ -1 +1 @@ -fe4c15f83c1bc84cf2099dd4f71382b0726ef620ea6740c8587ff6797067281b INSTALL.sh +803115d518c0ef187b041b942b0298ceaf329f4add07db405b508d6e7abb5d45 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha384 b/INSTALL/INSTALL.sh.sha384 index e8167e5a1..a79546494 100644 --- a/INSTALL/INSTALL.sh.sha384 +++ b/INSTALL/INSTALL.sh.sha384 @@ -1 +1 @@ -37e30e0d3c58745f0e4f82aa7770d69c30750b809dda210aa371e6557c07004972dfbe44775b31192171e56da0641292 INSTALL.sh +6b8019972e761edc6e8cd1335e8b280376315dd762f6c98ae36c149d7960a0fc5d3c6e3f228dcb2949bdcb747942b557 INSTALL.sh diff --git a/INSTALL/INSTALL.sh.sha512 b/INSTALL/INSTALL.sh.sha512 index ef01f65d7..7ddd6a4bb 100644 --- a/INSTALL/INSTALL.sh.sha512 +++ b/INSTALL/INSTALL.sh.sha512 @@ -1 +1 @@ -376336e61df47afa4dc701ed58658092580a7a2f501dd63994066c919b3e90e351631a33761887373529d88bc37029ce135942d15be3bbb07d004a47d61a6c4c INSTALL.sh +abe5d6541d23895e863bab8be306e54058897823ea703e6a8a225097ed8ca24f365441e8e98851576e5289b34d81c351c1a1a2520a0865bfe0d37ad77d018282 INSTALL.sh diff --git a/INSTALL/INSTALL.tpl.sh b/INSTALL/INSTALL.tpl.sh index 8cebd541d..ace8d0fc3 100755 --- a/INSTALL/INSTALL.tpl.sh +++ b/INSTALL/INSTALL.tpl.sh @@ -284,10 +284,10 @@ installSupported () { # Install PHP 7.2 Dependencies - functionLocation('INSTALL.ubuntu1804.md') [[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp72 elif [[ "$PHP_VER" == 7.3 ]]; then - # Install PHP 7.4 Dependencies - functionLocation('INSTALL.ubuntu2004.md') + # Install PHP 7.3 Dependencies - functionLocation('generic/supportFunctions.md') [[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp73 elif [[ "$PHP_VER" == 7.4 ]]; then - # Install PHP 7.3 Dependencies - functionLocation('generic/supportFunctions.md') + # Install PHP 7.4 Dependencies - functionLocation('INSTALL.ubuntu2004.md') [[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp74 elif [[ "$PHP_VER" == 7.0 ]]; then # Install PHP 7.0 Dependencies - functionLocation('generic/supportFunctions.md') @@ -389,7 +389,7 @@ installSupported () { # Main Kali Install function installMISPonKali () { - # Kali might have a bug on installs where libc6 is not up to date, this forces bash and libc to update - functionLocation('') + # Kali might have a bug on installs where libc6 is not up to date, this forces bash and libc to update - functionLocation('generic/supportFunctions.md') kaliUpgrade # Set locale if not set - functionLocation('generic/supportFunctions.md') @@ -398,8 +398,8 @@ installMISPonKali () { # Set Base URL - functionLocation('generic/supportFunctions.md') setBaseURL - # Install PHP 7.3 Dependencies - functionLocation('generic/supportFunctions.md') - installDepsPhp73 + # Install PHP 7.4 Dependencies - functionLocation('INSTALL.ubuntu2004.md') + installDepsPhp74 # Set custom Kali only variables and tweaks space @@ -417,12 +417,12 @@ installMISPonKali () { installCoreDeps debug "Enabling redis and gnupg modules" - sudo phpenmod -v 7.3 redis - sudo phpenmod -v 7.3 gnupg + sudo phpenmod -v 7.4 redis + sudo phpenmod -v 7.4 gnupg debug "Apache2 ops: dismod: status - dissite: 000-default enmod: ssl rewrite headers php7.3 ensite: default-ssl" sudo a2dismod status - sudo a2enmod ssl rewrite headers php7.3 + sudo a2enmod ssl rewrite headers php7.4 sudo a2dissite 000-default sudo a2ensite default-ssl @@ -525,26 +525,18 @@ installMISPonKali () { debug "Setting up database" if [[ ! -e /var/lib/mysql/misp/users.ibd ]]; then - echo " - set timeout 10 - spawn sudo mysql_secure_installation - expect \"Enter current password for root (enter for none):\" - send -- \"\r\" - expect \"Set root password?\" - send -- \"y\r\" - expect \"New password:\" - send -- \"${DBPASSWORD_ADMIN}\r\" - expect \"Re-enter new password:\" - send -- \"${DBPASSWORD_ADMIN}\r\" - expect \"Remove anonymous users?\" - send -- \"y\r\" - expect \"Disallow root login remotely?\" - send -- \"y\r\" - expect \"Remove test database and access to it?\" - send -- \"y\r\" - expect \"Reload privilege tables now?\" - send -- \"y\r\" - expect eof" | expect -f - + # Kill the anonymous users + sudo mysql -h $DBHOST -e "DROP USER IF EXISTS ''@'localhost'" + # Because our hostname varies we'll use some Bash magic here. + sudo mysql -h $DBHOST -e "DROP USER IF EXISTS ''@'$(hostname)'" + # Kill off the demo database + sudo mysql -h $DBHOST -e "DROP DATABASE IF EXISTS test" + # No root remote logins + sudo mysql -h $DBHOST -e "DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1')" + # Make sure that NOBODY can access the server without a password + sudo mysqladmin -h $DBHOST -u "${DBUSER_ADMIN}" password "${DBPASSWORD_ADMIN}" + # Make our changes take effect + sudo mysql -h $DBHOST -e "FLUSH PRIVILEGES" sudo mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "CREATE DATABASE $DBNAME;" sudo mysql -u $DBUSER_ADMIN -p$DBPASSWORD_ADMIN -e "GRANT USAGE ON *.* TO $DBUSER_MISP@localhost IDENTIFIED BY '$DBPASSWORD_MISP';" @@ -795,9 +787,6 @@ x86_64-debian-stretch x86_64-debian-buster x86_64-ubuntu-bionic x86_64-ubuntu-focal -x86_64-kali-2020.1 -x86_64-kali-2020.2 -x86_64-kali-2020.3 x86_64-kali-2020.4 armv6l-raspbian-stretch armv7l-raspbian-stretch diff --git a/INSTALL/README.md b/INSTALL/README.md index eecf5cd20..3b7c38cb9 100644 --- a/INSTALL/README.md +++ b/INSTALL/README.md @@ -1,4 +1,4 @@ -# Main INSTALL Documentation for the MISP Project. +# INSTALL Documentation for the MISP Project. To have a more web friendly view please visit the mkdocs generated gh-pages site [here](https://misp.github.io/MISP/) @@ -6,21 +6,185 @@ The text files in this folder are symlink to ../docs - Which is the actual sourc Currently the following install guides are being tested on a regular basis: ``` -INSTALL.kali.txt -INSTALL.ubuntu1804.txt +INSTALL.ubuntu1804.md +INSTALL.ubuntu2004.md +INSTALL.kali.md +INSTALL.rhel7.md +INSTALL.rhel8.md ``` -A folder of interest might be 'old'. In old you will find previous INSTALL guides. - Files prefixed with 'CONFIG.' are CONFIGuration guides and not full blown INSTALL guides. -UPDATE.txt give you a brief overview on how to update MISP to the latest version, as well as some other core dependencies that can be updated. +UPDATE.md gives you a brief overview on how to update MISP to the latest version, as well as some other core dependencies that can be updated. Install guides with the 'x' prefix, are marked as Experimental. The following are tested on a semi-regular basis: ``` -xINSTALL.centos7.txt -xINSTALL.debian_testing.txt -xINSTALL.Arch.txt +xINSTALL.centos7.md +xINSTALL.debian10.md +``` + +# INSTALL.sh hacking + +First of all, please read the *INSTALL.sh* script. Running a random piece of shell script that randomly invokes sudo left right and center is dangerous. (Without a sword) + +Now read *INSTALL.tpl.sh*. This is the generator for *INSTALL.sh*. + +If for example you want to modify *INSTALL.sh*, NEVER EVER touch *INSTALL.sh*. This will break the checksum and I will be very, very angry. + +*INSTALL.tpl.sh* will source the various Markdown files and generate the main installer. Meaning, if changes happen they mostly happen in the .md files. +The advantage being that when the manual documentation is up to date the installer is up to date. + + +There are 2 scenarios here. + +1. There is an issue or improvement to be made in *INSTALL.ubuntu2004.md* for example. +2. A core *INSTALL.sh* issue or improvement needs to be done. + +You will need *xsnippet* that extracts bits of shell code from the .md files: + +```bash +mkdir -p ~/bin +git clone https://github.com/SteveClement/xsnippet.git +cd xsnippet; cp xsnippet ~/bin/ +export PATH="$PATH:~/bin" # By now you are aware that this needs to be in your $PATH, aren't you. #PAAF +``` + +You need *rhash* too: +```bash +sudo apt install rhash +``` + +Now you are ready. + +To test if you are really ready, do the following: + +``` +git clone https://github.com/MISP/MISP.git +cd MISP/INSTALL ; ./INSTALL.tpl.sh +``` + +The only file that should have been changed is: *INSTALL.sh.sfv* +And nothing on *stdout* should have been displayed, and the exit code would have been obviously 0. + +## Scenario 1 + +The easiest scenario. Everythin between *# GitHub version - Travis - + CI Action + Gitter diff --git a/VERSION.json b/VERSION.json index 9210fc980..e981bf379 100644 --- a/VERSION.json +++ b/VERSION.json @@ -1 +1 @@ -{"major":2, "minor":4, "hotfix":135} +{"major":2, "minor":4, "hotfix":137} diff --git a/app/Config/config.default.php b/app/Config/config.default.php index c1f944dba..ab3688427 100644 --- a/app/Config/config.default.php +++ b/app/Config/config.default.php @@ -6,6 +6,7 @@ $config = array( 'level' => 'medium', 'salt' => '', 'cipherSeed' => '', + 'require_password_confirmation' => true //'auth'=>array('CertAuth.Certificate'), // additional authentication methods //'auth'=>array('ShibbAuth.ApacheShibb'), ), diff --git a/app/Console/Command/AdminShell.php b/app/Console/Command/AdminShell.php index e77575d56..c6c9a757d 100644 --- a/app/Console/Command/AdminShell.php +++ b/app/Console/Command/AdminShell.php @@ -214,19 +214,24 @@ class AdminShell extends AppShell if (empty($user)) { echo 'User with ID: ' . $userId . ' not found' . PHP_EOL; $result = $this->ObjectTemplate->update(); - if ($result) { - echo 'Object templates updated' . PHP_EOL; - } else { - echo 'Could not update object templates' . PHP_EOL; - } } else { $result = $this->ObjectTemplate->update($user, false,false); - if ($result) { - echo 'Object templates updated' . PHP_EOL; - } else { - echo 'Could not update object templates' . PHP_EOL; + } + + $successes = count(!empty($result['success']) ? $result['success'] : []); + $fails = count(!empty($result['fails']) ? $result['fails'] : []); + $message = ''; + if ($successes == 0 && $fails == 0) { + $message = __('All object templates are up to date already.'); + } elseif ($successes == 0 && $fails > 0) { + $message = __('Could not update any of the object templates.'); + } elseif ($successes > 0 ) { + $message = __('Successfully updated %s object templates.', $successes); + if ($fails != 0) { + $message .= __(' However, could not update %s object templates.', $fails); } } + echo $message . PHP_EOL; } } @@ -655,4 +660,15 @@ class AdminShell extends AppShell echo $result . PHP_EOL; } } + + public function cleanExcludedCorrelations() + { + $jobId = $this->args[0]; + $this->CorrelationExclusion = ClassRegistry::init('CorrelationExclusion'); + $this->CorrelationExclusion->clean($jobId); + $this->Job->id = $jobId; + $this->Job->saveField('progress', 100); + $this->Job->saveField('message', 'Job done.'); + $this->Job->saveField('status', 4); + } } diff --git a/app/Console/Command/EventShell.php b/app/Console/Command/EventShell.php index 915183adb..69d7c5bc0 100644 --- a/app/Console/Command/EventShell.php +++ b/app/Console/Command/EventShell.php @@ -13,6 +13,70 @@ class EventShell extends AppShell public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Allowedlist', 'Server', 'Organisation'); public $tasks = array('ConfigLoad'); + public function getOptionParser() + { + $parser = parent::getOptionParser(); + $parser->addSubcommand('import', array( + 'help' => __('Import event from file into MISP.'), + 'parser' => array( + 'arguments' => array( + 'user_id' => ['help' => __('User ID that will owner of uploaded event.'), 'required' => true], + 'file' => ['help' => __('Path to JSON MISP file, can be gzipped or bz2 compressed.'), 'required' => true], + ), + 'options' => [ + 'take-ownership' => ['boolean' => true], + 'publish' => ['boolean' => true], + ], + ) + )); + return $parser; + } + + public function import() + { + list($userId, $path) = $this->args; + $user = $this->User->getAuthUser($userId); + if (empty($user)) { + $this->error("User with ID $userId does not exists."); + } + + if (!file_exists($path)) { + $this->error("File '$path' does not exists."); + } + if (!is_readable($path)) { + $this->error("File '$path' is not readable."); + } + + $pathInfo = pathinfo($path); + if ($pathInfo['extension'] === 'gz') { + $content = file_get_contents("compress.zlib://$path"); + $extension = pathinfo($pathInfo['filename'], PATHINFO_EXTENSION); + } else if ($pathInfo['extension'] === 'bz2') { + $content = file_get_contents("compress.bzip2://$path"); + $extension = pathinfo($pathInfo['filename'], PATHINFO_EXTENSION); + } else { + $content = file_get_contents($path); + $extension = $pathInfo['extension']; + } + + if ($content === false) { + $this->error("Could not read content from '$path'."); + } + + $isXml = $extension === 'xml'; + $takeOwnership = $this->param('take_ownership'); + $publish = $this->param('publish'); + $results = $this->Event->addMISPExportFile($user, $content, $isXml, $takeOwnership, $publish); + + foreach ($results as $result) { + if (is_numeric($result['result'])) { + $this->out("Event #{$result['id']}: {$result['info']} imported."); + } else { + $this->out("Could not import event because of validation errors: " . json_encode($result['validationIssues'])); + } + } + } + public function doPublish() { $this->ConfigLoad->execute(); diff --git a/app/Console/Command/StatisticsShell.php b/app/Console/Command/StatisticsShell.php index 5a476c962..8b943825e 100644 --- a/app/Console/Command/StatisticsShell.php +++ b/app/Console/Command/StatisticsShell.php @@ -1,7 +1,7 @@ out(json_encode([ + 'events' => $this->Event->find('count'), + 'attributes' => $this->Event->Attribute->find('count', + ['conditions' => ['Attribute.deleted' => 0], 'recursive' => -1] + ), + 'objects' => $this->Event->Object->find('count', + ['conditions' => ['Object.deleted' => 0], 'recursive' => -1] + ), + 'correlations' => $this->Correlation->find('count') / 2, + 'users' => $this->User->find('count', + ['conditions' => ['User.disabled' => 0], 'recursive' => -1] + ), + 'local_organisations' => $this->Organisation->find('count', + ['conditions' => ['Organisation.local' => 1], 'recursive' => -1] + ), + 'external_organisations' => $this->Organisation->find('count', + ['conditions' => ['Organisation.local' => 0], 'recursive' => -1] + ) + ], JSON_PRETTY_PRINT)); + } } diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index f8b72956b..b80557c4b 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -1,27 +1,4 @@ Auth->loginRedirect = Configure::read('MISP.baseurl') . '/users/routeafterlogin'; + $this->_setupBaseurl(); + $this->Auth->loginRedirect = $this->baseurl. '/users/routeafterlogin'; $customLogout = Configure::read('Plugin.CustomAuth_custom_logout'); - if ($customLogout) { - $this->Auth->logoutRedirect = $customLogout; - } else { - $this->Auth->logoutRedirect = Configure::read('MISP.baseurl') . '/users/login'; - } + $this->Auth->logoutRedirect = $customLogout ?: ($this->baseurl . '/users/login'); + $this->__sessionMassage(); if (Configure::read('Security.allow_cors')) { // Add CORS headers @@ -152,8 +128,8 @@ class AppController extends Controller $this->sql_dump = intval($this->params['named']['sql']); } - $this->_setupDatabaseConnection(); $this->_setupDebugMode(); + $this->_setupDatabaseConnection(); $this->set('ajax', $this->request->is('ajax')); $this->set('queryVersion', $this->__queryVersion); @@ -166,17 +142,19 @@ class AppController extends Controller Configure::write('Config.language', 'eng'); } - //if fresh installation (salt empty) generate a new salt + // For fresh installation (salt empty) generate a new salt if (!Configure::read('Security.salt')) { $this->loadModel('Server'); $this->Server->serverSettingsSaveValue('Security.salt', $this->User->generateRandomPassword(32)); } + // Check if the instance has a UUID, if not assign one. if (!Configure::read('MISP.uuid')) { $this->loadModel('Server'); $this->Server->serverSettingsSaveValue('MISP.uuid', CakeText::uuid()); } - // check if Apache provides kerberos authentication data + + // Check if Apache provides kerberos authentication data $authUserFields = $this->User->describeAuthFields(); $envvar = Configure::read('ApacheSecureAuth.apacheEnv'); if ($envvar && isset($_SERVER[$envvar])) { @@ -196,10 +174,9 @@ class AppController extends Controller } Configure::write('CurrentController', $this->params['controller']); Configure::write('CurrentAction', $this->params['action']); - $versionArray = $this->{$this->modelClass}->checkMISPVersion(); + $versionArray = $this->User->checkMISPVersion(); $this->mispVersion = implode('.', array_values($versionArray)); $this->Security->blackHoleCallback = 'blackHole'; - $this->_setupBaseurl(); // send users away that are using ancient versions of IE // Make sure to update this if IE 20 comes out :) @@ -213,6 +190,27 @@ class AppController extends Controller $userLoggedIn = $this->__customAuthentication($_SERVER); } if ($this->_isRest()) { + $jsonDecode = function ($dataToDecode) { + if (empty($dataToDecode)) { + return null; + } + try { + if (defined('JSON_THROW_ON_ERROR')) { + // JSON_THROW_ON_ERROR is supported since PHP 7.3 + return json_decode($dataToDecode, true, 512, JSON_THROW_ON_ERROR); + } else { + $decoded = json_decode($dataToDecode, true); + if ($decoded === null) { + throw new UnexpectedValueException('Could not parse JSON: ' . json_last_error_msg(), json_last_error()); + } + return $decoded; + } + } catch (Exception $e) { + throw new HttpException('Invalid JSON input. Make sure that the JSON input is a correctly formatted JSON string. This request has been blocked to avoid an unfiltered request.', 405, $e); + } + }; + // Throw exception if JSON in request is invalid. Default CakePHP behaviour would just ignore that error. + $this->RequestHandler->addInputType('json', [$jsonDecode]); $this->Security->unlockedActions = array($this->action); } @@ -227,232 +225,68 @@ class AppController extends Controller // REST authentication if ($this->_isRest() || $this->_isAutomation()) { // disable CSRF for REST access - if (array_key_exists('Security', $this->components)) { + if (isset($this->components['Security'])) { $this->Security->csrfCheck = false; } - // If enabled, allow passing the API key via a named parameter (for crappy legacy systems only) - $namedParamAuthkey = false; - if (Configure::read('Security.allow_unsafe_apikey_named_param') && !empty($this->params['named']['apikey'])) { - $namedParamAuthkey = $this->params['named']['apikey']; - } - // Authenticate user with authkey in Authorization HTTP header - if (!empty($_SERVER['HTTP_AUTHORIZATION']) || !empty($namedParamAuthkey)) { - $found_misp_auth_key = false; - $authentication = explode(',', $_SERVER['HTTP_AUTHORIZATION']); - if (!empty($namedParamAuthkey)) { - $authentication[] = $namedParamAuthkey; - } - $user = false; - foreach ($authentication as $auth_key) { - if (preg_match('/^[a-zA-Z0-9]{40}$/', trim($auth_key))) { - $found_misp_auth_key = true; - $temp = $this->checkAuthUser(trim($auth_key)); - if ($temp) { - $user['User'] = $temp; - } - } - } - if ($found_misp_auth_key) { - if ($user) { - unset($user['User']['gpgkey']); - unset($user['User']['certif_public']); - // User found in the db, add the user info to the session - if (Configure::read('MISP.log_auth')) { - $this->Log = ClassRegistry::init('Log'); - $this->Log->create(); - $log = array( - 'org' => $user['User']['Organisation']['name'], - 'model' => 'User', - 'model_id' => $user['User']['id'], - 'email' => $user['User']['email'], - 'action' => 'auth', - 'title' => 'Successful authentication using API key', - 'change' => 'HTTP method: ' . $_SERVER['REQUEST_METHOD'] . PHP_EOL . 'Target: ' . $this->here, - ); - $this->Log->save($log); - } - $this->Session->renew(); - $this->Session->write(AuthComponent::$sessionKey, $user['User']); - $this->isApiAuthed = true; - } else { - // User not authenticated correctly - // reset the session information - $redis = $this->{$this->modelClass}->setupRedis(); - if ($redis && !$redis->exists('misp:auth_fail_throttling:' . trim($auth_key))) { - $redis->set('misp:auth_fail_throttling:' . trim($auth_key), 1); - $redis->expire('misp:auth_fail_throttling:' . trim($auth_key), 3600); - $this->Session->destroy(); - $this->Log = ClassRegistry::init('Log'); - $this->Log->create(); - $log = array( - 'org' => 'SYSTEM', - 'model' => 'User', - 'model_id' => 0, - 'email' => 'SYSTEM', - 'action' => 'auth_fail', - 'title' => 'Failed authentication using API key (' . trim($auth_key) . ')', - 'change' => null, - ); - $this->Log->save($log); - } - throw new ForbiddenException('Authentication failed. Please make sure you pass the API key of an API enabled user along in the Authorization header.'); - } - unset($user); - } - } - if ($this->Auth->user() == null) { + if ($this->__loginByAuthKey() === false || $this->Auth->user() === null) { throw new ForbiddenException('Authentication failed. Please make sure you pass the API key of an API enabled user along in the Authorization header.'); } } elseif (!$this->Session->read(AuthComponent::$sessionKey)) { $this->_loadAuthenticationPlugins(); } } - $this->set('externalAuthUser', $userLoggedIn); - // user must accept terms - // - // grab the base path from our base url for use in the following checks - $base_dir = parse_url($this->baseurl, PHP_URL_PATH); - // if MISP is running out of the web root already, just set this variable to blank so we don't wind up with '//' in the following if statements - if ($base_dir == '/') { - $base_dir = ''; - } + $user = $this->Auth->user(); + if ($user) { + Configure::write('CurrentUserId', $user['id']); + $this->__logAccess($user); - if ($this->Auth->user()) { - Configure::write('CurrentUserId', $this->Auth->user('id')); - $this->User->setMonitoring($this->Auth->user()); - if (Configure::read('MISP.log_user_ips')) { - $redis = $this->{$this->modelClass}->setupRedis(); - if ($redis) { - $redis->set('misp:ip_user:' . trim($_SERVER['REMOTE_ADDR']), $this->Auth->user('id')); - $redis->expire('misp:ip_user:' . trim($_SERVER['REMOTE_ADDR']), 60*60*24*30); - $redis->sadd('misp:user_ip:' . $this->Auth->user('id'), trim($_SERVER['REMOTE_ADDR'])); + // Try to run updates + if ($user['Role']['perm_site_admin'] || (Configure::read('MISP.live') && !$this->_isRest())) { + $this->User->runUpdates(); + } + + // Put username to response header for webserver or proxy logging + if (Configure::read('Security.username_in_response_header')) { + $headerValue = $user['email']; + if (isset($user['logged_by_authkey']) && $user['logged_by_authkey']) { + $headerValue .= isset($user['authkey_id']) ? "/API/{$user['authkey_id']}" : '/API/default'; } + $this->response->header('X-Username', $headerValue); + $this->RestResponse->setHeader('X-Username', $headerValue); } - // update script - if ($this->Auth->user('Role')['perm_site_admin'] || (Configure::read('MISP.live') && !$this->_isRest())) { - $this->{$this->modelClass}->runUpdates(); + + if (!$this->__verifyUser($user)) { + $this->_stop(); // just for sure } - $user = $this->Auth->user(); - if (!isset($user['force_logout']) || $user['force_logout']) { - $this->loadModel('User'); - $this->User->id = $this->Auth->user('id'); - $this->User->saveField('force_logout', false); + + if (isset($user['logged_by_authkey']) && $user['logged_by_authkey'] && !($this->_isRest() || $this->_isAutomation())) { + throw new ForbiddenException("When user is authenticated by authkey, just REST request can be processed"); } - if ($this->Auth->user('disabled')) { - $this->Log = ClassRegistry::init('Log'); - $this->Log->create(); - $log = array( - 'org' => $this->Auth->user('Organisation')['name'], - 'model' => 'User', - 'model_id' => $this->Auth->user('id'), - 'email' => $this->Auth->user('email'), - 'action' => 'auth_fail', - 'title' => 'Login attempt by disabled user.', - 'change' => null, - ); - $this->Log->save($log); - $this->Auth->logout(); - if ($this->_isRest()) { - throw new ForbiddenException('Authentication failed. Your user account has been disabled.'); - } else { - $this->Flash->error('Your user account has been disabled.', array('key' => 'error')); - $this->_redirectToLogin(); - } + + // Put token expiration time to response header that can be processed by automation tool + if (isset($user['authkey_expiration']) && $user['authkey_expiration']) { + $expiration = date('c', $user['authkey_expiration']); + $this->response->header('X-Auth-Key-Expiration', $expiration); + $this->RestResponse->setHeader('X-Auth-Key-Expiration', $expiration); } + $this->set('default_memory_limit', ini_get('memory_limit')); - if (isset($this->Auth->user('Role')['memory_limit'])) { - if ($this->Auth->user('Role')['memory_limit'] !== '') { - ini_set('memory_limit', $this->Auth->user('Role')['memory_limit']); - } + if (isset($user['Role']['memory_limit']) && $user['Role']['memory_limit'] !== '') { + ini_set('memory_limit', $user['Role']['memory_limit']); } $this->set('default_max_execution_time', ini_get('max_execution_time')); - if (isset($this->Auth->user('Role')['max_execution_time'])) { - if ($this->Auth->user('Role')['max_execution_time'] !== '') { - ini_set('max_execution_time', $this->Auth->user('Role')['max_execution_time']); - } + if (isset($user['Role']['max_execution_time']) && $user['Role']['max_execution_time'] !== '') { + ini_set('max_execution_time', $user['Role']['max_execution_time']); } - } else { - $pre_auth_actions = array('login', 'register'); - if (!empty(Configure::read('Security.email_otp_enabled'))) { - $pre_auth_actions[] = 'email_otp'; - } - if ($this->params['controller'] !== 'users' || !in_array($this->params['action'], $pre_auth_actions)) { - if (!$this->request->is('ajax')) { - $this->Session->write('pre_login_requested_url', $this->here); - } - $this->_redirectToLogin(); - } - } - // check if MISP is live - if ($this->Auth->user() && !Configure::read('MISP.live')) { - $role = $this->getActions(); - if (!$role['perm_site_admin']) { - $message = Configure::read('MISP.maintenance_message'); - if (empty($message)) { - $this->loadModel('Server'); - $message = $this->Server->serverSettings['MISP']['maintenance_message']['value']; - } - if (strpos($message, '$email') && Configure::read('MISP.email')) { - $email = Configure::read('MISP.email'); - $message = str_replace('$email', $email, $message); - } - $this->Flash->info($message); - $this->Auth->logout(); - throw new MethodNotAllowedException($message);//todo this should pb be removed? - } else { - $this->Flash->error(__('Warning: MISP is currently disabled for all users. Enable it in Server Settings (Administration -> Server Settings -> MISP tab -> live). An update might also be in progress, you can see the progress in ') , array('params' => array('url' => $this->baseurl . '/servers/updateProgress/', 'urlName' => __('Update Progress')), 'clear' => 1)); - } - } - if ($this->Session->check(AuthComponent::$sessionKey)) { - if ($this->action !== 'checkIfLoggedIn' || $this->request->params['controller'] !== 'users') { - $this->User->id = $this->Auth->user('id'); - if (!$this->User->exists()) { - $message = __('Something went wrong. Your user account that you are authenticated with doesn\'t exist anymore.'); - if ($this->_isRest) { - echo $this->RestResponse->throwException( - 401, - $message - ); - } else { - $this->Flash->info($message); - } - $this->Auth->logout(); - $this->_redirectToLogin(); - } - if (!empty(Configure::read('MISP.terms_file')) && !$this->Auth->user('termsaccepted') && (!in_array($this->request->here, array($base_dir.'/users/terms', $base_dir.'/users/logout', $base_dir.'/users/login', $base_dir.'/users/downloadTerms')))) { - //if ($this->_isRest()) throw new MethodNotAllowedException('You have not accepted the terms of use yet, please log in via the web interface and accept them.'); - if (!$this->_isRest()) { - $this->redirect(array('controller' => 'users', 'action' => 'terms', 'admin' => false)); - } - } elseif ($this->Auth->user('change_pw') && (!in_array($this->request->here, array($base_dir.'/users/terms', $base_dir.'/users/change_pw', $base_dir.'/users/logout', $base_dir.'/users/login')))) { - //if ($this->_isRest()) throw new MethodNotAllowedException('Your user account is expecting a password change, please log in via the web interface and change it before proceeding.'); - if (!$this->_isRest()) { - $this->redirect(array('controller' => 'users', 'action' => 'change_pw', 'admin' => false)); - } - } elseif (!$this->_isRest() && !($this->params['controller'] == 'news' && $this->params['action'] == 'index') && (!in_array($this->request->here, array($base_dir.'/users/terms', $base_dir.'/users/change_pw', $base_dir.'/users/logout', $base_dir.'/users/login')))) { - $newsread = $this->User->field('newsread', array('User.id' => $this->Auth->user('id'))); - $this->loadModel('News'); - $latest_news = $this->News->field('date_created', array(), 'date_created DESC'); - if ($latest_news && $newsread < $latest_news) { - $this->redirect(array('controller' => 'news', 'action' => 'index', 'admin' => false)); - } - } - } - } - unset($base_dir); - // We don't want to run these role checks before the user is logged in, but we want them available for every view once the user is logged on - // instead of using checkAction(), like we normally do from controllers when trying to find out about a permission flag, we can use getActions() - // getActions returns all the flags in a single SQL query - if ($this->Auth->user()) { - $this->set('mispVersion', implode('.', array($versionArray['major'], $versionArray['minor'], 0))); + $this->set('mispVersion', "{$versionArray['major']}.{$versionArray['minor']}.0"); $this->set('mispVersionFull', $this->mispVersion); - $role = $this->getActions(); - $this->set('me', $this->Auth->user()); + $this->set('me', $user); + $role = $user['Role']; $this->set('isAdmin', $role['perm_admin']); $this->set('isSiteAdmin', $role['perm_site_admin']); - $this->set('hostOrgUser', $this->Auth->user('org_id') == Configure::read('MISP.host_org_id')); + $this->set('hostOrgUser', $user['org_id'] == Configure::read('MISP.host_org_id')); $this->set('isAclAdd', $role['perm_add']); $this->set('isAclModify', $role['perm_modify']); $this->set('isAclModifyOrg', $role['perm_modify_org']); @@ -475,48 +309,26 @@ class AppController extends Controller $this->set('aclComponent', $this->ACL); $this->userRole = $role; - $this->set('loggedInUserName', $this->__convertEmailToName($this->Auth->user('email'))); + $this->set('loggedInUserName', $this->__convertEmailToName($user['email'])); + $this->__accessMonitor($user); - if ( - Configure::read('MISP.log_paranoid') || - !empty(Configure::read('Security.monitored')) - ) { - $this->Log = ClassRegistry::init('Log'); - $this->Log->create(); - $change = 'HTTP method: ' . $_SERVER['REQUEST_METHOD'] . PHP_EOL . 'Target: ' . $this->here; - if ( - ( - $this->request->is('post') || - $this->request->is('put') - ) && - ( - !empty(Configure::read('MISP.log_paranoid_include_post_body')) || - !empty(Configure::read('Security.monitored')) - ) - ) { - $payload = $this->request->input(); - if (!empty($payload['_Token'])) { - unset($payload['_Token']); - } - $change .= PHP_EOL . 'Request body: ' . json_encode($payload); - } - $log = array( - 'org' => $this->Auth->user('Organisation')['name'], - 'model' => 'User', - 'model_id' => $this->Auth->user('id'), - 'email' => $this->Auth->user('email'), - 'action' => 'request', - 'title' => 'Paranoid log entry', - 'change' => $change, - ); - $this->Log->save($log); - } } else { + $pre_auth_actions = array('login', 'register', 'getGpgPublicKey'); + if (!empty(Configure::read('Security.email_otp_enabled'))) { + $pre_auth_actions[] = 'email_otp'; + } + if (!$this->_isControllerAction(['users' => $pre_auth_actions])) { + if (!$this->request->is('ajax')) { + $this->Session->write('pre_login_requested_url', $this->here); + } + $this->_redirectToLogin(); + } + $this->set('me', false); } if ($this->Auth->user() && $this->_isSiteAdmin()) { - if (Configure::read('Session.defaults') == 'database') { + if (Configure::read('Session.defaults') === 'database') { $db = ConnectionManager::getDataSource('default'); $sqlResult = $db->query('SELECT COUNT(id) AS session_count FROM cake_sessions WHERE expires < ' . time() . ';'); if (isset($sqlResult[0][0]['session_count']) && $sqlResult[0][0]['session_count'] > 1000) { @@ -545,28 +357,329 @@ class AppController extends Controller } } } - $this->components['RestResponse']['sql_dump'] = $this->sql_dump; // Notifications and homepage is not necessary for AJAX or REST requests if ($this->Auth->user() && !$this->_isRest() && !$this->request->is('ajax')) { if ($this->request->params['controller'] === 'users' && $this->request->params['action'] === 'dashboard') { - $notifications = $this->{$this->modelClass}->populateNotifications($this->Auth->user()); + $notifications = $this->User->populateNotifications($this->Auth->user()); } else { - $notifications = $this->{$this->modelClass}->populateNotifications($this->Auth->user(), 'fast'); + $notifications = $this->User->populateNotifications($this->Auth->user(), 'fast'); } $this->set('notifications', $notifications); - $this->loadModel('UserSetting'); - $homepage = $this->UserSetting->find('first', array( - 'recursive' => -1, - 'conditions' => array( - 'UserSetting.user_id' => $this->Auth->user('id'), - 'UserSetting.setting' => 'homepage' - ), - 'contain' => array('User.id', 'User.org_id') - )); + $homepage = $this->User->UserSetting->getValueForUser($this->Auth->user('id'), 'homepage'); if (!empty($homepage)) { - $this->set('homepage', $homepage['UserSetting']['value']); + $this->set('homepage', $homepage); + } + if (version_compare(phpversion(), '8.0') >= 0) { + $this->Flash->error(__('WARNING: MISP is currently running under PHP 8.0, which is unsupported. Background jobs will fail, so please contact your administrator to run a supported PHP version (such as 7.4)')); + } + } + } + + /** + * @return null|bool True if authkey was correct, False if incorrect and Null if not provided + * @throws Exception + */ + private function __loginByAuthKey() + { + if (Configure::read('Security.authkey_keep_session') && $this->Auth->user()) { + // Do not check authkey if session is establish and correct, just close session to allow multiple requests + session_write_close(); + return true; + } + + // If enabled, allow passing the API key via a named parameter (for crappy legacy systems only) + $namedParamAuthkey = false; + if (Configure::read('Security.allow_unsafe_apikey_named_param') && !empty($this->params['named']['apikey'])) { + $namedParamAuthkey = $this->params['named']['apikey']; + } + // Authenticate user with authkey in Authorization HTTP header + if (!empty($_SERVER['HTTP_AUTHORIZATION']) || !empty($namedParamAuthkey)) { + $foundMispAuthKey = false; + $authentication = explode(',', $_SERVER['HTTP_AUTHORIZATION']); + if (!empty($namedParamAuthkey)) { + $authentication[] = $namedParamAuthkey; + } + $user = false; + foreach ($authentication as $authKey) { + $authKey = trim($authKey); + if (preg_match('/^[a-zA-Z0-9]{40}$/', $authKey)) { + $foundMispAuthKey = true; + $temp = $this->checkAuthUser($authKey); + if ($temp) { + $user = $temp; + break; + } + } + } + if ($foundMispAuthKey) { + $authKeyToStore = substr($authKey, 0, 4) + . str_repeat('*', 32) + . substr($authKey, -4); + if ($user) { + unset($user['gpgkey']); + unset($user['certif_public']); + // User found in the db, add the user info to the session + if (Configure::read('MISP.log_auth')) { + $this->loadModel('Log'); + $this->Log->create(); + $log = array( + 'org' => $user['Organisation']['name'], + 'model' => 'User', + 'model_id' => $user['id'], + 'email' => $user['email'], + 'action' => 'auth', + 'title' => "Successful authentication using API key ($authKeyToStore)", + 'change' => 'HTTP method: ' . $_SERVER['REQUEST_METHOD'] . PHP_EOL . 'Target: ' . $this->here, + ); + $this->Log->save($log); + } + $this->Session->renew(); + $this->Session->write(AuthComponent::$sessionKey, $user); + $this->isApiAuthed = true; + return true; + } else { + // User not authenticated correctly + // reset the session information + $redis = $this->User->setupRedis(); + // Do not log every fail, but just once per hour + if ($redis && !$redis->exists('misp:auth_fail_throttling:' . $authKeyToStore)) { + $redis->setex('misp:auth_fail_throttling:' . $authKeyToStore, 3600, 1); + $this->loadModel('Log'); + $this->Log->create(); + $log = array( + 'org' => 'SYSTEM', + 'model' => 'User', + 'model_id' => 0, + 'email' => 'SYSTEM', + 'action' => 'auth_fail', + 'title' => "Failed authentication using API key ($authKeyToStore)", + 'change' => null, + ); + $this->Log->save($log); + } + $this->Session->destroy(); + } + } + return false; + } + return null; + } + + /** + * Check if: + * - user exists in database + * - is not disabled + * - need to force logout + * - accepted terms and conditions + * - must change password + * - reads latest news + * + * @param array $user + * @return bool + */ + private function __verifyUser(array $user) + { + // Skip these checks for 'checkIfLoggedIn' action to make that call fast + if ($this->_isControllerAction(['users' => ['checkIfLoggedIn']])) { + return true; + } + + // Load last user profile modification from database + $userFromDb = $this->User->find('first', [ + 'conditions' => ['id' => $user['id']], + 'recursive' => -1, + 'fields' => ['date_modified'], + ]); + + // Check if user with given ID exists + if (!$userFromDb) { + $message = __('Something went wrong. Your user account that you are authenticated with doesn\'t exist anymore.'); + if ($this->_isRest()) { + // TODO: Why not exception? + $response = $this->RestResponse->throwException(401, $message); + $response->send(); + $this->_stop(); + } else { + $this->Flash->info($message); + $this->Auth->logout(); + $this->_redirectToLogin(); + } + return false; + } + + // Check if session data contain latest changes from db + if ((int)$user['date_modified'] < (int)$userFromDb['User']['date_modified']) { + $user = $this->_refreshAuth(); // session data are old, reload from database + } + + // Check if MISP access is enabled + if (!Configure::read('MISP.live')) { + if (!$user['Role']['perm_site_admin']) { + $message = Configure::read('MISP.maintenance_message'); + if (empty($message)) { + $this->loadModel('Server'); + $message = $this->Server->serverSettings['MISP']['maintenance_message']['value']; + } + if (strpos($message, '$email') && Configure::read('MISP.email')) { + $email = Configure::read('MISP.email'); + $message = str_replace('$email', $email, $message); + } + $this->Flash->info($message); + $this->Auth->logout(); + throw new MethodNotAllowedException($message);//todo this should pb be removed? + } else { + $this->Flash->error(__('Warning: MISP is currently disabled for all users. Enable it in Server Settings (Administration -> Server Settings -> MISP tab -> live). An update might also be in progress, you can see the progress in ') , array('params' => array('url' => $this->baseurl . '/servers/updateProgress/', 'urlName' => __('Update Progress')), 'clear' => 1)); + } + } + + // Force logout doesn't make sense for API key authentication + if (!$this->isApiAuthed && $user['force_logout']) { + $this->User->id = $user['id']; + $this->User->saveField('force_logout', false); + $this->Auth->logout(); + $this->_redirectToLogin(); + return false; + } + + if ($user['disabled']) { + $this->Log = ClassRegistry::init('Log'); + $this->Log->createLogEntry($user, 'auth_fail', 'User', $user['id'], 'Login attempt by disabled user.'); + + $this->Auth->logout(); + if ($this->_isRest()) { + throw new ForbiddenException('Authentication failed. Your user account has been disabled.'); + } else { + $this->Flash->error(__('Your user account has been disabled.')); + $this->_redirectToLogin(); + } + return false; + } + + // Check if auth key is not expired. Make sense when Security.authkey_keep_session is enabled. + if (isset($user['authkey_expiration']) && $user['authkey_expiration']) { + $time = isset($_SERVER['REQUEST_TIME']) ? $_SERVER['REQUEST_TIME'] : time(); + if ($user['authkey_expiration'] < $time) { + $this->Auth->logout(); + throw new ForbiddenException('Auth key is expired'); + } + } + + $isUserRequest = !$this->_isRest() && !$this->request->is('ajax') && !$this->_isAutomation(); + // Next checks makes sense just for user direct HTTP request, so skip REST and AJAX calls + if (!$isUserRequest) { + return true; + } + + // Check if user accepted terms and conditions + if (!$user['termsaccepted'] && !empty(Configure::read('MISP.terms_file')) && !$this->_isControllerAction(['users' => ['terms', 'logout', 'login', 'downloadTerms']])) { + //if ($this->_isRest()) throw new MethodNotAllowedException('You have not accepted the terms of use yet, please log in via the web interface and accept them.'); + $this->redirect(array('controller' => 'users', 'action' => 'terms', 'admin' => false)); + return false; + } + + // Check if user must change password + if ($user['change_pw'] && !$this->_isControllerAction(['users' => ['terms', 'change_pw', 'logout', 'login']])) { + //if ($this->_isRest()) throw new MethodNotAllowedException('Your user account is expecting a password change, please log in via the web interface and change it before proceeding.'); + $this->redirect(array('controller' => 'users', 'action' => 'change_pw', 'admin' => false)); + return false; + } + + // Check if user must read news + if (!$this->_isControllerAction(['news' => ['index'], 'users' => ['terms', 'change_pw', 'login', 'logout']])) { + $this->loadModel('News'); + $latestNewsCreated = $this->News->field('date_created', array(), 'date_created DESC'); + if ($latestNewsCreated && $user['newsread'] < $latestNewsCreated) { + $this->redirect(array('controller' => 'news', 'action' => 'index', 'admin' => false)); + return false; + } + } + + return true; + } + + /** + * @param array $actionsToCheck + * @return bool + */ + private function _isControllerAction($actionsToCheck = []) + { + $controller = Inflector::variable($this->request->params['controller']); + if (!isset($actionsToCheck[$controller])) { + return false; + } + return in_array($this->action, $actionsToCheck[$controller], true); + } + + /** + * User access monitoring + * @param array $user + */ + private function __logAccess(array $user) + { + $logUserIps = Configure::read('MISP.log_user_ips'); + if (!$logUserIps) { + return; + } + + $redis = $this->User->setupRedis(); + if (!$redis) { + return; + } + + $remoteAddress = trim($_SERVER['REMOTE_ADDR']); + + $pipe = $redis->multi(Redis::PIPELINE); + // keep for 30 days + $pipe->setex('misp:ip_user:' . $remoteAddress, 60 * 60 * 24 * 30, $user['id']); + $pipe->sadd('misp:user_ip:' . $user['id'], $remoteAddress); + + // Log key usage if enabled + if (isset($user['authkey_id']) && Configure::read('MISP.log_user_ips_authkeys')) { + // Use request time if defined + $time = isset($_SERVER['REQUEST_TIME']) ? $_SERVER['REQUEST_TIME'] : time(); + $hashKey = date("Y-m-d", $time) . ":$remoteAddress"; + $pipe->hIncrBy("misp:authkey_usage:{$user['authkey_id']}", $hashKey, 1); + // delete after one year of inactivity + $pipe->expire("misp:authkey_usage:{$user['authkey_id']}", 3600 * 24 * 365); + $pipe->set("misp:authkey_last_usage:{$user['authkey_id']}", $time); + } + $pipe->exec(); + } + + /** + * @param array $user + * @throws Exception + */ + private function __accessMonitor(array $user) + { + $userMonitoringEnabled = Configure::read('Security.user_monitoring_enabled'); + if ($userMonitoringEnabled) { + $redis = $this->User->setupRedis(); + $userMonitoringEnabled = $redis && $redis->sismember('misp:monitored_users', $user['id']); + } + + if (Configure::read('MISP.log_paranoid') || $userMonitoringEnabled) { + $change = 'HTTP method: ' . $_SERVER['REQUEST_METHOD'] . PHP_EOL . 'Target: ' . $this->here; + if ( + ( + $this->request->is('post') || + $this->request->is('put') + ) && + ( + !empty(Configure::read('MISP.log_paranoid_include_post_body')) || + $userMonitoringEnabled + ) + ) { + $payload = $this->request->input(); + $change .= PHP_EOL . 'Request body: ' . $payload; + } + $this->Log = ClassRegistry::init('Log'); + try { + $this->Log->createLogEntry($user, 'request', 'User', $user['id'], 'Paranoid log entry', $change); + } catch (Exception $e) { + // When `MISP.log_skip_db_logs_completely` is enabled, Log::createLogEntry method throws exception } } } @@ -601,7 +714,7 @@ class AppController extends Controller public function afterFilter() { - if ($this->isApiAuthed && $this->_isRest() && $this->Session->started()) { + if ($this->isApiAuthed && $this->_isRest() && !Configure::read('Security.authkey_keep_session')) { $this->Session->destroy(); } } @@ -648,16 +761,17 @@ class AppController extends Controller /* * Sanitize the configured `MISP.baseurl` and expose it to the view as `baseurl`. */ - protected function _setupBaseurl() { + protected function _setupBaseurl() + { // Let us access $baseurl from all views $baseurl = Configure::read('MISP.baseurl'); - if (substr($baseurl, -1) == '/') { + if (substr($baseurl, -1) === '/') { // if the baseurl has a trailing slash, remove it. It can lead to issues with the CSRF protection $baseurl = rtrim($baseurl, '/'); $this->loadModel('Server'); $this->Server->serverSettingsSaveValue('MISP.baseurl', $baseurl); } - if (trim($baseurl) == 'http://') { + if (trim($baseurl) === 'http://') { $this->Server->serverSettingsSaveValue('MISP.baseurl', ''); } $this->baseurl = $baseurl; @@ -683,8 +797,6 @@ class AppController extends Controller throw new BadRequestException('The request has been black-holed'); } - public $userRole = null; - protected function _isRest() { return $this->IndexFilter->isRest(); @@ -692,12 +804,7 @@ class AppController extends Controller protected function _isAutomation() { - foreach ($this->automationArray as $controllerName => $controllerActions) { - if ($this->params['controller'] == $controllerName && in_array($this->params['action'], $controllerActions)) { - return true; - } - } - return false; + return $this->IndexFilter->isApiFunction($this->params['controller'], $this->params['action']); } /** @@ -829,34 +936,12 @@ class AppController extends Controller return $data; } - // pass an action to this method for it to check the active user's access to the action - public function checkAction($action = 'perm_sync') - { - $this->loadModel('Role'); - $this->Role->recursive = -1; - $role = $this->Role->findById($this->Auth->user('role_id')); - if ($role['Role'][$action]) { - return true; - } - return false; - } - - // returns the role of the currently authenticated user as an array, used to set the permission variables for views in the AppController's beforeFilter() method - public function getActions() - { - $this->loadModel('Role'); - $this->Role->recursive = -1; - $role = $this->Role->findById($this->Auth->user('role_id')); - return $role['Role']; - } - public function checkAuthUser($authkey) { if (Configure::read('Security.advanced_authkeys')) { $this->loadModel('AuthKey'); $user = $this->AuthKey->getAuthUserByAuthKey($authkey); } else { - $this->loadModel('User'); $user = $this->User->getAuthUserByAuthKey($authkey); } @@ -866,22 +951,16 @@ class AppController extends Controller if (!$user['Role']['perm_auth']) { return false; } - if ($user['Role']['perm_site_admin']) { - $user['siteadmin'] = true; - } + $user['logged_by_authkey'] = true; return $user; } public function checkExternalAuthUser($authkey) { - $this->loadModel('User'); $user = $this->User->getAuthUserByExternalAuth($authkey); if (empty($user)) { return false; } - if ($user['Role']['perm_site_admin']) { - $user['siteadmin'] = true; - } return $user; } @@ -1270,7 +1349,7 @@ class AppController extends Controller $final = $this->$scope->restSearch($user, $returnFormat, $filters, false, false, $elementCounter, $renderView); if (!empty($renderView) && !empty($final)) { $this->layout = false; - $final = json_decode($final, true); + $final = json_decode($final->intoString(), true); foreach ($final as $key => $data) { $this->set($key, $data); } @@ -1331,4 +1410,30 @@ class AppController extends Controller } return false; } + + /** + * Refresh user data in session, but keep information about authkey. + * @return array User data in auth format + */ + protected function _refreshAuth() + { + $sessionUser = $this->Auth->user(); + $user = $this->User->getAuthUser($sessionUser['id']); + if (!$user) { + throw new RuntimeException("User with ID {$sessionUser['id']} not exists."); + } + if (isset($sessionUser['authkey_id'])) { + $this->loadModel('AuthKey'); + if (!$this->AuthKey->exists($sessionUser['authkey_id'])) { + throw new RuntimeException("Auth key with ID {$sessionUser['authkey_id']} not exists."); + } + } + foreach (['authkey_id', 'authkey_expiration', 'logged_by_authkey'] as $copy) { + if (isset($sessionUser[$copy])) { + $user[$copy] = $sessionUser[$copy]; + } + } + $this->Auth->login($user); + return $user; + } } diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index f90a51243..9424792be 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -83,20 +83,20 @@ class AttributesController extends AppController } return $this->RestResponse->viewData($attributes, $this->response->type()); } - $orgTable = $this->Attribute->Event->Orgc->find('list', array( - 'fields' => array('Orgc.id', 'Orgc.name') - )); + + $orgTable = $this->Attribute->Event->Orgc->find('all', [ + 'fields' => ['Orgc.id', 'Orgc.name', 'Orgc.uuid'], + ]); + $orgTable = Hash::combine($orgTable, '{n}.Orgc.id', '{n}.Orgc'); foreach ($attributes as &$attribute) { if (isset($orgTable[$attribute['Event']['orgc_id']])) { - $attribute['Event']['Orgc'] = [ - 'id' => $attribute['Event']['orgc_id'], - 'name' => $orgTable[$attribute['Event']['orgc_id']], - ]; + $attribute['Event']['Orgc'] = $orgTable[$attribute['Event']['orgc_id']]; } } + list($attributes, $sightingsData) = $this->__searchUI($attributes); $this->set('sightingsData', $sightingsData); - $this->set('orgTable', $orgTable); + $this->set('orgTable', array_column($orgTable, 'name', 'id')); $this->set('shortDist', $this->Attribute->shortDist); $this->set('attributes', $attributes); $this->set('attrDescriptions', $this->Attribute->fieldDescriptions); @@ -115,7 +115,7 @@ class AttributesController extends AppController if (!$this->userRole['perm_add']) { throw new MethodNotAllowedException(__('You do not have permissions to create attributes')); } - $event = $this->Attribute->Event->fetchSimpleEvent($this->Auth->user(), $eventId); + $event = $this->Attribute->Event->fetchSimpleEvent($this->Auth->user(), $eventId, ['contain' => ['Orgc']]); if (!$event) { throw new NotFoundException(__('Invalid event')); } @@ -355,7 +355,7 @@ class AttributesController extends AppController public function add_attachment($eventId = null) { - $event = $this->Attribute->Event->fetchSimpleEvent($this->Auth->user(), $eventId); + $event = $this->Attribute->Event->fetchSimpleEvent($this->Auth->user(), $eventId, ['contain' => ['Orgc']]); if (empty($event)) { throw new NotFoundException(__('Invalid Event.')); } @@ -998,6 +998,7 @@ class AttributesController extends AppController 'includeAllTags' => false, 'includeAttributeUuid' => true, 'flatten' => true, + 'deleted' => [0, 1] ); if ($this->_isRest()) { @@ -1505,7 +1506,6 @@ class AttributesController extends AppController 'request' => $this->request, 'named_params' => $this->params['named'], 'paramArray' => $paramArray, - 'ordered_url_params' => @compact($paramArray), 'additional_delimiters' => PHP_EOL ); $exception = false; @@ -1591,21 +1591,16 @@ class AttributesController extends AppController ); $attributes = $this->paginate(); - $orgTable = $this->Attribute->Event->Orgc->find('list', array( - 'fields' => ['Orgc.id', 'Orgc.name'], - )); + $orgTable = $this->Attribute->Event->Orgc->find('all', [ + 'fields' => ['Orgc.id', 'Orgc.name', 'Orgc.uuid'], + ]); + $orgTable = Hash::combine($orgTable, '{n}.Orgc.id', '{n}.Orgc'); foreach ($attributes as &$attribute) { if (isset($orgTable[$attribute['Event']['orgc_id']])) { - $attribute['Event']['Orgc'] = [ - 'id' => $attribute['Event']['orgc_id'], - 'name' => $orgTable[$attribute['Event']['orgc_id']], - ]; + $attribute['Event']['Orgc'] = $orgTable[$attribute['Event']['orgc_id']]; } if (isset($orgTable[$attribute['Event']['org_id']])) { - $attribute['Event']['Org'] = [ - 'id' => $attribute['Event']['org_id'], - 'name' => $orgTable[$attribute['Event']['org_id']], - ]; + $attribute['Event']['Org'] = $orgTable[$attribute['Event']['org_id']]; } } if ($this->_isRest()) { @@ -1634,7 +1629,7 @@ class AttributesController extends AppController } } } - $this->set('orgTable', $orgTable); + $this->set('orgTable', array_column($orgTable, 'name', 'id')); $this->set('filters', $filters); $this->set('attributes', $attributes); $this->set('isSearch', 1); diff --git a/app/Controller/AuthKeysController.php b/app/Controller/AuthKeysController.php index a721a5b3d..b90bb2fa4 100644 --- a/app/Controller/AuthKeysController.php +++ b/app/Controller/AuthKeysController.php @@ -1,6 +1,9 @@ set('user_id', $id); $conditions['AND'][] = ['AuthKey.user_id' => $id]; } + $keyUsageEnabled = Configure::read('MISP.log_user_ips') && Configure::read('MISP.log_user_ips_authkeys'); $this->CRUD->index([ - 'filters' => ['User.username', 'authkey', 'comment', 'User.id'], - 'quickFilters' => ['authkey', 'comment'], - 'contain' => ['User'], + 'filters' => ['User.email', 'authkey_start', 'authkey_end', 'comment', 'User.id'], + 'quickFilters' => ['comment', 'authkey_start', 'authkey_end', 'User.email'], + 'contain' => ['User.id', 'User.email'], 'conditions' => $conditions, - 'afterFind' => function (array $authKeys) { + 'afterFind' => function (array $authKeys) use ($keyUsageEnabled) { + if ($keyUsageEnabled) { + $keyIds = Hash::extract($authKeys, "{n}.AuthKey.id"); + $lastUsedById = $this->AuthKey->getLastUsageForKeys($keyIds); + } foreach ($authKeys as &$authKey) { + if ($keyUsageEnabled) { + $lastUsed = $lastUsedById[$authKey['AuthKey']['id']]; + $authKey['AuthKey']['last_used'] = $lastUsed; + } unset($authKey['AuthKey']['authkey']); } return $authKeys; @@ -38,8 +50,12 @@ class AuthKeysController extends AppController if ($this->IndexFilter->isRest()) { return $this->restResponsePayload; } - $this->set('metaGroup', $this->_isAdmin ? 'admin' : 'globalActions'); - $this->set('metaAction', 'authkeys_index'); + $this->set('title_for_layout', __('Auth Keys')); + $this->set('keyUsageEnabled', $keyUsageEnabled); + $this->set('menuData', [ + 'menuList' => $this->_isSiteAdmin() ? 'admin' : 'globalActions', + 'menuItem' => 'authkeys_index', + ]); } public function delete($id) @@ -61,18 +77,22 @@ class AuthKeysController extends AppController public function add($user_id = false) { - $this->set('menuData', array('menuList' => $this->_isSiteAdmin() ? 'admin' : 'globalActions', 'menuItem' => 'authKeyAdd')); $params = [ 'displayOnSuccess' => 'authkey_display', - 'saveModelVariable' => ['authkey_raw'] + 'saveModelVariable' => ['authkey_raw'], + 'override' => ['authkey' => null], // do not allow to use own key, always generate random one + 'afterFind' => function ($authKey) { // remove hashed key from response + unset($authKey['AuthKey']['authkey']); + return $authKey; + } ]; $selectConditions = []; if (!$this->_isSiteAdmin()) { $selectConditions['AND'][] = ['User.id' => $this->Auth->user('id')]; - $params['override'] = ['user_id' => $this->Auth->user('id')]; + $params['override']['user_id'] = $this->Auth->user('id'); } else if ($user_id) { $selectConditions['AND'][] = ['User.id' => $user_id]; - $params['override'] = ['user_id' => $user_id]; + $params['override']['user_id'] = $user_id; } $this->CRUD->add($params); if ($this->IndexFilter->isRest()) { @@ -86,6 +106,11 @@ class AuthKeysController extends AppController ]) ]; $this->set(compact('dropdownData')); + $this->set('menuData', [ + 'menuList' => $this->_isSiteAdmin() ? 'admin' : 'globalActions', + 'menuItem' => 'authKeyAdd', + ]); + $this->set('validity', Configure::read('Security.advanced_authkeys_validity')); } public function view($id = false) @@ -101,6 +126,15 @@ class AuthKeysController extends AppController if ($this->IndexFilter->isRest()) { return $this->restResponsePayload; } + + if (Configure::read('MISP.log_user_ips') && Configure::read('MISP.log_user_ips_authkeys')) { + list($keyUsage, $lastUsed, $uniqueIps) = $this->AuthKey->getKeyUsage($id); + $this->set('keyUsage', $keyUsage); + $this->set('lastUsed', $lastUsed); + $this->set('uniqueIps', $uniqueIps); + } + + $this->set('title_for_layout', __('Auth Key')); $this->set('menuData', [ 'menuList' => $this->_isSiteAdmin() ? 'admin' : 'globalActions', 'menuItem' => 'authKeyView', diff --git a/app/Controller/CerebratesController.php b/app/Controller/CerebratesController.php index d05c1de93..07e801ac4 100644 --- a/app/Controller/CerebratesController.php +++ b/app/Controller/CerebratesController.php @@ -30,13 +30,11 @@ class CerebratesController extends AppController public function add() { - $this->set('menuData', array('menuList' => 'sync', 'menuItem' => 'add_cerebrate')); $params = []; $this->CRUD->add($params); - if ($this->IndexFilter->isRest()) { + if ($this->restResponsePayload) { return $this->restResponsePayload; } - $this->set('permFlags', $this->Role->permFlags); $this->loadModel('Organisation'); $orgs = $this->Organisation->find('list', [ @@ -48,6 +46,7 @@ class CerebratesController extends AppController 'org_id' => $orgs ]; $this->set(compact('dropdownData')); + $this->set('menuData', array('menuList' => 'sync', 'menuItem' => 'add_cerebrate')); } public function edit($id) @@ -59,7 +58,6 @@ class CerebratesController extends AppController if ($this->IndexFilter->isRest()) { return $this->restResponsePayload; } - $this->set('permFlags', $this->Role->permFlags); $this->loadModel('Organisation'); $orgs = $this->Organisation->find('list', [ diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php index bcdd16ae7..73c481740 100644 --- a/app/Controller/Component/ACLComponent.php +++ b/app/Controller/Component/ACLComponent.php @@ -15,13 +15,11 @@ class ACLComponent extends Component private $__aclList = array( '*' => array( 'blackhole' => array(), - 'checkAction' => array(), 'checkAuthUser' => array(), 'checkExternalAuthUser' => array(), 'cleanModelCaches' => array(), 'debugACL' => array(), 'generateCount' => array(), - 'getActions' => array(), 'pruneDuplicateUUIDs' => array(), 'queryACL' => array(), 'removeDuplicateEvents' => array(), @@ -89,6 +87,13 @@ class ACLComponent extends Component 'pull_orgs' => [], 'view' => [] ], + 'correlationExclusions' => [ + 'add' => [], + 'clean' => [], + 'delete' => [], + 'index' => [], + 'view' => [] + ], 'dashboards' => array( 'getForm' => array('*'), 'index' => array('*'), @@ -161,11 +166,11 @@ class ACLComponent extends Component ] ), 'eventDelegations' => array( - 'acceptDelegation' => array('perm_add'), - 'delegateEvent' => array('perm_delegate'), - 'deleteDelegation' => array('perm_add'), - 'index' => array('*'), - 'view' => array('*'), + 'acceptDelegation' => array('AND' => ['delegation_enabled', 'perm_add']), + 'delegateEvent' => array('AND' => ['delegation_enabled', 'perm_delegate']), + 'deleteDelegation' => array('AND' => ['delegation_enabled', 'perm_add']), + 'index' => array('delegation_enabled'), + 'view' => array('delegation_enabled'), ), 'eventReports' => array( 'add' => array('perm_add'), @@ -409,6 +414,7 @@ class ACLComponent extends Component 'edit' => array('perm_object_template'), 'delete' => array('perm_object_template'), 'getToggleField' => array(), + 'getRaw' => array('perm_object_template'), 'objectChoice' => array('*'), 'objectMetaChoice' => array('perm_add'), 'view' => array('*'), @@ -463,7 +469,6 @@ class ACLComponent extends Component 'admin_add' => array(), 'admin_delete' => array(), 'admin_edit' => array(), - 'admin_index' => array('perm_admin'), 'admin_set_default' => array(), 'index' => array('*'), 'view' => array('*'), @@ -501,6 +506,7 @@ class ACLComponent extends Component 'postTest' => array('perm_sync'), 'previewEvent' => array(), 'previewIndex' => array(), + 'compareServers' => [], 'pull' => array(), 'purgeSessions' => array(), 'push' => array(), @@ -539,6 +545,7 @@ class ACLComponent extends Component 'generateCorrelation' => array(), 'index' => array('*'), 'view' => array('*'), + 'viewPicture' => array('*'), ), 'sharingGroups' => array( 'add' => array('perm_sharing_group'), @@ -617,6 +624,7 @@ class ACLComponent extends Component 'taxonomyMassUnhide' => array('perm_tagger'), 'toggleRequired' => array('perm_site_admin'), 'update' => array(), + 'import' => [], 'view' => array('*'), 'unhideTag' => array('perm_tagger'), 'hideTag' => array('perm_tagger'), @@ -687,6 +695,7 @@ class ACLComponent extends Component 'verifyCertificate' => array(), 'verifyGPG' => array(), 'view' => array('*'), + 'getGpgPublicKey' => array('*'), ), 'userSettings' => array( 'index' => array('*'), @@ -748,6 +757,9 @@ class ACLComponent extends Component } return true; }; + $this->dynamicChecks['delegation_enabled'] = function (array $user) { + return (bool)Configure::read('MISP.delegation'); + }; } private function __checkLoggedActions($user, $controller, $action) diff --git a/app/Controller/Component/CRUDComponent.php b/app/Controller/Component/CRUDComponent.php index 01355d3c7..1e682c197 100644 --- a/app/Controller/Component/CRUDComponent.php +++ b/app/Controller/Component/CRUDComponent.php @@ -2,6 +2,7 @@ class CRUDComponent extends Component { + /** @var AppController */ public $Controller = null; public function initialize(Controller $controller, $settings=array()) { @@ -15,7 +16,7 @@ class CRUDComponent extends Component } } - public function index($options) + public function index(array $options) { $this->prepareResponse(); if (!empty($options['quickFilters'])) { @@ -75,8 +76,6 @@ class CRUDComponent extends Component $input[$modelName][$field] = $value; } } - if (isset($input[$modelName]['id'])) { - } unset($input[$modelName]['id']); if (!empty($params['fields'])) { $data = []; @@ -86,20 +85,25 @@ class CRUDComponent extends Component } else { $data = $input; } - if ($this->Controller->{$modelName}->save($data)) { - $data = $this->Controller->{$modelName}->find('first', [ + /** @var Model $model */ + $model = $this->Controller->{$modelName}; + if ($model->save($data)) { + $data = $model->find('first', [ 'recursive' => -1, 'conditions' => [ - 'id' => $this->Controller->{$modelName}->id + 'id' => $model->id ] ]); if (!empty($params['saveModelVariable'])) { foreach ($params['saveModelVariable'] as $var) { - if (isset($this->Controller->{$modelName}->$var)) { - $data[$modelName][$var] = $this->Controller->{$modelName}->$var; + if (isset($model->$var)) { + $data[$modelName][$var] = $model->$var; } } } + if (isset($params['afterFind'])) { + $data = $params['afterFind']($data); + } $message = __('%s added.', $modelName); if ($this->Controller->IndexFilter->isRest()) { $this->Controller->restResponsePayload = $this->Controller->RestResponse->viewData($data, 'json'); @@ -111,12 +115,22 @@ class CRUDComponent extends Component $this->Controller->render($params['displayOnSuccess']); return; } - $this->Controller->redirect(['action' => 'index']); + + $redirect = isset($params['redirect']) ? $params['redirect'] : ['action' => 'index']; + // For AJAX requests doesn't make sense to redirect, redirect must be done on javascript side in `submitGenericFormInPlace` + if ($this->Controller->request->is('ajax')) { + $redirect = Router::url($redirect); + $this->Controller->restResponsePayload = $this->Controller->RestResponse->viewData(['redirect' => $redirect], 'json'); + } else { + $this->Controller->redirect($redirect); + } } } else { $message = __('%s could not be added.', $modelName); if ($this->Controller->IndexFilter->isRest()) { - + $controllerName = $this->Controller->params['controller']; + $actionName = $this->Controller->params['action']; + $this->Controller->restResponsePayload = $this->Controller->RestResponse->saveFailResponse($controllerName, $actionName, false, $model->validationErrors, 'json'); } else { $this->Controller->Flash->error($message); } @@ -125,7 +139,7 @@ class CRUDComponent extends Component $this->Controller->set('entity', $data); } - public function edit(int $id, array $params = []): void + public function edit(int $id, array $params = []) { $modelName = $this->Controller->defaultModel; if (empty($id)) { @@ -161,9 +175,10 @@ class CRUDComponent extends Component $message = __('%s updated.', $modelName); if ($this->Controller->IndexFilter->isRest()) { $this->Controller->restResponsePayload = $this->Controller->RestResponse->viewData($data, 'json'); + return; } else { $this->Controller->Flash->success($message); - $this->Controller->redirect(['action' => 'index']); + $this->Controller->redirect(isset($params['redirect']) ? $params['redirect'] : ['action' => 'index']); } } else { if ($this->Controller->IndexFilter->isRest()) { @@ -176,7 +191,7 @@ class CRUDComponent extends Component $this->Controller->set('entity', $data); } - public function view(int $id, array $params = []): void + public function view(int $id, array $params = []) { $modelName = $this->Controller->defaultModel; if (empty($id)) { @@ -204,7 +219,7 @@ class CRUDComponent extends Component } } - public function delete(int $id, array $params = []): void + public function delete(int $id, array $params = []) { $this->prepareResponse(); $modelName = $this->Controller->defaultModel; @@ -224,7 +239,18 @@ class CRUDComponent extends Component if (empty($data)) { throw new NotFoundException(__('Invalid %s.', $modelName)); } - if ($this->Controller->request->is('post') || $this->Controller->request->is('delete')) { + $validationError = null; + if (isset($params['validate'])) { + try { + $params['validate']($data); + } catch (Exception $e) { + $validationError = $e->getMessage(); + if ($this->Controller->IndexFilter->isRest()) { + $this->Controller->restResponsePayload = $this->Controller->RestResponse->saveFailResponse($modelName, 'delete', $id, $validationError); + } + } + } + if ($validationError === null && $this->Controller->request->is('post') || $this->Controller->request->is('delete')) { if (!empty($params['modelFunction'])) { $result = $this->Controller->$modelName->{$params['modelFunction']}($id); } else { @@ -234,36 +260,39 @@ class CRUDComponent extends Component $message = __('%s deleted.', $modelName); if ($this->Controller->IndexFilter->isRest()) { $this->Controller->restResponsePayload = $this->Controller->RestResponse->saveSuccessResponse($modelName, 'delete', $id, 'json', $message); + return; } else { $this->Controller->Flash->success($message); $this->Controller->redirect($this->Controller->referer()); } } } + $this->Controller->set('validationError', $validationError); $this->Controller->set('id', $data[$modelName]['id']); $this->Controller->set('data', $data); $this->Controller->layout = 'ajax'; $this->Controller->render('/genericTemplates/delete'); } - - protected function setQuickFilters($params, $query, $quickFilterFields) + protected function setQuickFilters($params, array $query, $quickFilterFields) { - $queryConditions = []; if (!empty($params['quickFilter']) && !empty($quickFilterFields)) { + $queryConditions = []; + $filter = '%' . strtolower($params['quickFilter']) . '%'; foreach ($quickFilterFields as $filterField) { - $queryConditions[$filterField] = $params['quickFilter']; + $queryConditions["LOWER($filterField) LIKE"] = $filter; } - $query['conditions']['OR'][] = $queryConditions; + $query['conditions']['OR'] = $queryConditions; } return $query; } - protected function setFilters($params, $query) + protected function setFilters(array $params, array $query) { - $params = $this->massageFilters($params); - if (!empty($params['simpleFilters'])) { - foreach ($params['simpleFilters'] as $filter => $filterValue) { + // For CakePHP 2, we don't need to distinguish between simpleFilters and relatedFilters + //$params = $this->massageFilters($params); + if (!empty($params)) { + foreach ($params as $filter => $filterValue) { if ($filter === 'quickFilter') { continue; } @@ -291,7 +320,7 @@ class CRUDComponent extends Component return $query; } - protected function massageFilters(array $params): array + protected function massageFilters(array $params) { $massagedFilters = [ 'simpleFilters' => [], diff --git a/app/Controller/Component/IndexFilterComponent.php b/app/Controller/Component/IndexFilterComponent.php index 4116d0ba4..9eb2837a6 100644 --- a/app/Controller/Component/IndexFilterComponent.php +++ b/app/Controller/Component/IndexFilterComponent.php @@ -6,7 +6,8 @@ class IndexFilterComponent extends Component { - public $Controller = false; + /** @var Controller */ + public $Controller; public $isRest = null; public function initialize(Controller $controller) { @@ -74,7 +75,7 @@ class IndexFilterComponent extends Component } } } - $this->Controller->set('passedArgs', json_encode($this->Controller->passedArgs, true)); + $this->Controller->set('passedArgs', json_encode($this->Controller->passedArgs)); return $data; } @@ -85,12 +86,7 @@ class IndexFilterComponent extends Component return $this->isRest; } $api = $this->isApiFunction($this->Controller->request->params['controller'], $this->Controller->request->params['action']); - if (isset($this->Controller->RequestHandler) && ($api || $this->Controller->RequestHandler->isXml() || $this->isJson() || $this->isCsv())) { - if ($this->isJson()) { - if (!empty($this->Controller->request->input()) && empty($this->Controller->request->input('json_decode'))) { - throw new MethodNotAllowedException('Invalid JSON input. Make sure that the JSON input is a correctly formatted JSON string. This request has been blocked to avoid an unfiltered request.'); - } - } + if (isset($this->Controller->RequestHandler) && ($api || $this->isJson() || $this->Controller->RequestHandler->isXml() || $this->isCsv())) { $this->isRest = true; return true; } else { @@ -99,11 +95,8 @@ class IndexFilterComponent extends Component } } - public function isJson($data=false) + public function isJson() { - if ($data) { - return (json_decode($data) != null) ? true : false; - } return $this->Controller->request->header('Accept') === 'application/json' || $this->Controller->RequestHandler->prefers() === 'json'; } @@ -117,12 +110,13 @@ class IndexFilterComponent extends Component } + /** + * @param string $controller + * @param string $action + * @return bool + */ public function isApiFunction($controller, $action) { - if (isset($this->Controller->automationArray[$controller]) && in_array($action, $this->Controller->automationArray[$controller])) { - return true; - } - return false; + return isset($this->Controller->automationArray[$controller]) && in_array($action, $this->Controller->automationArray[$controller], true); } - } diff --git a/app/Controller/Component/RestResponseComponent.php b/app/Controller/Component/RestResponseComponent.php index 43faa2339..44558d8b9 100644 --- a/app/Controller/Component/RestResponseComponent.php +++ b/app/Controller/Component/RestResponseComponent.php @@ -1,5 +1,8 @@ array('type', 'source', 'timestamp', 'date', 'time') ), 'restSearch' => array( - 'description' => "Search MISP sightings using a list of filter parameters and return the data in the JSON format. The search is available on an event, attribute or instance level, just select the scope via the URL (/sighting/restSearch/event vs /sighting/restSearch/attribute vs /sighting/restSearch/). id MUST be provided if context is set.", + 'description' => "Search MISP sightings using a list of filter parameters and return the data in the JSON format. The search is available on an event, attribute or instance level, just select the scope via the URL (/sighting/restSearch/event vs /sighting/restSearch/attribute vs /sighting/restSearch/). id or uuid MUST be provided if context is set.", 'mandatory' => array('returnFormat'), - 'optional' => array('id', 'type', 'from', 'to', 'last', 'org_id', 'source', 'includeAttribute', 'includeEvent'), + 'optional' => array('id', 'uuid', 'type', 'from', 'to', 'last', 'org_id', 'source', 'includeAttribute', 'includeEvent'), 'params' => array('context') ), ), @@ -343,7 +346,8 @@ class RestResponseComponent extends Component $this->__setup(); $result = array(); foreach ($this->__scopedFieldsConstraint as $controller => $actions) { - $controller = Inflector::tableize($controller); + // EventGraph controller has different rule + $controller = $controller === 'EventGraph' ? 'event_graph' : Inflector::tableize($controller); foreach ($actions as $action => $data) { if ($this->ACL->canUserAccess($user, $controller, $action)) { $admin_routing = ''; @@ -364,7 +368,8 @@ class RestResponseComponent extends Component $this->__setup(); $result = array(); foreach ($this->__descriptions as $controller => $actions) { - $controller = Inflector::tableize($controller); + // EventGraph controller has different rule + $controller = $controller === 'EventGraph' ? 'event_graph' : Inflector::tableize($controller); foreach ($actions as $action => $data) { if ($this->ACL->canUserAccess($user, $controller, $action)) { $admin_routing = ''; @@ -514,11 +519,12 @@ class RestResponseComponent extends Component } else { $type = $format; } + $dumpSql = !empty($this->Controller->sql_dump) && Configure::read('debug') > 1; if (!$raw) { if (is_string($response)) { $response = array('message' => $response); } - if (Configure::read('debug') > 1 && !empty($this->Controller->sql_dump)) { + if ($dumpSql) { $this->Log = ClassRegistry::init('Log'); if ($this->Controller->sql_dump === 2) { $response = array('sql_dump' => $this->Log->getDataSource()->getLog(false, false)); @@ -526,9 +532,11 @@ class RestResponseComponent extends Component $response['sql_dump'] = $this->Log->getDataSource()->getLog(false, false); } } - $response = json_encode($response, JSON_PRETTY_PRINT); + // Do not pretty print response for automatic tools + $flags = $this->isAutomaticTool() ? JSON_UNESCAPED_UNICODE : JSON_PRETTY_PRINT; + $response = json_encode($response, $flags); } else { - if (Configure::read('debug') > 1 && !empty($this->Controller->sql_dump)) { + if ($dumpSql) { $this->Log = ClassRegistry::init('Log'); if ($this->Controller->sql_dump === 2) { $response = json_encode(array('sql_dump' => $this->Log->getDataSource()->getLog(false, false))); @@ -542,7 +550,15 @@ class RestResponseComponent extends Component } } } - $cakeResponse = new CakeResponse(array('body' => $response, 'status' => $code, 'type' => $type)); + + App::uses('TmpFileTool', 'Tools'); + if ($response instanceof TmpFileTool) { + App::uses('CakeResponseTmp', 'Tools'); + $cakeResponse = new CakeResponseTmp(['status' => $code, 'type' => $type]); + $cakeResponse->file($response); + } else { + $cakeResponse = new CakeResponse(array('body' => $response, 'status' => $code, 'type' => $type)); + } if (Configure::read('Security.allow_cors')) { $headers["Access-Control-Allow-Headers"] = "Origin, Content-Type, Authorization, Accept"; @@ -568,6 +584,16 @@ class RestResponseComponent extends Component return $cakeResponse; } + /** + * Detect if request comes from automatic tool, like other MISP instance or PyMISP + * @return bool + */ + public function isAutomaticTool() + { + $userAgent = CakeRequest::header('User-Agent'); + return $userAgent && (substr($userAgent, 0, 6) === 'PyMISP' || substr($userAgent, 0, 4) === 'MISP'); + } + private function __generateURL($action, $controller, $id) { $controller = Inflector::underscore(Inflector::pluralize($controller)); @@ -1792,8 +1818,6 @@ class RestResponseComponent extends Component private function __overwriteReturnFormat($scope, $action, &$field) { switch($scope) { case "Attribute": - $field['values'] = array_keys(ClassRegistry::init($scope)->validFormats); - break; case "Event": $field['values'] = array_keys(ClassRegistry::init($scope)->validFormats); break; @@ -1837,33 +1861,36 @@ class RestResponseComponent extends Component $field['values'][] = array('label' => h($model_name), 'value' => $i); } } - private function __overwriteTags($scope, $action, &$field) { - $this->{$scope} = ClassRegistry::init("Tag"); - $tags = $this->{$scope}->find('list', array( - 'recursive' => -1, - 'fields' => array('name') - )); - foreach($tags as $i => $tag) { - $tagname = htmlspecialchars($tag); - $tags[$tagname] = $tagname; - unset($tags[$i]); + + private function __overwriteTags($scope, $action, &$field) + { + static $values; + if ($values === null) { + $tagModel = ClassRegistry::init("Tag"); + $tags = $tagModel->find('column', array( + 'fields' => array('Tag.name') + )); + $values = []; + foreach ($tags as $tag) { + $tagname = htmlspecialchars($tag); + $values[$tagname] = $tagname; + } } - $field['values'] = $tags; + $field['values'] = $values; if ($action == 'attachTagToObject') { $field['help'] = __('Also supports array of tags'); } } private function __overwriteNationality($scope, $action, &$field) { - $field['values'] = ClassRegistry::init("Organisation")->countries; + $field['values'] = ClassRegistry::init("Organisation")->getCountries(); } private function __overwriteAction($scope, $action, &$field) { $field['values'] = array_keys(ClassRegistry::init("Log")->actionDefinitions); } private function __overwriteRoleId($scope, $action, &$field) { $this->{$scope} = ClassRegistry::init("Role"); - $roles = $this->{$scope}->find('list', array( - 'recursive' => -1, + $roles = $this->{$scope}->find('column', array( 'fields' => array('name') )); $field['values'] = $roles; diff --git a/app/Controller/CorrelationExclusionsController.php b/app/Controller/CorrelationExclusionsController.php new file mode 100644 index 000000000..b261c517d --- /dev/null +++ b/app/Controller/CorrelationExclusionsController.php @@ -0,0 +1,94 @@ + 60, + 'order' => array( + 'CorrelationExclusion.value' => 'ASC', + ) + ); + + public function index($id = false) + { + $this->CRUD->index([ + 'filters' => ['value'], + 'quickFilters' => ['value'] + ]); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + $this->set('title_for_layout', __('Correlation Exclusions index')); + $this->set('menuData', [ + 'menuList' => 'correlationExclusions', + 'menuItem' => 'index' + ]); + } + + public function delete($id) + { + $this->CRUD->delete($id); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + } + + public function add($user_id = false) + { + $params = []; + $this->CRUD->add($params); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + $dropdownData = []; + $this->set(compact('dropdownData')); + $this->set('menuData', [ + 'menuList' => 'correlationExclusions', + 'menuItem' => 'add', + ]); + } + + public function view($id = false) + { + $this->CRUD->view($id); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + + $this->set('title_for_layout', __('Correlation Exclusion')); + $this->set('menuData', [ + 'menuList' => 'correlationExclusions', + 'menuItem' => 'view', + ]); + } + + public function clean() + { + if ($this->request->is('post')) { + $this->CorrelationExclusion->cleanRouter($this->Auth->user()); + $message = __('Correlations cleanup initiated, based on the exclusion rules.'); + if ($this->_isRest()) { + return $this->RestResponse->saveSuccessResponse('CorrelationExclusion', 'clean', false, false, $message); + } else { + $this->Flash->success($message); + $this->redirect($this->referer()); + } + } else { + $this->set('title', __('Clean up correlations')); + $this->set('question', __('Execute the cleaning of all correlations that are at odds with the exclusion rules? This will delete all matching correlations.')); + $this->set('actionName', 'clean');; + $this->layout = 'ajax'; + $this->render('/genericTemplates/confirm'); + } + } +} diff --git a/app/Controller/DashboardsController.php b/app/Controller/DashboardsController.php index 432cfcb7f..792596032 100644 --- a/app/Controller/DashboardsController.php +++ b/app/Controller/DashboardsController.php @@ -1,6 +1,9 @@ request->is('post')) { - if (empty($this->request->data['data'])) { - $this->request->data = array('data' => $this->request->data); - - } - if (empty($this->request->data['data'])) { - throw new MethodNotAllowedException(__('You need to specify the widget to use along with the configuration.')); - } - $value = $this->request->data['data']; - $dashboardWidget = $this->Dashboard->loadWidget($this->Auth->user(), $value['widget']); - $this->layout = false; - $this->set('title', $dashboardWidget->title); - $redis = $this->Dashboard->setupRedis(); - $org_scope = $this->_isSiteAdmin() ? 0 : $this->Auth->user('org_id'); - $lookup_hash = hash('sha256', $value['widget'] . $value['config']); - $data = $redis->get('misp:dashboard:' . $org_scope . ':' . $lookup_hash); - if (!isset($dashboardWidget->cacheLifetime)) { - $dashboardWidget->cacheLifetime = false; - } - if (empty($dashboardWidget->cacheLifetime) || empty($data)) { - $data = $dashboardWidget->handler($this->Auth->user(), json_decode($value['config'], true)); - if (!empty($dashboardWidget->cacheLifetime)) { - $redis->set('misp:dashboard:' . $org_scope . ':' . $lookup_hash, json_encode(array('data' => $data))); - $redis->expire('misp:dashboard:' . $org_scope . ':' . $lookup_hash, $dashboardWidget->cacheLifetime); - } - } else { - $data = json_decode($data, true)['data']; - } - $valueConfig = json_decode($value['config'], true); - $config = array( - 'render' => $dashboardWidget->render, - 'autoRefreshDelay' => empty($dashboardWidget->autoRefreshDelay) ? false : $dashboardWidget->autoRefreshDelay, - 'widget_config' => empty($valueConfig['widget_config']) ? array() : $valueConfig['widget_config'] - ); - $this->set('widget_id', $widget_id); - $this->set('data', $data); - $this->set('config', $config); - $this->render('widget_loader'); - } else { + if (!$this->request->is('post')) { throw new MethodNotAllowedException(__('This endpoint can only be reached via POST requests.')); } + + @session_write_close(); // allow concurrent AJAX requests (session hold lock by default) + + if (empty($this->request->data['data'])) { + $this->request->data = array('data' => $this->request->data); + } + if (empty($this->request->data['data'])) { + throw new MethodNotAllowedException(__('You need to specify the widget to use along with the configuration.')); + } + $value = $this->request->data['data']; + $valueConfig = json_decode($value['config'], true); + $dashboardWidget = $this->Dashboard->loadWidget($this->Auth->user(), $value['widget']); + + $redis = $this->Dashboard->setupRedis(); + $org_scope = $this->_isSiteAdmin() ? 0 : $this->Auth->user('org_id'); + $lookup_hash = hash('sha256', $value['widget'] . $value['config']); + $cacheKey = 'misp:dashboard:' . $org_scope . ':' . $lookup_hash; + $data = $redis->get($cacheKey); + if (!isset($dashboardWidget->cacheLifetime)) { + $dashboardWidget->cacheLifetime = false; + } + if (empty($dashboardWidget->cacheLifetime) || empty($data)) { + $data = $dashboardWidget->handler($this->Auth->user(), $valueConfig); + if (!empty($dashboardWidget->cacheLifetime)) { + $redis->setex($cacheKey, $dashboardWidget->cacheLifetime, json_encode(array('data' => $data))); + } + } else { + $data = json_decode($data, true)['data']; + } + $config = array( + 'render' => $dashboardWidget->render, + 'autoRefreshDelay' => empty($dashboardWidget->autoRefreshDelay) ? false : $dashboardWidget->autoRefreshDelay, + 'widget_config' => empty($valueConfig['widget_config']) ? array() : $valueConfig['widget_config'] + ); + + $this->layout = false; + $this->set('title', $dashboardWidget->title); + $this->set('widget_id', $widget_id); + $this->set('data', $data); + $this->set('config', $config); + $this->render('widget_loader'); } public function import() @@ -230,6 +236,7 @@ class DashboardsController extends AppController public function saveTemplate($update = false) { + $this->loadModel('UserSetting'); if (!empty($update)) { $conditions = array('Dashboard.id' => $update); if (Validation::uuid($update)) { @@ -391,7 +398,7 @@ class DashboardsController extends AppController $element['User']['email'] = ''; } } - $this->set('passedArgs', $this->passedArgs); + $this->set('passedArgs', json_encode($this->passedArgs)); $this->set('data', $data); } } diff --git a/app/Controller/EventDelegationsController.php b/app/Controller/EventDelegationsController.php index e0b31069d..91ba32e2c 100644 --- a/app/Controller/EventDelegationsController.php +++ b/app/Controller/EventDelegationsController.php @@ -124,7 +124,7 @@ class EventDelegationsController extends AppController 'order' => array('lower(name) ASC') )); $distribution = $this->EventDelegation->Event->distributionLevels; - $sgs = $this->EventDelegation->Event->SharingGroup->fetchAllAuthorised($this->Auth->User, 'name', true); + $sgs = $this->EventDelegation->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', true); if (empty($sgs)) { unset($distribution[4]); } diff --git a/app/Controller/EventReportsController.php b/app/Controller/EventReportsController.php index 0c79c39cb..3d9bf0343 100644 --- a/app/Controller/EventReportsController.php +++ b/app/Controller/EventReportsController.php @@ -124,12 +124,14 @@ class EventReportsController extends AppController { $report = $this->EventReport->fetchIfAuthorized($this->Auth->user(), $id, 'delete', $throwErrors=true, $full=false); if ($this->request->is('post')) { - $errors = $this->EventReport->deleteReport($this->Auth->user(), $report, $hard=$hard); + if (!empty($this->request->data['hard'])) { + $hard = true; + } + $errors = $this->EventReport->deleteReport($this->Auth->user(), $report, $hard); $redirectTarget = $this->referer(); if (empty($errors)) { $successMessage = __('Event Report %s %s deleted', $id, $hard ? __('hard') : __('soft')); - $report = $hard ? null : $this->EventReport->simpleFetchById($this->Auth->user(), $id); - return $this->__getSuccessResponseBasedOnContext($successMessage, $report, 'delete', $id, $redirectTarget); + return $this->__getSuccessResponseBasedOnContext($successMessage, null, 'delete', $id, $redirectTarget); } else { $errorMessage = __('Event Report %s could not be %s deleted.%sReasons: %s', $id, $hard ? __('hard') : __('soft'), PHP_EOL, json_encode($errors)); return $this->__getFailResponseBasedOnContext($errorMessage, array(), 'edit', $id, $redirectTarget); @@ -153,8 +155,7 @@ class EventReportsController extends AppController $redirectTarget = $this->referer(); if (empty($errors)) { $successMessage = __('Event Report %s restored', $id); - $report = $this->EventReport->simpleFetchById($this->Auth->user(), $id); - return $this->__getSuccessResponseBasedOnContext($successMessage, $report, 'restore', $id, $redirectTarget); + return $this->__getSuccessResponseBasedOnContext($successMessage, null, 'restore', $id, $redirectTarget); } else { $errorMessage = __('Event Report %s could not be %s restored.%sReasons: %s', $id, PHP_EOL, json_encode($errors)); return $this->__getFailResponseBasedOnContext($errorMessage, array(), 'restore', $id, $redirectTarget); @@ -209,31 +210,30 @@ class EventReportsController extends AppController { if (!$this->request->is('ajax')) { throw new MethodNotAllowedException(__('This function can only be reached via AJAX.')); - } else { - if ($this->request->is('post')) { - $report = $this->EventReport->fetchIfAuthorized($this->Auth->user(), $reportId, 'edit', $throwErrors=true, $full=false); - $results = $this->EventReport->getComplexTypeToolResultWithReplacements($this->Auth->user(), $report); - $report['EventReport']['content'] = $results['replacementResult']['contentWithReplacements']; - $contextResults = $this->EventReport->extractWithReplacements($this->Auth->user(), $report, ['replace' => true]); - $suggestionResult = $this->EventReport->transformFreeTextIntoSuggestion($contextResults['contentWithReplacements'], $results['complexTypeToolResult']); - $errors = $this->EventReport->applySuggestions($this->Auth->user(), $report, $suggestionResult['contentWithSuggestions'], $suggestionResult['suggestionsMapping']); - if (empty($errors)) { - if (!empty($this->data['EventReport']['tag_event'])) { - $this->EventReport->attachTagsAfterReplacements($this->Auth->User(), $contextResults['replacedContext'], $report['EventReport']['event_id']); - } - $report = $this->EventReport->simpleFetchById($this->Auth->user(), $reportId); - $data = [ 'report' => $report ]; - $successMessage = __('Automatic extraction applied to Event Report %s', $reportId); - return $this->__getSuccessResponseBasedOnContext($successMessage, $data, 'applySuggestions', $reportId); - } else { - $errorMessage = __('Automatic extraction could not be applied to Event Report %s.%sReasons: %s', $reportId, PHP_EOL, json_encode($errors)); - return $this->__getFailResponseBasedOnContext($errorMessage, array(), 'applySuggestions', $reportId); - } - } - $this->layout = 'ajax'; - $this->set('reportId', $reportId); - $this->render('ajax/extractAllFromReport'); } + if ($this->request->is('post')) { + $report = $this->EventReport->fetchIfAuthorized($this->Auth->user(), $reportId, 'edit', $throwErrors=true, $full=false); + $results = $this->EventReport->getComplexTypeToolResultWithReplacements($this->Auth->user(), $report); + $report['EventReport']['content'] = $results['replacementResult']['contentWithReplacements']; + $contextResults = $this->EventReport->extractWithReplacements($this->Auth->user(), $report, ['replace' => true]); + $suggestionResult = $this->EventReport->transformFreeTextIntoSuggestion($contextResults['contentWithReplacements'], $results['complexTypeToolResult']); + $errors = $this->EventReport->applySuggestions($this->Auth->user(), $report, $suggestionResult['contentWithSuggestions'], $suggestionResult['suggestionsMapping']); + if (empty($errors)) { + if (!empty($this->data['EventReport']['tag_event'])) { + $this->EventReport->attachTagsAfterReplacements($this->Auth->User(), $contextResults['replacedContext'], $report['EventReport']['event_id']); + } + $report = $this->EventReport->simpleFetchById($this->Auth->user(), $reportId); + $data = [ 'report' => $report ]; + $successMessage = __('Automatic extraction applied to Event Report %s', $reportId); + return $this->__getSuccessResponseBasedOnContext($successMessage, $data, 'applySuggestions', $reportId); + } else { + $errorMessage = __('Automatic extraction could not be applied to Event Report %s.%sReasons: %s', $reportId, PHP_EOL, json_encode($errors)); + return $this->__getFailResponseBasedOnContext($errorMessage, array(), 'applySuggestions', $reportId); + } + } + $this->layout = 'ajax'; + $this->set('reportId', $reportId); + $this->render('ajax/extractAllFromReport'); } public function extractFromReport($reportId) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 0ee5546f4..f7e871e3a 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -136,12 +136,12 @@ class EventsController extends AppController $includeConditions['OR'][] = array('lower(Attribute.value2) LIKE' => $i); } - $includeIDs = array_values($this->Event->Attribute->fetchAttributes($this->Auth->user(), array( + $includeIDs = $this->Event->Attribute->fetchAttributes($this->Auth->user(), array( 'conditions' => $includeConditions, 'flatten' => true, 'event_ids' => true, 'list' => true, - ))); + )); } if (!empty($exclude)) { @@ -151,12 +151,12 @@ class EventsController extends AppController $excludeConditions['OR'][] = array('lower(Attribute.value2) LIKE' => $e); } - $excludeIDs = array_values($this->Event->Attribute->fetchAttributes($this->Auth->user(), array( + $excludeIDs = $this->Event->Attribute->fetchAttributes($this->Auth->user(), array( 'conditions' => $excludeConditions, 'flatten' => true, 'event_ids' => true, 'list' => true, - ))); + )); } } // return -1 as the only value in includedIDs if both arrays are empty. This will mean that no events will be shown if there was no hit @@ -191,15 +191,13 @@ class EventsController extends AppController $conditions = array( 'OR' => $subconditions, ); - $attributeHits = $this->Event->Attribute->fetchAttributes($this->Auth->user(), array( + $result = $this->Event->Attribute->fetchAttributes($this->Auth->user(), array( 'conditions' => $conditions, 'flatten' => 1, 'event_ids' => true, 'list' => true, )); - $result = array_values($attributeHits); - // we now have a list of event IDs that match on an attribute level, and the user can see it. Let's also find all of the events that match on other criteria! // What is interesting here is that we no longer have to worry about the event's releasability. With attributes this was a different case, // because we might run into a situation where a user can see an event but not a specific attribute @@ -234,10 +232,9 @@ class EventsController extends AppController foreach ($values as $v) { $subconditions[] = array('lower(name) LIKE' => $v); } - $orgs = $this->Event->Org->find('list', array( + $orgs = $this->Event->Org->find('column', array( 'conditions' => $subconditions, - 'recursive' => -1, - 'fields' => array('id') + 'fields' => array('Org.id') )); $conditions = empty($result) ? [] : ['NOT' => ['id' => $result]]; // Do not include events that we already found @@ -246,11 +243,10 @@ class EventsController extends AppController $conditions['OR'][] = array('lower(uuid) LIKE' => $v); } if (!empty($orgs)) { - $conditions['OR']['orgc_id'] = array_values($orgs); + $conditions['OR']['orgc_id'] = $orgs; } - $otherEvents = $this->Event->find('list', array( - 'recursive' => -1, - 'fields' => array('id'), + $otherEvents = $this->Event->find('column', array( + 'fields' => array('Event.id'), 'conditions' => $conditions, )); foreach ($otherEvents as $eventId) { @@ -311,7 +307,6 @@ class EventsController extends AppController } else { $pieces = explode('|', $v); } - $temp = array(); $eventidConditions = array(); foreach ($pieces as $piece) { $piece = trim($piece); @@ -454,9 +449,9 @@ class EventsController extends AppController $test = array(); foreach ($pieces as $piece) { if ($piece[0] == '!') { - $this->paginate['conditions']['AND'][] = array('lower(Event.info)' . ' NOT LIKE' => '%' . strtolower(substr($piece, 1)) . '%'); + $this->paginate['conditions']['AND'][] = array('lower(Event.info) NOT LIKE' => '%' . strtolower(substr($piece, 1)) . '%'); } else { - $test['OR'][] = array('lower(Event.info)' . ' LIKE' => '%' . strtolower($piece) . '%'); + $test['OR'][] = array('lower(Event.info) LIKE' => '%' . strtolower($piece) . '%'); } } $this->paginate['conditions']['AND'][] = $test; @@ -495,17 +490,13 @@ class EventsController extends AppController $filterString .= '!' . $piece; continue; } - $block = $this->Event->EventTag->find('all', array( - 'conditions' => array('EventTag.tag_id' => $tagName['Tag']['id']), - 'fields' => 'event_id', - 'recursive' => -1, + $block = $this->Event->EventTag->find('column', array( + 'conditions' => array('EventTag.tag_id' => $tagName['Tag']['id']), + 'fields' => ['EventTag.event_id'], )); if (!empty($block)) { - $sqlSubQuery = 'Event.id NOT IN ('; - foreach ($block as $b) { - $sqlSubQuery .= $b['EventTag']['event_id'] . ','; - } - $tagRules['AND'][] = substr($sqlSubQuery, 0, -1) . ')'; + $sqlSubQuery = 'Event.id NOT IN (' . implode(",", $block) . ')'; + $tagRules['AND'][] = $sqlSubQuery; } if ($filterString != "") { $filterString .= "|"; @@ -532,18 +523,14 @@ class EventsController extends AppController continue; } - $allow = $this->Event->EventTag->find('all', array( - 'conditions' => array('EventTag.tag_id' => $tagName['Tag']['id']), - 'fields' => 'event_id', - 'recursive' => -1, + $allow = $this->Event->EventTag->find('column', array( + 'conditions' => array('EventTag.tag_id' => $tagName['Tag']['id']), + 'fields' => ['EventTag.event_id'], )); if (!empty($allow)) { - $sqlSubQuery = 'Event.id IN ('; - foreach ($allow as $a) { - $setOR = true; - $sqlSubQuery .= $a['EventTag']['event_id'] . ','; - } - $tagRules['OR'][] = substr($sqlSubQuery, 0, -1) . ')'; + $setOR = true; + $sqlSubQuery = 'Event.id IN ('. implode(",", $allow) . ')'; + $tagRules['OR'][] = $sqlSubQuery; } if ($filterString != "") { $filterString .= "|"; @@ -687,22 +674,9 @@ class EventsController extends AppController } } } - $this->set('passedArgs', json_encode($passedArgs)); + // check each of the passed arguments whether they're a filter (could also be a sort for example) and if yes, add it to the pagination conditions $passedArgsArray = $this->__setIndexFilterConditions($passedArgs, $urlparams); - if (!$this->_isRest()) { - $this->paginate['contain'] = array_merge($this->paginate['contain'], array('User.email', 'EventTag')); - } else { - $this->paginate['contain'] = array_merge($this->paginate['contain'], array('User.email')); - } - $this->set('urlparams', $urlparams); - $this->set('passedArgsArray', $passedArgsArray); - $this->paginate = Set::merge($this->paginate, array('contain' => array( - 'ThreatLevel' => array( - 'fields' => array( - 'ThreatLevel.name')) - ), - )); $this->loadModel('GalaxyCluster'); // for REST, don't use the pagination. With this, we'll escape the limit of events shown on the index. @@ -721,11 +695,13 @@ class EventsController extends AppController if (isset($this->paginate['conditions'])) { $rules['conditions'] = $this->paginate['conditions']; } - if (!empty($passedArgs['searchminimal']) || !empty($passedArgs['minimal'])) { - unset($rules['contain']); + $minimal = !empty($passedArgs['searchminimal']) || !empty($passedArgs['minimal']); + if ($minimal) { $rules['recursive'] = -1; $rules['fields'] = array('id', 'timestamp', 'sighting_timestamp', 'published', 'uuid'); $rules['contain'] = array('Orgc.uuid'); + } else { + $rules['contain'][] = 'EventTag'; } $paginationRules = array('page', 'limit', 'sort', 'direction', 'order'); foreach ($paginationRules as $paginationRule) { @@ -733,86 +709,72 @@ class EventsController extends AppController $rules[$paginationRule] = $passedArgs[$paginationRule]; } } - $counting_rules = $rules; - if (!empty($counting_rules['limit'])) { - unset($counting_rules['limit']); - } - if (!empty($counting_rules['page'])) { - unset($counting_rules['page']); - } - $absolute_total = $this->Event->find('count', $counting_rules); + if (empty($rules['limit'])) { $events = array(); $i = 1; - $continue = true; $rules['limit'] = 20000; - while ($continue) { + while (true) { $rules['page'] = $i; $temp = $this->Event->find('all', $rules); - if (!empty($temp)) { + $resultCount = count($temp); + if ($resultCount !== 0) { $events = array_merge($events, $temp); - } else { - $continue = false; + } + if ($resultCount < $rules['limit']) { + break; } $i += 1; } + $absolute_total = count($events); } else { - $events = $this->Event->find('all', $rules); + $counting_rules = $rules; + unset($counting_rules['limit']); + unset($counting_rules['page']); + $absolute_total = $this->Event->find('count', $counting_rules); + + $events = $absolute_total === 0 ? [] : $this->Event->find('all', $rules); } - $total_events = count($events); - foreach ($events as $k => $event) { - if (empty($event['SharingGroup']['name'])) { - unset($events[$k]['SharingGroup']); + + if (!$minimal) { + $tagIds = []; + foreach (array_column($events, 'EventTag') as $eventTags) { + foreach (array_column($eventTags, 'tag_id') as $tagId) { + $tagIds[$tagId] = true; + } } - } - if (empty($passedArgs['searchminimal']) && empty($passedArgs['minimal'])) { - $passes = ceil($total_events / 1000); - for ($i = 0; $i < $passes; $i++) { - $event_tag_objects = array(); - $event_tag_ids = array(); - $elements = 1000; - if ($i == ($passes-1)) { - $elements = ($total_events % 1000); - } - for ($j = 0; $j < $elements; $j++) { - $event_tag_ids[$events[($i*1000) + $j]['Event']['id']] = true; - } - $eventTags = $this->Event->EventTag->find('all', array( + if (!empty($tagIds)) { + $tags = $this->Event->EventTag->Tag->find('all', [ + 'conditions' => [ + 'Tag.id' => array_keys($tagIds), + 'Tag.exportable' => 1, + ], 'recursive' => -1, - 'conditions' => array( - 'EventTag.event_id' => array_keys($event_tag_ids) - ), - 'contain' => array( - 'Tag' => array( - 'conditions' => array('Tag.exportable' => 1), - 'fields' => array('Tag.id', 'Tag.name', 'Tag.colour', 'Tag.is_galaxy') - ) - ) - )); - foreach ($eventTags as $ket => $et) { - if (empty($et['Tag']['id'])) { - unset($eventTags[$ket]); - } else { - $et['EventTag']['Tag'] = $et['Tag']; - unset($et['Tag']); - if (empty($event_tag_objects[$et['EventTag']['event_id']])) { - $event_tag_objects[$et['EventTag']['event_id']] = array($et['EventTag']); + 'fields' => ['Tag.id', 'Tag.name', 'Tag.colour', 'Tag.is_galaxy'], + ]); + unset($tagIds); + $tags = array_column(array_column($tags, 'Tag'), null, 'id'); + + foreach ($events as $k => $event) { + if (empty($event['EventTag'])) { + continue; + } + foreach ($event['EventTag'] as $k2 => $et) { + if (!isset($tags[$et['tag_id']])) { + unset($events[$k]['EventTag'][$k2]); // tag not exists or is not exportable } else { - $event_tag_objects[$et['EventTag']['event_id']][] = $et['EventTag']; + $events[$k]['EventTag'][$k2]['Tag'] = $tags[$et['tag_id']]; } } } - for ($j = 0; $j < $elements; $j++) { - if (!empty($event_tag_objects[$events[($i*1000) + $j]['Event']['id']])) { - $events[($i*1000) + $j]['EventTag'] = $event_tag_objects[$events[($i*1000) + $j]['Event']['id']]; - } else { - $events[($i*1000) + $j]['EventTag'] = array(); - } - } + $events = $this->GalaxyCluster->attachClustersToEventIndex($this->Auth->user(), $events, false, false); } - $events = $this->GalaxyCluster->attachClustersToEventIndex($this->Auth->user(), $events); foreach ($events as $key => $event) { - $temp = $events[$key]['Event']; + if (empty($event['SharingGroup']['name'])) { + unset($event['SharingGroup']); + } + + $temp = $event['Event']; $temp['Org'] = $event['Org']; $temp['Orgc'] = $event['Orgc']; unset($temp['user_id']); @@ -827,39 +789,53 @@ class EventsController extends AppController if ($this->response->type() === 'application/xml') { $events = array('Event' => $events); } - return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, array('X-Result-Count' => $absolute_total)); } else { foreach ($events as $key => $event) { $event['Event']['orgc_uuid'] = $event['Orgc']['uuid']; $events[$key] = $event['Event']; } - return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, array('X-Result-Count' => $absolute_total)); } - } else { - $events = $this->paginate(); - foreach ($events as $k => $event) { - if (empty($event['SharingGroup']['name'])) { - unset($events[$k]['SharingGroup']); - } + return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, ['X-Result-Count' => $absolute_total]); + } + + $this->paginate['contain']['ThreatLevel'] = [ + 'fields' => array('ThreatLevel.name') + ]; + $this->paginate['contain'][] = 'EventTag'; + if ($this->_isSiteAdmin()) { + $this->paginate['contain'][] = 'User.email'; + } + + $events = $this->paginate(); + + if (count($events) === 1 && isset($this->passedArgs['searchall'])) { + $this->redirect(array('controller' => 'events', 'action' => 'view', $events[0]['Event']['id'])); + } + + foreach ($events as $k => $event) { + if (empty($event['SharingGroup']['name'])) { + unset($events[$k]['SharingGroup']); } - if (count($events) == 1 && isset($this->passedArgs['searchall'])) { - $this->redirect(array('controller' => 'events', 'action' => 'view', $events[0]['Event']['id'])); - } - $events = $this->Event->attachTagsToEvents($events); - if (Configure::read('MISP.showCorrelationsOnIndex')) { - $events = $this->Event->attachCorrelationCountToEvents($this->Auth->user(), $events); - } - if (Configure::read('MISP.showSightingsCountOnIndex')) { - $events = $this->Event->attachSightingsCountToEvents($this->Auth->user(), $events); - } - if (Configure::read('MISP.showProposalsCountOnIndex')) { - $events = $this->Event->attachProposalsCountToEvents($this->Auth->user(), $events); - } - if (Configure::read('MISP.showDiscussionsCountOnIndex')) { - $events = $this->Event->attachDiscussionsCountToEvents($this->Auth->user(), $events); - } - $events = $this->GalaxyCluster->attachClustersToEventIndex($this->Auth->user(), $events, true, false); - $this->set('events', $events); + } + $events = $this->Event->attachTagsToEvents($events); + if (Configure::read('MISP.showCorrelationsOnIndex')) { + $events = $this->Event->attachCorrelationCountToEvents($this->Auth->user(), $events); + } + if (Configure::read('MISP.showSightingsCountOnIndex')) { + $events = $this->Event->attachSightingsCountToEvents($this->Auth->user(), $events); + } + if (Configure::read('MISP.showProposalsCountOnIndex')) { + $events = $this->Event->attachProposalsCountToEvents($this->Auth->user(), $events); + } + if (Configure::read('MISP.showDiscussionsCountOnIndex')) { + $events = $this->Event->attachDiscussionsCountToEvents($this->Auth->user(), $events); + } + $events = $this->GalaxyCluster->attachClustersToEventIndex($this->Auth->user(), $events, true, false); + + if ($this->params['ext'] === 'csv') { + App::uses('CsvExport', 'Export'); + $export = new CsvExport(); + return $this->RestResponse->viewData($export->eventIndex($events), 'csv'); } $user = $this->Auth->user(); @@ -882,16 +858,17 @@ class EventsController extends AppController $this->Flash->info(__('No GnuPG key set in your profile. To receive attributes in emails, submit your public key in your profile.')); } } + + $this->set('events', $events); $this->set('eventDescriptions', $this->Event->fieldDescriptions); $this->set('analysisLevels', $this->Event->analysisLevels); $this->set('distributionLevels', $this->Event->distributionLevels); $this->set('shortDist', $this->Event->shortDist); $this->set('distributionData', $this->genDistributionGraph(-1)); - if ($this->params['ext'] === 'csv') { - App::uses('CsvExport', 'Export'); - $export = new CsvExport(); - return $this->RestResponse->viewData($export->eventIndex($events), 'csv'); - } + $this->set('urlparams', $urlparams); + $this->set('passedArgsArray', $passedArgsArray); + $this->set('passedArgs', json_encode($passedArgs)); + if ($this->request->is('ajax')) { $this->autoRender = false; $this->layout = false; @@ -962,17 +939,10 @@ class EventsController extends AppController } } $this->set('filtering', json_encode($filtering)); - $tags = $this->Event->EventTag->Tag->find('all', array('recursive' => -1)); - $tagNames = array(); + $tagNames = $this->Event->EventTag->Tag->find('list', array('recursive' => -1, 'fields' => ['Tag.id', 'Tag.name'])); $tagJSON = array(); - foreach ($tags as $k => $v) { - $tagNames[$v['Tag']['id']] = $v['Tag']['name']; - $tagJSON[] = array('id' => $v['Tag']['id'], 'value' => h($v['Tag']['name'])); - } - $conditions = array(); - if (!$this->_isSiteAdmin()) { - $eIds = $this->Event->fetchEventIds($this->Auth->user(), false, false, false, true); - $conditions['AND'][] = array('Event.id' => $eIds); + foreach ($tagNames as $tagId => $tagName) { + $tagJSON[] = array('id' => $tagId, 'value' => h($tagName)); } $rules = array('published', 'eventid', 'tag', 'date', 'eventinfo', 'threatlevel', 'distribution', 'sharinggroup', 'analysis', 'attribute', 'hasproposal'); if ($this->_isSiteAdmin()) { @@ -997,7 +967,6 @@ class EventsController extends AppController $this->set('tags', $tagNames); $this->set('tagJSON', json_encode($tagJSON)); $this->set('rules', $rules); - $this->set('baseurl', Configure::read('MISP.baseurl')); $this->layout = 'ajax'; } @@ -1643,17 +1612,17 @@ class EventsController extends AppController } if ($this->_isRest()) { - $this->set('event', $event); - } else { - $this->set('deleted', isset($deleted) ? ($deleted == 2 ? 0 : 1) : 0); - $this->set('includeRelatedTags', (!empty($this->params['named']['includeRelatedTags'])) ? 1 : 0); - $this->set('includeDecayScore', (!empty($this->params['named']['includeDecayScore'])) ? 1 : 0); - - if ($this->_isSiteAdmin() && $event['Event']['orgc_id'] !== $this->Auth->user('org_id')) { - $this->Flash->info(__('You are currently logged in as a site administrator and about to edit an event not belonging to your organisation. This goes against the sharing model of MISP. Use a normal user account for day to day work.')); - } - $this->__viewUI($event, $continue, $fromEvent); + return $this->__restResponse($event); } + + $this->set('deleted', isset($deleted) ? ($deleted == 2 ? 0 : 1) : 0); + $this->set('includeRelatedTags', (!empty($this->params['named']['includeRelatedTags'])) ? 1 : 0); + $this->set('includeDecayScore', (!empty($this->params['named']['includeDecayScore'])) ? 1 : 0); + + if ($this->_isSiteAdmin() && $event['Event']['orgc_id'] !== $this->Auth->user('org_id')) { + $this->Flash->info(__('You are currently logged in as a site administrator and about to edit an event not belonging to your organisation. This goes against the sharing model of MISP. Use a normal user account for day to day work.')); + } + $this->__viewUI($event, $continue, $fromEvent); } private function __startPivoting($id, $info, $date) @@ -1945,9 +1914,7 @@ class EventsController extends AppController if (!empty($validationErrors)) { $event['errors'] = $validationErrors; } - $this->set('event', $event); - $this->render('view'); - return true; + return $this->__restResponse($event); } else { // redirect to the view of the newly created event $this->Flash->success(__('The event has been saved')); @@ -2143,11 +2110,11 @@ class EventsController extends AppController $this->Flash->error(__('Could not import STIX document: ' . $result)); } } else { - $max_size = intval(ini_get('post_max_size')); - if (intval(ini_get('upload_max_filesize')) < $max_size) { - $max_size = intval(ini_get('upload_max_filesize')); + $maxUploadSize = intval(ini_get('post_max_size')); + if (intval(ini_get('upload_max_filesize')) < $maxUploadSize) { + $maxUploadSize = intval(ini_get('upload_max_filesize')); } - $this->Flash->error(__('File upload failed. Make sure that you select a stix file to be uploaded and that the file doesn\'t exceed the maximum file size of ' . $max_size . '.')); + $this->Flash->error(__('File upload failed. Make sure that you select a STIX file to be uploaded and that the file doesn\'t exceed the maximum file size of %s MB.', $maxUploadSize)); } } } @@ -2179,7 +2146,7 @@ class EventsController extends AppController } } } - $target_event = $this->Event->fetchSimpleEvent($this->Auth->user(), $target_id); + $target_event = $this->Event->fetchSimpleEvent($this->Auth->user(), $target_id, ['contain' => ['Orgc']]); if (empty($target_event)) { throw new NotFoundException(__('Invalid target event.')); } @@ -2271,7 +2238,7 @@ class EventsController extends AppController if ($this->request->is('get') && $this->_isRest()) { return $this->RestResponse->describe('Events', 'edit', false, $this->response->type()); } - $event = $this->Event->fetchSimpleEvent($this->Auth->user(), $id); + $event = $this->Event->fetchSimpleEvent($this->Auth->user(), $id, ['contain' => ['Orgc']]); if (!$event) { throw new NotFoundException(__('Invalid event')); } @@ -2310,9 +2277,7 @@ class EventsController extends AppController $metadata = $this->request->param('named.metadata'); $results = $this->Event->fetchEvent($this->Auth->user(), ['eventid' => $id, 'metadata' => $metadata]); $event = $results[0]; - $this->set('event', $event); - $this->render('view'); - return true; + return $this->__restResponse($event); } else { $message = 'Error'; if ($this->_isRest()) { @@ -2531,19 +2496,12 @@ class EventsController extends AppController public function publishSightings($id = null) { - $id = $this->Toolbox->findIdByUuid($this->Event, $id); - $event = $this->Event->fetchEvent( - $this->Auth->user(), - array( - 'eventid' => $id, - 'metadata' => 1 - ) - ); + $event = $this->Event->fetchSimpleEvent($this->Auth->user(), $id); if (empty($event)) { throw new NotFoundException(__('Invalid event')); } if ($this->request->is('post') || $this->request->is('put')) { - $result = $this->Event->publishRouter($id, null, $this->Auth->user(), 'sightings'); + $result = $this->Event->publishRouter($event['Event']['id'], null, $this->Auth->user(), 'sightings'); if (!Configure::read('MISP.background_jobs')) { if (!is_array($result)) { // redirect to the view event page @@ -2568,12 +2526,12 @@ class EventsController extends AppController if (!empty($errors)) { $this->set('errors', $errors); } - $this->set('url', $this->baseurl . '/events/publishSightings/' . $id); - $this->set('id', $id); + $this->set('url', $this->baseurl . '/events/publishSightings/' . $event['Event']['id']); + $this->set('id', $event['Event']['id']); $this->set('_serialize', array('name', 'message', 'url', 'id', 'errors')); } else { $this->Flash->success($message); - $this->redirect(array('action' => 'view', $id)); + $this->redirect(array('action' => 'view', $event['Event']['id'])); } } else { $this->set('id', $id); @@ -2742,8 +2700,8 @@ class EventsController extends AppController // Users with a GnuPG key will get the mail encrypted, other users will get the mail unencrypted public function contact($id = null) { - $events = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $id)); - if (empty($events)) { + $event = $this->Event->fetchSimpleEvent($this->Auth->user(), $id, ['contain' => ['Orgc']]); + if (empty($event)) { throw new NotFoundException(__('Invalid event')); } // User has filled in his contact form, send out the email. @@ -2758,7 +2716,7 @@ class EventsController extends AppController throw new MethodNotAllowedException($error); } else { $this->Flash->error($error); - $this->redirect(array('action' => 'contact', $id)); + $this->redirect(array('action' => 'contact', $event['Event']['id'])); } } @@ -2769,31 +2727,29 @@ class EventsController extends AppController $user = $this->Auth->user(); $user = $this->Event->User->fillKeysToUser($user); - $success = $this->Event->sendContactEmailRouter($id, $message, $creator_only, $user); + $success = $this->Event->sendContactEmailRouter($event['Event']['id'], $message, $creator_only, $user); if ($success) { $return_message = __('Email sent to the reporter.'); if ($this->_isRest()) { - return $this->RestResponse->saveSuccessResponse('Events', 'contact', $id, $this->response->type(), $return_message); + return $this->RestResponse->saveSuccessResponse('Events', 'contact', $event['Event']['id'], $this->response->type(), $return_message); } else { $this->Flash->success($return_message); // redirect to the view event page - $this->redirect(array('action' => 'view', $id)); + $this->redirect(array('action' => 'view', $event['Event']['id'])); } } else { $return_message = __('Sending of email failed.'); if ($this->_isRest()) { - return $this->RestResponse->saveFailResponse('Events', 'contact', $id, $return_message, $this->response->type()); + return $this->RestResponse->saveFailResponse('Events', 'contact', $event['Event']['id'], $return_message, $this->response->type()); } else { $this->Flash->error($return_message, 'default', array(), 'error'); // redirect to the view event page - $this->redirect(array('action' => 'view', $id)); + $this->redirect(array('action' => 'view', $event['Event']['id'])); } } } - // User didn't see the contact form yet. Present it to him. - if (empty($this->data)) { - $this->data = $events[0]; - } + $this->set('event', $event); + $this->set('mayModify', $this->__canModifyEvent($event)); } public function automation($legacy = false) @@ -3618,8 +3574,7 @@ class EventsController extends AppController $this->set('sgs', $sgs); $this->set('event', $event); $this->set('mayModify', $this->__canModifyEvent($event)); - $this->set('typeList', array_keys($this->Event->Attribute->typeDefinitions)); - $this->set('defaultCategories', $this->Event->Attribute->defaultCategories); + $this->set('typeDefinitions', $this->Event->Attribute->typeDefinitions); $this->set('typeCategoryMapping', $typeCategoryMapping); foreach ($typeCategoryMapping as $k => $v) { $typeCategoryMapping[$k] = array_values($v); @@ -3656,7 +3611,7 @@ class EventsController extends AppController $attribute['category'] = $attribute['default_category']; unset($attribute['default_category']); } else { - $attribute['category'] = $this->Event->Attribute->defaultCategories[$attribute['type']]; + $attribute['category'] = $this->Event->Attribute->typeDefinitions[$attribute['type']]['default_category']; } $attribute['distribution'] = $distribution; $attribute['event_id'] = $event['Event']['id']; @@ -3857,104 +3812,105 @@ class EventsController extends AppController if (empty($event)) { throw new NotFoundException(__('Event not found or you are not authorised to view it.')); } + $id = $event['Event']['id']; // #TODO i18n $exports = array( 'xml' => array( - 'url' => $this->baseurl . '/events/restSearch/xml/eventid:' . $id . '.xml', - 'text' => 'MISP XML (metadata + all attributes)', - 'requiresPublished' => false, - 'checkbox' => true, - 'checkbox_text' => 'Encode Attachments', - 'checkbox_set' => $this->baseurl . '/events/restSearch/xml/eventid:' . $id . '/withAttachments:1.xml', - 'checkbox_default' => true + 'url' => $this->baseurl . '/events/restSearch/xml/eventid:' . $id . '.xml', + 'text' => 'MISP XML (metadata + all attributes)', + 'requiresPublished' => false, + 'checkbox' => true, + 'checkbox_text' => 'Encode Attachments', + 'checkbox_set' => $this->baseurl . '/events/restSearch/xml/eventid:' . $id . '/withAttachments:1.xml', + 'checkbox_default' => true ), 'json' => array( - 'url' => $this->baseurl . '/events/restSearch/json/eventid:' . $id . '.json', - 'text' => 'MISP JSON (metadata + all attributes)', - 'requiresPublished' => false, - 'checkbox' => true, - 'checkbox_text' => 'Encode Attachments', - 'checkbox_set' => $this->baseurl . '/events/restSearch/json/withAttachments:1/eventid:' . $id . '.json', - 'checkbox_default' => true + 'url' => $this->baseurl . '/events/restSearch/json/eventid:' . $id . '.json', + 'text' => 'MISP JSON (metadata + all attributes)', + 'requiresPublished' => false, + 'checkbox' => true, + 'checkbox_text' => 'Encode Attachments', + 'checkbox_set' => $this->baseurl . '/events/restSearch/json/withAttachments:1/eventid:' . $id . '.json', + 'checkbox_default' => true ), 'openIOC' => array( - 'url' => $this->baseurl . '/events/restSearch/openioc/to_ids:1/published:1/eventid:' . $id . '.json', - 'text' => 'OpenIOC (all indicators marked to IDS)', - 'requiresPublished' => false, - 'checkbox' => false, + 'url' => $this->baseurl . '/events/restSearch/openioc/to_ids:1/published:1/eventid:' . $id . '.json', + 'text' => 'OpenIOC (all indicators marked to IDS)', + 'requiresPublished' => false, + 'checkbox' => false, ), 'csv' => array( - 'url' => $this->baseurl . '/events/restSearch/returnFormat:csv/to_ids:1/published:1/includeContext:0/eventid:' . $id, - 'text' => 'CSV', - 'requiresPublished' => false, - 'checkbox' => true, - 'checkbox_text' => 'Include non-IDS marked attributes', - 'checkbox_set' => $this->baseurl . '/events/restSearch/returnFormat:csv/to_ids:1||0/published:1||0/includeContext:0/eventid:' . $id + 'url' => $this->baseurl . '/events/restSearch/returnFormat:csv/to_ids:1/published:1/includeContext:0/eventid:' . $id, + 'text' => 'CSV', + 'requiresPublished' => false, + 'checkbox' => true, + 'checkbox_text' => 'Include non-IDS marked attributes', + 'checkbox_set' => $this->baseurl . '/events/restSearch/returnFormat:csv/to_ids:1||0/published:1||0/includeContext:0/eventid:' . $id ), 'csv_with_context' => array( - 'url' => $this->baseurl . '/events/restSearch/returnFormat:csv/to_ids:1/published:1/includeContext:1/eventid:' . $id, - 'text' => 'CSV with additional context', - 'requiresPublished' => false, - 'checkbox' => true, - 'checkbox_text' => 'Include non-IDS marked attributes', - 'checkbox_set' => $this->baseurl . '/events/restSearch/returnFormat:csv/to_ids:1||0/published:1||0/includeContext:1/eventid:' . $id + 'url' => $this->baseurl . '/events/restSearch/returnFormat:csv/to_ids:1/published:1/includeContext:1/eventid:' . $id, + 'text' => 'CSV with additional context', + 'requiresPublished' => false, + 'checkbox' => true, + 'checkbox_text' => 'Include non-IDS marked attributes', + 'checkbox_set' => $this->baseurl . '/events/restSearch/returnFormat:csv/to_ids:1||0/published:1||0/includeContext:1/eventid:' . $id ), 'stix_xml' => array( - 'url' => $this->baseurl . '/events/restSearch/stix/eventid:' . $id, - 'text' => 'STIX XML (metadata + all attributes)', - 'requiresPublished' => false, - 'checkbox' => true, - 'checkbox_text' => 'Encode Attachments', - 'checkbox_set' => $this->baseurl . '/events/restSearch/stix/eventid:' . $id . '/withAttachments:1' + 'url' => $this->baseurl . '/events/restSearch/stix/eventid:' . $id, + 'text' => 'STIX 1 XML (metadata + all attributes)', + 'requiresPublished' => false, + 'checkbox' => true, + 'checkbox_text' => 'Encode Attachments', + 'checkbox_set' => $this->baseurl . '/events/restSearch/stix/eventid:' . $id . '/withAttachments:1' ), 'stix_json' => array( - 'url' => $this->baseurl . '/events/restSearch/stix-json/eventid:' . $id, - 'text' => 'STIX JSON (metadata + all attributes)', - 'requiresPublished' => false, - 'checkbox' => true, - 'checkbox_text' => 'Encode Attachments', - 'checkbox_set' => $this->baseurl . '/events/restSearch/stix-json/withAttachments:1/eventid:' . $id + 'url' => $this->baseurl . '/events/restSearch/stix-json/eventid:' . $id, + 'text' => 'STIX 1 JSON (metadata + all attributes)', + 'requiresPublished' => false, + 'checkbox' => true, + 'checkbox_text' => 'Encode Attachments', + 'checkbox_set' => $this->baseurl . '/events/restSearch/stix-json/withAttachments:1/eventid:' . $id ), 'stix2_json' => array( - 'url' => $this->baseurl . '/events/restSearch/stix2/eventid:' . $id, - 'text' => 'STIX2 (requires the STIX 2 library)', - 'requiresPublished' => false, - 'checkbox' => true, - 'checkbox_text' => 'Encode Attachments', - 'checkbox_set' => $this->baseurl . '/events/restSearch/stix2/eventid:' . $id . '/withAttachments:1' + 'url' => $this->baseurl . '/events/restSearch/stix2/eventid:' . $id, + 'text' => 'STIX 2', + 'requiresPublished' => false, + 'checkbox' => true, + 'checkbox_text' => 'Encode Attachments', + 'checkbox_set' => $this->baseurl . '/events/restSearch/stix2/eventid:' . $id . '/withAttachments:1' ), 'rpz' => array( - 'url' => $this->baseurl . '/attributes/restSearch/returnFormat:rpz/published:1||0/eventid:' . $id, - 'text' => 'RPZ Zone file', - 'requiresPublished' => false, - 'checkbox' => false, + 'url' => $this->baseurl . '/attributes/restSearch/returnFormat:rpz/published:1||0/eventid:' . $id, + 'text' => 'RPZ Zone file', + 'requiresPublished' => false, + 'checkbox' => false, ), 'suricata' => array( - 'url' => $this->baseurl . '/events/restSearch/returnFormat:suricata/published:1||0/eventid:' . $id, - 'text' => 'Download Suricata rules', - 'requiresPublished' => false, - 'checkbox' => false, + 'url' => $this->baseurl . '/events/restSearch/returnFormat:suricata/published:1||0/eventid:' . $id, + 'text' => 'Download Suricata rules', + 'requiresPublished' => false, + 'checkbox' => false, ), 'snort' => array( - 'url' => $this->baseurl . '/events/restSearch/returnFormat:snort/published:1||0/eventid:' . $id, - 'text' => 'Download Snort rules', - 'requiresPublished' => false, - 'checkbox' => false, + 'url' => $this->baseurl . '/events/restSearch/returnFormat:snort/published:1||0/eventid:' . $id, + 'text' => 'Download Snort rules', + 'requiresPublished' => false, + 'checkbox' => false, ), 'bro' => array( - 'url' => $this->baseurl . '/attributes/bro/download/all/false/' . $id, - // 'url' => '/attributes/restSearch/returnFormat:bro/published:1||0/eventid:' . $id, - 'text' => 'Download Bro rules', - 'requiresPublished' => false, - 'checkbox' => false + 'url' => $this->baseurl . '/attributes/bro/download/all/false/' . $id, + // 'url' => '/attributes/restSearch/returnFormat:bro/published:1||0/eventid:' . $id, + 'text' => 'Download Bro rules', + 'requiresPublished' => false, + 'checkbox' => false ), 'text' => array( - 'text' => 'Export all attribute values as a text file', - 'url' => $this->baseurl . '/attributes/restSearch/returnFormat:text/published:1||0/eventid:' . $id, - 'requiresPublished' => false, - 'checkbox' => true, - 'checkbox_text' => 'Include non-IDS marked attributes', - 'checkbox_set' => $this->baseurl . '/attributes/restSearch/returnFormat:text/published:1||0/to_ids:1||0/eventid:' . $id + 'text' => 'Export all attribute values as a text file', + 'url' => $this->baseurl . '/attributes/restSearch/returnFormat:text/published:1||0/eventid:' . $id, + 'requiresPublished' => false, + 'checkbox' => true, + 'checkbox_text' => 'Include non-IDS marked attributes', + 'checkbox_set' => $this->baseurl . '/attributes/restSearch/returnFormat:text/published:1||0/to_ids:1||0/eventid:' . $id ), ); if ($event['Event']['published'] == 0) { @@ -3975,10 +3931,10 @@ class EventsController extends AppController if (is_array($modules) && !empty($modules)) { foreach ($modules['modules'] as $module) { $exports[$module['name']] = array( - 'url' => $this->baseurl . '/events/exportModule/' . $module['name'] . '/' . $id, - 'text' => Inflector::humanize($module['name']), - 'requiresPublished' => true, - 'checkbox' => false, + 'url' => $this->baseurl . '/events/exportModule/' . $module['name'] . '/' . $id, + 'text' => Inflector::humanize($module['name']), + 'requiresPublished' => true, + 'checkbox' => false, ); } } @@ -4293,9 +4249,10 @@ class EventsController extends AppController public function viewGraph($id) { + // Event data are fetched by 'updateGraph', here we need just metadata. $event = $this->Event->fetchEvent($this->Auth->user(), array( 'eventid' => $id, - 'includeGranularCorrelations' => 1 + 'metadata' => true, )); if (empty($event)) { throw new MethodNotAllowedException(__('Invalid Event.')); @@ -4349,8 +4306,8 @@ class EventsController extends AppController $grapher = new DistributionGraphTool(); $this->loadModel('Server'); - $servers = $this->Server->find('list', array( - 'fields' => array('name'), + $servers = $this->Server->find('column', array( + 'fields' => array('Server.name'), )); $grapher->construct($this->Event, $servers, $this->Auth->user(), $extended); $json = $grapher->get_distributions_graph($id); @@ -4906,8 +4863,7 @@ class EventsController extends AppController $this->set('event', array('Event' => $attribute[0]['Event'])); } $this->set('resultArray', $resultArray); - $this->set('typeList', array_keys($this->Event->Attribute->typeDefinitions)); - $this->set('defaultCategories', $this->Event->Attribute->defaultCategories); + $this->set('typeDefinitions', $this->Event->Attribute->typeDefinitions); $this->set('typeCategoryMapping', $typeCategoryMapping); $this->set('title', 'Enrichment Results'); $this->set('importComment', $importComment); @@ -5096,8 +5052,7 @@ class EventsController extends AppController } $this->set('event', $event); $this->set('resultArray', $resultArray); - $this->set('typeList', array_keys($this->Event->Attribute->typeDefinitions)); - $this->set('defaultCategories', $this->Event->Attribute->defaultCategories); + $this->set('typeDefinitions', $this->Event->Attribute->typeDefinitions); $this->set('typeCategoryMapping', $typeCategoryMapping); $render_name = 'resolved_attributes'; } @@ -5121,6 +5076,7 @@ class EventsController extends AppController $this->set('module', $module); $this->set('eventId', $eventId); $this->set('event', $event); + $this->set('mayModify', $this->__canModifyEvent($event)); } public function exportModule($module, $id, $standard = false) @@ -5721,4 +5677,37 @@ class EventsController extends AppController } return $this->RestResponse->viewData($allConflicts); } + + /** + * @param array $event + * @return CakeResponseTmp + * @throws Exception + */ + private function __restResponse(array $event) + { + $tmpFile = new TmpFileTool(); + + if ($this->request->is('json')) { + App::uses('JSONConverterTool', 'Tools'); + $converter = new JSONConverterTool(); + if ($this->RestResponse->isAutomaticTool()) { + foreach ($converter->streamConvert($event) as $part) { + $tmpFile->write($part); + } + } else { + $tmpFile->write($converter->convert($event)); + } + $format = 'json'; + } elseif ($this->request->is('xml')) { + App::uses('XMLConverterTool', 'Tools'); + $converter = new XMLConverterTool(); + foreach ($converter->frameCollection($converter->convert($event)) as $chunk) { + $tmpFile->write($chunk); + } + $format = 'xml'; + } else { + throw new Exception("Invalid format, only JSON or XML is supported."); + } + return $this->RestResponse->viewData($tmpFile, $format, false, true); + } } diff --git a/app/Controller/FeedsController.php b/app/Controller/FeedsController.php index b7be783d7..f25cf37ca 100644 --- a/app/Controller/FeedsController.php +++ b/app/Controller/FeedsController.php @@ -773,8 +773,10 @@ class FeedsController extends AppController return $this->RestResponse->viewData($event, $this->response->type()); } if (is_array($event)) { - $this->loadModel('Warninglist'); - $this->Warninglist->attachWarninglistToAttributes($event['Event']['Attribute']); + if (isset($event['Event']['Attribute'])) { + $this->loadModel('Warninglist'); + $this->Warninglist->attachWarninglistToAttributes($event['Event']['Attribute']); + } $this->loadModel('Event'); $params = $this->Event->rearrangeEventForView($event, $this->passedArgs, $all); diff --git a/app/Controller/GalaxiesController.php b/app/Controller/GalaxiesController.php index c59a1ebe1..630cde267 100644 --- a/app/Controller/GalaxiesController.php +++ b/app/Controller/GalaxiesController.php @@ -19,11 +19,7 @@ class GalaxiesController extends AppController public function index() { $aclConditions = array(); - $filters = $this->IndexFilter->harvestParameters(array('context', 'value')); - $contextConditions = array(); - if (empty($filters['context'])) { - $filters['context'] = 'all'; - } + $filters = $this->IndexFilter->harvestParameters(array('value')); $searchConditions = array(); if (empty($filters['value'])) { $filters['value'] = ''; @@ -45,18 +41,16 @@ class GalaxiesController extends AppController array( 'recursive' => -1, 'conditions' => array( - 'AND' => array($contextConditions, $searchConditions, $aclConditions) + 'AND' => array($searchConditions, $aclConditions) ) ) ); return $this->RestResponse->viewData($galaxies, $this->response->type()); } else { - $this->paginate['conditions']['AND'][] = $contextConditions; $this->paginate['conditions']['AND'][] = $searchConditions; $this->paginate['conditions']['AND'][] = $aclConditions; $galaxies = $this->paginate(); $this->set('galaxyList', $galaxies); - $this->set('context', $filters['context']); $this->set('searchall', $filters['value']); } } diff --git a/app/Controller/LogsController.php b/app/Controller/LogsController.php index 538aab069..ed5370ee3 100644 --- a/app/Controller/LogsController.php +++ b/app/Controller/LogsController.php @@ -66,7 +66,7 @@ class LogsController extends AppController $conditions['AND'][] = array('created <= ' => date("Y-m-d H:i:s", $tempData[0])); $conditions['AND'][] = array('created >= ' => date("Y-m-d H:i:s", $tempData[1])); } - } else { + } else if ($filter !== 'limit' && $filter !== 'page') { $data = array('OR' => $data); $conditions = $this->Log->generic_add_filter($conditions, $data, 'Log.' . $filter); } @@ -88,20 +88,13 @@ class LogsController extends AppController $log_entries = $this->Log->find('all', $params); return $this->RestResponse->viewData($log_entries, 'json'); } else { - if (!$this->userRole['perm_audit']) { - $this->redirect(array('controller' => 'events', 'action' => 'index', 'admin' => false)); - } $this->set('isSearch', 0); $this->recursive = 0; $validFilters = $this->Log->logMeta; if (!$this->_isSiteAdmin()) { $orgRestriction = $this->Auth->user('Organisation')['name']; $conditions['Log.org'] = $orgRestriction; - $this->paginate = array( - 'limit' => 60, - 'conditions' => $conditions, - 'order' => array('Log.id' => 'DESC') - ); + $this->paginate['conditions'] = $conditions; } else { $validFilters = array_merge_recursive($validFilters, $this->Log->logMetaAdmin); } @@ -115,17 +108,16 @@ class LogsController extends AppController } // Shows a minimalistic history for the currently selected event - public function event_index($id) + public function event_index($id, $org = null) { - // check if the user has access to this event... - $mayModify = false; $this->loadModel('Event'); $event = $this->Event->fetchEvent($this->Auth->user(), array( 'eventid' => $id, - 'includeAllTags' => 1, 'sgReferenceOnly' => 1, 'deleted' => [0, 1], - 'deleted_proposals' => 1 + 'deleted_proposals' => 1, + 'noSightings' => true, + 'noEventReports' => true, )); if (empty($event)) { throw new NotFoundException('Invalid event.'); @@ -190,57 +182,54 @@ class LogsController extends AppController ) ); } - // send unauthorised people away. Only site admins and users of the same org may see events that are "your org only". Everyone else can proceed for all other levels of distribution - $mineOrAdmin = true; - if (!$this->_isSiteAdmin() && $event['Event']['org_id'] != $this->Auth->user('org_id')) { - $mineOrAdmin = false; - } - $this->set('published', $event['Event']['published']); - if ($mineOrAdmin && $this->userRole['perm_modify']) { - $mayModify = true; + + if ($org) { + $conditions['org'] = $org; } - $fieldList = array('title', 'created', 'model', 'model_id', 'action', 'change', 'org', 'email'); - $this->paginate = array( - 'limit' => 60, - 'conditions' => $conditions, - 'order' => array('Log.id' => 'DESC'), - 'fields' => $fieldList - ); + $this->paginate['fields'] = array('title', 'created', 'model', 'model_id', 'action', 'change', 'org', 'email'); + $this->paginate['conditions'] = $conditions; + $list = $this->paginate(); if (!$this->_isSiteAdmin()) { $this->loadModel('User'); - $emails = $this->User->find('list', array( - 'conditions' => array('User.org_id' => $this->Auth->user('org_id')), - 'fields' => array('User.id', 'User.email') + $orgEmails = $this->User->find('column', array( + 'conditions' => array('User.org_id' => $this->Auth->user('org_id')), + 'fields' => array('User.email') )); foreach ($list as $k => $item) { - if (!in_array($item['Log']['email'], $emails)) { + if (!in_array($item['Log']['email'], $orgEmails, true)) { $list[$k]['Log']['email'] = ''; } } } if ($this->_isRest()) { - foreach ($list as $k => $item) { - $list[$k] = $item['Log']; - } - $list = array('Log' => $list); + $list = array('Log' => array_column($list, 'Log')); return $this->RestResponse->viewData($list, $this->response->type()); - } else { - $this->set('event', $event); - $this->set('list', $list); - $this->set('eventId', $id); - $this->set('mayModify', $mayModify); } + + // send unauthorised people away. Only site admins and users of the same org may see events that are "your org only". Everyone else can proceed for all other levels of distribution + $mineOrAdmin = true; + if (!$this->_isSiteAdmin() && $event['Event']['org_id'] != $this->Auth->user('org_id')) { + $mineOrAdmin = false; + } + + $mayModify = false; + if ($mineOrAdmin && $this->userRole['perm_modify']) { + $mayModify = true; + } + + $this->set('published', $event['Event']['published']); + $this->set('event', $event); + $this->set('list', $list); + $this->set('eventId', $id); + $this->set('mayModify', $mayModify); } public $helpers = array('Js' => array('Jquery'), 'Highlight'); public function admin_search($new = false) { - if (!$this->userRole['perm_audit']) { - $this->redirect(array('controller' => 'events', 'action' => 'index', 'admin' => false)); - } $orgRestriction = null; if ($this->_isSiteAdmin()) { $orgRestriction = false; diff --git a/app/Controller/NewsController.php b/app/Controller/NewsController.php index 6a6ad0f7f..2601a3edd 100755 --- a/app/Controller/NewsController.php +++ b/app/Controller/NewsController.php @@ -17,24 +17,19 @@ class NewsController extends AppController { $this->paginate['contain'] = array('User' => array('fields' => array('User.email'))); $newsItems = $this->paginate(); - $this->loadModel('User'); - $currentUser = $this->User->find('first', array( - 'recursive' => -1, - 'conditions' => array('User.id' => $this->Auth->user('id')), - 'fields' => array('User.newsread') - )); + + $newsread = $this->Auth->user('newsread'); foreach ($newsItems as $key => $item) { - if ($item['News']['date_created'] > $currentUser['User']['newsread']) { + if ($item['News']['date_created'] > $newsread) { $newsItems[$key]['News']['new'] = true; } else { $newsItems[$key]['News']['new'] = false; } } - $this->User->id = $this->Auth->user('id'); - //if ($this->User->exists()) { - $this->User->saveField('newsread', time()); $this->set('newsItems', $newsItems); - //} + + $this->loadModel('User'); + $this->User->updateField($this->Auth->user(), 'newsread', time()); } public function add() diff --git a/app/Controller/ObjectTemplatesController.php b/app/Controller/ObjectTemplatesController.php index 11046e82a..8d498a6be 100644 --- a/app/Controller/ObjectTemplatesController.php +++ b/app/Controller/ObjectTemplatesController.php @@ -1,7 +1,9 @@ -1 ); - public function objectMetaChoice($event_id) { - $metas = $this->ObjectTemplate->find('list', array( - 'recursive' => -1, + public function objectMetaChoice($event_id) + { + $metas = $this->ObjectTemplate->find('column', array( 'conditions' => array('ObjectTemplate.active' => 1), - 'fields' => array('meta-category', 'meta-category'), - 'group' => array('ObjectTemplate.meta-category'), - 'order' => array('ObjectTemplate.meta-category asc') + 'fields' => array('ObjectTemplate.meta-category'), + 'order' => array('ObjectTemplate.meta-category asc'), + 'unique' => true, )); - $items = array(); - $items[] = array( + $eventId = h($event_id); + $items = [[ 'name' => __('All Objects'), - 'value' => $this->baseurl . "/ObjectTemplates/objectChoice/" . h($event_id) . "/" . "0" - ); - foreach($metas as $meta) { + 'value' => $this->baseurl . "/ObjectTemplates/objectChoice/$eventId/0" + ]]; + foreach ($metas as $meta) { $items[] = array( 'name' => $meta, - 'value' => $this->baseurl . "/ObjectTemplates/objectChoice/" . h($event_id) . "/" . h($meta) + 'value' => $this->baseurl . "/ObjectTemplates/objectChoice/$eventId/" . h($meta) ); } @@ -134,19 +136,20 @@ class ObjectTemplatesController extends AppController } $this->ObjectTemplate->id = $id; if (!$this->ObjectTemplate->exists()) { - throw new NotFoundException('Invalid ObjectTemplate'); + throw new NotFoundException('Invalid Object Template'); } if ($this->ObjectTemplate->delete()) { if ($this->_isRest()) { return $this->RestResponse->saveSuccessResponse('ObjectTemplates', 'admin_delete', $id, $this->response->type()); } else { - $this->Flash->success(__('ObjectTemplate deleted')); + $this->Flash->success(__('Object Template deleted')); } - } - if ($this->_isRest()) { - return $this->RestResponse->saveFailResponse('ObjectTemplates', 'admin_delete', $id, $this->ObjectTemplate->validationErrors, $this->response->type()); } else { - $this->Flash->error('ObjectTemplate could not be deleted'); + if ($this->_isRest()) { + return $this->RestResponse->saveFailResponse('ObjectTemplates', 'admin_delete', $id, $this->ObjectTemplate->validationErrors, $this->response->type()); + } else { + $this->Flash->error('Object Template could not be deleted'); + } } $this->redirect($this->referer()); } @@ -295,4 +298,13 @@ class ObjectTemplatesController extends AppController $this->layout = 'ajax'; $this->render('ajax/getToggleField'); } + + public function getRaw($uuidOrName) + { + $template = $this->ObjectTemplate->getRawFromDisk($uuidOrName); + if (empty($template)) { + throw new NotFoundException(__('Template not found')); + } + return $this->RestResponse->viewData($template, $this->response->type()); + } } diff --git a/app/Controller/ObjectsController.php b/app/Controller/ObjectsController.php index 1a22c6caf..5a85b2a50 100644 --- a/app/Controller/ObjectsController.php +++ b/app/Controller/ObjectsController.php @@ -39,7 +39,7 @@ class ObjectsController extends AppController 'ObjectTemplateElement' ) )); - $event = $this->MispObject->Event->fetchSimpleEvent($this->Auth->user(), $event_id); + $event = $this->MispObject->Event->fetchSimpleEvent($this->Auth->user(), $event_id, ['contain' => ['Orgc']]); if (empty($event)) { throw new NotFoundException(__('Invalid event.')); } @@ -169,7 +169,7 @@ class ObjectsController extends AppController } } // Find the event that is to be updated - $event = $this->MispObject->Event->fetchSimpleEvent($this->Auth->user(), $eventId); + $event = $this->MispObject->Event->fetchSimpleEvent($this->Auth->user(), $eventId, ['contain' => ['Orgc']]); if (empty($event)) { throw new NotFoundException(__('Invalid event.')); } @@ -808,6 +808,9 @@ class ObjectsController extends AppController $this->MispObject->Event->insertLock($this->Auth->user(), $eventId); } if ($this->request->is('post') || $this->request->is('delete')) { + if (!empty($this->request->data['hard'])) { + $hard = true; + } if ($this->__delete($object['Object']['id'], $hard)) { $message = 'Object deleted.'; if ($this->request->is('ajax')) { diff --git a/app/Controller/OrganisationsController.php b/app/Controller/OrganisationsController.php index 9e9cd509f..6cf0f9a04 100644 --- a/app/Controller/OrganisationsController.php +++ b/app/Controller/OrganisationsController.php @@ -47,57 +47,62 @@ class OrganisationsController extends AppController $searchall = $this->passedArgs['searchall']; } - if (isset($searchall) && !empty($searchall)) { $passedArgs['searchall'] = $searchall; $allSearchFields = array('name', 'description', 'nationality', 'sector', 'type', 'contacts', 'restricted_to_domain', 'uuid'); + $searchTerm = '%' . strtolower($passedArgs['searchall']) . '%'; foreach ($allSearchFields as $field) { - $conditions['OR'][] = array('LOWER(Organisation.' . $field . ') LIKE' => '%' . strtolower($passedArgs['searchall']) . '%'); + $conditions['OR'][] = array('LOWER(Organisation.' . $field . ') LIKE' => $searchTerm); } } - $this->set('passedArgs', json_encode($passedArgs)); + $this->paginate['conditions'] = $conditions; - $usersPerOrg = $this->User->getMembersCount(); + + $this->Organisation->addCountField('user_count', $this->User, ['User.org_id = Organisation.id']); if ($this->_isRest()) { unset($this->paginate['limit']); $orgs = $this->Organisation->find('all', $this->paginate); } else { - if (isset($this->params['named']['viewall']) && $this->params['named']['viewall']) { - $orgCount = $this->Organisation->find('count'); - $this->paginate['limit'] = $orgCount; + $viewAll = isset($this->params['named']['viewall']) && $this->params['named']['viewall']; + if ($viewAll) { + unset($this->paginate['limit']); } - $this->set('viewall', isset($this->params['named']['viewall']) ? $this->params['named']['viewall'] : false); + $this->set('viewall', $viewAll); $orgs = $this->paginate(); } + $this->loadModel('User'); $org_creator_ids = array(); foreach ($orgs as $k => $org) { - if (isset($usersPerOrg[$org['Organisation']['id']])) { - $orgs[$k]['Organisation']['user_count'] = $usersPerOrg[$org['Organisation']['id']]; - } if ($this->_isSiteAdmin()) { - if (!in_array($org['Organisation']['created_by'], array_keys($org_creator_ids))) { - $email = $this->User->find('first', array('recursive' => -1, 'fields' => array('id', 'email'), 'conditions' => array('id' => $org['Organisation']['created_by']))); + if (!isset($org_creator_ids[$org['Organisation']['created_by']])) { + $email = $this->User->find('first', array( + 'recursive' => -1, + 'fields' => array('id', 'email'), + 'conditions' => array('id' => $org['Organisation']['created_by'])) + ); if (!empty($email)) { $org_creator_ids[$org['Organisation']['created_by']] = $email['User']['email']; } else { - $org_creator_ids[$org['Organisation']['created_by']] = 'Unknown'; + $org_creator_ids[$org['Organisation']['created_by']] = __('Unknown'); } } $orgs[$k]['Organisation']['created_by_email'] = $org_creator_ids[$org['Organisation']['created_by']]; + } else { + unset($orgs[$k]['Organisation']['created_by']); } } if ($this->_isRest()) { return $this->RestResponse->viewData($orgs, $this->response->type()); - } else { - foreach ($orgs as &$org) { - $org['Organisation']['country_code'] = $this->Organisation->getCountryCode($org['Organisation']['nationality']); - } - - $this->set('named', $this->params['named']); - $this->set('scope', $scope); - $this->set('orgs', $orgs); } + foreach ($orgs as &$org) { + $org['Organisation']['country_code'] = $this->Organisation->getCountryCode($org['Organisation']['nationality']); + } + + $this->set('named', $this->params['named']); + $this->set('scope', $scope); + $this->set('orgs', $orgs); + $this->set('passedArgs', json_encode($passedArgs)); } public function admin_add() @@ -122,14 +127,7 @@ class OrganisationsController extends AppController } } if ($this->Organisation->save($this->request->data)) { - if (isset($this->request->data['Organisation']['logo']['size']) && $this->request->data['Organisation']['logo']['size'] > 0 && $this->request->data['Organisation']['logo']['error'] == 0) { - $filename = basename($this->Organisation->id . '.png'); - if (preg_match("/^[0-9a-z\-\_\.]*\.(png)$/i", $filename)) { - if (!empty($this->request->data['Organisation']['logo']['tmp_name']) && is_uploaded_file($this->request->data['Organisation']['logo']['tmp_name'])) { - $result = move_uploaded_file($this->request->data['Organisation']['logo']['tmp_name'], APP . 'webroot/img/orgs/' . $filename); - } - } - } + $this->__uploadLogo($this->Organisation->id); if ($this->_isRest()) { $org = $this->Organisation->find('first', array( 'conditions' => array('Organisation.id' => $this->Organisation->id), @@ -198,14 +196,7 @@ class OrganisationsController extends AppController } $this->request->data['Organisation']['id'] = $id; if ($this->Organisation->save($this->request->data)) { - if (isset($this->request->data['Organisation']['logo']['size']) && $this->request->data['Organisation']['logo']['size'] > 0 && $this->request->data['Organisation']['logo']['error'] == 0) { - $filename = basename($this->request->data['Organisation']['id'] . '.png'); - if (preg_match("/^[0-9a-z\-\_\.]*\.(png)$/i", $filename)) { - if (!empty($this->request->data['Organisation']['logo']['tmp_name']) && is_uploaded_file($this->request->data['Organisation']['logo']['tmp_name'])) { - $result = move_uploaded_file($this->request->data['Organisation']['logo']['tmp_name'], APP . 'webroot/img/orgs/' . $filename); - } - } - } + $this->__uploadLogo($this->Organisation->id); if ($this->_isRest()) { $org = $this->Organisation->find('first', array( 'conditions' => array('Organisation.id' => $this->Organisation->id), @@ -308,76 +299,63 @@ class OrganisationsController extends AppController public function view($id) { - if (Validation::uuid($id)) { - $temp = $this->Organisation->find('first', array('recursive' => -1, 'fields' => array('Organisation.id'), 'conditions' => array('Organisation.uuid' => $id))); - if (empty($temp)) { - throw new NotFoundException(__('Invalid organisation.')); - } - $id = $temp['Organisation']['id']; - } elseif (!is_numeric($id)) { - $temp = $this->Organisation->find('first', array('recursive' => -1, 'fields' => array('Organisation.id'), 'conditions' => array('Organisation.name' => urldecode($id)))); - if (empty($temp)) { - throw new NotFoundException(__('Invalid organisation.')); - } - $id = $temp['Organisation']['id']; + if (is_numeric($id)) { + $conditions = ['Organisation.id' => $id]; + } else if (Validation::uuid($id)) { + $conditions = ['Organisation.uuid' => $id]; + } else { + $conditions = ['Organisation.name' => urldecode($id)]; } - $this->Organisation->id = $id; - if (!$this->Organisation->exists()) { + + if ($this->request->is('head')) { // Just check if org exists and user can access it + $org = $this->Organisation->find('first', array( + 'conditions' => $conditions, + 'recursive' => -1, + 'fields' => ['id'], + )); + $exists = $org && $this->Organisation->canSee($this->Auth->user(), $org['Organisation']['id']); + return new CakeResponse(['status' => $exists ? 200 : 404]); + } + + $fields = ['id', 'name', 'date_created', 'date_modified', 'type', 'nationality', 'sector', 'contacts', 'description', 'local', 'uuid', 'restricted_to_domain', 'created_by']; + if ($this->_isRest()) { + $this->Organisation->addCountField('user_count', $this->User, ['User.org_id = Organisation.id']); + $fields[] = 'user_count'; + } + + $org = $this->Organisation->find('first', array( + 'conditions' => $conditions, + 'recursive' => -1, + 'fields' => $fields, + )); + if (!$org || !$this->Organisation->canSee($this->Auth->user(), $org['Organisation']['id'])) { throw new NotFoundException(__('Invalid organisation')); } - $fullAccess = false; - $fields = array('id', 'name', 'date_created', 'date_modified', 'type', 'nationality', 'sector', 'contacts', 'description', 'local', 'uuid', 'restricted_to_domain'); - if ($this->_isSiteAdmin() || ($this->_isAdmin() && $this->Auth->user('Organisation')['id'] == $id)) { - $fullAccess = true; - $fields = array_merge($fields, array('created_by')); - } - $org = $this->Organisation->find('first', array( - 'conditions' => array('id' => $id), - 'fields' => $fields, - 'recursive' => -1 - )); - if (!$this->Auth->user('Role')['perm_sharing_group'] && Configure::read('Security.hide_organisation_index_from_users')) { - $this->loadModel('Event'); - $event = $this->Event->find('first', array( - 'fields' => array('Event.id'), - 'recursive' => -1, - 'conditions' => array('Event.orgc_id' => $org['Organisation']['id']) - )); - if (empty($event)) { - $proposal = $this->Event->ShadowAttribute->find('first', array( - 'fields' => array('ShadowAttribute.id'), - 'recursive' => -1, - 'conditions' => array('ShadowAttribute.org_id' => $org['Organisation']['id']) - )); - if (empty($proposal)) { - throw new NotFoundException(__('Invalid organisation')); - } - } - } - $this->set('local', $org['Organisation']['local']); + $fullAccess = $this->_isSiteAdmin() || ($this->_isAdmin() && $this->Auth->user('Organisation')['id'] == $org['Organisation']['id']); if ($fullAccess) { - $creator = $this->Organisation->User->find( - 'first', - array( - 'conditions' => array('User.id' => $org['Organisation']['created_by']), - 'fields' => array('email'), - 'recursive' => -1 - ) - ); + $creator = $this->Organisation->User->find('first', array( + 'conditions' => array('User.id' => $org['Organisation']['created_by']), + 'fields' => array('email'), + 'recursive' => -1 + )); if (!empty($creator)) { $org['Organisation']['created_by_email'] = $creator['User']['email']; } - } - if ($this->_isRest()) { - $org['Organisation']['user_count'] = $this->Organisation->User->getMembersCount($org['Organisation']['id']); - return $this->RestResponse->viewData($org, $this->response->type()); } else { - $org['Organisation']['country_code'] = $this->Organisation->getCountryCode($org['Organisation']['nationality']); - $this->set('fullAccess', $fullAccess); - $this->set('org', $org); - $this->set('id', $id); + unset($org['Organisation']['created_by']); } + + if ($this->_isRest()) { + return $this->RestResponse->viewData($org, $this->response->type()); + } + + $org['Organisation']['country_code'] = $this->Organisation->getCountryCode($org['Organisation']['nationality']); + $this->set('local', $org['Organisation']['local']); + $this->set('fullAccess', $fullAccess); + $this->set('org', $org); + $this->set('id', $org['Organisation']['id']); + $this->set('title_for_layout', __('Organisation %s', $org['Organisation']['name'])); } public function fetchOrgsForSG($idList = '{}', $type) @@ -487,4 +465,25 @@ class OrganisationsController extends AppController $this->render('ajax/merge'); } } + + /** + * @return bool + */ + private function __uploadLogo($orgId) + { + if (!isset($this->request->data['Organisation']['logo']['size'])) { + return false; + } + + $logo = $this->request->data['Organisation']['logo']; + if ($logo['size'] > 0 && $logo['error'] == 0) { + $extension = pathinfo($logo['name'], PATHINFO_EXTENSION); + $filename = $orgId . '.' . ($extension === 'svg' ? 'svg' : 'png'); + if (!empty($logo['tmp_name']) && is_uploaded_file($logo['tmp_name'])) { + return move_uploaded_file($logo['tmp_name'], APP . 'webroot/img/orgs/' . $filename); + } + } + + return false; + } } diff --git a/app/Controller/RolesController.php b/app/Controller/RolesController.php index 1a1e9d4a5..e579d0c27 100644 --- a/app/Controller/RolesController.php +++ b/app/Controller/RolesController.php @@ -9,8 +9,6 @@ App::uses('AppController', 'Controller'); */ class RolesController extends AppController { - public $options = array('0' => 'Read Only', '1' => 'Manage My Own Events', '2' => 'Manage Organization Events', '3' => 'Manage & Publish Organization Events'); // FIXME move this to Role Model - public $components = array( 'Security', 'Session', @@ -28,36 +26,32 @@ class RolesController extends AppController public function view($id=false) { - $this->set('menuData', ['menuList' => 'globalActions', 'menuItem' => 'roles']); $this->CRUD->view($id); if ($this->IndexFilter->isRest()) { return $this->restResponsePayload; } $this->set('permissionLevelName', $this->Role->premissionLevelName); $this->set('permFlags', $this->Role->permFlags); + $this->set('menuData', ['menuList' => 'globalActions', 'menuItem' => 'roles']); } public function admin_add() { - $this->set('menuData', array('menuList' => 'admin', 'menuItem' => 'addRole')); - $params = []; - $selectConditions = []; + $params = ['redirect' => ['action' => 'index', 'admin' => false]]; $this->CRUD->add($params); - if ($this->IndexFilter->isRest()) { + if ($this->restResponsePayload) { return $this->restResponsePayload; } $this->set('permFlags', $this->Role->permFlags); $dropdownData = [ - 'options' => $this->options + 'options' => $this->Role->premissionLevelName, ]; $this->set(compact('dropdownData')); + $this->set('menuData', array('menuList' => 'admin', 'menuItem' => 'addRole')); } public function admin_edit($id = null) { - if (!$this->_isSiteAdmin()) { - $this->redirect(array('controller' => 'roles', 'action' => 'index', 'admin' => false)); - } $this->Role->id = $id; if (!$this->Role->exists() && !$this->request->is('get')) { throw new NotFoundException(__('Invalid Role')); @@ -76,7 +70,7 @@ class RolesController extends AppController return $this->RestResponse->viewData($role, $this->response->type()); } else { $this->Flash->success(__('The Role has been saved')); - $this->redirect(array('action' => 'index')); + $this->redirect(array('action' => 'index', 'admin' => false)); } } else { if ($this->_isRest()) { @@ -94,12 +88,30 @@ class RolesController extends AppController $this->request->data['Role']['id'] = $id; $this->request->data = $this->Role->read(null, $id); } - $this->set('options', $this->options); + $this->set('options', $this->Role->premissionLevelName); $this->set('permFlags', $this->Role->permFlags); $this->set('id', $id); } - public function admin_index($id = false) + public function admin_delete($id = null) + { + $this->CRUD->delete($id, [ + 'validate' => function (array $role) { + $usersWithRole = $this->User->find('count', [ + 'conditions' => ['role_id' => $role['Role']['id']], + 'recursive' => -1, + ]); + if ($usersWithRole) { + throw new Exception(__("It is not possible to delete role that is assigned to users.")); + } + } + ]); + if ($this->IndexFilter->isRest()) { + return $this->restResponsePayload; + } + } + + public function index() { $params = [ 'filters' => ['name'], @@ -108,43 +120,21 @@ class RolesController extends AppController $this->loadModel('AdminSetting'); $default_setting = $this->AdminSetting->getSetting('default_role'); foreach ($elements as &$role) { - $role['Role']['default'] = ($role['Role']['id'] == $default_setting) ? true : false; + $role['Role']['default'] = $role['Role']['id'] == $default_setting; } return $elements; } ]; - //$this->paginate['fields'] = ['id', 'name']; $this->CRUD->index($params); if ($this->IndexFilter->isRest()) { return $this->restResponsePayload; } + $this->set('options', $this->Role->premissionLevelName); $this->set('permFlags', $this->Role->permFlags); - $this->set('menuData', array('menuList' => 'globalActions', 'menuItem' => 'roles')); - } - - public function admin_delete($id = null) - { - $this->CRUD->delete($id); - if ($this->IndexFilter->isRest()) { - return $this->restResponsePayload; - } - } - - public function index() - { - $this->recursive = 0; - if ($this->_isRest()) { - $roles = $this->Role->find('all', array( - 'recursive' => -1 - )); - return $this->RestResponse->viewData($roles, $this->response->type()); - } else { - $this->set('list', $this->paginate()); - $this->set('permFlags', $this->Role->permFlags); - $this->loadModel('AdminSetting'); - $this->set('default_role_id', $this->AdminSetting->getSetting('default_role')); - $this->set('options', $this->options); - } + $this->set('menuData', $this->_isAdmin() ? + ['menuList' => 'admin', 'menuItem' => 'indexRole'] : + ['menuList' => 'globalActions', 'menuItem' => 'roles'] + ); } public function admin_set_default($role_id = false) diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php index d82fda6a9..070b855b5 100644 --- a/app/Controller/ServersController.php +++ b/app/Controller/ServersController.php @@ -86,10 +86,8 @@ class ServersController extends AppController { $urlparams = ''; $passedArgs = array(); - if (!$this->_isSiteAdmin()) { - throw new MethodNotAllowedException('You are not authorised to do that.'); - } - $server = $this->Server->find('first', array('conditions' => array('Server.id' => $id), 'recursive' => -1, 'fields' => array('Server.id', 'Server.url', 'Server.name'))); + + $server = $this->Server->find('first', array('conditions' => array('Server.id' => $id), 'recursive' => -1)); if (empty($server)) { throw new NotFoundException('Invalid server ID.'); } @@ -115,15 +113,15 @@ class ServersController extends AppController $combinedArgs['limit'] = 60; } try { - list($events, $total_count) = $this->Server->previewIndex($id, $this->Auth->user(), $combinedArgs); + list($events, $total_count) = $this->Server->previewIndex($server, $this->Auth->user(), $combinedArgs); } catch (Exception $e) { $this->Flash->error(__('Download failed.') . ' ' . $e->getMessage()); $this->redirect(array('action' => 'index')); } $this->loadModel('Event'); - $threat_levels = $this->Event->ThreatLevel->find('all'); - $this->set('threatLevels', Set::combine($threat_levels, '{n}.ThreatLevel.id', '{n}.ThreatLevel.name')); + $threat_levels = $this->Event->ThreatLevel->find('list', ['fields' => ['id', 'name']]); + $this->set('threatLevels', $threat_levels); App::uses('CustomPaginationTool', 'Tools'); $customPagination = new CustomPaginationTool(); $params = $customPagination->createPaginationRules($events, $this->passedArgs, $this->alias); @@ -150,19 +148,15 @@ class ServersController extends AppController public function previewEvent($serverId, $eventId, $all = false) { - if (!$this->_isSiteAdmin()) { - throw new MethodNotAllowedException('You are not authorised to do that.'); - } $server = $this->Server->find('first', array( 'conditions' => array('Server.id' => $serverId), 'recursive' => -1, - 'fields' => array('Server.id', 'Server.url', 'Server.name')) - ); + )); if (empty($server)) { throw new NotFoundException('Invalid server ID.'); } try { - $event = $this->Server->previewEvent($serverId, $eventId); + $event = $this->Server->previewEvent($server, $eventId); } catch (NotFoundException $e) { throw new NotFoundException(__("Event '%s' not found.", $eventId)); } catch (Exception $e) { @@ -170,6 +164,10 @@ class ServersController extends AppController $this->redirect(array('action' => 'previewIndex', $serverId)); } + if ($this->_isRest()) { + return $this->RestResponse->viewData($event, $this->response->type()); + } + $this->loadModel('Warninglist'); if (isset($event['Event']['Attribute'])) { $this->Warninglist->attachWarninglistToAttributes($event['Event']['Attribute']); @@ -200,8 +198,17 @@ class ServersController extends AppController $this->set($alias, $currentModel->{$variable}); } } - $threat_levels = $this->Event->ThreatLevel->find('all'); - $this->set('threatLevels', Set::combine($threat_levels, '{n}.ThreatLevel.id', '{n}.ThreatLevel.name')); + $threat_levels = $this->Event->ThreatLevel->find('list', ['fields' => ['id', 'name']]); + $this->set('threatLevels', $threat_levels); + $this->set('title_for_layout', __('Remote event preview')); + } + + public function compareServers() + { + list($servers, $overlap) = $this->Server->serverEventsOverlap(); + $this->set('servers', $servers); + $this->set('overlap', $overlap); + $this->set('title_for_layout', __('Server overlap analysis matrix')); } public function filterEventIndex($id) @@ -929,27 +936,6 @@ class ServersController extends AppController $this->render('/Elements/healthElements/settings_row'); } - private function __loadAvailableLanguages() - { - return $this->Server->loadAvailableLanguages(); - } - - private function __loadTagCollections() - { - return $this->Server->loadTagCollections($this->Auth->user()); - } - - private function __loadLocalOrgs() - { - $this->loadModel('Organisation'); - $local_orgs = $this->Organisation->find('list', array( - 'conditions' => array('local' => 1), - 'recursive' => -1, - 'fields' => array('Organisation.id', 'Organisation.name') - )); - return array_replace(array(0 => __('No organisation selected.')), $local_orgs); - } - public function serverSettings($tab=false) { if (!$this->_isSiteAdmin()) { @@ -975,7 +961,6 @@ class ServersController extends AppController $mixboxVersion = array(0 => __('Incorrect mixbox version installed, found $current, expecting $expected'), 1 => __('OK')); $maecVersion = array(0 => __('Incorrect maec version installed, found $current, expecting $expected'), 1 => __('OK')); $pymispVersion = array(0 => __('Incorrect PyMISP version installed, found $current, expecting $expected'), 1 => __('OK')); - $plyaraVersion = array(0 => __('Incorrect plyara version installed, found $current, expecting $expected'), 1 => __('OK')); $sessionErrors = array(0 => __('OK'), 1 => __('High'), 2 => __('Alternative setting used'), 3 => __('Test failed')); $moduleErrors = array(0 => __('OK'), 1 => __('System not enabled'), 2 => __('No modules found')); @@ -1015,8 +1000,8 @@ class ServersController extends AppController $tabs[$result['tab']]['severity'] = $result['level']; } } - if (isset($result['optionsSource']) && !empty($result['optionsSource'])) { - $result['options'] = $this->{'__load' . $result['optionsSource']}(); + if (isset($result['optionsSource']) && is_callable($result['optionsSource'])) { + $result['options'] = $result['optionsSource'](); } $dumpResults[] = $result; if ($result['tab'] == $tab) { @@ -1032,13 +1017,12 @@ class ServersController extends AppController $diagnostic_errors = 0; App::uses('File', 'Utility'); App::uses('Folder', 'Utility'); - $additionalViewVars = array(); - if ($tab == 'files') { + if ($tab === 'files') { $files = $this->__manageFiles(); $this->set('files', $files); } // Only run this check on the diagnostics tab - if ($tab == 'diagnostics' || $tab == 'download' || $this->_isRest()) { + if ($tab === 'diagnostics' || $tab === 'download' || $this->_isRest()) { $php_ini = php_ini_loaded_file(); $this->set('php_ini', $php_ini); @@ -1058,36 +1042,35 @@ class ServersController extends AppController $this->set('branch', $gitStatus['branch']); $this->set('commit', $gitStatus['commit']); $this->set('latestCommit', $gitStatus['latestCommit']); - $phpSettings = array( - 'max_execution_time' => array( - 'explanation' => 'The maximum duration that a script can run (does not affect the background workers). A too low number will break long running scripts like comprehensive API exports', - 'recommended' => 300, - 'unit' => false - ), - 'memory_limit' => array( - 'explanation' => 'The maximum memory that PHP can consume. It is recommended to raise this number since certain exports can generate a fair bit of memory usage', - 'recommended' => 2048, - 'unit' => 'M' - ), - 'upload_max_filesize' => array( - 'explanation' => 'The maximum size that an uploaded file can be. It is recommended to raise this number to allow for the upload of larger samples', - 'recommended' => 50, - 'unit' => 'M' - ), - 'post_max_size' => array( - 'explanation' => 'The maximum size of a POSTed message, this has to be at least the same size as the upload_max_filesize setting', - 'recommended' => 50, - 'unit' => 'M' - ) + $phpSettings = array( + 'max_execution_time' => array( + 'explanation' => 'The maximum duration that a script can run (does not affect the background workers). A too low number will break long running scripts like comprehensive API exports', + 'recommended' => 300, + 'unit' => 'seconds', + ), + 'memory_limit' => array( + 'explanation' => 'The maximum memory that PHP can consume. It is recommended to raise this number since certain exports can generate a fair bit of memory usage', + 'recommended' => 2048, + 'unit' => 'MB' + ), + 'upload_max_filesize' => array( + 'explanation' => 'The maximum size that an uploaded file can be. It is recommended to raise this number to allow for the upload of larger samples', + 'recommended' => 50, + 'unit' => 'MB' + ), + 'post_max_size' => array( + 'explanation' => 'The maximum size of a POSTed message, this has to be at least the same size as the upload_max_filesize setting', + 'recommended' => 50, + 'unit' => 'MB' + ) ); foreach ($phpSettings as $setting => $settingArray) { - $phpSettings[$setting]['value'] = ini_get($setting); - if ($settingArray['unit']) { - $phpSettings[$setting]['value'] = intval(rtrim($phpSettings[$setting]['value'], $phpSettings[$setting]['unit'])); - } else { - $phpSettings[$setting]['value'] = intval($phpSettings[$setting]['value']); + $phpSettings[$setting]['value'] = $this->Server->getIniSetting($setting); + if ($phpSettings[$setting]['value'] && $settingArray['unit'] && $settingArray['unit'] === 'MB') { + // convert basic unit to M + $phpSettings[$setting]['value'] = (int) floor($phpSettings[$setting]['value'] / 1024 / 1024); } } $this->set('phpSettings', $phpSettings); @@ -1133,7 +1116,9 @@ class ServersController extends AppController $attachmentScan = ['status' => false, 'error' => $e->getMessage()]; } - $additionalViewVars = array('gpgStatus', 'sessionErrors', 'proxyStatus', 'sessionStatus', 'zmqStatus', 'stixVersion', 'cyboxVersion', 'mixboxVersion', 'maecVersion', 'stix2Version', 'pymispVersion', 'moduleStatus', 'yaraStatus', 'gpgErrors', 'proxyErrors', 'zmqErrors', 'stixOperational', 'stix', 'moduleErrors', 'moduleTypes', 'dbDiagnostics', 'dbSchemaDiagnostics', 'redisInfo', 'attachmentScan'); + $view = compact('gpgStatus', 'sessionErrors', 'proxyStatus', 'sessionStatus', 'zmqStatus', 'stixVersion', 'cyboxVersion', 'mixboxVersion', 'maecVersion', 'stix2Version', 'pymispVersion', 'moduleStatus', 'yaraStatus', 'gpgErrors', 'proxyErrors', 'zmqErrors', 'stixOperational', 'stix', 'moduleErrors', 'moduleTypes', 'dbDiagnostics', 'dbSchemaDiagnostics', 'redisInfo', 'attachmentScan'); + } else { + $view = []; } // check whether the files are writeable $writeableDirs = $this->Server->writeableDirsDiagnostics($diagnostic_errors); @@ -1144,13 +1129,8 @@ class ServersController extends AppController // check if the encoding is not set to utf8 $dbEncodingStatus = $this->Server->databaseEncodingDiagnostics($diagnostic_errors); - $viewVars = array( - 'diagnostic_errors', 'tabs', 'tab', 'issues', 'finalSettings', 'writeableErrors', 'readableErrors', 'writeableDirs', 'writeableFiles', 'readableFiles', 'extensions', 'dbEncodingStatus' - ); - $viewVars = array_merge($viewVars, $additionalViewVars); - foreach ($viewVars as $viewVar) { - $this->set($viewVar, ${$viewVar}); - } + $view = array_merge($view, compact('diagnostic_errors', 'tabs', 'tab', 'issues', 'finalSettings', 'writeableErrors', 'readableErrors', 'writeableDirs', 'writeableFiles', 'readableFiles', 'extensions', 'dbEncodingStatus')); + $this->set($view); $workerIssueCount = 4; $worker_array = array(); @@ -1200,6 +1180,7 @@ class ServersController extends AppController $this->set('phpversion', phpversion()); $this->set('phpmin', $this->phpmin); $this->set('phprec', $this->phprec); + $this->set('phptoonew', $this->phptoonew); $this->set('pythonmin', $this->pythonmin); $this->set('pythonrec', $this->pythonrec); $this->set('pymisp', $this->pymisp); @@ -1296,47 +1277,48 @@ class ServersController extends AppController } } - public function idTranslator() { - - // The id translation feature is limited to people from the host org - if (!$this->_isSiteAdmin() && $this->Auth->user('org_id') != Configure::read('MISP.host_org_id')) { - throw new MethodNotAllowedException(__('You don\'t have the required privileges to do that.')); - } - - //We retrieve the list of remote servers that we can query - $options = array(); - $options['conditions'] = array("pull" => true); - $servers = $this->Server->find('all', $options); + public function idTranslator($localId = null) + { + // We retrieve the list of remote servers that we can query + $servers = $this->Server->find('all', [ + 'conditions' => ['OR' => ['pull' => true, 'push' => true]], + 'recursive' => -1, + 'order' => ['Server.priority ASC'], + ]); // We generate the list of servers for the dropdown $displayServers = array(); - foreach($servers as $s) { - $displayServers[] = array('name' => $s['Server']['name'], - 'value' => $s['Server']['id']); + foreach ($servers as $s) { + $displayServers[] = [ + 'name' => $s['Server']['name'], + 'value' => $s['Server']['id'], + ]; } $this->set('servers', $displayServers); - if ($this->request->is('post')) { + if ($localId || $this->request->is('post')) { + if ($localId && $this->request->is('get')) { + $this->request->data['Event']['local'] = 'local'; + $this->request->data['Event']['uuid'] = $localId; + } $remote_events = array(); - if(!empty($this->request->data['Event']['uuid']) && $this->request->data['Event']['local'] == "local") { + if (!empty($this->request->data['Event']['uuid']) && $this->request->data['Event']['local'] === "local") { $local_event = $this->Event->fetchSimpleEvent($this->Auth->user(), $this->request->data['Event']['uuid']); - } else if (!empty($this->request->data['Event']['uuid']) && $this->request->data['Event']['local'] == "remote" && !empty($this->request->data['Server']['id'])) { + } else if (!empty($this->request->data['Event']['uuid']) && $this->request->data['Event']['local'] === "remote" && !empty($this->request->data['Server']['id'])) { //We check on the remote server for any event with this id and try to find a match locally $conditions = array('AND' => array('Server.id' => $this->request->data['Server']['id'], 'Server.pull' => true)); $remote_server = $this->Server->find('first', array('conditions' => $conditions)); - if(!empty($remote_server)) { + if (!empty($remote_server)) { try { $remote_event = $this->Event->downloadEventFromServer($this->request->data['Event']['uuid'], $remote_server, null, true); } catch (Exception $e) { - $error_msg = __("Issue while contacting the remote server to retrieve event information"); - $this->logException($error_msg, $e); - $this->Flash->error($error_msg); + $this->Flash->error(__("Issue while contacting the remote server to retrieve event information")); return; } $local_event = $this->Event->fetchSimpleEvent($this->Auth->user(), $remote_event[0]['uuid']); - //we record it to avoid re-querying the same server in the 2nd phase - if(!empty($local_event)) { + // we record it to avoid re-querying the same server in the 2nd phase + if (!empty($local_event)) { $remote_events[] = array( "server_id" => $remote_server['Server']['id'], "server_name" => $remote_server['Server']['name'], @@ -1346,38 +1328,40 @@ class ServersController extends AppController } } } - if(empty($local_event)) { - $this->Flash->error( __("This event could not be found or you don't have permissions to see it.")); + if (empty($local_event)) { + $this->Flash->error(__("This event could not be found or you don't have permissions to see it.")); return; } else { $this->Flash->success(__('The event has been found.')); } // In the second phase, we query all configured sync servers to get their info on the event - foreach($servers as $s) { + foreach ($servers as $server) { // We check if the server was not already contacted in phase 1 - if(count($remote_events) > 0 && $remote_events[0]['server_id'] == $s['Server']['id']) { + if (count($remote_events) > 0 && $remote_events[0]['server_id'] == $server['Server']['id']) { continue; } try { - $remote_event = $this->Event->downloadEventFromServer($local_event['Event']['uuid'], $s, null, true); - $remote_event_id = $remote_event[0]['id']; + $remoteEvent = $this->Event->downloadEventFromServer($local_event['Event']['uuid'], $server, null, true); } catch (Exception $e) { - $this->logException("Couldn't download event from server", $e); - $remote_event_id = null; + $remoteEvent = null; + } + if ($remoteEvent) { + $remoteEventId = $remoteEvent[0]['id']; } $remote_events[] = array( - "server_id" => $s['Server']['id'], - "server_name" => $s['Server']['name'], - "url" => isset($remote_event_id) ? $s['Server']['url']."/events/view/".$remote_event_id : $s['Server']['url'], - "remote_id" => isset($remote_event_id) ? $remote_event_id : false + "server_id" => $server['Server']['id'], + "server_name" => $server['Server']['name'], + "url" => isset($remoteEventId) ? $server['Server']['url'] . "/events/view/" . $remoteEventId : $server['Server']['url'], + "remote_id" => isset($remoteEventId) ? $remoteEventId : false ); } $this->set('local_event', $local_event); $this->set('remote_events', $remote_events); } + $this->set('title_for_layout', __('Event ID translator')); } public function getSubmodulesStatus() { @@ -1428,8 +1412,8 @@ class ServersController extends AppController $setting['value'] = $value; } $setting['setting'] = $setting['name']; - if (isset($setting['optionsSource']) && !empty($setting['optionsSource'])) { - $setting['options'] = $this->{'__load' . $setting['optionsSource']}(); + if (isset($setting['optionsSource']) && is_callable($setting['optionsSource'])) { + $setting['options'] = $setting['optionsSource'](); } $subGroup = explode('.', $setting['name']); if ($subGroup[0] === 'Plugin') { @@ -1448,7 +1432,7 @@ class ServersController extends AppController if (!isset($this->request->data['Server'])) { $this->request->data = array('Server' => $this->request->data); } - if (!isset($this->request->data['Server']['value'])) { + if (!isset($this->request->data['Server']['value']) || !is_scalar($this->request->data['Server']['value'])) { if ($this->_isRest()) { return $this->RestResponse->saveFailResponse('Servers', 'serverSettingsEdit', false, 'Invalid input. Expected: {"value": "new_setting"}', $this->response->type()); } @@ -1491,7 +1475,7 @@ class ServersController extends AppController return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => 'Field updated.')), 'status'=>200, 'type' => 'json')); } } else { - if ($this->_isRest) { + if ($this->_isRest()) { return $this->RestResponse->saveFailResponse('Servers', 'serverSettingsEdit', false, $result, $this->response->type()); } else { return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => $result)), 'status'=>200, 'type' => 'json')); @@ -1627,28 +1611,31 @@ class ServersController extends AppController public function postTest() { - if ($this->request->is('post')) { - // Fix for PHP-FPM / Nginx / etc - // Fix via https://www.popmartian.com/tipsntricks/2015/07/14/howto-use-php-getallheaders-under-fastcgi-php-fpm-nginx-etc/ - if (!function_exists('getallheaders')) { - $headers = []; - foreach ($_SERVER as $name => $value) { - if (substr($name, 0, 5) == 'HTTP_') { - $headers[str_replace(' ', '-', ucwords(strtolower(str_replace('_', ' ', substr($name, 5)))))] = $value; - } - } - } else { - $headers = getallheaders(); - } - $result = array(); - $result['body'] = $this->request->data; - $result['headers']['Content-type'] = isset($headers['Content-type']) ? $headers['Content-type'] : 0; - $result['headers']['Accept'] = isset($headers['Accept']) ? $headers['Accept'] : 0; - $result['headers']['Authorization'] = isset($headers['Authorization']) ? 'OK' : 0; - return new CakeResponse(array('body'=> json_encode($result), 'type' => 'json')); - } else { + if (!$this->request->is('post')) { throw new MethodNotAllowedException('Invalid request, expecting a POST request.'); } + // Fix for PHP-FPM / Nginx / etc + // Fix via https://www.popmartian.com/tipsntricks/2015/07/14/howto-use-php-getallheaders-under-fastcgi-php-fpm-nginx-etc/ + if (!function_exists('getallheaders')) { + $headers = []; + foreach ($_SERVER as $name => $value) { + if (substr($name, 0, 5) === 'HTTP_') { + $headers[strtolower(str_replace('_', '-', substr($name, 5)))] = $value; + } + } + } else { + $headers = getallheaders(); + $headers = array_change_key_case($headers, CASE_LOWER); + } + $result = [ + 'body' => $this->request->data, + 'headers' => [ + 'Content-type' => isset($headers['content-type']) ? $headers['content-type'] : 0, + 'Accept' => isset($headers['accept']) ? $headers['accept'] : 0, + 'Authorization' => isset($headers['authorization']) ? 'OK' : 0, + ], + ]; + return new CakeResponse(array('body'=> json_encode($result), 'type' => 'json')); } public function getRemoteUser($id) @@ -1667,10 +1654,6 @@ class ServersController extends AppController public function testConnection($id = false) { - if (!$this->Auth->user('Role')['perm_sync'] && !$this->Auth->user('Role')['perm_site_admin']) { - throw new MethodNotAllowedException('You don\'t have permission to do that.'); - } - $server = $this->Server->find('first', ['conditions' => ['Server.id' => $id]]); if (!$server) { throw new NotFoundException(__('Invalid server')); @@ -1728,7 +1711,8 @@ class ServersController extends AppController 'version' => implode('.', $version), 'mismatch' => $mismatch, 'newer' => $newer, - 'post' => isset($post) ? $post : 'too old', + 'post' => isset($post) ? $post['status'] : 'too old', + 'response_encoding' => isset($post['content-encoding']) ? $post['content-encoding'] : null, 'client_certificate' => $result['client_certificate'], ) ), @@ -1838,9 +1822,19 @@ class ServersController extends AppController $result = $this->Server->checkoutMain(); } - public function update() + public function update($branch = false) { if ($this->request->is('post')) { + $branch = false; + $filterData = array( + 'request' => $this->request, + 'named_params' => $this->params['named'], + 'paramArray' => ['branch'], + 'ordered_url_params' => @compact($paramArray), + 'additional_delimiters' => PHP_EOL + ); + $exception = false; + $settings = $this->_harvestParameters($filterData, $exception); $status = $this->Server->getCurrentGitStatus(); $raw = array(); if (empty($status['branch'])) { // do not try to update if you are not on branch @@ -1848,7 +1842,10 @@ class ServersController extends AppController $raw[] = $msg; $update = $msg; } else { - $update = $this->Server->update($status, $raw); + if ($settings === false) { + $settings = []; + } + $update = $this->Server->update($status, $raw, $settings); } if ($this->_isRest()) { return $this->RestResponse->viewData(array('results' => $raw), $this->response->type()); diff --git a/app/Controller/ShadowAttributesController.php b/app/Controller/ShadowAttributesController.php index 214bd9daf..06faa8b5d 100644 --- a/app/Controller/ShadowAttributesController.php +++ b/app/Controller/ShadowAttributesController.php @@ -828,6 +828,52 @@ class ShadowAttributesController extends AppController $this->set('_serialize', array('ShadowAttribute')); } + public function viewPicture($id, $thumbnail=false) + { + $conditions['ShadowAttribute.id'] = $id; + $conditions['ShadowAttribute.type'] = 'attachment'; + $options = array( + 'conditions' => $conditions, + 'includeAllTags' => false, + 'includeAttributeUuid' => true, + 'flatten' => true, + 'deleted' => [0, 1] + ); + + + $sa = $this->ShadowAttribute->find('first', array( + 'recursive' => -1, + 'contain' => ['Event', 'Attribute'], // required because of conditions + 'fields' => array( + 'ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.first_seen', 'ShadowAttribute.last_seen', + ), + 'conditions' => $conditions, + )); + if (empty($sa)) { + throw new NotFoundException(__('Invalid proposal.')); + } + + if (!$this->ShadowAttribute->Attribute->isImage($sa['ShadowAttribute'])) { + throw new NotFoundException("ShadowAttribute is not an image."); + } + if ($this->_isRest()) { + if ($this->ShadowAttribute->typeIsAttachment($sa['ShadowAttribute']['type'])) { + $encodedFile = $this->ShadowAttribute->base64EncodeAttachment($sa['ShadowAttribute']); + $sa['ShadowAttribute']['data'] = $encodedFile; + } + } + + if ($this->_isRest()) { + return $this->RestResponse->viewData($sa['ShadowAttribute']['data'], $this->response->type()); + } else { + $width = isset($this->request->params['named']['width']) ? $this->request->params['named']['width'] : 200; + $height = isset($this->request->params['named']['height']) ? $this->request->params['named']['height'] : 200; + $imageData = $this->ShadowAttribute->getPictureData($sa, $thumbnail, $width, $height); + $extension = pathinfo($sa['ShadowAttribute']['value'], PATHINFO_EXTENSION); + return new CakeResponse(array('body' => $imageData, 'type' => strtolower($extension))); + } + } + public function index($eventId = false) { $conditions = array(); diff --git a/app/Controller/SharingGroupsController.php b/app/Controller/SharingGroupsController.php index 409782735..e9c50caae 100644 --- a/app/Controller/SharingGroupsController.php +++ b/app/Controller/SharingGroupsController.php @@ -1,6 +1,9 @@ request->params['admin']) && !$this->_isSiteAdmin()) { $this->redirect('/'); } - $sgs = $this->SharingGroup->fetchAllAuthorised($this->Auth->user()); - $this->paginate = Set::merge($this->paginate, array('conditions' => array('SharingGroup.id' => $sgs))); } public $paginate = array( - 'limit' => 60, - 'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page. - 'order' => array( - 'SharingGroup.name' => 'ASC' + 'limit' => 60, + 'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page. + 'order' => array( + 'SharingGroup.name' => 'ASC' + ), + 'fields' => array('SharingGroup.id', 'SharingGroup.uuid', 'SharingGroup.name', 'SharingGroup.description', 'SharingGroup.releasability', 'SharingGroup.local', 'SharingGroup.active'), + 'contain' => array( + 'SharingGroupOrg' => array( + 'Organisation' => array('fields' => array('Organisation.name', 'Organisation.id', 'Organisation.uuid')) ), - 'fields' => array('SharingGroup.id', 'SharingGroup.uuid', 'SharingGroup.name', 'SharingGroup.description', 'SharingGroup.releasability', 'SharingGroup.local', 'SharingGroup.active'), - 'contain' => array( - 'SharingGroupOrg' => array( - 'Organisation' => array('fields' => array('Organisation.name', 'Organisation.id', 'Organisation.uuid')) - ), - 'Organisation' => array( - 'fields' => array('Organisation.id', 'Organisation.name', 'Organisation.uuid'), - ), - 'SharingGroupServer' => array( - 'fields' => array('SharingGroupServer.all_orgs'), - 'Server' => array( - 'fields' => array('Server.name', 'Server.id') - ) - ) + 'Organisation' => array( + 'fields' => array('Organisation.id', 'Organisation.name', 'Organisation.uuid'), ), + 'SharingGroupServer' => array( + 'fields' => array('SharingGroupServer.all_orgs'), + 'Server' => array( + 'fields' => array('Server.name', 'Server.id') + ) + ) + ), ); public function add() @@ -43,12 +44,6 @@ class SharingGroupsController extends AppController if (!$this->userRole['perm_sharing_group']) { throw new MethodNotAllowedException('You don\'t have the required privileges to do that.'); } - $orgs = $this->SharingGroup->Organisation->find('all', array( - 'conditions' => array('local' => 1), - 'recursive' => -1, - 'fields' => array('id', 'name', 'uuid') - )); - if ($this->request->is('post')) { if ($this->_isRest()) { if (isset($this->request->data['SharingGroup'])) { @@ -129,6 +124,12 @@ class SharingGroupsController extends AppController } elseif ($this->_isRest()) { return $this->RestResponse->describe('SharingGroup', 'add', false, $this->response->type()); } + + $orgs = $this->SharingGroup->Organisation->find('all', array( + 'conditions' => array('local' => 1), + 'recursive' => -1, + 'fields' => array('id', 'name', 'uuid') + )); $this->set('orgs', $orgs); $this->set('localInstance', empty(Configure::read('MISP.external_baseurl')) ? Configure::read('MISP.baseurl') : Configure::read('MISP.external_baseurl')); // We just pass true and allow the user to edit, since he/she is just about to create the SG. This is needed to reuse the view for the edit @@ -143,33 +144,28 @@ class SharingGroupsController extends AppController if (empty($id)) { throw new NotFoundException('Invalid sharing group.'); } - // add check for perm_sharing_group - $this->SharingGroup->id = $id; - if (!$this->SharingGroup->exists()) { - throw new NotFoundException('Invalid sharing group.'); - } - if (!$this->_isSiteAdmin() && !$this->SharingGroup->checkIfAuthorisedExtend($this->Auth->user(), $id)) { - throw new MethodNotAllowedException('Action not allowed.'); - } // check if the user is eligible to edit the SG (original creator or extend) $sharingGroup = $this->SharingGroup->find('first', array( - 'conditions' => array('SharingGroup.id' => $id), + 'conditions' => Validation::uuid($id) ? ['SharingGroup.uuid' => $id] : ['SharingGroup.id' => $id], 'recursive' => -1, 'contain' => array( - 'SharingGroupOrg' => array( - 'Organisation' => array('name', 'local', 'id') - ), - 'SharingGroupServer' => array( - 'Server' => array( - 'fields' => array('name', 'url', 'id') - ) - ), - 'Organisation' => array( - 'fields' => array('name', 'local', 'id') - ), + 'SharingGroupOrg' => array( + 'Organisation' => array('name', 'local', 'id') + ), + 'SharingGroupServer' => array( + 'Server' => array( + 'fields' => array('name', 'url', 'id') + ) + ), + 'Organisation' => array( + 'fields' => array('name', 'local', 'id') + ), ), )); + if (!$this->SharingGroup->checkIfAuthorisedExtend($this->Auth->user(), $sharingGroup['SharingGroup']['id'])) { + throw new MethodNotAllowedException('Action not allowed.'); + } if ($this->request->is('post')) { if ($this->_isRest()) { if (isset($this->request->data['SharingGroup'])) { @@ -179,24 +175,24 @@ class SharingGroupsController extends AppController $id = $this->SharingGroup->captureSG($this->request->data, $this->Auth->user()); if ($id) { $sg = $this->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'simplified', false, $id); - return $this->RestResponse->viewData($sg, $this->response->type()); + return $this->RestResponse->viewData($sg[0], $this->response->type()); } else { return $this->RestResponse->saveFailResponse('SharingGroup', 'add', false, 'Could not save sharing group.', $this->response->type()); } } else { $json = json_decode($this->request->data['SharingGroup']['json'], true); $sg = $json['sharingGroup']; - $sg['id'] = $id; + $sg['id'] = $sharingGroup['SharingGroup']['id']; $fields = array('name', 'releasability', 'description', 'active', 'roaming'); - $existingSG = $this->SharingGroup->find('first', array('recursive' => -1, 'conditions' => array('SharingGroup.id' => $id))); + $existingSG = $this->SharingGroup->find('first', array('recursive' => -1, 'conditions' => array('SharingGroup.id' => $sharingGroup['SharingGroup']['id']))); foreach ($fields as $field) { $existingSG['SharingGroup'][$field] = $sg[$field]; } unset($existingSG['SharingGroup']['modified']); if ($this->SharingGroup->save($existingSG)) { - $this->SharingGroup->SharingGroupOrg->updateOrgsForSG($id, $json['organisations'], $sharingGroup['SharingGroupOrg'], $this->Auth->user()); - $this->SharingGroup->SharingGroupServer->updateServersForSG($id, $json['servers'], $sharingGroup['SharingGroupServer'], $json['sharingGroup']['roaming'], $this->Auth->user()); - $this->redirect('/SharingGroups/view/' . $id); + $this->SharingGroup->SharingGroupOrg->updateOrgsForSG($sharingGroup['SharingGroup']['id'], $json['organisations'], $sharingGroup['SharingGroupOrg'], $this->Auth->user()); + $this->SharingGroup->SharingGroupServer->updateServersForSG($sharingGroup['SharingGroup']['id'], $json['servers'], $sharingGroup['SharingGroupServer'], $json['sharingGroup']['roaming'], $this->Auth->user()); + $this->redirect('/SharingGroups/view/' . $sharingGroup['SharingGroup']['id']); } else { $validationReplacements = array( 'notempty' => 'This field cannot be left empty.', @@ -221,7 +217,7 @@ class SharingGroupsController extends AppController 'fields' => array('id', 'name') )); $this->set('sharingGroup', $sharingGroup); - $this->set('id', $id); + $this->set('id', $sharingGroup['SharingGroup']['id']); $this->set('orgs', $orgs); $this->set('localInstance', empty(Configure::read('MISP.external_baseurl')) ? Configure::read('MISP.baseurl') : Configure::read('MISP.external_baseurl')); // We just pass true and allow the user to edit, since he/she is just about to create the SG. This is needed to reuse the view for the edit @@ -236,15 +232,15 @@ class SharingGroupsController extends AppController if (!$this->request->is('post') && !$this->request->is('delete')) { throw new MethodNotAllowedException(__('Action not allowed, post or delete request expected.')); } - if (!$this->SharingGroup->checkIfOwner($this->Auth->user(), $id)) { + $deletedSg = $this->SharingGroup->find('first', array( + 'conditions' => Validation::uuid($id) ? ['uuid' => $id] : ['id' => $id], + 'recursive' => -1, + 'fields' => ['id', 'active'], + )); + if (empty($deletedSg) || !$this->SharingGroup->checkIfOwner($this->Auth->user(), $deletedSg['SharingGroup']['id'])) { throw new MethodNotAllowedException('Action not allowed.'); } - $deletedSg = $this->SharingGroup->find('first', array( - 'conditions' => array('id' => $id), - 'recursive' => -1, - 'fields' => array('active') - )); - if ($this->SharingGroup->delete($id)) { + if ($this->SharingGroup->delete($deletedSg['SharingGroup']['id'])) { if ($this->_isRest()) { return $this->RestResponse->saveSuccessResponse('SharingGroups', 'delete', $id, $this->response->type()); } @@ -265,62 +261,142 @@ class SharingGroupsController extends AppController public function index($passive = false) { - if ($passive === 'true') { - $passive = true; - } - if ($passive === true) { - $this->paginate['conditions'][] = array('SharingGroup.active' => 0); - } else { - $this->paginate['conditions'][] = array('SharingGroup.active' => 1); - } - $result = $this->paginate(); - // check if the current user can modify or delete the SG - foreach ($result as $k => $sg) { - if ($sg['Organisation']['uuid'] == $this->Auth->user('Organisation')['uuid'] && $this->userRole['perm_sharing_group']) { - $result[$k]['editable'] = true; + $passive = $passive === 'true'; + $authorizedSgIds = $this->SharingGroup->fetchAllAuthorised($this->Auth->user()); + $this->paginate['conditions'][] = array('SharingGroup.id' => $authorizedSgIds); + $this->paginate['conditions'][] = array('SharingGroup.active' => $passive === true ? 0 : 1); + + if (isset($this->params['named']['value'])) { + $term = '%' . strtolower($this->params['named']['value']) . '%'; + if ($this->__showOrgs()) { + $sgIds = $this->SharingGroup->SharingGroupOrg->find('column', [ + 'conditions' => [ + 'OR' => [ + 'Organisation.uuid LIKE' => $term, + 'LOWER(Organisation.name) LIKE' => $term, + ], + 'SharingGroupOrg.sharing_group_id' => $authorizedSgIds, + ], + 'contain' => ['Organisation'], + 'fields' => ['SharingGroupOrg.sharing_group_id'], + ]); } else { - $result[$k]['editable'] = false; + $sgIds = []; + } + $this->paginate['conditions'][]['OR'] = [ + 'SharingGroup.id' => $sgIds, + 'SharingGroup.uuid LIKE' => $term, + 'LOWER(SharingGroup.name) LIKE' => $term, + 'LOWER(SharingGroup.description) LIKE' => $term, + 'LOWER(SharingGroup.releasability) LIKE' => $term, + 'LOWER(Organisation.name) LIKE' => $term, + ]; + } + + if ($this->__showOrgs() && isset($this->params['named']['searchorg'])) { + $orgs = explode('|', $this->params['named']['searchorg']); + $conditions = []; + foreach ($orgs as $org) { + $exclude = $org[0] === '!'; + if ($exclude) { + $org = substr($org, 1); + } + $org = $this->SharingGroup->Organisation->fetchOrg($org); + if ($org) { + if ($exclude) { + $conditions['AND'][] = ['org_id !=' => $org['id']]; + } else { + $conditions['OR'][] = ['org_id' => $org['id']]; + } + } + } + $sgIds = $this->SharingGroup->SharingGroupOrg->find('column', [ + 'conditions' => $conditions, + 'fields' => ['SharingGroupOrg.sharing_group_id'], + ]); + if (empty($sgIds)) { + $sgIds = -1; + } + $this->paginate['conditions'][] = ['SharingGroup.id' => $sgIds]; + } + + // To allow sort sharing group by number of organisation and also show org count when user don't have permission ot see them + $this->SharingGroup->addCountField('org_count', $this->SharingGroup->SharingGroupOrg, ['SharingGroupOrg.sharing_group_id = SharingGroup.id']); + $this->paginate['fields'][] = 'SharingGroup.org_count'; + + if (!$this->__showOrgs()) { + unset($this->paginate['contain']['SharingGroupOrg']); + unset($this->paginate['contain']['SharingGroupServer']); + } + + $result = $this->paginate(); + + // check if the current user can modify or delete the SG + $userOrganisationUuid = $this->Auth->user()['Organisation']['uuid']; + foreach ($result as $k => $sg) { + $editable = false; + $deletable = false; + + if ($this->userRole['perm_site_admin'] || ($this->userRole['perm_sharing_group'] && $sg['Organisation']['uuid'] === $userOrganisationUuid)) { + $editable = true; + $deletable = true; + } else if ($this->userRole['perm_sharing_group']) { if (!empty($sg['SharingGroupOrg'])) { foreach ($sg['SharingGroupOrg'] as $sgo) { - if ($sgo['org_id'] == $this->Auth->user('org_id') && $sgo['extend']) { - $result[$k]['editable'] = true; + if ($sgo['extend'] && $sgo['org_id'] == $this->Auth->user('org_id')) { + $editable = true; + break; } } } } + + $result[$k]['editable'] = $editable; + $result[$k]['deletable'] = $deletable; + } + if ($this->_isRest()) { + return $this->RestResponse->viewData(['response' => $result], $this->response->type()); // 'response' to keep BC } $this->set('passive', $passive); - if ($this->_isRest()) { - $this->set('response', $result); - $this->set('_serialize', array('response')); - } else { - $this->set('sharingGroups', $result); - } + $this->set('sharingGroups', $result); + $this->set('passedArgs', $passive ? 'true' : '[]'); + $this->set('title_for_layout', __('Sharing Groups')); } public function view($id) { + if ($this->request->is('head')) { // Just check if sharing group exists and user can access it + $exists = $this->SharingGroup->checkIfAuthorised($this->Auth->user(), $id); + return new CakeResponse(['status' => $exists ? 200 : 404]); + } if (!$this->SharingGroup->checkIfAuthorised($this->Auth->user(), $id)) { throw new MethodNotAllowedException('Sharing group doesn\'t exist or you do not have permission to access it.'); } - $this->SharingGroup->id = $id; - $this->SharingGroup->contain( - array( - 'SharingGroupOrg' => array( - 'Organisation' => array( - 'fields' => array('id', 'name', 'uuid', 'local') - ) - ), - 'Organisation', - 'SharingGroupServer' => array( - 'Server' => array( - 'fields' => array('id', 'name', 'url') - ) + + $contain = array( + 'Organisation', + 'SharingGroupOrg' => array( + 'Organisation' => array( + 'fields' => array('id', 'name', 'uuid', 'local') + ) + ), + 'SharingGroupServer' => array( + 'Server' => array( + 'fields' => array('id', 'name', 'url') ) ) ); - $this->SharingGroup->read(); - $sg = $this->SharingGroup->data; + + if (!$this->__showOrgs()) { + unset($contain['SharingGroupOrg']); + unset($contain['SharingGroupServer']); + $this->SharingGroup->addCountField('org_count', $this->SharingGroup->SharingGroupOrg, ['SharingGroupOrg.sharing_group_id = SharingGroup.id']); + } + + $sg = $this->SharingGroup->find('first', [ + 'conditions' => Validation::uuid($id) ? ['SharingGroup.uuid' => $id] : ['SharingGroup.id' => $id], + 'contain' => $contain, + ]); if (isset($sg['SharingGroupServer'])) { foreach ($sg['SharingGroupServer'] as $key => $sgs) { if ($sgs['server_id'] == 0) { @@ -334,25 +410,38 @@ class SharingGroupsController extends AppController } if ($sg['SharingGroup']['sync_user_id']) { $this->loadModel('User'); - $sync_user = $this->User->find('first', array( - 'conditions' => array('User.id' => $sg['SharingGroup']['sync_user_id']), - 'recursive' => -1, - 'fields' => array('User.id', 'User.org_id'), - 'contain' => array('Organisation' => array( - 'fields' => array('Organisation.name') - )) + $syncUser = $this->User->find('first', array( + 'conditions' => array('User.id' => $sg['SharingGroup']['sync_user_id']), + 'recursive' => -1, + 'fields' => array('User.id'), + 'contain' => array('Organisation' => array( + 'fields' => array('Organisation.id', 'Organisation.name', 'Organisation.uuid'), + )) )); - if (empty($sync_user)) { + if (empty($syncUser)) { $sg['SharingGroup']['sync_org_name'] = 'N/A'; + } else { + $sg['SharingGroup']['sync_org_name'] = $syncUser['Organisation']['name']; + $sg['SharingGroup']['sync_org'] = $syncUser['Organisation']; } - $sg['SharingGroup']['sync_org_name'] = $sync_user['Organisation']['name']; } if ($this->_isRest()) { return $this->RestResponse->viewData($sg, $this->response->type()); } - $this->set('mayModify', $this->SharingGroup->checkIfAuthorisedExtend($this->Auth->user(), $id)); - $this->set('id', $id); + + $this->loadModel('Event'); + $conditions = $this->Event->createEventConditions($this->Auth->user()); + $conditions['AND']['sharing_group_id'] = $sg['SharingGroup']['id']; + $sg['SharingGroup']['event_count'] = $this->Event->find('count', [ + 'conditions' => $conditions, + 'recursive' => -1, + 'callbacks' => false, + ]); + + $this->set('mayModify', $this->SharingGroup->checkIfAuthorisedExtend($this->Auth->user(), $sg['SharingGroup']['id'])); + $this->set('id', $sg['SharingGroup']['id']); $this->set('sg', $sg); + $this->set('title_for_layout', __('Sharing Group %s', $sg['SharingGroup']['name'])); } private function __initialiseSGQuickEdit($id, $request) @@ -538,4 +627,12 @@ class SharingGroupsController extends AppController return $this->RestResponse->saveFailResponse('SharingGroup', $action, false, $object_type . ' could not be ' . $actionType . ' the sharing group.', $this->response->type()); } } + + /** + * @return bool + */ + private function __showOrgs() + { + return $this->Auth->user()['Role']['perm_sharing_group'] || !Configure::read('Security.hide_organisations_in_sharing_groups'); + } } diff --git a/app/Controller/SightingsController.php b/app/Controller/SightingsController.php index 3ae3e2e79..7e55fd3ab 100644 --- a/app/Controller/SightingsController.php +++ b/app/Controller/SightingsController.php @@ -334,24 +334,23 @@ class SightingsController extends AppController // Save sightings synced over, restricted to sync users public function bulkSaveSightings($eventId = false) { - if ($this->request->is('post')) { - if (empty($this->request->data['Sighting'])) { - $sightings = $this->request->data; - } else { - $sightings = $this->request->data['Sighting']; - } - $saved = $this->Sighting->bulkSaveSightings($eventId, $sightings, $this->Auth->user()); - if (is_numeric($saved)) { - if ($saved > 0) { - return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => $saved . ' sightings added.')), 'status' => 200, 'type' => 'json')); - } else { - return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'success' => 'No sightings added.')), 'status' => 200, 'type' => 'json')); - } - } else { - throw new MethodNotAllowedException($saved); - } - } else { + if (!$this->request->is('post')) { throw new MethodNotAllowedException('This method is only accessible via POST requests.'); } + if (empty($this->request->data['Sighting'])) { + $sightings = $this->request->data; + } else { + $sightings = $this->request->data['Sighting']; + } + try { + $saved = $this->Sighting->bulkSaveSightings($eventId, $sightings, $this->Auth->user()); + if ($saved > 0) { + return new CakeResponse(array('body' => json_encode(array('saved' => true, 'success' => $saved . ' sightings added.')), 'status' => 200, 'type' => 'json')); + } else { + return new CakeResponse(array('body' => json_encode(array('saved' => false, 'success' => 'No sightings added.')), 'status' => 200, 'type' => 'json')); + } + } catch (NotFoundException $e) { + throw new MethodNotAllowedException($e->getMessage()); + } } } diff --git a/app/Controller/TagsController.php b/app/Controller/TagsController.php index e00dcf69a..08a725aba 100644 --- a/app/Controller/TagsController.php +++ b/app/Controller/TagsController.php @@ -41,7 +41,6 @@ class TagsController extends AppController 'request' => $this->request, 'named_params' => $this->params['named'], 'paramArray' => ['favouritesOnly', 'filter', 'searchall', 'name', 'search', 'exclude_statistics'], - 'ordered_url_params' => @compact($paramArray) ); $exception = false; $passedArgsArray = $this->_harvestParameters($filterData, $exception); @@ -574,9 +573,6 @@ class TagsController extends AppController public function selectTag($id, $taxonomy_id, $scope = 'event', $filterData = '') { - if (!$this->_isSiteAdmin() && !$this->userRole['perm_tagger']) { - throw new NotFoundException('You don\'t have permission to do that.'); - } $this->loadModel('Taxonomy'); $expanded = array(); $this->set('taxonomy_id', $taxonomy_id); @@ -629,12 +625,14 @@ class TagsController extends AppController $expanded = $tags; } } elseif ($taxonomy_id === 'all') { - $conditions = []; + $conditions = [ + 'Tag.name NOT LIKE' => 'misp-galaxy:%', + 'Tag.hide_tag' => 0, + ]; if (!$this->_isSiteAdmin()) { - $conditions[] = array('Tag.org_id' => array(0, $this->Auth->user('org_id'))); - $conditions[] = array('Tag.user_id' => array(0, $this->Auth->user('id'))); + $conditions['Tag.org_id'] = array(0, $this->Auth->user('org_id')); + $conditions['Tag.user_id'] = array(0, $this->Auth->user('id')); } - $conditions['Tag.hide_tag'] = 0; $allTags = $this->Tag->find('all', array( 'conditions' => $conditions, 'recursive' => -1, @@ -643,10 +641,7 @@ class TagsController extends AppController )); $tags = array(); foreach ($allTags as $tag) { - $isGalaxyTag = strpos($tag['Tag']['name'], 'misp-galaxy:') === 0; - if (!$isGalaxyTag) { - $tags[$tag['Tag']['id']] = $tag['Tag']; - } + $tags[$tag['Tag']['id']] = $tag['Tag']; } unset($allTags); $expanded = $tags; @@ -654,45 +649,28 @@ class TagsController extends AppController $taxonomies = $this->Taxonomy->getTaxonomy($taxonomy_id); $tags = array(); if (!empty($taxonomies['entries'])) { + $isSiteAdmin = $this->_isSiteAdmin(); foreach ($taxonomies['entries'] as $entry) { if (!empty($entry['existing_tag']['Tag'])) { - $tags[$entry['existing_tag']['Tag']['id']] = $entry['existing_tag']['Tag']; - $expanded[$entry['existing_tag']['Tag']['id']] = $entry['expanded']; + $tag = $entry['existing_tag']['Tag']; + if ($tag['hide_tag']) { + continue; // do not include hidden tags + } + if (!$isSiteAdmin) { + // Skip all tags that this user cannot use for tagging, determined by the org restriction on tags + if ($tag['org_id'] != '0' && $tag['org_id'] != $this->Auth->user('org_id')) { + continue; + } + if ($tag['user_id'] != '0' && $tag['user_id'] != $this->Auth->user('id')) { + continue; + } + } + + $tags[$tag['id']] = $tag; + $expanded[$tag['id']] = $entry['expanded']; } } } - - // Unset all tags that this user cannot use for tagging, determined by the org restriction on tags - if (!$this->_isSiteAdmin()) { - $banned_tags = $this->Tag->find('list', array( - 'conditions' => array( - 'NOT' => array( - 'Tag.org_id' => array( - 0, - $this->Auth->user('org_id') - ), - 'Tag.user_id' => array( - 0, - $this->Auth->user('id') - ) - ) - ), - 'fields' => array('Tag.id') - )); - foreach ($banned_tags as $banned_tag) { - unset($tags[$banned_tag]); - unset($expanded[$banned_tag]); - } - } - - $hidden_tags = $this->Tag->find('list', array( - 'conditions' => array('Tag.hide_tag' => 1), - 'fields' => array('Tag.id') - )); - foreach ($hidden_tags as $hidden_tag) { - unset($tags[$hidden_tag]); - unset($expanded[$hidden_tag]); - } } } @@ -1108,18 +1086,25 @@ class TagsController extends AppController $conditions['OR'][] = array('LOWER(Tag.name) LIKE' => $t); } } else { - foreach ($tag as $k => $t) { - $conditions['OR'][] = array('Tag.name' => $t); + foreach ($tag as $t) { + if (is_numeric($t)) { + $conditions['OR'][] = ['Tag.id' => $t]; + } else { + $conditions['OR'][] = array('Tag.name' => $t); + } } } $tags = $this->Tag->find('all', array( 'conditions' => $conditions, 'recursive' => -1 )); - if (!$searchIfTagExists && empty($tags)) { - $tags = []; - foreach ($tag as $i => $tagName) { - $tags[] = ['Tag' => ['name' => $tagName], 'simulatedTag' => true]; + if (!$searchIfTagExists) { + $foundTagNames = Hash::extract($tags, "{n}.Tag.name"); + foreach ($tag as $tagName) { + if (!in_array($tagName, $foundTagNames, true)) { + // Tag not found, insert simulated tag + $tags[] = ['Tag' => ['name' => $tagName], 'simulatedTag' => true]; + } } } $this->loadModel('Taxonomy'); diff --git a/app/Controller/TaxonomiesController.php b/app/Controller/TaxonomiesController.php index ff4482904..e1385b869 100644 --- a/app/Controller/TaxonomiesController.php +++ b/app/Controller/TaxonomiesController.php @@ -1,6 +1,9 @@ paginate['recursive'] = -1; + + if (!empty($this->passedArgs['value'])) { + $this->paginate['conditions']['id'] = $this->__search($this->passedArgs['value']); + } + + if (isset($this->passedArgs['enabled'])) { + $this->paginate['conditions']['enabled'] = $this->passedArgs['enabled'] ? 1 : 0; + } + if ($this->_isRest()) { $keepFields = array('conditions', 'contain', 'recursive', 'sort'); $searchParams = array(); @@ -41,13 +53,17 @@ class TaxonomiesController extends AppController $total += empty($predicate['TaxonomyEntry']) ? 1 : count($predicate['TaxonomyEntry']); } $taxonomies[$key]['total_count'] = $total; - $taxonomies[$key]['current_count'] = $this->Tag->find('count', array('conditions' => array('lower(Tag.name) LIKE ' => strtolower($taxonomy['Taxonomy']['namespace']) . ':%', 'hide_tag' => 0))); + $taxonomies[$key]['current_count'] = $this->Tag->find('count', array( + 'conditions' => array('lower(Tag.name) LIKE ' => strtolower($taxonomy['Taxonomy']['namespace']) . ':%', 'hide_tag' => 0), + 'recursive' => -1, + )); unset($taxonomies[$key]['TaxonomyPredicate']); } if ($this->_isRest()) { return $this->RestResponse->viewData($taxonomies, $this->response->type()); } else { $this->set('taxonomies', $taxonomies); + $this->set('passedArgsArray', $this->passedArgs); } } @@ -69,18 +85,18 @@ class TaxonomiesController extends AppController } $this->loadModel('EventTag'); $this->loadModel('AttributeTag'); + + $tagIds = array_column(array_column(array_column($taxonomy['entries'], 'existing_tag'), 'Tag'), 'id'); + $eventCount = $this->EventTag->countForTags($tagIds, $this->Auth->user()); + $attributeTags = $this->AttributeTag->countForTags($tagIds, $this->Auth->user()); + foreach ($taxonomy['entries'] as $key => $value) { $count = 0; $count_a = 0; if (!empty($value['existing_tag'])) { - foreach ($value['existing_tag'] as $et) { - $count = $this->EventTag->find('count', array( - 'conditions' => array('EventTag.tag_id' => $et['id']) - )); - $count_a = $this->AttributeTag->find('count', array( - 'conditions' => array('AttributeTag.tag_id' => $et['id']) - )); - } + $tagId = $value['existing_tag']['Tag']['id']; + $count = isset($eventCount[$tagId]) ? $eventCount[$tagId] : 0; + $count_a = isset($attributeTags[$tagId]) ? $attributeTags[$tagId] : 0; } $taxonomy['entries'][$key]['events'] = $count; $taxonomy['entries'][$key]['attributes'] = $count_a; @@ -168,11 +184,21 @@ class TaxonomiesController extends AppController } } + public function import() + { + if (!$this->request->is('post')) { + throw new MethodNotAllowedException('This endpoint requires a POST request.'); + } + try { + $id = $this->Taxonomy->import($this->request->data); + return $this->view($id); + } catch (Exception $e) { + return $this->RestResponse->saveFailResponse('Taxonomy', 'import', false, $e->getMessage()); + } + } + public function update() { - if (!$this->_isSiteAdmin()) { - throw new MethodNotAllowedException(__('You don\'t have permission to do that.')); - } $result = $this->Taxonomy->update(); $this->Log = ClassRegistry::init('Log'); $fails = 0; @@ -380,27 +406,18 @@ class TaxonomiesController extends AppController public function taxonomyMassConfirmation($id) { - if (!$this->_isSiteAdmin() && !$this->userRole['perm_tagger']) { - throw new NotFoundException(__('You don\'t have permission to do that.')); - } $this->set('id', $id); $this->render('ajax/taxonomy_mass_confirmation'); } public function taxonomyMassHide($id) { - if (!$this->_isSiteAdmin() && !$this->userRole['perm_tagger']) { - throw new NotFoundException(__('You don\'t have permission to do that.')); - } $this->set('id', $id); $this->render('ajax/taxonomy_mass_hide'); } public function taxonomyMassUnhide($id) { - if (!$this->_isSiteAdmin() && !$this->userRole['perm_tagger']) { - throw new NotFoundException(__('You don\'t have permission to do that.')); - } $this->set('id', $id); $this->render('ajax/taxonomy_mass_unhide'); } @@ -410,7 +427,7 @@ class TaxonomiesController extends AppController if ($this->request->is('post')) { $result = $this->Taxonomy->delete($id, true); if ($result) { - $this->Flash->success(__('Taxonomy successfuly deleted.')); + $this->Flash->success(__('Taxonomy successfully deleted.')); $this->redirect(array('controller' => 'taxonomies', 'action' => 'index')); } else { $this->Flash->error(__('Taxonomy could not be deleted.')); @@ -443,12 +460,47 @@ class TaxonomiesController extends AppController } else { return $this->RestResponse->saveFailResponse('Taxonomy', 'toggleRequired', $id, $this->validationError, $this->response->type()); } - } else { - $this->set('required', !$taxonomy['Taxonomy']['required']); - $this->set('id', $id); - $this->autoRender = false; - $this->layout = 'ajax'; - $this->render('ajax/toggle_required'); } + + $this->set('required', !$taxonomy['Taxonomy']['required']); + $this->set('id', $id); + $this->autoRender = false; + $this->layout = 'ajax'; + $this->render('ajax/toggle_required'); + } + + private function __search($value) + { + $value = mb_strtolower(trim($value)); + $searchTerm = "%$value%"; + $taxonomyPredicateIds = $this->Taxonomy->TaxonomyPredicate->TaxonomyEntry->find('column', [ + 'fields' => ['TaxonomyEntry.taxonomy_predicate_id'], + 'conditions' => ['OR' => [ + 'LOWER(value) LIKE' => $searchTerm, + 'LOWER(expanded) LIKE' => $searchTerm, + ]], + 'unique' => true, + ]); + + $taxonomyIds = $this->Taxonomy->TaxonomyPredicate->find('column', [ + 'fields' => ['TaxonomyPredicate.taxonomy_id'], + 'conditions' => ['OR' => [ + 'id' => $taxonomyPredicateIds, + 'LOWER(value) LIKE' => $searchTerm, + 'LOWER(expanded) LIKE' => $searchTerm, + ]], + 'unique' => true, + ]); + + $taxonomyIds = $this->Taxonomy->find('column', [ + 'fields' => ['Taxonomy.id'], + 'conditions' => ['OR' => [ + 'id' => $taxonomyIds, + 'LOWER(namespace) LIKE' => $searchTerm, + 'LOWER(description) LIKE' => $searchTerm, + ]], + ]); + + return $taxonomyIds; } } diff --git a/app/Controller/ThreadsController.php b/app/Controller/ThreadsController.php index 44df77968..be98633cd 100644 --- a/app/Controller/ThreadsController.php +++ b/app/Controller/ThreadsController.php @@ -1,7 +1,9 @@ paginate = array( - 'limit' => 10, - 'conditions' => array('Post.thread_id' => $thread_id), - 'contain' => array( - 'User' => array( - 'fields' => array('User.email', 'User.id'), - 'Organisation' => array( - 'fields' => array('id', 'name') - ), - ), + 'limit' => 10, + 'conditions' => array('Post.thread_id' => $thread_id), + 'contain' => array( + 'User' => array( + 'fields' => array('User.email', 'User.id'), + 'Organisation' => array( + 'fields' => array('id', 'uuid', 'name') + ), ), + ), ); if ($this->_isRest()) { $posts = $this->Thread->Post->find('all', array( @@ -114,7 +116,14 @@ class ThreadsController extends AppController $posts = $this->paginate('Post'); } foreach ($posts as $k => $post) { - $posts[$k]['Post']['org_name'] = empty($post['User']['id']) ? 'Deactivated user' : $post['User']['Organisation']['name']; + if (!empty($post['User']['id'])) { + $posts[$k]['Post']['org_id'] = $post['User']['Organisation']['id']; + $posts[$k]['Post']['org_uuid'] = $post['User']['Organisation']['uuid']; + $posts[$k]['Post']['org_name'] = $post['User']['Organisation']['name']; + } else { + $posts[$k]['Post']['org_name'] = 'Deactivated user'; // to keep BC + } + if ($this->_isSiteAdmin() || $this->Auth->user('org_id') == $post['User']['org_id']) { $posts[$k]['Post']['user_email'] = empty($post['User']['id']) ? 'Unavailable' : $post['User']['email']; } diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php index fc807c31d..5ba9cef94 100644 --- a/app/Controller/UsersController.php +++ b/app/Controller/UsersController.php @@ -15,15 +15,15 @@ class UsersController extends AppController ); public $paginate = array( - 'limit' => 60, - 'recursive' => -1, - 'order' => array( - 'Organisation.name' => 'ASC' - ), - 'contain' => array( - 'Organisation' => array('id', 'name'), - 'Role' => array('id', 'name', 'perm_auth', 'perm_site_admin') - ) + 'limit' => 60, + 'recursive' => -1, + 'order' => array( + 'Organisation.name' => 'ASC' + ), + 'contain' => array( + 'Organisation' => array('id', 'uuid', 'name'), + 'Role' => array('id', 'name', 'perm_auth', 'perm_site_admin') + ) ); public $helpers = array('Js' => array('Jquery')); @@ -33,7 +33,7 @@ class UsersController extends AppController parent::beforeFilter(); // what pages are allowed for non-logged-in users - $allowedActions = array('login', 'logout'); + $allowedActions = array('login', 'logout', 'getGpgPublicKey'); if(!empty(Configure::read('Security.email_otp_enabled'))) { $allowedActions[] = 'email_otp'; } @@ -51,14 +51,6 @@ class UsersController extends AppController if (!$this->_isSiteAdmin() && $this->Auth->user('id') != $id) { throw new NotFoundException(__('Invalid user or not authorised.')); } - if (!is_numeric($id) && !empty($id)) { - $userId = $this->User->find('first', array( - 'conditions' => array('email' => $id), - 'fields' => array('id') - )); - $id = $userid['User']['id']; - } - $user = $this->User->read(null, $id); $user = $this->User->find('first', array( 'recursive' => -1, 'conditions' => array('User.id' => $id), @@ -182,7 +174,7 @@ class UsersController extends AppController } if (!$abortPost) { // What fields should be saved (allowed to be saved) - $fieldList = array('autoalert', 'gpgkey', 'certif_public', 'nids_sid', 'contactalert', 'disabled'); + $fieldList = array('autoalert', 'gpgkey', 'certif_public', 'nids_sid', 'contactalert', 'disabled', 'date_modified'); if ($this->__canChangeLogin()) { $fieldList[] = 'email'; } @@ -217,7 +209,6 @@ class UsersController extends AppController return $this->RestResponse->viewData($this->__massageUserObject($user), $this->response->type()); } else { $this->Flash->success(__('The profile has been updated')); - $this->_refreshAuth(); $this->redirect(array('action' => 'view', $id)); } } else { @@ -305,7 +296,6 @@ class UsersController extends AppController return $this->RestResponse->saveSuccessResponse('User', 'change_pw', false, $this->response->type(), $message); } $this->Flash->success($message); - $this->_refreshAuth(); $this->redirect(array('action' => 'view', $id)); } else { $message = __('The password could not be updated. Make sure you meet the minimum password length / complexity requirements.'); @@ -341,7 +331,7 @@ class UsersController extends AppController $this->User->virtualFields['org_ci'] = 'UPPER(Organisation.name)'; $urlParams = ""; $passedArgsArray = array(); - $booleanFields = array('autoalert', 'contactalert', 'termsaccepted'); + $booleanFields = array('autoalert', 'contactalert', 'termsaccepted', 'disabled'); $textFields = array('role', 'email', 'all', 'authkey'); // org admins can't see users of other orgs if ($this->_isSiteAdmin()) { @@ -494,8 +484,11 @@ class UsersController extends AppController public function admin_filterUserIndex() { $passedArgsArray = array(); - $booleanFields = array('autoalert', 'contactalert', 'termsaccepted'); - $textFields = array('role', 'email', 'authkey'); + $booleanFields = array('autoalert', 'contactalert', 'termsaccepted', 'disabled'); + $textFields = array('role', 'email'); + if (empty(Configure::read('Security.advanced_authkeys'))) { + $textFields[] = 'authkey'; + } $showOrg = 0; // org admins can't see users of other orgs if ($this->_isSiteAdmin()) { @@ -542,17 +535,15 @@ class UsersController extends AppController $roleNames[$v['Role']['id']] = $v['Role']['name']; $roleJSON[] = array('id' => $v['Role']['id'], 'value' => $v['Role']['name']); } - $temp = $this->User->Organisation->find('all', array( - 'conditions' => array('local' => 1), - 'recursive' => -1, - 'fields' => array('id', 'name'), - 'order' => array('LOWER(name) ASC') - )); - $orgs = array(); - foreach ($temp as $org) { - $orgs[$org['Organisation']['id']] = $org['Organisation']['name']; + if ($showOrg) { + $orgs = $this->User->Organisation->find('list', array( + 'conditions' => array('local' => 1), + 'recursive' => -1, + 'fields' => array('id', 'name'), + 'order' => array('LOWER(name) ASC') + )); + $this->set('orgs', $orgs); } - $this->set('orgs', $orgs); $this->set('roles', $roleNames); $this->set('roleJSON', json_encode($roleJSON)); $rules = $this->_arrayToValuesIndexArray($rules); @@ -563,29 +554,21 @@ class UsersController extends AppController public function admin_view($id = null) { - $contain = [ - 'UserSetting', - 'Role', - 'Organisation' - ]; - if (!empty(Configure::read('Security.advanced_authkeys'))) { - $contain['AuthKey'] = [ - 'conditions' => [ - 'OR' => [ - 'AuthKey.expiration' => 0, - 'AuthKey.expiration <' => time() - ] - ] - ]; - } $user = $this->User->find('first', array( 'recursive' => -1, 'conditions' => array('User.id' => $id), - 'contain' => $contain + 'contain' => [ + 'UserSetting', + 'Role', + 'Organisation' + ] )); if (empty($user)) { throw new NotFoundException(__('Invalid user')); } + if (!$this->_isSiteAdmin() && !($this->_isAdmin() && $this->Auth->user('org_id') == $user['User']['org_id'])) { + throw new MethodNotAllowedException(); + } if (!empty($user['User']['gpgkey'])) { $pgpDetails = $this->User->verifySingleGPG($user); $user['User']['pgp_status'] = isset($pgpDetails[2]) ? $pgpDetails[2] : 'OK'; @@ -598,10 +581,6 @@ class UsersController extends AppController if (!empty(Configure::read('Security.advanced_authkeys'))) { unset($user['User']['authkey']); } - $this->set('user', $user); - if (!$this->_isSiteAdmin() && !($this->_isAdmin() && $this->Auth->user('org_id') == $user['User']['org_id'])) { - throw new MethodNotAllowedException(); - } if ($this->_isRest()) { $user['User']['password'] = '*****'; $temp = array(); @@ -614,14 +593,13 @@ class UsersController extends AppController 'Role' => $user['Role'], 'UserSetting' => $user['UserSetting'] ), $this->response->type()); - return $this->RestResponse->viewData(array('User' => $user['User']), $this->response->type()); - } else { - $user2 = $this->User->find('first', array('conditions' => array('User.id' => $user['User']['invited_by']), 'recursive' => -1)); - $this->set('id', $id); - $this->set('user2', $user2); - $this->set('admin_view', true); - $this->render('view'); } + $this->set('user', $user); + $user2 = $this->User->find('first', array('conditions' => array('User.id' => $user['User']['invited_by']), 'recursive' => -1)); + $this->set('id', $id); + $this->set('user2', $user2); + $this->set('admin_view', true); + $this->render('view'); } public function admin_add() @@ -927,8 +905,8 @@ class UsersController extends AppController if (isset($this->request->data['User']['role_id']) && !array_key_exists($this->request->data['User']['role_id'], $syncRoles)) { $this->request->data['User']['server_id'] = 0; } - $fields = array(); - $blockedFields = array('id', 'invited_by'); + $fields = []; + $blockedFields = array('id', 'invited_by', 'date_modified'); if (!$this->_isSiteAdmin()) { $blockedFields[] = 'org_id'; } @@ -979,11 +957,15 @@ class UsersController extends AppController throw new Exception('You are not authorised to assign that role to a user.'); } } - if (!empty($fields) && $this->User->save($this->request->data, true, $fields)) { + $fields[] = 'date_modified'; // time will be inserted in `beforeSave` action + if ($this->User->save($this->request->data, true, $fields)) { // newValues to array $fieldsNewValues = array(); foreach ($fields as $field) { - if ($field != 'confirm_password') { + if ($field === 'date_modified') { + continue; + } + if ($field !== 'confirm_password') { $newValue = $this->data['User'][$field]; if (gettype($newValue) == 'array') { $newValueStr = ''; @@ -1026,7 +1008,6 @@ class UsersController extends AppController return $this->RestResponse->viewData($user, $this->response->type()); } else { $this->Flash->success(__('The user has been saved')); - $this->_refreshAuth(); // in case we modify ourselves $this->redirect(array('action' => 'index')); } } else { @@ -1134,6 +1115,7 @@ class UsersController extends AppController public function login() { + $oldHash = false; if ($this->request->is('post') || $this->request->is('put')) { $this->Bruteforce = ClassRegistry::init('Bruteforce'); if (!empty($this->request->data['User']['email'])) { @@ -1142,6 +1124,17 @@ class UsersController extends AppController throw new ForbiddenException('You have reached the maximum number of login attempts. Please wait ' . $expire . ' seconds and try again.'); } } + // Check the length of the user's authkey match old format. This can be removed in future. + $userPass = $this->User->find('first', [ + 'conditions' => ['User.email' => $this->request->data['User']['email']], + 'fields' => ['User.password'], + 'recursive' => -1, + ]); + if (!empty($userPass) && strlen($userPass['User']['password']) === 40) { + $oldHash = true; + unset($this->Auth->authenticate['Form']['passwordHasher']); // use default password hasher + $this->Auth->constructAuthenticate(); + } } if ($this->request->is('post') && Configure::read('Security.email_otp_enabled')) { $user = $this->Auth->identify($this->request, $this->response); @@ -1154,6 +1147,12 @@ class UsersController extends AppController $this->set('formLoginEnabled', $formLoginEnabled); if ($this->Auth->login()) { + if ($oldHash) { + // Convert old style password hash to blowfish + $passwordToSave = $this->request->data['User']['password']; + // Password is converted to hashed form automatically + $this->User->save(['id' => $this->Auth->user('id'), 'password' => $passwordToSave], false, ['password']); + } $this->_postlogin(); } else { $dataSourceConfig = ConnectionManager::getDataSource('default')->config; @@ -1264,16 +1263,9 @@ class UsersController extends AppController // Events list $url = $this->Session->consume('pre_login_requested_url'); if (empty($url)) { - $homepage = $this->User->UserSetting->find('first', array( - 'recursive' => -1, - 'conditions' => array( - 'UserSetting.user_id' => $this->Auth->user('id'), - 'UserSetting.setting' => 'homepage' - ), - 'contain' => array('User.id', 'User.org_id') - )); + $homepage = $this->User->UserSetting->getValueForUser($this->Auth->user('id'), 'homepage'); if (!empty($homepage)) { - $url = $homepage['UserSetting']['value']['path']; + $url = $homepage['path']; } else { $url = array('controller' => 'events', 'action' => 'index'); } @@ -1321,7 +1313,6 @@ class UsersController extends AppController } if (!$this->_isRest()) { $this->Flash->success(__('New authkey generated.', true)); - $this->_refreshAuth(); $this->redirect($this->referer()); } else { return $this->RestResponse->saveSuccessResponse('User', 'resetauthkey', $id, $this->response->type(), 'Authkey updated: ' . $newkey); @@ -1448,9 +1439,7 @@ class UsersController extends AppController public function terms() { if ($this->request->is('post') || $this->request->is('put')) { - $this->User->id = $this->Auth->user('id'); - $this->User->saveField('termsaccepted', true); - $this->_refreshAuth(); // refresh auth info + $this->User->updateField($this->Auth->user(), 'termsaccepted', true); $this->Flash->success(__('You accepted the Terms and Conditions.')); $this->redirect(array('action' => 'routeafterlogin')); } @@ -1834,9 +1823,11 @@ class UsersController extends AppController $params['conditions'] = array('Organisation.id' => $this->Auth->user('org_id')); } $orgs = $this->User->Organisation->find('all', $params); + $local_orgs_params = $params; $local_orgs_params['conditions']['Organisation.local'] = 1; - $local_orgs = $this->User->Organisation->find('all', $local_orgs_params); + $local_orgs_count = $this->User->Organisation->find('count', $local_orgs_params); + $this->loadModel('Log'); $year = date('Y'); $month = date('n'); @@ -1864,7 +1855,7 @@ class UsersController extends AppController $stats['user_count'] = $this->User->find('count', array('recursive' => -1)); $stats['user_count_pgp'] = $this->User->find('count', array('recursive' => -1, 'conditions' => array('User.gpgkey !=' => ''))); $stats['org_count'] = count($orgs); - $stats['local_org_count'] = count($local_orgs); + $stats['local_org_count'] = $local_orgs_count; $stats['contributing_org_count'] = $this->User->Event->find('count', array('recursive' => -1, 'group' => array('Event.orgc_id'))); $stats['average_user_per_org'] = round($stats['user_count'] / $stats['local_org_count'], 1); @@ -1875,7 +1866,6 @@ class UsersController extends AppController $stats['post_count'] = $this->Thread->Post->find('count', array('recursive' => -1)); $stats['post_count_month'] = $this->Thread->Post->find('count', array('conditions' => array('Post.date_created >' => date("Y-m-d H:i:s", $this_month)), 'recursive' => -1)); - if ($this->_isRest()) { $data = array( 'stats' => $stats @@ -1954,7 +1944,6 @@ class UsersController extends AppController } elseif ($params['scope'] == 'external') { $conditions['Organisation.local'] = 0; } - $orgs = array(); $orgs = $this->Organisation->find('all', array( 'recursive' => -1, 'conditions' => $conditions, @@ -2111,8 +2100,6 @@ class UsersController extends AppController private function __statisticsTags($params = array()) { - $trending_tags = array(); - $all_tags = array(); if ($this->_isRest()) { return $this->tagStatisticsGraph(); } else { @@ -2130,25 +2117,27 @@ class UsersController extends AppController } else { $galaxy_id = $mitre_galaxy_id; } - $organisations = $this->User->Organisation->find('all', array( - 'recursive' => -1, + + $organisations = $this->User->Organisation->find('list', array( + 'recursive' => -1, + 'fields' => ['id', 'name'], )); - array_unshift($organisations, array('Organisation' => array('id' => 0, 'name' => 'All'))); + foreach ($organisations as $id => $foo) { + if (!$this->User->Organisation->canSee($this->Auth->user(), $id)) { + unset($organisations[$id]); + } + } + $organisations = array_merge([0 => __('All')], $organisations); $this->set('organisations', $organisations); - $picked_organisation = 0; + if (isset($params['organisation']) && $params['organisation'] != 0) { - $org = $this->User->Organisation->find('first', array( - 'recursive' => -1, - 'conditions' => array('id' => $params['organisation']), - )); - if (!empty($org)) { - $picked_organisation = $org; - $this->set('picked_organisation', $picked_organisation); + if (isset($organisations[$params['organisation']])) { + $this->set('picked_organisation_id', $params['organisation']); } else { - $this->set('picked_organisation', array('Organisation' => array('id' => ''))); + throw new NotFoundException(__("Invalid organisation")); } } else { - $this->set('picked_organisation', array('Organisation' => array('id' => ''))); + $this->set('picked_organisation_id', -1); } $rest_response_empty = true; @@ -2290,18 +2279,6 @@ class UsersController extends AppController $this->set('users', $user_results); } - // Refreshes the Auth session with new/updated data - protected function _refreshAuth() - { - $oldUser = $this->Auth->user(); - $newUser = $this->User->find('first', array('conditions' => array('User.id' => $oldUser['id']), 'recursive' => -1,'contain' => array('Organisation', 'Role'))); - // Rearrange it a bit to match the Auth object created during the login - $newUser['User']['Role'] = $newUser['Role']; - $newUser['User']['Organisation'] = $newUser['Organisation']; - unset($newUser['Organisation'], $newUser['Role']); - $this->Auth->login($newUser['User']); - } - public function searchGpgKey($email = false) { if (!$email) { @@ -2329,6 +2306,26 @@ class UsersController extends AppController return new CakeResponse(array('body' => $key)); } + public function getGpgPublicKey() + { + if (!Configure::read("MISP.download_gpg_from_homedir")) { + throw new MethodNotAllowedException("Downloading GPG public key from homedir is not allowed."); + } + + $key = $this->User->getGpgPublicKey(); + if (!$key) { + throw new NotFoundException("Public key not found."); + } + + list($fingeprint, $publicKey) = $key; + $response = new CakeResponse(array( + 'body' => $publicKey, + 'type' => 'text/plain', + )); + $response->download($fingeprint . '.asc'); + return $response; + } + public function checkIfLoggedIn() { return new CakeResponse(array('body'=> 'OK','status' => 200)); diff --git a/app/Lib/Dashboard/MispSystemResourceWidget.php b/app/Lib/Dashboard/MispSystemResourceWidget.php index 0f2b7cf8d..656335631 100644 --- a/app/Lib/Dashboard/MispSystemResourceWidget.php +++ b/app/Lib/Dashboard/MispSystemResourceWidget.php @@ -7,41 +7,43 @@ class MispSystemResourceWidget public $width = 3; public $height = 3; public $params = array( - 'treshold' => 'Treshold for disk space' + 'threshold' => 'Threshold for disk space' ); public $description = 'Basic widget showing some system server statistics.'; public $cacheLifetime = false; public $autoRefreshDelay = 30; public $placeholder = '{ - "treshold": "85" + "threshold": "85" }'; - public function handler($user, $options = array()) + public function handler(array $user, $options = array()) { + // Keep BC with typo value + $threshold = isset($options['threshold']) ? $options['threshold'] : (isset($options['treshold']) ? $options['treshold'] : 85); - $drive = round((1 - disk_free_space(getcwd())/disk_total_space(getcwd()))*100,2); + $cwd = getcwd(); + $drive = round((1 - disk_free_space($cwd)/disk_total_space($cwd))*100,2); $driveFree = $drive . "%"; $driveFreeClass = ""; - if ($drive > intval($options['treshold'])) { - $driveFree = $drive . "% - [Above Treshhold]"; + if ($drive > intval($threshold)) { + $driveFree = $drive . "% - [Above Threshold]"; $driveFreeClass = "red"; } - $sysload = sys_getloadavg(); - - preg_match('#MemFree:[\s\t]+([\d]+)\s+kB#', file_get_contents('/proc/meminfo'), $matches); + $meminfo = file_get_contents('/proc/meminfo'); + preg_match('#MemFree:[\s\t]+([\d]+)\s+kB#', $meminfo, $matches); $memoryFree = $matches[1]; - preg_match('#MemTotal:[\s\t]+([\d]+)\s+kB#', file_get_contents('/proc/meminfo'), $matches); + preg_match('#MemTotal:[\s\t]+([\d]+)\s+kB#', $meminfo, $matches); $memoryTotal = $matches[1]; $data = array( - array( 'title' => __('User'), 'value' => $user['email']), - array( 'title' => __('System'), 'value' => php_uname()), - array( 'title' => __('Disk usage'), 'value' => h($driveFree), 'class' => $driveFreeClass), - array( 'title' => __('Load'), 'value' => h($sysload[0] . " - " . $sysload[1] . " - " . $sysload[2])), - array( 'title' => __('Memory'), 'value' => h(round($memoryFree/1024,2) . "M free (" . round((1 - $memoryFree/$memoryTotal)*100,2) . "% used)")), - ); + array( 'title' => __('User'), 'value' => $user['email']), + array( 'title' => __('System'), 'value' => php_uname()), + array( 'title' => __('Disk usage'), 'value' => h($driveFree), 'class' => $driveFreeClass), + array( 'title' => __('Load'), 'value' => h(implode(" - ", sys_getloadavg()))), + array( 'title' => __('Memory'), 'value' => h(round($memoryFree / 1024,2) . " MB free (" . round((1 - $memoryFree/$memoryTotal)*100,2) . " % used)")), + ); return $data; } diff --git a/app/Lib/Dashboard/TrendingTagsWidget.php b/app/Lib/Dashboard/TrendingTagsWidget.php index 0a1f6b25a..b9bdc9b7f 100644 --- a/app/Lib/Dashboard/TrendingTagsWidget.php +++ b/app/Lib/Dashboard/TrendingTagsWidget.php @@ -20,36 +20,41 @@ class TrendingTagsWidget "include": ["misp-galaxy:", "my-internal-taxonomy"] }'; public $description = 'Widget showing the trending tags over the past x seconds, along with the possibility to include/exclude tags.'; + public $cacheLifetime = 600; public function handler($user, $options = array()) { - $this->Event = ClassRegistry::init('Event'); - $params = array( - 'metadata' => 1, - 'timestamp' => time() - (empty($options['time_window']) ? 8640000 : $options['time_window']) - ); + /** @var Event $eventModel */ + $eventModel = ClassRegistry::init('Event'); $threshold = empty($options['threshold']) ? 10 : $options['threshold']; - $eventIds = $this->Event->filterEventIds($user, $params); - $params['eventid'] = $eventIds; - $events = array(); + $params = [ + 'timestamp' => time() - (empty($options['time_window']) ? 8640000 : $options['time_window']), + ]; + $eventIds = $eventModel->filterEventIds($user, $params); + + $tags = []; + $tagColours = []; if (!empty($eventIds)) { - $events = $this->Event->fetchEvent($user, $params); - } - $tags = array(); - $tagColours = array(); - foreach ($events as $event) { - foreach ($event['EventTag'] as $et) { - if ($this->checkTag($options, $et['Tag']['name'])) { - if (empty($tags[$et['Tag']['name']])) { - $tags[$et['Tag']['name']] = 1; - $tagColours[$et['Tag']['name']] = $et['Tag']['colour']; - } else { - $tags[$et['Tag']['name']] += 1; - } + $eventTags = $eventModel->EventTag->find('all', [ + 'conditions' => ['EventTag.event_id' => $eventIds], + 'contain' => ['Tag' => ['fields' => ['name', 'colour']]], + 'recursive' => -1, + 'fields' => ['id'], + ]); + + foreach ($eventTags as $eventTag) { + $tagName = $eventTag['Tag']['name']; + if (isset($tags[$tagName])) { + $tags[$tagName]++; + } else if ($this->checkTag($options, $tagName)) { + $tags[$tagName] = 1; + $tagColours[$tagName] = $eventTag['Tag']['colour']; } } + + arsort($tags); } - arsort($tags); + $data['data'] = array_slice($tags, 0, $threshold); $data['colours'] = $tagColours; return $data; diff --git a/app/Lib/Export/BroExport.php b/app/Lib/Export/BroExport.php index e9f299fdc..5d7b37f63 100644 --- a/app/Lib/Export/BroExport.php +++ b/app/Lib/Export/BroExport.php @@ -27,6 +27,7 @@ class BroExport 'domain|ip' => array('brotype' => 'DOMAIN', 'composite' => 'ADDR'), 'url' => array('brotype' => 'URL', 'replace' => array('#^https?://#', '')), 'user-agent' => array('brotype' => 'SOFTWARE'), + 'ja3-fingerprint-md5' => array('brotype' => 'JA3'), 'md5' => array('brotype' => 'FILE_HASH'), 'malware-sample' => array('brotype' => 'FILE_NAME', 'composite' => 'FILE_HASH'), 'filename|md5' => array('brotype' => 'FILE_NAME', 'composite' => 'FILE_HASH'), @@ -79,6 +80,9 @@ class BroExport array('domain', 1), array('domain|ip', 1) ), + 'ja3-fingerprint-md5' => array( + array('ja3-fingerprint-md5', 1) + ), 'email' => array( array('email', 1), array('email-src', 1), diff --git a/app/Lib/Export/CsvExport.php b/app/Lib/Export/CsvExport.php index a4c531387..37dfcc0da 100644 --- a/app/Lib/Export/CsvExport.php +++ b/app/Lib/Export/CsvExport.php @@ -6,6 +6,7 @@ class CsvExport public $default_fields = array('uuid', 'event_id', 'category', 'type', 'value', 'comment', 'to_ids', 'timestamp', 'object_relation', 'attribute_tag'); public $default_obj_fields = array('object_uuid', 'object_name', 'object_meta-category'); public $requested_fields = array(); + public $decaying_fields = array('decay_score_score', 'decay_score_decayed'); public $non_restrictive_export = true; public function handler($data, $options = array()) @@ -22,6 +23,9 @@ class CsvExport public function modify_params($user, $params) { + if (!empty($params['includeDecayScore'])) { + $this->enable_decaying(); + } if (empty($params['contain'])) { $params['contain'] = array(); } @@ -36,6 +40,11 @@ class CsvExport return $params; } + public function enable_decaying() + { + $this->default_fields = array_merge($this->default_fields, $this->decaying_fields); + } + private function __attributesHandler($attribute, $options) { $attribute = $this->__addMetadataToAttributeAtomic($attribute); @@ -44,6 +53,17 @@ class CsvExport $attribute['object_name'] = $attribute['Object']['name']; $attribute['object_meta-category'] = $attribute['Object']['meta-category']; } + if (!empty($attribute['decay_score'])) { + $all_scores = Hash::extract($attribute, 'decay_score.{n}.score'); + $all_decayed = Hash::extract($attribute, 'decay_score.{n}.decayed'); + $avg_score = array_sum($all_scores)/count($all_scores); + $avg_decayed = count(array_intersect([true], $all_decayed)) > 0; + $attribute['decay_score_score'] = $avg_score; + $attribute['decay_score_decayed'] = $avg_decayed; + } else { + $attribute['decay_score_score'] = 0; + $attribute['decay_score_decayed'] = false; + } return $this->__addLine($attribute, $options); } diff --git a/app/Lib/Export/NidsExport.php b/app/Lib/Export/NidsExport.php index 9e836669a..2b691314d 100644 --- a/app/Lib/Export/NidsExport.php +++ b/app/Lib/Export/NidsExport.php @@ -189,6 +189,12 @@ class NidsExport case 'user-agent': $this->userAgentRule($ruleFormat, $item['Attribute'], $sid); break; + case 'ja3-fingerprint-md5': + $this->ja3Rule($ruleFormat, $item['Attribute'], $sid); + break; + case 'ja3s-fingerprint-md5': // Atribute type doesn't exists yet (2020-12-10) but ready when created. + $this->ja3sRule($ruleFormat, $item['Attribute'], $sid); + break; case 'snort': $this->snortRule($ruleFormat, $item['Attribute'], $sid, $ruleFormatMsg, $ruleFormatReference); // no break @@ -503,6 +509,16 @@ class NidsExport ); } + public function ja3Rule($ruleFormat, $attribute, &$sid) + { + //Empty because Snort doesn't support JA3 Rules + } + + public function ja3sRule($ruleFormat, $attribute, &$sid) + { + //Empty because Snort doesn't support JA3S Rules + } + public function snortRule($ruleFormat, $attribute, &$sid, $ruleFormatMsg, $ruleFormatReference) { // LATER nids - test using lots of snort rules, some rules don't contain all the necessary to be a valid rule. diff --git a/app/Lib/Export/NidsSuricataExport.php b/app/Lib/Export/NidsSuricataExport.php index dc42eecad..49847b8fb 100644 --- a/app/Lib/Export/NidsSuricataExport.php +++ b/app/Lib/Export/NidsSuricataExport.php @@ -229,4 +229,49 @@ class NidsSuricataExport extends NidsExport 1 // rev ); } + + public function ja3Rule($ruleFormat, $attribute, &$sid) + { + $overruled = $this->checkWhitelist($attribute['value']); + $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule + $content = 'ja3.hash; content:"' . $attribute['value'] . '"; fast_pattern;'; + $this->rules[] = sprintf( + $ruleFormat, + ($overruled) ? '#OVERRULED BY WHITELIST# ' : '', + 'tls', // proto + 'any', // src_ip + 'any', // src_port + '->', // direction + 'any', // dst_ip + 'any', // dst_port + 'JA3 Hash: ' . $attribute['value'], // msg + $content, // rule_content + 'tag:session,600,seconds;', // tag + $sid, // sid + 1 // rev + ); + } + + // For Future use once JA3S Hash Attribute type is created + public function ja3sRule($ruleFormat, $attribute, &$sid) + { + $overruled = $this->checkWhitelist($attribute['value']); + $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']); // substitute chars not allowed in rule + $content = 'ja3s.hash; content:"' . $attribute['value'] . '"; fast_pattern;'; + $this->rules[] = sprintf( + $ruleFormat, + ($overruled) ? '#OVERRULED BY WHITELIST# ' : '', + 'tls', // proto + 'any', // src_ip + 'any', // src_port + '->', // direction + 'any', // dst_ip + 'any', // dst_port + 'JA3S Hash: ' . $attribute['value'], // msg + $content, // rule_content + 'tag:session,600,seconds;', // tag + $sid, // sid + 1 // rev + ); + } } diff --git a/app/Lib/Tools/CakeResponseTmp.php b/app/Lib/Tools/CakeResponseTmp.php new file mode 100644 index 000000000..12952868b --- /dev/null +++ b/app/Lib/Tools/CakeResponseTmp.php @@ -0,0 +1,40 @@ +header('Content-Length', $path->size()); + $this->_clearBuffer(); + $this->_file = $path; + } else { + parent::file($path, $options); + } + } + + /** + * @param File|TmpFileTool $file + * @param array $range + * @return bool + * @throws Exception + */ + protected function _sendFile($file, $range) + { + if ($file instanceof TmpFileTool) { + set_time_limit(0); + session_write_close(); + + foreach ($file->intoChunks() as $chunk) { + if (!$this->_isActive()) { + $file->close(); + return false; + } + echo $chunk; + $this->_flushBuffer(); + } + return true; + } else { + return parent::_sendFile($file, $range); + } + } +} diff --git a/app/Lib/Tools/CidrTool.php b/app/Lib/Tools/CidrTool.php new file mode 100644 index 000000000..faf978f3d --- /dev/null +++ b/app/Lib/Tools/CidrTool.php @@ -0,0 +1,125 @@ +filterInputList($list); + } + + /** + * @param string $value IPv4 or IPv6 address or range + * @return false|string + */ + public function contains($value) + { + $valueMask = null; + if (strpos($value, '/') !== false) { + list($value, $valueMask) = explode('/', $value); + } + + $match = false; + if (filter_var($value, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) { + // This code converts IP address to all possible CIDRs that can contains given IP address + // and then check if given hash table contains that CIDR. + $ip = ip2long($value); + // Start from 1, because doesn't make sense to check 0.0.0.0/0 match + for ($bits = $this->minimumIpv4Mask; $bits <= 32; $bits++) { + $mask = -1 << (32 - $bits); + $needle = long2ip($ip & $mask) . "/$bits"; + if (isset($this->ipv4[$needle])) { + $match = $needle; + break; + } + } + + } elseif (filter_var($value, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) { + $value = unpack('n*', inet_pton($value)); + foreach ($this->ipv6 as $netmask => $lv) { + foreach ($lv as $l) { + if ($this->ipv6InCidr($value, $l, $netmask)) { + $match = inet_ntop($l) . "/$netmask"; + break; + } + } + } + } + + if ($match && $valueMask) { + $matchMask = explode('/', $match)[1]; + if ($valueMask < $matchMask) { + return false; + } + } + + return $match; + } + + /** + * Using solution from https://github.com/symfony/symfony/blob/master/src/Symfony/Component/HttpFoundation/IpUtils.php + * + * @param array $ip + * @param string $cidr + * @param int $netmask + * @return bool + */ + private function ipv6InCidr($ip, $cidr, $netmask) + { + $bytesAddr = unpack('n*', $cidr); + for ($i = 1, $ceil = ceil($netmask / 16); $i <= $ceil; ++$i) { + $left = $netmask - 16 * ($i - 1); + $left = ($left <= 16) ? $left : 16; + $mask = ~(0xffff >> $left) & 0xffff; + if (($bytesAddr[$i] & $mask) != ($ip[$i] & $mask)) { + return false; + } + } + + return true; + } + + /** + * Filter out invalid IPv4 or IPv4 CIDR and append maximum netmask if no netmask is given. + * @param array $list + */ + private function filterInputList(array $list) + { + foreach ($list as $v) { + $parts = explode('/', $v, 2); + $ipBytes = inet_pton($parts[0]); + if ($ipBytes === false) { + continue; // IP address part of CIDR is invalid + } + $maximumNetmask = strlen($ipBytes) === 4 ? 32 : 128; + + if (isset($parts[1]) && ($parts[1] > $maximumNetmask || $parts[1] < 0)) { + // Netmask part of CIDR is invalid + continue; + } + + $mask = isset($parts[1]) ? $parts[1] : $maximumNetmask; + if ($maximumNetmask === 32) { + if ($mask < $this->minimumIpv4Mask) { + $this->minimumIpv4Mask = (int)$mask; + } + if (!isset($parts[1])) { + $v = "$v/$maximumNetmask"; // If CIDR doesnt contains '/', we will consider CIDR as /32 + } + $this->ipv4[$v] = true; + } else { + $this->ipv6[$mask][] = $ipBytes; + } + } + } +} diff --git a/app/Lib/Tools/ComplexTypeTool.php b/app/Lib/Tools/ComplexTypeTool.php index 65a73e2ff..71569b10c 100644 --- a/app/Lib/Tools/ComplexTypeTool.php +++ b/app/Lib/Tools/ComplexTypeTool.php @@ -177,7 +177,7 @@ class ComplexTypeTool unset($input); $iocArray = []; - foreach ($tmpFile->csv($delimiter) as $row) { + foreach ($tmpFile->intoParsedCsv($delimiter) as $row) { if (!empty($row[0][0]) && $row[0][0] === '#') { // Comment continue; } @@ -234,6 +234,7 @@ class ComplexTypeTool if (isset($resultArray[$typeArray['value']])) { continue; } + $typeArray['original_value'] = $ioc; $resultArray[$typeArray['value']] = $typeArray; } return array_values($resultArray); diff --git a/app/Lib/Tools/CorrelationGraphTool.php b/app/Lib/Tools/CorrelationGraphTool.php index bc6ba9700..0e18512be 100644 --- a/app/Lib/Tools/CorrelationGraphTool.php +++ b/app/Lib/Tools/CorrelationGraphTool.php @@ -4,13 +4,14 @@ private $__lookupTables = array(); private $__related_events = array(); private $__related_attributes = array(); - private $__eventModel = false; + /** @var Event */ + private $__eventModel; private $__taxonomyModel = false; private $__galaxyClusterModel = false; private $__user = false; private $__json = array(); - public function construct($eventModel, $taxonomyModel, $galaxyClusterModel, $user, $json) + public function construct(Event $eventModel, $taxonomyModel, $galaxyClusterModel, $user, $json) { $this->__eventModel = $eventModel; $this->__taxonomyModel = $taxonomyModel; @@ -26,7 +27,15 @@ private function __expandEvent($id) { - $event = $this->__eventModel->fetchEvent($this->__user, array('eventid' => $id, 'flatten' => 0, 'includeTagRelations' => 1, 'includeGalaxy' => 1, 'includeGranularCorrelations' => 1)); + $event = $this->__eventModel->fetchEvent($this->__user, array( + 'eventid' => $id, + 'flatten' => 0, + 'includeTagRelations' => 1, + 'includeGalaxy' => 1, + 'includeGranularCorrelations' => 1, + 'noSightings' => true, + 'sgReferenceOnly' => true, + )); if (empty($event)) { return $this->__json; } @@ -91,11 +100,11 @@ { foreach ($objects as $k => $object) { $include = $full; - if (!$include) { + if (!$include && isset($object['Attribute'])) { foreach ($object['Attribute'] as $attribute) { if (isset($this->__related_attributes[$attribute['id']])) { $include = true; - continue; + break; } } } diff --git a/app/Lib/Tools/CryptGpgExtended.php b/app/Lib/Tools/CryptGpgExtended.php index 15ba96c9c..3ea777b43 100644 --- a/app/Lib/Tools/CryptGpgExtended.php +++ b/app/Lib/Tools/CryptGpgExtended.php @@ -73,7 +73,8 @@ class CryptGpgExtended extends Crypt_GPG } /** - * Return key info without importing it. + * Return key info without importing it when GPG supports --import-options show-only, otherwise just import and + * then return details. * * @param string $key * @return Crypt_GPG_Key[] @@ -82,6 +83,18 @@ class CryptGpgExtended extends Crypt_GPG */ public function keyInfo($key) { + $version = $this->engine->getVersion(); + if (version_compare($version, '2.1.23', 'le')) { + $importResult = $this->importKey($key); + $keys = []; + foreach ($importResult['fingerprints'] as $fingerprint) { + foreach ($this->getKeys($fingerprint) as $key) { + $keys[] = $key; + } + } + return $keys; + } + $input = $this->_prepareInput($key, false, false); $output = ''; diff --git a/app/Lib/Tools/DistributionGraphTool.php b/app/Lib/Tools/DistributionGraphTool.php index d0523a822..2f44debbc 100644 --- a/app/Lib/Tools/DistributionGraphTool.php +++ b/app/Lib/Tools/DistributionGraphTool.php @@ -1,12 +1,15 @@ __extended_view = $extended_view; // construct distribution info - $this->__json['distributionInfo'] = array(); - $sgs = $this->__eventModel->SharingGroup->fetchAllAuthorised($this->__user, 'simplified', 1); + $sgs = $this->__eventModel->SharingGroup->fetchAllAuthorised($this->__user, 'distribution_graph', true); $this->__json['allSharingGroup'] = h($sgs); - $distributionLevels = $this->__eventModel->distributionLevels; - foreach ($distributionLevels as $key => $value) { - $this->__json['distributionInfo'][$key] = array('key' => h($value), 'desc' => h($this->__eventModel->distributionDescriptions[$key]['formdesc']), 'value' => h($key)); + + $this->__json['distributionInfo'] = array(); + foreach ($this->__eventModel->distributionLevels as $key => $value) { + $this->__json['distributionInfo'][$key] = [ + 'key' => h($value), + 'desc' => h($this->__eventModel->distributionDescriptions[$key]['formdesc']), + 'value' => h($key) + ]; } $this->__json['distributionInfo'][5] = ""; // inherit event. Will be deleted afterward @@ -72,27 +79,26 @@ class DistributionGraphTool $this->__addAdditionalDistributionInfo(3, "All other communities"); // add current community // connected - $servers = $this->__serverList; $this->__addAdditionalDistributionInfo(2, "This community"); // add current community - foreach ($servers as $server) { + foreach ($this->__serverList as $server) { $this->__addAdditionalDistributionInfo(2, $server); } // community - $orgs = $this->__organisationModel->find('list', array( - 'fields' => array('name'), - 'conditions' => array('local' => true) + $orgConditions = $this->__organisationModel->createConditions($this->__user); + $orgConditions['local'] = true; + $orgConditions['id !='] = $this->__user['Organisation']['id']; + $orgs = $this->__organisationModel->find('column', array( + 'fields' => ['name'], + 'conditions' => $orgConditions, )); $thisOrg = $this->__user['Organisation']['name']; $this->__addAdditionalDistributionInfo(1, $thisOrg); // add current community - foreach ($orgs as $org) { - if ($thisOrg != $org) { - $this->__addAdditionalDistributionInfo(1, $org); - } + foreach ($orgs as $orgName) { + $this->__addAdditionalDistributionInfo(1, $orgName); } // org only - $thisOrg = $this->__user['Organisation']['name']; $this->__addAdditionalDistributionInfo(0, $thisOrg); // add current community } diff --git a/app/Lib/Tools/EventGraphTool.php b/app/Lib/Tools/EventGraphTool.php index adda8c2e0..ecb4eebf7 100644 --- a/app/Lib/Tools/EventGraphTool.php +++ b/app/Lib/Tools/EventGraphTool.php @@ -77,6 +77,11 @@ if (!($check1 && $check2)) { unset($event['Object'][$i]); } + foreach($obj['ObjectReference'] as $j => $rel) { + if ($rel['deleted']) { + unset($event['Object'][$i]['ObjectReference'][$j]); + } + } } foreach ($event['Attribute'] as $i => $attr) { $check1 = $this->__satisfy_val_filtering($attr, false); @@ -519,7 +524,7 @@ public function get_reference_data($uuid) { $objectReference = $this->__refModel->ObjectReference->find('all', array( - 'conditions' => array('ObjectReference.uuid' => $uuid), + 'conditions' => array('ObjectReference.uuid' => $uuid, 'ObjectReference.deleted' => false), 'recursive' => -1, //'fields' => array('ObjectReference.id', 'relationship_type', 'comment', 'referenced_uuid') )); diff --git a/app/Lib/Tools/GpgTool.php b/app/Lib/Tools/GpgTool.php index 009ef03ca..406b9b6e0 100644 --- a/app/Lib/Tools/GpgTool.php +++ b/app/Lib/Tools/GpgTool.php @@ -46,7 +46,7 @@ class GpgTool */ public function searchGpgKey($search) { - $uri = 'https://pgp.circl.lu/pks/lookup?search=' . urlencode($search) . '&op=index&fingerprint=on&options=mr'; + $uri = 'https://openpgp.circl.lu/pks/lookup?search=' . urlencode($search) . '&op=index&fingerprint=on&options=mr'; $response = $this->keyServerLookup($uri); if ($response->code == 404) { return array(); // no keys found @@ -63,7 +63,7 @@ class GpgTool */ public function fetchGpgKey($fingerprint) { - $uri = 'https://pgp.circl.lu/pks/lookup?search=0x' . urlencode($fingerprint) . '&op=get&options=mr'; + $uri = 'https://openpgp.circl.lu/pks/lookup?search=0x' . urlencode($fingerprint) . '&op=get&options=mr'; $response = $this->keyServerLookup($uri); if ($response->code == 404) { return null; // key with given fingerprint not found diff --git a/app/Lib/Tools/HttpSocketExtended.php b/app/Lib/Tools/HttpSocketExtended.php new file mode 100644 index 000000000..cee948df9 --- /dev/null +++ b/app/Lib/Tools/HttpSocketExtended.php @@ -0,0 +1,132 @@ +response = $response; + parent::__construct($message, 0, $previous); + } + + /** + * @return HttpSocketResponse + */ + public function getResponse() + { + return $this->response; + } +} + +class HttpSocketResponseExtended extends HttpSocketResponse +{ + /** + * @param string $message + * @throws SocketException + */ + public function parseResponse($message) + { + parent::parseResponse($message); + + $contentEncoding = $this->getHeader('Content-Encoding'); + if ($contentEncoding === 'gzip' && function_exists('gzdecode')) { + $this->body = gzdecode($this->body); + if ($this->body === false) { + throw new SocketException("Response should be gzip encoded, but gzip decoding failed."); + } + } else if ($contentEncoding === 'br' && function_exists('brotli_uncompress')) { + $this->body = brotli_uncompress($this->body); + if ($this->body === false) { + throw new SocketException("Response should be brotli encoded, but brotli decoding failed."); + } + } else if ($contentEncoding) { + throw new SocketException("Remote server returns unsupported content encoding '$contentEncoding'"); + } + } + + /** + * Decodes JSON string and throws exception if string is not valid JSON. + * + * @return array + * @throws HttpClientJsonException + */ + public function json() + { + try { + if (defined('JSON_THROW_ON_ERROR')) { + // JSON_THROW_ON_ERROR is supported since PHP 7.3 + $decoded = json_decode($this->body, true, 512, JSON_THROW_ON_ERROR); + } else { + $decoded = json_decode($this->body, true); + if ($decoded === null) { + throw new UnexpectedValueException('Could not parse JSON: ' . json_last_error_msg(), json_last_error()); + } + } + return $decoded; + } catch (Exception $e) { + throw new HttpClientJsonException('Could not parse response as JSON.', $this, $e); + } + } +} + +/** + * Supports response compression and also decodes response as JSON + */ +class HttpSocketExtended extends HttpSocket +{ + public $responseClass = 'HttpSocketResponseExtended'; + + public function __construct($config = array()) + { + parent::__construct($config); + if (isset($config['compress']) && $config['compress']) { + $acceptEncoding = $this->acceptedEncodings(); + if (!empty($acceptEncoding)) { + $this->config['request']['header']['Accept-Encoding'] = implode(', ', $this->acceptedEncodings()); + } + } + } + + /** + * @param array $request + * @return HttpSocketResponseExtended + */ + public function request($request = array()) + { + // Reset last error + $this->lastError = []; + + /** @var HttpSocketResponseExtended $response */ + $response = parent::request($request); + if ($response === false) { + throw new InvalidArgumentException("Invalid argument provided."); + } + // Convert connection timeout to SocketException + if (!empty($this->lastError)) { + throw new SocketException($this->lastError['msg']); + } + return $response; + } + + /** + * Returns accepted content encodings (compression algorithms) + * @return string[] + */ + private function acceptedEncodings() + { + $supportedEncoding = []; + // Enable brotli compressed responses if PHP has 'brotli_uncompress' method + if (function_exists('brotli_uncompress')) { + $supportedEncoding[] = 'br'; + } + // Enable gzipped responses if PHP has 'gzdecode' method + if (function_exists('gzdecode')) { + $supportedEncoding[] = 'gzip'; + } + return $supportedEncoding; + } +} diff --git a/app/Lib/Tools/JSONConverterTool.php b/app/Lib/Tools/JSONConverterTool.php index 74eb5154f..9cd63b954 100644 --- a/app/Lib/Tools/JSONConverterTool.php +++ b/app/Lib/Tools/JSONConverterTool.php @@ -105,18 +105,19 @@ class JSONConverterTool } /** - * Event to JSON stream convertor. + * Event to JSON convertor, but that is intended for machine to machine communication * @param array $event * @return Generator */ public function streamConvert(array $event) { $event = $this->convert($event, false, true); + // Fast and inaccurate way how to check if event is too big for to convert in one call. This can be changed in future. $isBigEvent = (isset($event['Event']['Attribute']) ? count($event['Event']['Attribute']) : 0) + (isset($event['Event']['Object']) ? count($event['Event']['Object']) : 0) > 100; if (!$isBigEvent) { - yield json_encode($event, JSON_PRETTY_PRINT); + yield json_encode($event, JSON_UNESCAPED_UNICODE); return; } @@ -127,11 +128,11 @@ class JSONConverterTool yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":["; $firstInnerKey = key($value); foreach ($value as $i => $attribute) { - yield ($firstInnerKey === $i ? '' : ',') . json_encode($attribute); + yield ($firstInnerKey === $i ? '' : ',') . json_encode($attribute, JSON_UNESCAPED_UNICODE); } yield "]"; } else { - yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":" . json_encode($value); + yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":" . json_encode($value, JSON_UNESCAPED_UNICODE); } } if (isset($event['errors'])) { diff --git a/app/Lib/Tools/SendEmail.php b/app/Lib/Tools/SendEmail.php index 0779feae4..a8b6e95f9 100644 --- a/app/Lib/Tools/SendEmail.php +++ b/app/Lib/Tools/SendEmail.php @@ -320,7 +320,7 @@ class SendEmail if (!$this->gpg) { throw new SendEmailException("GPG encryption is enabled, but GPG is not configured."); } - + try { $fingerprint = $this->importAndValidateGpgPublicKey($params['gpgkey']); } catch (Crypt_GPG_NoDataException $e) { @@ -415,7 +415,7 @@ class SendEmail if (!$this->gpg) { throw new SendEmailException("GPG signing is enabled, but GPG is not initialized. Check debug log why GPG could not be initialized."); } - + try { $fingerprint = $this->importAndValidateGpgPublicKey($user['User']['gpgkey']); } catch (Crypt_GPG_NoDataException $e) { @@ -446,7 +446,9 @@ class SendEmail } if (!$canEncryptGpg && $canEncryptSmime) { - $this->signBySmime($email); + if (!empty(Configure::read('SMIME.cert_public_sign')) && !empty(Configure::read('SMIME.key_sign'))) { + $this->signBySmime($email); + } $this->encryptBySmime($email, $user['User']['certif_public']); $encrypted = true; } @@ -727,14 +729,14 @@ class SendEmail } list($inputFile, $outputFile) = $this->createInputOutputFiles($body); - $result = openssl_pkcs7_sign($inputFile->pwd(), $outputFile->pwd(), $certPublicSign, $keySign, array(), 0); + $result = openssl_pkcs7_sign($inputFile->pwd(), $outputFile->pwd(), $certPublicSign, $keySign, array(), PKCS7_DETACHED); $inputFile->delete(); if ($result) { $data = $outputFile->read(); $outputFile->delete(); $parts = explode("\n\n", $data); - return $parts[1] . "\n"; + return $parts[4] . "\n"; } else { $outputFile->delete(); diff --git a/app/Lib/Tools/SyncTool.php b/app/Lib/Tools/SyncTool.php index 94d8d308e..4f3d87974 100644 --- a/app/Lib/Tools/SyncTool.php +++ b/app/Lib/Tools/SyncTool.php @@ -2,10 +2,17 @@ class SyncTool { - // take a server as parameter and return a HttpSocket object using the ssl options defined in the server settings + /** + * Take a server as parameter and return a HttpSocket object using the ssl options defined in the server settings + * @param array|null $server + * @param false $timeout + * @param string $model + * @return HttpSocketExtended + * @throws Exception + */ public function setupHttpSocket($server = null, $timeout = false, $model = 'Server') { - $params = array(); + $params = ['compress' => true]; if (!empty($server)) { if (!empty($server[$model]['cert_file'])) { $params['ssl_cafile'] = APP . "files" . DS . "certs" . DS . $server[$model]['id'] . '.pem'; @@ -33,12 +40,12 @@ class SyncTool public function setupHttpSocketFeed($feed = null) { - return $this->setupHttpSocket(); + return $this->createHttpSocket(['compress' => true]); } /** * @param array $params - * @return HttpSocket + * @return HttpSocketExtended * @throws Exception */ public function createHttpSocket($params = array()) @@ -52,8 +59,8 @@ class SyncTool $params['ssl_cafile'] = $caPath; } - App::uses('HttpSocket', 'Network/Http'); - $HttpSocket = new HttpSocket($params); + App::uses('HttpSocketExtended', 'Tools'); + $HttpSocket = new HttpSocketExtended($params); $proxy = Configure::read('Proxy'); if (empty($params['skip_proxy']) && isset($proxy['host']) && !empty($proxy['host'])) { $HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']); diff --git a/app/Lib/Tools/TmpFileTool.php b/app/Lib/Tools/TmpFileTool.php index 9b85aebf1..86e25e5cf 100644 --- a/app/Lib/Tools/TmpFileTool.php +++ b/app/Lib/Tools/TmpFileTool.php @@ -14,7 +14,7 @@ class TmpFileTool public function __construct($maxInMemory = null) { if ($maxInMemory === null) { - $maxInMemory = 2 * 1024 * 1024; + $maxInMemory = 5 * 1024 * 1024; } $this->tmpfile = fopen("php://temp/maxmemory:$maxInMemory", "w+"); if ($this->tmpfile === false) { @@ -68,7 +68,7 @@ class TmpFileTool } /** - * Get one line from file parsed as CSV. + * Returns generator of parsed CSV line from file. * * @param string $delimiter * @param string $enclosure @@ -76,7 +76,7 @@ class TmpFileTool * @return Generator * @throws Exception */ - public function csv($delimiter = ',', $enclosure = '"', $escape = "\\") + public function intoParsedCsv($delimiter = ',', $enclosure = '"', $escape = "\\") { $this->rewind(); $line = 0; @@ -88,15 +88,16 @@ class TmpFileTool $line++; yield $result; } - fclose($this->tmpfile); - $this->tmpfile = null; + $this->close(); } /** + * Returns generator of line from file. + * * @return Generator * @throws Exception */ - public function lines() + public function intoLines() { $this->rewind(); while (!feof($this->tmpfile)) { @@ -106,24 +107,64 @@ class TmpFileTool } yield $result; } - fclose($this->tmpfile); - $this->tmpfile = null; + $this->close(); + } + + /** + * @param int $chunkSize In bytes + * @return Generator + * @throws Exception + */ + public function intoChunks($chunkSize = 8192) + { + $this->rewind(); + while (!feof($this->tmpfile)) { + $result = fread($this->tmpfile, $chunkSize); + if ($result === false) { + throw new Exception('Could not read from temporary file.'); + } + yield $result; + } + $this->close(); } /** * @return string * @throws Exception */ - public function finish() + public function intoString() { $this->rewind(); - $final = stream_get_contents($this->tmpfile); - if ($final === false) { + $string = stream_get_contents($this->tmpfile); + if ($string === false) { throw new Exception('Could not read from temporary file.'); } - fclose($this->tmpfile); - $this->tmpfile = null; - return $final; + $this->close(); + return $string; + } + + /** + * Pass data to output. + * + * @throws Exception + */ + public function intoOutput() + { + $this->rewind(); + if (fpassthru($this->tmpfile) === false) { + throw new Exception('Could not pass temporary file to output.'); + } + $this->close(); + } + + /** + * @return int + * @throws Exception + */ + public function size() + { + $this->isOpen(); + return fstat($this->tmpfile)['size']; } /** @@ -132,7 +173,30 @@ class TmpFileTool */ public function __toString() { - return $this->finish(); + return $this->intoString(); + } + + /** + * @return bool + */ + public function close() + { + if ($this->tmpfile) { + $result = fclose($this->tmpfile); + $this->tmpfile = null; + return $result; + } + return true; + } + + /** + * @throws Exception + */ + private function isOpen() + { + if ($this->tmpfile === null) { + throw new Exception('Temporary file is already closed.'); + } } /** @@ -142,6 +206,7 @@ class TmpFileTool */ private function rewind() { + $this->isOpen(); if (fseek($this->tmpfile, 0) === -1) { throw new Exception('Could not seek to start of temporary file.'); } diff --git a/app/Lib/Tools/XMLConverterTool.php b/app/Lib/Tools/XMLConverterTool.php index e70032437..ef7653db5 100644 --- a/app/Lib/Tools/XMLConverterTool.php +++ b/app/Lib/Tools/XMLConverterTool.php @@ -198,13 +198,18 @@ class XMLConverterTool $field = str_replace($this->__toEscape, $this->__escapeWith, $field); } + /** + * @param string $input + * @param false $mispVersion + * @return Generator + */ public function frameCollection($input, $mispVersion = false) { - $result = '' . PHP_EOL . '' . PHP_EOL; - $result .= $input; + yield '' . PHP_EOL . '' . PHP_EOL; + yield $input . PHP_EOL; if ($mispVersion) { - $result .= '' . $mispVersion . ''; + yield '' . $mispVersion . ''; } - return $result . '' . PHP_EOL; + yield '' . PHP_EOL; } } diff --git a/app/Lib/cakephp b/app/Lib/cakephp index d0c51b374..cf14e6546 160000 --- a/app/Lib/cakephp +++ b/app/Lib/cakephp @@ -1 +1 @@ -Subproject commit d0c51b37422d0d2c99be74045d4439a674259308 +Subproject commit cf14e6546ec44e3369e3531add11fdb946656280 diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 509cceead..af8202449 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -55,6 +55,7 @@ class AppModel extends Model parent::__construct($id, $table, $ds); $this->name = get_class($this); + $this->findMethods['column'] = true; } // deprecated, use $db_changes @@ -87,7 +88,7 @@ class AppModel extends Model 45 => false, 46 => false, 47 => false, 48 => false, 49 => false, 50 => false, 51 => false, 52 => false, 53 => false, 54 => false, 55 => false, 56 => false, 57 => false, 58 => false, 59 => false, 60 => false, 61 => false, 62 => false, - 63 => true, 64 => false + 63 => true, 64 => false, 65 => false ); public $advanced_updates_description = array( @@ -1557,6 +1558,15 @@ class AppModel extends Model KEY `org_id` (`org_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"; break; + case 65: + $sqlArray[] = "CREATE TABLE IF NOT EXISTS `correlation_exclusions` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `value` text NOT NULL, + `from_json` tinyint(1) default 0, + PRIMARY KEY (`id`), + INDEX `value` (`value`(255)) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"; + break; case 'fixNonEmptySharingGroupID': $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; @@ -1627,10 +1637,8 @@ class AppModel extends Model break; default: return false; - break; } - $now = new DateTime(); // switch MISP instance live to false if ($liveOff) { $this->Server = Classregistry::init('Server'); @@ -1643,7 +1651,7 @@ class AppModel extends Model $this->__setUpdateProgress(0, $total_update_count, $command); $str_index_array = array(); foreach($indexArray as $toIndex) { - $str_index_array[] = __('Indexing ') . sprintf('%s -> %s', $toIndex[0], $toIndex[1]); + $str_index_array[] = __('Indexing %s -> %s', $toIndex[0], $toIndex[1]); } $this->__setUpdateCmdMessages(array_merge($sqlArray, $str_index_array)); $flagStop = false; @@ -1679,10 +1687,10 @@ class AppModel extends Model 'email' => 'SYSTEM', 'action' => 'update_database', 'user_id' => 0, - 'title' => __('Successfuly executed the SQL query for ') . $command, + 'title' => __('Successfully executed the SQL query for ') . $command, 'change' => sprintf(__('The executed SQL query was: %s'), $sql) )); - $this->__setUpdateResMessages($i, sprintf(__('Successfuly executed the SQL query for %s'), $command)); + $this->__setUpdateResMessages($i, sprintf(__('Successfully executed the SQL query for %s'), $command)); } catch (Exception $e) { $errorMessage = $e->getMessage(); $this->Log->create(); @@ -1736,14 +1744,13 @@ class AppModel extends Model } } } - $this->__setUpdateProgress(count($sqlArray)+count($indexArray), false); + $this->__setUpdateProgress(count($sqlArray) + count($indexArray), false); } if ($clean) { $this->cleanCacheFiles(); } if ($liveOff) { - $liveSetting = 'MISP.live'; - $this->Server->serverSettingsSaveValue($liveSetting, true); + $this->Server->serverSettingsSaveValue('MISP.live', true); } if (!$flagStop && $errorCount == 0) { $this->__postUpdate($command); @@ -2132,11 +2139,20 @@ class AppModel extends Model } } if ($requiresLogout) { - $this->updateDatabase('destroyAllSessions'); + $this->refreshSessions(); } return true; } + /** + * Update date_modified for all users, this will ensure that all users will refresh their session data. + */ + private function refreshSessions() + { + $this->User = ClassRegistry::init('User'); + $this->User->updateAll(['date_modified' => time()]); + } + private function __setUpdateProgress($current, $total=false, $toward_db_version=false) { $updateProgress = $this->getUpdateProgress(); @@ -2735,7 +2751,7 @@ class AppModel extends Model { static $versionArray; if ($versionArray === null) { - $file = new File(ROOT . DS . 'VERSION.json', true); + $file = new File(ROOT . DS . 'VERSION.json'); $versionArray = $this->jsonDecode($file->read()); $file->close(); } @@ -3011,6 +3027,66 @@ class AppModel extends Model } } + /** + * Find method that allows to fetch just one column from database. + * @param $state + * @param $query + * @param array $results + * @return array + * @throws Exception + */ + protected function _findColumn($state, $query, $results = array()) + { + if ($state === 'before') { + if (count($query['fields']) === 1) { + if (strpos($query['fields'][0], '.') === false) { + $query['fields'][0] = $this->alias . '.' . $query['fields'][0]; + } + + $query['column'] = $query['fields'][0]; + if (isset($query['unique']) && $query['unique']) { + $query['fields'] = array("DISTINCT {$query['fields'][0]}"); + } else { + $query['fields'] = array($query['fields'][0]); + } + } else { + throw new Exception("Invalid number of column, expected one, " . count($query['fields']) . " given"); + } + + if (!isset($query['recursive'])) { + $query['recursive'] = -1; + } + + return $query; + } + + // Faster version of `Hash::extract` + foreach (explode('.', $query['column']) as $part) { + $results = array_column($results, $part); + } + return $results; + } + + /** + * @param string $field + * @param AppModel $model + * @param array $conditions + */ + public function addCountField($field, AppModel $model, array $conditions) + { + $db = $this->getDataSource(); + $subQuery = $db->buildStatement( + array( + 'fields' => ['COUNT(*)'], + 'table' => $db->fullTableName($model), + 'alias' => $model->alias, + 'conditions' => $conditions, + ), + $model + ); + $this->virtualFields[$field] = $subQuery; + } + /** * Log exception with backtrace and with nested exceptions. * diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index e7f543e24..c7806acf6 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -12,6 +12,9 @@ App::uses('ComplexTypeTool', 'Tools'); /** * @property Event $Event * @property AttributeTag $AttributeTag + * @property Sighting $Sighting + * @property-read array $typeDefinitions + * @property-read array $categoryDefinitions */ class Attribute extends AppModel { @@ -60,12 +63,10 @@ class Attribute extends AppModel public $shortDist = array(0 => 'Organisation', 1 => 'Community', 2 => 'Connected', 3 => 'All', 4 => ' Sharing Group', 5 => 'Inherit'); - public $categoryDefinitions = array(); - - public $typeDefinitions = array(); - - public function __construct($id = false, $table = null, $ds = null) { + private $exclusions = null; + public function __construct($id = false, $table = null, $ds = null) + { parent::__construct($id, $table, $ds); $this->distributionLevels = array( @@ -76,282 +77,6 @@ class Attribute extends AppModel 4 => __('Sharing group'), 5 => __('Inherit event') ); - - // - // NOTE WHEN MODIFYING: please ensure to run the script 'tools/gen_misp_types_categories.py' to update the new definitions everywhere. (docu, website, RFC, ...) - // - $this->categoryDefinitions = array( - 'Internal reference' => array( - 'desc' => __('Reference used by the publishing party (e.g. ticket number)'), - 'types' => array('text', 'link', 'comment', 'other', 'hex', 'anonymised', 'git-commit-id') - ), - 'Targeting data' => array( - 'desc' => __('Internal Attack Targeting and Compromise Information'), - 'formdesc' => __('Targeting information to include recipient email, infected machines, department, and or locations.'), - 'types' => array('target-user', 'target-email', 'target-machine', 'target-org', 'target-location', 'target-external', 'comment', 'anonymised') - ), - 'Antivirus detection' => array( - 'desc' => __('All the info about how the malware is detected by the antivirus products'), - 'formdesc' => __('List of anti-virus vendors detecting the malware or information on detection performance (e.g. 13/43 or 67%). Attachment with list of detection or link to VirusTotal could be placed here as well.'), - 'types' => array('link', 'comment', 'text', 'hex', 'attachment', 'other', 'anonymised') - ), - 'Payload delivery' => array( - 'desc' => __('Information about how the malware is delivered'), - 'formdesc' => __('Information about the way the malware payload is initially delivered, for example information about the email or web-site, vulnerability used, originating IP etc. Malware sample itself should be attached here.'), - 'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash','filename|impfuzzy', 'filename|pehash', 'mac-address', 'mac-eui-64', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'hostname', 'domain', 'email', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'email-body', 'url', 'user-agent', 'AS', 'pattern-in-file', 'pattern-in-traffic', 'filename-pattern', 'stix2-pattern', 'yara', 'sigma', 'mime-type', 'attachment', 'malware-sample', 'link', 'malware-type', 'comment', 'text', 'hex', 'vulnerability', 'cpe', 'weakness', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'jarm-fingerprint', 'hassh-md5', 'hasshserver-md5', 'other', 'hostname|port', 'email-dst-display-name', 'email-src-display-name', 'email-header', 'email-reply-to', 'email-x-mailer', 'email-mime-boundary', 'email-thread-index', 'email-message-id', 'mobile-application-id', 'chrome-extension-id', 'whois-registrant-email', 'anonymised') - ), - 'Artifacts dropped' => array( - 'desc' => __('Any artifact (files, registry keys etc.) dropped by the malware or other modifications to the system'), - 'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy','filename|pehash', 'regkey', 'regkey|value', 'pattern-in-file', 'pattern-in-memory', 'filename-pattern', 'pdb', 'stix2-pattern', 'yara', 'sigma', 'attachment', 'malware-sample', 'named pipe', 'mutex', 'process-state','windows-scheduled-task', 'windows-service-name', 'windows-service-displayname', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'cookie', 'gene', 'kusto-query', 'mime-type', 'anonymised', 'pgp-public-key', 'pgp-private-key') - ), - 'Payload installation' => array( - 'desc' => __('Info on where the malware gets installed in the system'), - 'formdesc' => __('Location where the payload was placed in the system and the way it was installed. For example, a filename|md5 type attribute can be added here like this: c:\\windows\\system32\\malicious.exe|41d8cd98f00b204e9800998ecf8427e.'), - 'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy', 'filename|pehash', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'filename-pattern', 'stix2-pattern', 'yara', 'sigma', 'vulnerability', 'cpe','weakness', 'attachment', 'malware-sample', 'malware-type', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'mobile-application-id', 'chrome-extension-id', 'other', 'mime-type', 'anonymised') - ), - 'Persistence mechanism' => array( - 'desc' => __('Mechanisms used by the malware to start at boot'), - 'formdesc' => __('Mechanisms used by the malware to start at boot. This could be a registry key, legitimate driver modification, LNK file in startup'), - 'types' => array('filename', 'regkey', 'regkey|value', 'comment', 'text', 'other', 'hex', 'anonymised') - ), - 'Network activity' => array( - 'desc' => __('Information about network traffic generated by the malware'), - 'types' => array('ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'port', 'hostname', 'domain', 'domain|ip', 'mac-address', 'mac-eui-64', 'email', 'email-dst', 'email-src', 'eppn', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'pattern-in-file', 'filename-pattern','stix2-pattern', 'pattern-in-traffic', 'attachment', 'comment', 'text', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'jarm-fingerprint', 'hassh-md5', 'hasshserver-md5', 'other', 'hex', 'cookie', 'hostname|port', 'bro', 'zeek', 'anonymised', 'community-id', 'email-subject') - ), - 'Payload type' => array( - 'desc' => __('Information about the final payload(s)'), - 'formdesc' => __('Information about the final payload(s). Can contain a function of the payload, e.g. keylogger, RAT, or a name if identified, such as Poison Ivy.'), - 'types' => array('comment', 'text', 'other', 'anonymised') - ), - 'Attribution' => array( - 'desc' => __('Identification of the group, organisation, or country behind the attack'), - 'types' => array('threat-actor', 'campaign-name', 'campaign-id', 'whois-registrant-phone', 'whois-registrant-email', 'whois-registrant-name', 'whois-registrant-org', 'whois-registrar', 'whois-creation-date','comment', 'text', 'x509-fingerprint-sha1','x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'dns-soa-email', 'anonymised', 'email') - ), - 'External analysis' => array( - 'desc' => __('Any other result from additional analysis of the malware like tools output'), - 'formdesc' => __('Any other result from additional analysis of the malware like tools output Examples: pdf-parser output, automated sandbox analysis, reverse engineering report.'), - 'types' => array('md5', 'sha1', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'mac-address', 'mac-eui-64', 'hostname', 'domain', 'domain|ip', 'url', 'user-agent', 'regkey', 'regkey|value', 'AS', 'snort', 'bro', 'zeek', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'filename-pattern','vulnerability', 'cpe', 'weakness', 'attachment', 'malware-sample', 'link', 'comment', 'text', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'jarm-fingerprint', 'hassh-md5', 'hasshserver-md5', 'github-repository', 'other', 'cortex', 'anonymised', 'community-id') - ), - 'Financial fraud' => array( - 'desc' => __('Financial Fraud indicators'), - 'formdesc' => __('Financial Fraud indicators, for example: IBAN Numbers, BIC codes, Credit card numbers, etc.'), - 'types' => array('btc', 'dash', 'xmr', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'phone-number', 'comment', 'text', 'other', 'hex', 'anonymised'), - ), - 'Support Tool' => array( - 'desc' => __('Tools supporting analysis or detection of the event'), - 'types' => array('link', 'text', 'attachment', 'comment', 'other', 'hex', 'anonymised') - ), - 'Social network' => array( - 'desc' => __('Social networks and platforms'), - // email-src and email-dst or should we go with a new email type that is neither / both? - 'types' => array('github-username', 'github-repository', 'github-organisation', 'jabber-id', 'twitter-id', 'email', 'email-src', 'email-dst', 'eppn','comment', 'text', 'other', 'whois-registrant-email', 'anonymised', 'pgp-public-key', 'pgp-private-key') - ), - 'Person' => array( - 'desc' => __('A human being - natural person'), - 'types' => array('first-name', 'middle-name', 'last-name', 'date-of-birth', 'place-of-birth', 'gender', 'passport-number', 'passport-country', 'passport-expiration', 'redress-number', 'nationality', 'visa-number', 'issue-date-of-the-visa', 'primary-residence', 'country-of-residence', 'special-service-request', 'frequent-flyer-number', 'travel-details', 'payment-details', 'place-port-of-original-embarkation', 'place-port-of-clearance', 'place-port-of-onward-foreign-destination', 'passenger-name-record-locator-number', 'comment', 'text', 'other', 'phone-number', 'identity-card-number', 'anonymised', 'email', 'pgp-public-key', 'pgp-private-key') - ), - 'Other' => array( - 'desc' => __('Attributes that are not part of any other category or are meant to be used as a component in MISP objects in the future'), - 'types' => array('comment', 'text', 'other', 'size-in-bytes', 'counter', 'datetime', 'cpe', 'port', 'float', 'hex', 'phone-number', 'boolean', 'anonymised', 'pgp-public-key', 'pgp-private-key') - ) - ); - - // - // NOTE WHEN MODIFYING: please ensure to run the script 'tools/gen_misp_types_categories.py' to update the new definitions everywhere. (docu, website, RFC, ...) - // - $this->typeDefinitions = array( - 'md5' => array('desc' => __('A checksum in md5 format'), 'formdesc' => __("You are encouraged to use filename|md5 instead. A checksum in md5 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'sha1' => array('desc' => __('A checksum in sha1 format'), 'formdesc' => __("You are encouraged to use filename|sha1 instead. A checksum in sha1 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'sha256' => array('desc' => __('A checksum in sha256 format'), 'formdesc' => __("You are encouraged to use filename|sha256 instead. A checksum in sha256 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'filename' => array('desc' => __('Filename'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'pdb' => array('desc' => __('Microsoft Program database (PDB) path information'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0), - 'filename|md5' => array('desc' => __('A filename and an md5 hash separated by a |'), 'formdesc' => __("A filename and an md5 hash separated by a | (no spaces)"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'filename|sha1' => array('desc' => __('A filename and an sha1 hash separated by a |'), 'formdesc' => __("A filename and an sha1 hash separated by a | (no spaces)"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'filename|sha256' => array('desc' => __('A filename and an sha256 hash separated by a |'), 'formdesc' => __("A filename and an sha256 hash separated by a | (no spaces)"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'ip-src' => array('desc' => __("A source IP address of the attacker"), 'default_category' => 'Network activity', 'to_ids' => 1), - 'ip-dst' => array('desc' => __('A destination IP address of the attacker or C&C server'), 'formdesc' => __("A destination IP address of the attacker or C&C server. Also set the IDS flag on when this IP is hardcoded in malware"), 'default_category' => 'Network activity', 'to_ids' => 1), - 'hostname' => array('desc' => __('A full host/dnsname of an attacker'), 'formdesc' => __("A full host/dnsname of an attacker. Also set the IDS flag on when this hostname is hardcoded in malware"), 'default_category' => 'Network activity', 'to_ids' => 1), - 'domain' => array('desc' => __('A domain name used in the malware'), 'formdesc' => __("A domain name used in the malware. Use this instead of hostname when the upper domain is important or can be used to create links between events."), 'default_category' => 'Network activity', 'to_ids' => 1), - 'domain|ip' => array('desc' => __('A domain name and its IP address (as found in DNS lookup) separated by a |'),'formdesc' => __("A domain name and its IP address (as found in DNS lookup) separated by a | (no spaces)"), 'default_category' => 'Network activity', 'to_ids' => 1), - 'email' => array('desc' => ('An e-mail address'), 'default_category' => 'Social network', 'to_ids' => 1), - 'email-src' => array('desc' => __("The source email address. Used to describe the sender when describing an e-mail."), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'eppn' => array('desc' => __("eduPersonPrincipalName - eppn - the NetId of the person for the purposes of inter-institutional authentication. Should be stored in the form of user@univ.edu, where univ.edu is the name of the local security domain."), 'default_category' => 'Network activity', 'to_ids' => 1), - 'email-dst' => array('desc' => __("The destination email address. Used to describe the recipient when describing an e-mail."), 'default_category' => 'Network activity', 'to_ids' => 1), - 'email-subject' => array('desc' => __("The subject of the email"), 'default_category' => 'Payload delivery', 'to_ids' => 0), - 'email-attachment' => array('desc' => __("File name of the email attachment."), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'email-body' => array('desc' => __('Email body'), 'default_category' => 'Payload delivery', 'to_ids' => 0), - 'float' => array('desc' => __("A floating point value."), 'default_category' => 'Other', 'to_ids' => 0), - 'git-commit-id' => array('desc' => __("A git commit ID."), 'default_category' => 'Internal reference', 'to_ids' => 0), - 'url' => array('desc' => __('url'), 'default_category' => 'Network activity', 'to_ids' => 1), - 'http-method' => array('desc' => __("HTTP method used by the malware (e.g. POST, GET, ...)."), 'default_category' => 'Network activity', 'to_ids' => 0), - 'user-agent' => array('desc' => __("The user-agent used by the malware in the HTTP request."), 'default_category' => 'Network activity', 'to_ids' => 0), - 'ja3-fingerprint-md5' => array('desc' => __("JA3 is a method for creating SSL/TLS client fingerprints that should be easy to produce on any platform and can be easily shared for threat intelligence."), 'default_category' => 'Network activity', 'to_ids' => 1), - 'jarm-fingerprint' => array('desc' => __("JARM is a method for creating SSL/TLS server fingerprints."), 'default_category' => 'Network activity', 'to_ids' => 1), - 'hassh-md5' => array('desc' => __("hassh is a network fingerprinting standard which can be used to identify specific Client SSH implementations. The fingerprints can be easily stored, searched and shared in the form of an MD5 fingerprint."), 'default_category' => 'Network activity', 'to_ids' => 1), - 'hasshserver-md5' => array('desc' => __("hasshServer is a network fingerprinting standard which can be used to identify specific Server SSH implementations. The fingerprints can be easily stored, searched and shared in the form of an MD5 fingerprint."), 'default_category' => 'Network activity', 'to_ids' => 1), - 'regkey' => array('desc' => __("Registry key or value"), 'default_category' => 'Persistence mechanism', 'to_ids' => 1), - 'regkey|value' => array('desc' => __("Registry value + data separated by |"), 'default_category' => 'Persistence mechanism', 'to_ids' => 1), - 'AS' => array('desc' => __('Autonomous system'), 'default_category' => 'Network activity', 'to_ids' => 0), - 'snort' => array('desc' => __('An IDS rule in Snort rule-format'), 'formdesc' => __("An IDS rule in Snort rule-format. This rule will be automatically rewritten in the NIDS exports."), 'default_category' => 'Network activity', 'to_ids' => 1), - 'bro' => array('desc' => __('An NIDS rule in the Bro rule-format'), 'formdesc' => __("An NIDS rule in the Bro rule-format."), 'default_category' => 'Network activity', 'to_ids' => 1), - 'zeek' => array('desc' => __('An NIDS rule in the Zeek rule-format'), 'formdesc' => __("An NIDS rule in the Zeek rule-format."), 'default_category' => 'Network activity', 'to_ids' => 1), - 'community-id' => array('desc' => __('a community ID flow hashing algorithm to map multiple traffic monitors into common flow id'), 'formdesc' => __("a community ID flow hashing algorithm to map multiple traffic monitors into common flow id"), 'default_category' => 'Network activity', 'to_ids' => 1), - 'pattern-in-file' => array('desc' => __('Pattern in file that identifies the malware'), 'default_category' => 'Payload installation', 'to_ids' => 1), - 'pattern-in-traffic' => array('desc' => __('Pattern in network traffic that identifies the malware'), 'default_category' => 'Network activity', 'to_ids' => 1), - 'pattern-in-memory' => array('desc' => __('Pattern in memory dump that identifies the malware'), 'default_category' => 'Payload installation', 'to_ids' => 1), - 'pattern-filename' => array('desc' => __('A pattern in the name of a file'), 'default_category' => 'Payload installation', 'to_ids' => 1), - 'pgp-public-key' => array('desc' => __('A PGP public key'), 'default_category' => 'Person', 'to_ids' => 0), - 'pgp-private-key' => array('desc' => __('A PGP private key'), 'default_category' => 'Person', 'to_ids' => 0), - 'yara' => array('desc' => __('Yara signature'), 'default_category' => 'Payload installation', 'to_ids' => 1), - 'stix2-pattern' => array('desc' => __('STIX 2 pattern'), 'default_category' => 'Payload installation', 'to_ids' => 1), - 'sigma' => array('desc' => __('Sigma - Generic Signature Format for SIEM Systems'), 'default_category' => 'Payload installation', 'to_ids' => 1), - 'gene' => array('desc' => __('GENE - Go Evtx sigNature Engine'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0), - 'kusto-query' => array('desc' => __('Kusto query - Kusto from Microsoft Azure is a service for storing and running interactive analytics over Big Data.'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0), - 'mime-type' => array('desc' => __('A media type (also MIME type and content type) is a two-part identifier for file formats and format contents transmitted on the Internet'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0), - 'identity-card-number' => array('desc' => __('Identity card number'), 'default_category' => 'Person', 'to_ids' => 0), - 'cookie' => array('desc' => __('HTTP cookie as often stored on the user web client. This can include authentication cookie or session cookie.'), 'default_category' => 'Network activity', 'to_ids' => 0), - 'vulnerability' => array('desc' => __('A reference to the vulnerability used in the exploit'), 'default_category' => 'External analysis', 'to_ids' => 0), - 'cpe' => array('desc' => __('Common Platform Enumeration - structured naming scheme for information technology systems, software, and packages.'), 'default_category' => 'External analysis', 'to_ids' => 0), - 'weakness' => array('desc'=> __('A reference to the weakness used in the exploit'), 'default_category' => 'External analysis', 'to_ids' => 0), - 'attachment' => array('desc' => __('Attachment with external information'), 'formdesc' => __("Please upload files using the Upload Attachment button."), 'default_category' => 'External analysis', 'to_ids' => 0), - 'malware-sample' => array('desc' => __('Attachment containing encrypted malware sample'), 'formdesc' => __("Please upload files using the Upload Attachment button."), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'link' => array('desc' => __('Link to an external information'), 'default_category' => 'External analysis', 'to_ids' => 0), - 'comment' => array('desc' => __('Comment or description in a human language'), 'formdesc' => __('Comment or description in a human language. This will not be correlated with other attributes'), 'default_category' => 'Other', 'to_ids' => 0), - 'text' => array('desc' => __('Name, ID or a reference'), 'default_category' => 'Other', 'to_ids' => 0), - 'hex' => array('desc' => __('A value in hexadecimal format'), 'default_category' => 'Other', 'to_ids' => 0), - 'other' => array('desc' => __('Other attribute'), 'default_category' => 'Other', 'to_ids' => 0), - 'named pipe' => array('desc' => __('Named pipe, use the format \\.\pipe\'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0), - 'mutex' => array('desc' => __('Mutex, use the format \BaseNamedObjects\'), 'default_category' => 'Artifacts dropped', 'to_ids' => 1), - 'process-state' => array('desc' => __('State of a process'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0), - 'target-user' => array('desc' => __('Attack Targets Username(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0), - 'target-email' => array('desc' => __('Attack Targets Email(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0), - 'target-machine' => array('desc' => __('Attack Targets Machine Name(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0), - 'target-org' => array('desc' => __('Attack Targets Department or Organization(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0), - 'target-location' => array('desc' => __('Attack Targets Physical Location(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0), - 'target-external' => array('desc' => __('External Target Organizations Affected by this Attack'), 'default_category' => 'Targeting data', 'to_ids' => 0), - 'btc' => array('desc' => __('Bitcoin Address'), 'default_category' => 'Financial fraud', 'to_ids' => 1), - 'dash' => array('desc' => __('Dash Address'), 'default_category' => 'Financial fraud', 'to_ids' => 1), - 'xmr' => array('desc' => __('Monero Address'), 'default_category' => 'Financial fraud', 'to_ids' => 1), - 'iban' => array('desc' => __('International Bank Account Number'), 'default_category' => 'Financial fraud', 'to_ids' => 1), - 'bic' => array('desc' => __('Bank Identifier Code Number also known as SWIFT-BIC, SWIFT code or ISO 9362 code'), 'default_category' => 'Financial fraud', 'to_ids' => 1), - 'bank-account-nr' => array('desc' => __('Bank account number without any routing number'), 'default_category' => 'Financial fraud', 'to_ids' => 1), - 'aba-rtn' => array('desc' => __('ABA routing transit number'), 'default_category' => 'Financial fraud', 'to_ids' => 1), - 'bin' => array('desc' => __('Bank Identification Number'), 'default_category' => 'Financial fraud', 'to_ids' => 1), - 'cc-number' => array('desc' => __('Credit-Card Number'), 'default_category' => 'Financial fraud', 'to_ids' => 1), - 'prtn' => array('desc' => __('Premium-Rate Telephone Number'), 'default_category' => 'Financial fraud', 'to_ids' => 1), - 'phone-number' => array('desc' => __('Telephone Number'), 'default_category' => 'Person', 'to_ids' => 0), - 'threat-actor' => array('desc' => __('A string identifying the threat actor'), 'default_category' => 'Attribution', 'to_ids' => 0), - 'campaign-name' => array('desc' => __('Associated campaign name'), 'default_category' => 'Attribution', 'to_ids' => 0), - 'campaign-id' => array('desc' => __('Associated campaign ID'), 'default_category' => 'Attribution', 'to_ids' => 0), - 'malware-type' => array('desc' => '', 'default_category' => 'Payload delivery', 'to_ids' => 0), - 'uri' => array('desc' => __('Uniform Resource Identifier'), 'default_category' => 'Network activity', 'to_ids' => 1), - 'authentihash' => array('desc' => __('Authenticode executable signature hash'), 'formdesc' => __("You are encouraged to use filename|authentihash instead. Authenticode executable signature hash, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'vhash' => array('desc' => __('A VirusTotal checksum'), 'formdesc' => __("You are encouraged to use filename|vhash instead. A checksum from VirusTotal, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'ssdeep' => array('desc' => __('A checksum in ssdeep format'), 'formdesc' => __("You are encouraged to use filename|ssdeep instead. A checksum in the SSDeep format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'imphash' => array('desc' => __('Import hash - a hash created based on the imports in the sample.'), 'formdesc' => __("You are encouraged to use filename|imphash instead. A hash created based on the imports in the sample, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'telfhash' => array('desc' => __('telfhash is symbol hash for ELF files, just like imphash is imports hash for PE files.'), 'formdesc' => __("You are encouraged to use a file object with telfash"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'pehash' => array('desc' => __('PEhash - a hash calculated based of certain pieces of a PE executable file'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'impfuzzy' => array('desc' => __('A fuzzy hash of import table of Portable Executable format'), 'formdesc' => __("You are encouraged to use filename|impfuzzy instead. A fuzzy hash created based on the imports in the sample, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'sha224' => array('desc' => __('A checksum in sha-224 format'), 'formdesc' => __("You are encouraged to use filename|sha224 instead. A checksum in sha224 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'sha384' => array('desc' => __('A checksum in sha-384 format'), 'formdesc' => __("You are encouraged to use filename|sha384 instead. A checksum in sha384 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'sha512' => array('desc' => __('A checksum in sha-512 format'), 'formdesc' => __("You are encouraged to use filename|sha512 instead. A checksum in sha512 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'sha512/224' => array('desc' => __('A checksum in the sha-512/224 format'), 'formdesc' => __("You are encouraged to use filename|sha512/224 instead. A checksum in sha512/224 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'sha512/256' => array('desc' => __('A checksum in the sha-512/256 format'), 'formdesc' => __("You are encouraged to use filename|sha512/256 instead. A checksum in sha512/256 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'sha3-224' => array('desc' => __('A checksum in sha3-224 format'), 'formdesc' => __("You are encouraged to use filename|sha3-224 instead. A checksum in sha3-224 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'sha3-256' => array('desc' => __('A checksum in sha3-256 format'), 'formdesc' => __("You are encouraged to use filename|sha3-256 instead. A checksum in sha3-256 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'sha3-384' => array('desc' => __('A checksum in sha3-384 format'), 'formdesc' => __("You are encouraged to use filename|sha3-384 instead. A checksum in sha3-384 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'sha3-512' => array('desc' => __('A checksum in sha3-512 format'), 'formdesc' => __("You are encouraged to use filename|sha3-512 instead. A checksum in sha3-512 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'tlsh' => array('desc' => __('A checksum in the Trend Micro Locality Sensitive Hash format'), 'formdesc' => __("You are encouraged to use filename|tlsh instead. A checksum in the Trend Micro Locality Sensitive Hash format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'cdhash' => array('desc' => __('An Apple Code Directory Hash, identifying a code-signed Mach-O executable file'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'filename|authentihash' => array('desc' => __('A checksum in md5 format'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'filename|vhash' => array('desc' => __('A filename and a VirusTotal hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'filename|ssdeep' => array('desc' => __('A checksum in ssdeep format'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'filename|imphash' => array('desc' => __('Import hash - a hash created based on the imports in the sample.'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'filename|impfuzzy' => array('desc' => __('Import fuzzy hash - a fuzzy hash created based on the imports in the sample.'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'filename|pehash' => array('desc' => __('A filename and a PEhash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'filename|sha224' => array('desc' => __('A filename and a sha-224 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'filename|sha384' => array('desc' => __('A filename and a sha-384 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'filename|sha512' => array('desc' => __('A filename and a sha-512 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'filename|sha512/224' => array('desc' => __('A filename and a sha-512/224 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'filename|sha512/256' => array('desc' => __('A filename and a sha-512/256 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'filename|sha3-224' => array('desc' => __('A filename and an sha3-224 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'filename|sha3-256' => array('desc' => __('A filename and an sha3-256 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'filename|sha3-384' => array('desc' => __('A filename and an sha3-384 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'filename|sha3-512' => array('desc' => __('A filename and an sha3-512 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'filename|tlsh' => array('desc' => __('A filename and a Trend Micro Locality Sensitive Hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'windows-scheduled-task' => array('desc' => __('A scheduled task in windows'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0), - 'windows-service-name' => array('desc' => __('A windows service name. This is the name used internally by windows. Not to be confused with the windows-service-displayname.'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0), - 'windows-service-displayname' => array('desc' => __('A windows service\'s displayname, not to be confused with the windows-service-name. This is the name that applications will generally display as the service\'s name in applications.'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0), - 'whois-registrant-email' => array('desc' => __('The e-mail of a domain\'s registrant, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0), - 'whois-registrant-phone' => array('desc' => __('The phone number of a domain\'s registrant, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0), - 'whois-registrant-name' => array('desc' => __('The name of a domain\'s registrant, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0), - 'whois-registrant-org' => array('desc' => __('The org of a domain\'s registrant, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0), - 'whois-registrar' => array('desc' => __('The registrar of the domain, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0), - 'whois-creation-date' => array('desc' => __('The date of domain\'s creation, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0), - // 'targeted-threat-index' => array('desc' => ''), // currently not mapped! - // 'mailslot' => array('desc' => 'MailSlot interprocess communication'), // currently not mapped! - // 'pipe' => array('desc' => 'Pipeline (for named pipes use the attribute type "named pipe")'), // currently not mapped! - // 'ssl-cert-attributes' => array('desc' => 'SSL certificate attributes'), // currently not mapped! - 'x509-fingerprint-sha1' => array('desc' => __('X509 fingerprint in SHA-1 format'), 'default_category' => 'Network activity', 'to_ids' => 1), - 'x509-fingerprint-md5' => array('desc' => __('X509 fingerprint in MD5 format'), 'default_category' => 'Network activity', 'to_ids' => 1), - 'x509-fingerprint-sha256' => array('desc' => __('X509 fingerprint in SHA-256 format'), 'default_category' => 'Network activity', 'to_ids' => 1), - 'dns-soa-email' => array('desc' => __('RFC1035 mandates that DNS zones should have a SOA (Statement Of Authority) record that contains an email address where a PoC for the domain could be contacted. This can sometimes be used for attribution/linkage between different domains even if protected by whois privacy'), 'default_category' => 'Attribution', 'to_ids' => 0), - 'size-in-bytes' => array('desc' => __('Size expressed in bytes'), 'default_category' => 'Other', 'to_ids' => 0), - 'counter' => array('desc' => __('An integer counter, generally to be used in objects'), 'default_category' => 'Other', 'to_ids' => 0), - 'datetime' => array('desc' => __('Datetime in the ISO 8601 format'), 'default_category' => 'Other', 'to_ids' => 0), - 'port' => array('desc' => __('Port number'), 'default_category' => 'Network activity', 'to_ids' => 0), - 'ip-dst|port' => array('desc' => __('IP destination and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1), - 'ip-src|port' => array('desc' => __('IP source and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1), - 'hostname|port' => array('desc' => __('Hostname and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1), - 'mac-address' => array('desc' => __('Mac address'), 'default_category' => 'Network activity', 'to_ids' => 0), - 'mac-eui-64' => array('desc' => __('Mac EUI-64 address'), 'default_category' => 'Network activity', 'to_ids' => 0), - // verify IDS flag defaults for these - 'email-dst-display-name' => array('desc' => __('Email destination display name'), 'default_category' => 'Payload delivery', 'to_ids' => 0), - 'email-src-display-name' => array('desc' => __('Email source display name'), 'default_category' => 'Payload delivery', 'to_ids' => 0), - 'email-header' => array('desc' => __('Email header'), 'default_category' => 'Payload delivery', 'to_ids' => 0), - 'email-reply-to' => array('desc' => __('Email reply to header'), 'default_category' => 'Payload delivery', 'to_ids' => 0), - 'email-x-mailer' => array('desc' => __('Email x-mailer header'), 'default_category' => 'Payload delivery', 'to_ids' => 0), - 'email-mime-boundary' => array('desc' => __('The email mime boundary separating parts in a multipart email'), 'default_category' => 'Payload delivery', 'to_ids' => 0), - 'email-thread-index' => array('desc' => __('The email thread index header'), 'default_category' => 'Payload delivery', 'to_ids' => 0), - 'email-message-id' => array('desc' => __('The email message ID'), 'default_category' => 'Payload delivery', 'to_ids' => 0), - 'github-username' => array('desc' => __('A github user name'), 'default_category' => 'Social network', 'to_ids' => 0), - 'github-repository' => array('desc' => __('A github repository'), 'default_category' => 'Social network', 'to_ids' => 0), - 'github-organisation' => array('desc' => __('A github organisation'), 'default_category' => 'Social network', 'to_ids' => 0), - 'jabber-id' => array('desc' => __('Jabber ID'), 'default_category' => 'Social network', 'to_ids' => 0), - 'twitter-id' => array('desc' => __('Twitter ID'), 'default_category' => 'Social network', 'to_ids' => 0), - 'first-name' => array('desc' => __('First name of a natural person'), 'default_category' => 'Person', 'to_ids' => 0), - 'middle-name' => array('desc' => __('Middle name of a natural person'), 'default_category' => 'Person', 'to_ids' => 0), - 'last-name' => array('desc' => __('Last name of a natural person'), 'default_category' => 'Person', 'to_ids' => 0), - 'date-of-birth' => array('desc' => __('Date of birth of a natural person (in YYYY-MM-DD format)'), 'default_category' => 'Person', 'to_ids' => 0), - 'place-of-birth' => array('desc' => __('Place of birth of a natural person'), 'default_category' => 'Person', 'to_ids' => 0), - 'gender' => array('desc' => __('The gender of a natural person (Male, Female, Other, Prefer not to say)'), 'default_category' => 'Person', 'to_ids' => 0), - 'passport-number' => array('desc' => __('The passport number of a natural person'), 'default_category' => 'Person', 'to_ids' => 0), - 'passport-country' => array('desc' => __('The country in which the passport was issued'), 'default_category' => 'Person', 'to_ids' => 0), - 'passport-expiration' => array('desc' => __('The expiration date of a passport'), 'default_category' => 'Person', 'to_ids' => 0), - 'redress-number' => array('desc' => __('The Redress Control Number is the record identifier for people who apply for redress through the DHS Travel Redress Inquiry Program (DHS TRIP). DHS TRIP is for travelers who have been repeatedly identified for additional screening and who want to file an inquiry to have erroneous information corrected in DHS systems'), 'default_category' => 'Person', 'to_ids' => 0), - 'nationality' => array('desc' => __('The nationality of a natural person'), 'default_category' => 'Person', 'to_ids' => 0), - 'visa-number' => array('desc' => __('Visa number'), 'default_category' => 'Person', 'to_ids' => 0), - 'issue-date-of-the-visa' => array('desc' => __('The date on which the visa was issued'), 'default_category' => 'Person', 'to_ids' => 0), - 'primary-residence' => array('desc' => __('The primary residence of a natural person'), 'default_category' => 'Person', 'to_ids' => 0), - 'country-of-residence' => array('desc' => __('The country of residence of a natural person'), 'default_category' => 'Person', 'to_ids' => 0), - 'special-service-request' => array('desc' => __('A Special Service Request is a function to an airline to provide a particular facility for A Passenger or passengers. '), 'default_category' => 'Person', 'to_ids' => 0), - 'frequent-flyer-number' => array('desc' => __('The frequent flyer number of a passenger'), 'default_category' => 'Person', 'to_ids' => 0), - // Do we really need remarks? Or just use comment/text for this? - //'remarks' => array('desc' => '', 'default_category' => 'Person', 'to_ids' => 0), - 'travel-details' => array('desc' => __('Travel details'), 'default_category' => 'Person', 'to_ids' => 0), - 'payment-details' => array('desc' => __('Payment details'), 'default_category' => 'Person', 'to_ids' => 0), - 'place-port-of-original-embarkation' => array('desc' => __('The orignal port of embarkation'), 'default_category' => 'Person', 'to_ids' => 0), - 'place-port-of-clearance' => array('desc' => __('The port of clearance'), 'default_category' => 'Person', 'to_ids' => 0), - 'place-port-of-onward-foreign-destination' => array('desc' => __('A Port where the passenger is transiting to'), 'default_category' => 'Person', 'to_ids' => 0), - 'passenger-name-record-locator-number' => array('desc' => __('The Passenger Name Record Locator is a key under which the reservation for a trip is stored in the system. The PNR contains, among other data, the name, flight segments and address of the passenger. It is defined by a combination of five or six letters and numbers.'), 'default_category' => 'Person', 'to_ids' => 0), - 'mobile-application-id' => array('desc' => __('The application id of a mobile application'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'chrome-extension-id' => array('desc' => __('Chrome extension id'), 'default_category' => 'Payload delivery', 'to_ids' => 1), - 'cortex' => array('desc' => __('Cortex analysis result'), 'default_category' => 'External analysis', 'to_ids' => 0), - 'boolean' => array('desc' => __('Boolean value - to be used in objects'), 'default_category' => 'Other', 'to_ids' => 0), - 'anonymised' => array('desc' => __('Anonymised value - described with the anonymisation object via a relationship'), 'formdesc' => __('Anonymised value - described with the anonymisation object via a relationship.'), 'default_category' => 'Other', 'to_ids' => 0) - // Not convinced about this. - //'url-regex' => array('desc' => '', 'default_category' => 'Person', 'to_ids' => 0), - ); } @@ -446,66 +171,13 @@ class Attribute extends AppModel 'yara-json' => array('json', 'YaraExport', 'json') ); - // FIXME we need a better way to list the defaultCategories knowing that new attribute types will continue to appear in the future. We should generate this dynamically or use a function using the default_category of the $typeDefinitions - public $defaultCategories = array( - 'md5' => 'Payload delivery', - 'sha1' => 'Payload delivery', - 'sha224' =>'Payload delivery', - 'sha256' => 'Payload delivery', - 'sha384' => 'Payload delivery', - 'sha512' => 'Payload delivery', - 'sha512/224' => 'Payload delivery', - 'sha512/256' => 'Payload delivery', - 'sha3-224' =>'Payload delivery', - 'sha3-256' =>'Payload delivery', - 'sha3-384' =>'Payload delivery', - 'sha3-512' =>'Payload delivery', - 'authentihash' => 'Payload delivery', - 'vhash' => 'Payload delivery', - 'imphash' => 'Payload delivery', - 'telfhash' => 'Payload delivery', - 'impfuzzy'=> 'Payload delivery', - 'pehash' => 'Payload delivery', - 'cdhash' => 'Payload delivery', - 'filename|md5' => 'Payload delivery', - 'filename|sha1' => 'Payload delivery', - 'filename|sha256' => 'Payload delivery', - 'regkey' => 'Persistence mechanism', - 'filename' => 'Payload delivery', - 'ip-src' => 'Network activity', - 'ip-dst' => 'Network activity', - 'ip-dst|port' => 'Network activity', - 'mac-address' => 'Network activity', - 'mac-eui-64' => 'Network activity', - 'hostname' => 'Network activity', - 'domain' => 'Network activity', - 'eppn' => 'Network activity', - 'git-commit-id' => 'Internal reference', - 'url' => 'Network activity', - 'ja3-fingerprint-md5' => 'Network activity', - 'jarm-fingerprint' => 'Network activity', - 'hassh-md5' => 'Network activity', - 'hasshserver-md5' => 'Network activity', - 'link' => 'External analysis', - 'email' => 'Social network', - 'email-src' => 'Payload delivery', - 'email-dst' => 'Payload delivery', - 'text' => 'Other', - 'hex' => 'Other', - 'attachment' => 'External analysis', - 'malware-sample' => 'Payload delivery', - 'cortex' => 'External analysis', - 'dns-soa-email' => 'Attribution', - 'boolean' => 'Other' - ); - // typeGroupings are a mapping to high level groups for attributes // for example, IP addresses, domain names, hostnames and e-mail addresses are network related attribute types // whilst filenames and hashes are file related attribute types // This helps generate quick filtering for the event view, but we may reuse this and enhance it in the future for other uses (such as the API?) public $typeGroupings = array( 'file' => array('attachment', 'pattern-in-file', 'filename-pattern', 'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|pehash', 'malware-sample', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'x509-fingerprint-md5'), - 'network' => array('ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port', 'mac-address', 'mac-eui-64', 'hostname', 'hostname|port', 'domain', 'domain|ip', 'email-dst', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'bro', 'zeek', 'pattern-in-traffic', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256','ja3-fingerprint-md5', 'jarm-fingerprint', 'hassh-md5', 'hasshserver-md5', 'community-id'), + 'network' => array('ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port', 'mac-address', 'mac-eui-64', 'hostname', 'hostname|port', 'domain', 'domain|ip', 'email-dst', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'bro', 'zeek', 'pattern-in-traffic', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256','ja3-fingerprint-md5', 'jarm-fingerprint', 'favicon-mmh3', 'hassh-md5', 'hasshserver-md5', 'community-id'), 'financial' => array('btc', 'xmr', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'phone-number') ); @@ -532,17 +204,17 @@ class Attribute extends AppModel 'stringNotEmpty' => array( 'rule' => array('stringNotEmpty') ), + 'validComposite' => array( + 'rule' => array('validComposite'), + 'message' => 'Composite type found but the value not in the composite (value1|value2) format.' + ), 'userdefined' => array( 'rule' => array('validateAttributeValue'), 'message' => 'Value not in the right type/format. Please double check the value or select type "other".' ), 'uniqueValue' => array( - 'rule' => array('valueIsUnique'), - 'message' => 'A similar attribute already exists for this event.' - ), - 'validComposite' => array( - 'rule' => array('validComposite'), - 'message' => 'Composite type found but the value not in the composite (value1|value2) format.' + 'rule' => array('valueIsUnique'), + 'message' => 'A similar attribute already exists for this event.' ), 'maxTextLength' => array( 'rule' => array('maxTextLength') @@ -932,7 +604,7 @@ class Attribute extends AppModel public function validComposite($fields) { $compositeTypes = $this->getCompositeTypes(); - if (in_array($this->data['Attribute']['type'], $compositeTypes)) { + if (in_array($this->data['Attribute']['type'], $compositeTypes, true)) { if (substr_count($fields['value'], '|') !== 1) { return false; } @@ -998,6 +670,7 @@ class Attribute extends AppModel 'recursive' => -1, 'fields' => array('id'), 'conditions' => $conditions, + 'order' => false, ); if (!empty($this->find('first', $params))) { // value isn't unique @@ -1011,7 +684,7 @@ class Attribute extends AppModel { $category = $this->data['Attribute']['category']; if (isset($this->categoryDefinitions[$category]['types'])) { - return in_array($fields['type'], $this->categoryDefinitions[$category]['types']); + return in_array($fields['type'], $this->categoryDefinitions[$category]['types'], true); } return false; } @@ -1424,6 +1097,7 @@ class Attribute extends AppModel case 'github-repository': case 'github-organisation': case 'twitter-id': + case 'favicon-mmh3': case 'chrome-extension-id': case 'mobile-application-id': if (strpos($value, "\n") !== false) { @@ -1709,13 +1383,6 @@ class Attribute extends AppModel return $compositeTypes; } - public function isOwnedByOrg($attributeId, $org) - { - $this->id = $attributeId; - $this->read(); - return $this->data['Event']['org_id'] === $org; - } - public function getRelatedAttributes($user, $attribute, $fields=array(), $includeEventData = false) { // LATER getRelatedAttributes($attribute) this might become a performance bottleneck @@ -1910,7 +1577,7 @@ class Attribute extends AppModel * @return string * @throws Exception */ - private function resizeImage($data, $maxWidth, $maxHeight) + public function resizeImage($data, $maxWidth, $maxHeight) { $image = imagecreatefromstring($data); if ($image === false) { @@ -2048,11 +1715,11 @@ class Attribute extends AppModel } } - $ipList = $this->find('list', array( + $ipList = $this->find('column', array( 'conditions' => $conditions, - 'group' => 'value1', // return just unique values - 'fields' => array('value1'), - 'order' => false + 'fields' => ['Attribute.value1'], + 'unique' => true, + 'order' => false, )); foreach ($ipList as $ipToCheck) { $ipToCheckVersion = filter_var($ipToCheck, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) ? 4 : 6; @@ -2093,17 +1760,61 @@ class Attribute extends AppModel return $extraConditions; } + private function __preventExcludedCorrelations($a) + { + $value = $a['value1']; + if (!empty($a['value2'])) { + $value .= '|' . $a['value2']; + } + if ($this->exclusions === null) { + try { + $redis = $this->setupRedisWithException(); + $this->exclusions = $redis->sMembers('misp:correlation_exclusions'); + } catch (Exception $e) { + $this->exclusions = []; + } + } + foreach ($this->exclusions as $exclusion) { + if (!empty($exclusion)) { + $firstChar = $exclusion[0]; + $lastChar = substr($exclusion, -1); + if ($firstChar === '%' && $lastChar === '%') { + $exclusion = substr($exclusion, 1, -1); + if (strpos($value, $exclusion) !== false) { + return true; + } + } else if ($firstChar === '%') { + $exclusion = substr($exclusion, 1); + if (substr($value, -strlen($exclusion)) === $exclusion) { + return true; + } + } else if ($lastChar === '%') { + $exclusion = substr($exclusion, 0, -1); + if (substr($value, 0, strlen($exclusion)) === $exclusion) { + return true; + } + } else { + if ($value === $exclusion) { + return true; + } + } + } + } + return false; + } + public function __afterSaveCorrelation($a, $full = false, $event = false) { if (!empty($a['disable_correlation']) || Configure::read('MISP.completely_disable_correlation')) { return true; } - // Don't do any correlation if the type is a non correlating type - if (in_array($a['type'], $this->nonCorrelatingTypes)) { + if (in_array($a['type'], $this->nonCorrelatingTypes, true)) { + return true; + } + if ($this->__preventExcludedCorrelations($a)) { return true; } - if (!$event) { $event = $this->Event->find('first', array( 'recursive' => -1, @@ -2851,8 +2562,7 @@ class Attribute extends AppModel // get all attributes.. if (!$eventId) { - $eventIds = $this->Event->find('list', [ - 'recursive' => -1, + $eventIds = $this->Event->find('column', [ 'fields' => ['Event.id'], 'conditions' => ['Event.disable_correlation' => 0], ]); @@ -2868,7 +2578,7 @@ class Attribute extends AppModel } else { $jobId = false; } - foreach (array_values($eventIds) as $j => $id) { + foreach ($eventIds as $j => $id) { if ($jobId) { if ($attributeId) { $message = 'Correlating Attribute ' . $attributeId; @@ -2878,10 +2588,10 @@ class Attribute extends AppModel $this->Job->saveProgress($jobId, $message, $startPercentage + ($j / $eventCount * (100 - $startPercentage))); } $event = $this->Event->find('first', array( - 'recursive' => -1, - 'fields' => array('Event.distribution', 'Event.id', 'Event.info', 'Event.org_id', 'Event.date', 'Event.sharing_group_id', 'Event.disable_correlation'), - 'conditions' => array('id' => $id), - 'order' => array() + 'recursive' => -1, + 'fields' => array('Event.distribution', 'Event.id', 'Event.info', 'Event.org_id', 'Event.date', 'Event.sharing_group_id', 'Event.disable_correlation'), + 'conditions' => array('id' => $id), + 'order' => false, )); $attributeConditions = array( 'Attribute.event_id' => $id, @@ -3207,19 +2917,6 @@ class Attribute extends AppModel return $conditions; } - public function listVisibleAttributes($user, $options = array()) - { - $params = array( - 'conditions' => $this->buildConditions($user), - 'recursive' => -1, - 'fields' => array('Attribute.id', 'Attribute.id'), - ); - if (isset($options['conditions'])) { - $params['conditions']['AND'][] = $options['conditions']; - } - return $this->find('list', $params); - } - /* * Unlike the other fetchers, this one foregoes any ACL checks. * the objective is simple: Fetch the given attribute with all related objects needed for the ZMQ output, @@ -3285,7 +2982,7 @@ class Attribute extends AppModel 'recursive' => -1, 'fields' => $params['fields'], 'contain' => $params['contain'], - 'sort' => false + 'order' => false, )); } @@ -3306,13 +3003,7 @@ class Attribute extends AppModel 'Event' => array( 'fields' => array('id', 'info', 'org_id', 'orgc_id', 'uuid'), ), - 'AttributeTag' => array( - 'Tag' => array( - 'fields' => array( - 'id', 'name', 'colour', 'numerical_value' - ) - ) - ), + 'AttributeTag', // tags are fetched separately, @see Attribute::__attachTagsToAttributes 'Object' => array( 'fields' => array('id', 'distribution', 'sharing_group_id') ) @@ -3351,9 +3042,6 @@ class Attribute extends AppModel "value" )); } - if (empty($options['includeAllTags'])) { - $params['contain']['AttributeTag']['Tag']['conditions']['exportable'] = 1; - } if (!empty($options['includeContext'])) { // include just event id for conditions, rest event data will be fetched later $params['contain']['Event']['fields'] = ['id']; @@ -3454,21 +3142,22 @@ class Attribute extends AppModel } if (!empty($options['list'])) { if (!empty($options['event_ids'])) { - $fields = array('Attribute.event_id', 'Attribute.event_id'); - $group = array('Attribute.event_id'); + return $this->find('column', [ + 'conditions' => $params['conditions'], + 'contain' => array('Event', 'Object'), + 'fields' => ['Attribute.event_id'], + 'unique' => true, + 'order' => false, + ]); } else { - $fields = array('Attribute.event_id'); - $group = false; + return $this->find('list', array( + 'conditions' => $params['conditions'], + 'recursive' => -1, + 'contain' => array('Event', 'Object'), + 'fields' => array('Attribute.event_id'), + 'order' => false + )); } - $results = $this->find('list', array( - 'conditions' => $params['conditions'], - 'recursive' => -1, - 'contain' => array('Event', 'Object'), - 'fields' => $fields, - 'group' => $group, - 'sort' => false - )); - return $results; } if (($options['enforceWarninglist'] || $options['includeWarninglistHits']) && !isset($this->Warninglist)) { @@ -3508,21 +3197,9 @@ class Attribute extends AppModel unset($eventsById, $result); // unset result is important, because it is reference } + $this->__attachTagsToAttributes($results, $options); + foreach ($results as $k => $result) { - if (!empty($result['AttributeTag'])) { - $tagCulled = false; - foreach ($result['AttributeTag'] as $k2 => $at) { - if (empty($at['Tag'])) { - unset($results[$k]['AttributeTag'][$k2]); - $tagCulled = true; - } else { - $results[$k]['AttributeTag'][$k2]['Tag']['local'] = $results[$k]['AttributeTag'][$k2]['local']; - } - } - if ($tagCulled) { - $results[$k]['AttributeTag'] = array_values($results[$k]['AttributeTag']); - } - } if (!empty($options['includeSightings'])) { $temp = $result['Attribute']; $temp['Event'] = $result['Event']; @@ -3649,6 +3326,53 @@ class Attribute extends AppModel return $eventsById; } + private function __attachTagsToAttributes(array &$attributes, array $options) + { + $tagIdsToFetch = []; + foreach ($attributes as $attribute) { + foreach ($attribute['AttributeTag'] as $at) { + $tagIdsToFetch[$at['tag_id']] = true; + } + } + + if (empty($tagIdsToFetch)) { + return; + } + + $conditions = ['Tag.id' => array_keys($tagIdsToFetch)]; + unset($tagIdsToFetch); + if (empty($options['includeAllTags'])) { + $conditions['Tag.exportable'] = 1; + } + + $tagsToModify = $this->AttributeTag->Tag->find('all', [ + 'conditions' => $conditions, + 'fields' => ['id', 'name', 'colour', 'numerical_value'], + 'recursive' => -1, + ]); + $tags = []; + foreach ($tagsToModify as $tag) { + $tags[$tag['Tag']['id']] = $tag['Tag']; + } + + foreach ($attributes as $k => $attribute) { + $tagCulled = false; + foreach ($attribute['AttributeTag'] as $k2 => $at) { + if (!isset($tags[$at['tag_id']])) { + unset($attributes[$k]['AttributeTag'][$k2]); + $tagCulled = true; + } else { + $tag = $tags[$at['tag_id']]; + $tag['local'] = $at['local']; + $attributes[$k]['AttributeTag'][$k2]['Tag'] = $tag; + } + } + if ($tagCulled) { + $attributes[$k]['AttributeTag'] = array_values($attributes[$k]['AttributeTag']); + } + } + } + private function __attachEventTagsToAttributes($eventTags, &$results, $key, $options) { if (!isset($eventTags[$results[$key]['Event']['id']])) { @@ -3981,13 +3705,13 @@ class Attribute extends AppModel private function __getCIDRList() { - return $this->find('list', array( + return $this->find('column', array( 'conditions' => array( 'type' => array('ip-src', 'ip-dst'), 'value1 LIKE' => '%/%' ), - 'fields' => array('value1'), - 'group' => array('value1', 'id'), // return just unique value + 'fields' => array('Attribute.value1'), + 'unique' => true, 'order' => false )); } @@ -4254,9 +3978,7 @@ class Attribute extends AppModel } } if (!empty($attribute['Sighting'])) { - foreach ($attribute['Sighting'] as $k => $sighting) { - $this->Sighting->captureSighting($sighting, $this->id, $eventId, $user); - } + $this->Sighting->captureSightings($attribute['Sighting'], $this->id, $eventId, $user); } } if (!empty($this->validationErrors)) { @@ -4697,7 +4419,7 @@ class Attribute extends AppModel $elementCounter = $this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams); } $tmpfile->write($exportTool->footer($exportToolParams)); - return $tmpfile->finish(); + return $tmpfile; } /** @@ -4842,4 +4564,311 @@ class Attribute extends AppModel } return $typeCategoryMapping; } + + public function __isset($name) + { + if ($name === 'typeDefinitions' || $name === 'categoryDefinitions') { + return true; + } + return parent::__isset($name); + } + + public function __get($name) + { + if ($name === 'typeDefinitions') { + $this->typeDefinitions = $this->generateTypeDefinitions(); + return $this->typeDefinitions; + } else if ($name === 'categoryDefinitions') { + $this->categoryDefinitions = $this->generateCategoryDefintions(); + return $this->categoryDefinitions; + } + return parent::__get($name); + } + + /** + * Generate just when really need + * NOTE WHEN MODIFYING: please ensure to run the script 'tools/gen_misp_types_categories.py' to update the new definitions everywhere. (docu, website, RFC, ... ) + * @return array[] + */ + private function generateCategoryDefintions() + { + return array( + 'Internal reference' => array( + 'desc' => __('Reference used by the publishing party (e.g. ticket number)'), + 'types' => array('text', 'link', 'comment', 'other', 'hex', 'anonymised', 'git-commit-id') + ), + 'Targeting data' => array( + 'desc' => __('Internal Attack Targeting and Compromise Information'), + 'formdesc' => __('Targeting information to include recipient email, infected machines, department, and or locations.'), + 'types' => array('target-user', 'target-email', 'target-machine', 'target-org', 'target-location', 'target-external', 'comment', 'anonymised') + ), + 'Antivirus detection' => array( + 'desc' => __('All the info about how the malware is detected by the antivirus products'), + 'formdesc' => __('List of anti-virus vendors detecting the malware or information on detection performance (e.g. 13/43 or 67%). Attachment with list of detection or link to VirusTotal could be placed here as well.'), + 'types' => array('link', 'comment', 'text', 'hex', 'attachment', 'other', 'anonymised') + ), + 'Payload delivery' => array( + 'desc' => __('Information about how the malware is delivered'), + 'formdesc' => __('Information about the way the malware payload is initially delivered, for example information about the email or web-site, vulnerability used, originating IP etc. Malware sample itself should be attached here.'), + 'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash','filename|impfuzzy', 'filename|pehash', 'mac-address', 'mac-eui-64', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'hostname', 'domain', 'email', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'email-body', 'url', 'user-agent', 'AS', 'pattern-in-file', 'pattern-in-traffic', 'filename-pattern', 'stix2-pattern', 'yara', 'sigma', 'mime-type', 'attachment', 'malware-sample', 'link', 'malware-type', 'comment', 'text', 'hex', 'vulnerability', 'cpe', 'weakness', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'jarm-fingerprint', 'hassh-md5', 'hasshserver-md5', 'other', 'hostname|port', 'email-dst-display-name', 'email-src-display-name', 'email-header', 'email-reply-to', 'email-x-mailer', 'email-mime-boundary', 'email-thread-index', 'email-message-id', 'mobile-application-id', 'chrome-extension-id', 'whois-registrant-email', 'anonymised') + ), + 'Artifacts dropped' => array( + 'desc' => __('Any artifact (files, registry keys etc.) dropped by the malware or other modifications to the system'), + 'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy','filename|pehash', 'regkey', 'regkey|value', 'pattern-in-file', 'pattern-in-memory', 'filename-pattern', 'pdb', 'stix2-pattern', 'yara', 'sigma', 'attachment', 'malware-sample', 'named pipe', 'mutex', 'process-state','windows-scheduled-task', 'windows-service-name', 'windows-service-displayname', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'cookie', 'gene', 'kusto-query', 'mime-type', 'anonymised', 'pgp-public-key', 'pgp-private-key') + ), + 'Payload installation' => array( + 'desc' => __('Info on where the malware gets installed in the system'), + 'formdesc' => __('Location where the payload was placed in the system and the way it was installed. For example, a filename|md5 type attribute can be added here like this: c:\\windows\\system32\\malicious.exe|41d8cd98f00b204e9800998ecf8427e.'), + 'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy', 'filename|pehash', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'filename-pattern', 'stix2-pattern', 'yara', 'sigma', 'vulnerability', 'cpe','weakness', 'attachment', 'malware-sample', 'malware-type', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'mobile-application-id', 'chrome-extension-id', 'other', 'mime-type', 'anonymised') + ), + 'Persistence mechanism' => array( + 'desc' => __('Mechanisms used by the malware to start at boot'), + 'formdesc' => __('Mechanisms used by the malware to start at boot. This could be a registry key, legitimate driver modification, LNK file in startup'), + 'types' => array('filename', 'regkey', 'regkey|value', 'comment', 'text', 'other', 'hex', 'anonymised') + ), + 'Network activity' => array( + 'desc' => __('Information about network traffic generated by the malware'), + 'types' => array('ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'port', 'hostname', 'domain', 'domain|ip', 'mac-address', 'mac-eui-64', 'email', 'email-dst', 'email-src', 'eppn', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'pattern-in-file', 'filename-pattern','stix2-pattern', 'pattern-in-traffic', 'attachment', 'comment', 'text', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'jarm-fingerprint', 'hassh-md5', 'hasshserver-md5', 'other', 'hex', 'cookie', 'hostname|port', 'bro', 'zeek', 'anonymised', 'community-id', 'email-subject', 'favicon-mmh3') + ), + 'Payload type' => array( + 'desc' => __('Information about the final payload(s)'), + 'formdesc' => __('Information about the final payload(s). Can contain a function of the payload, e.g. keylogger, RAT, or a name if identified, such as Poison Ivy.'), + 'types' => array('comment', 'text', 'other', 'anonymised') + ), + 'Attribution' => array( + 'desc' => __('Identification of the group, organisation, or country behind the attack'), + 'types' => array('threat-actor', 'campaign-name', 'campaign-id', 'whois-registrant-phone', 'whois-registrant-email', 'whois-registrant-name', 'whois-registrant-org', 'whois-registrar', 'whois-creation-date','comment', 'text', 'x509-fingerprint-sha1','x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'dns-soa-email', 'anonymised', 'email') + ), + 'External analysis' => array( + 'desc' => __('Any other result from additional analysis of the malware like tools output'), + 'formdesc' => __('Any other result from additional analysis of the malware like tools output Examples: pdf-parser output, automated sandbox analysis, reverse engineering report.'), + 'types' => array('md5', 'sha1', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'mac-address', 'mac-eui-64', 'hostname', 'domain', 'domain|ip', 'url', 'user-agent', 'regkey', 'regkey|value', 'AS', 'snort', 'bro', 'zeek', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'filename-pattern','vulnerability', 'cpe', 'weakness', 'attachment', 'malware-sample', 'link', 'comment', 'text', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'jarm-fingerprint', 'hassh-md5', 'hasshserver-md5', 'github-repository', 'other', 'cortex', 'anonymised', 'community-id') + ), + 'Financial fraud' => array( + 'desc' => __('Financial Fraud indicators'), + 'formdesc' => __('Financial Fraud indicators, for example: IBAN Numbers, BIC codes, Credit card numbers, etc.'), + 'types' => array('btc', 'dash', 'xmr', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'phone-number', 'comment', 'text', 'other', 'hex', 'anonymised'), + ), + 'Support Tool' => array( + 'desc' => __('Tools supporting analysis or detection of the event'), + 'types' => array('link', 'text', 'attachment', 'comment', 'other', 'hex', 'anonymised') + ), + 'Social network' => array( + 'desc' => __('Social networks and platforms'), + // email-src and email-dst or should we go with a new email type that is neither / both? + 'types' => array('github-username', 'github-repository', 'github-organisation', 'jabber-id', 'twitter-id', 'email', 'email-src', 'email-dst', 'eppn','comment', 'text', 'other', 'whois-registrant-email', 'anonymised', 'pgp-public-key', 'pgp-private-key') + ), + 'Person' => array( + 'desc' => __('A human being - natural person'), + 'types' => array('first-name', 'middle-name', 'last-name', 'date-of-birth', 'place-of-birth', 'gender', 'passport-number', 'passport-country', 'passport-expiration', 'redress-number', 'nationality', 'visa-number', 'issue-date-of-the-visa', 'primary-residence', 'country-of-residence', 'special-service-request', 'frequent-flyer-number', 'travel-details', 'payment-details', 'place-port-of-original-embarkation', 'place-port-of-clearance', 'place-port-of-onward-foreign-destination', 'passenger-name-record-locator-number', 'comment', 'text', 'other', 'phone-number', 'identity-card-number', 'anonymised', 'email', 'pgp-public-key', 'pgp-private-key') + ), + 'Other' => array( + 'desc' => __('Attributes that are not part of any other category or are meant to be used as a component in MISP objects in the future'), + 'types' => array('comment', 'text', 'other', 'size-in-bytes', 'counter', 'datetime', 'cpe', 'port', 'float', 'hex', 'phone-number', 'boolean', 'anonymised', 'pgp-public-key', 'pgp-private-key') + ) + ); + } + + /** + * Generate just when really need + * NOTE WHEN MODIFYING: please ensure to run the script 'tools/gen_misp_types_categories.py' to update the new definitions everywhere. (docu, website, RFC, ... ) + * @return array[] + */ + private function generateTypeDefinitions() + { + return array( + 'md5' => array('desc' => __('A checksum in md5 format'), 'formdesc' => __("You are encouraged to use filename|md5 instead. A checksum in md5 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'sha1' => array('desc' => __('A checksum in sha1 format'), 'formdesc' => __("You are encouraged to use filename|sha1 instead. A checksum in sha1 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'sha256' => array('desc' => __('A checksum in sha256 format'), 'formdesc' => __("You are encouraged to use filename|sha256 instead. A checksum in sha256 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'filename' => array('desc' => __('Filename'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'pdb' => array('desc' => __('Microsoft Program database (PDB) path information'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0), + 'filename|md5' => array('desc' => __('A filename and an md5 hash separated by a |'), 'formdesc' => __("A filename and an md5 hash separated by a | (no spaces)"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'filename|sha1' => array('desc' => __('A filename and an sha1 hash separated by a |'), 'formdesc' => __("A filename and an sha1 hash separated by a | (no spaces)"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'filename|sha256' => array('desc' => __('A filename and an sha256 hash separated by a |'), 'formdesc' => __("A filename and an sha256 hash separated by a | (no spaces)"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'ip-src' => array('desc' => __("A source IP address of the attacker"), 'default_category' => 'Network activity', 'to_ids' => 1), + 'ip-dst' => array('desc' => __('A destination IP address of the attacker or C&C server'), 'formdesc' => __("A destination IP address of the attacker or C&C server. Also set the IDS flag on when this IP is hardcoded in malware"), 'default_category' => 'Network activity', 'to_ids' => 1), + 'hostname' => array('desc' => __('A full host/dnsname of an attacker'), 'formdesc' => __("A full host/dnsname of an attacker. Also set the IDS flag on when this hostname is hardcoded in malware"), 'default_category' => 'Network activity', 'to_ids' => 1), + 'domain' => array('desc' => __('A domain name used in the malware'), 'formdesc' => __("A domain name used in the malware. Use this instead of hostname when the upper domain is important or can be used to create links between events."), 'default_category' => 'Network activity', 'to_ids' => 1), + 'domain|ip' => array('desc' => __('A domain name and its IP address (as found in DNS lookup) separated by a |'),'formdesc' => __("A domain name and its IP address (as found in DNS lookup) separated by a | (no spaces)"), 'default_category' => 'Network activity', 'to_ids' => 1), + 'email' => array('desc' => ('An e-mail address'), 'default_category' => 'Social network', 'to_ids' => 1), + 'email-src' => array('desc' => __("The source email address. Used to describe the sender when describing an e-mail."), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'eppn' => array('desc' => __("eduPersonPrincipalName - eppn - the NetId of the person for the purposes of inter-institutional authentication. Should be stored in the form of user@univ.edu, where univ.edu is the name of the local security domain."), 'default_category' => 'Network activity', 'to_ids' => 1), + 'email-dst' => array('desc' => __("The destination email address. Used to describe the recipient when describing an e-mail."), 'default_category' => 'Network activity', 'to_ids' => 1), + 'email-subject' => array('desc' => __("The subject of the email"), 'default_category' => 'Payload delivery', 'to_ids' => 0), + 'email-attachment' => array('desc' => __("File name of the email attachment."), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'email-body' => array('desc' => __('Email body'), 'default_category' => 'Payload delivery', 'to_ids' => 0), + 'float' => array('desc' => __("A floating point value."), 'default_category' => 'Other', 'to_ids' => 0), + 'git-commit-id' => array('desc' => __("A git commit ID."), 'default_category' => 'Internal reference', 'to_ids' => 0), + 'url' => array('desc' => __('url'), 'default_category' => 'Network activity', 'to_ids' => 1), + 'http-method' => array('desc' => __("HTTP method used by the malware (e.g. POST, GET, ...)."), 'default_category' => 'Network activity', 'to_ids' => 0), + 'user-agent' => array('desc' => __("The user-agent used by the malware in the HTTP request."), 'default_category' => 'Network activity', 'to_ids' => 0), + 'ja3-fingerprint-md5' => array('desc' => __("JA3 is a method for creating SSL/TLS client fingerprints that should be easy to produce on any platform and can be easily shared for threat intelligence."), 'default_category' => 'Network activity', 'to_ids' => 1), + 'jarm-fingerprint' => array('desc' => __("JARM is a method for creating SSL/TLS server fingerprints."), 'default_category' => 'Network activity', 'to_ids' => 1), + 'favicon-mmh3' => array('desc' => __("favicon-mmh3 is the murmur3 hash of a favicon as used in Shodan."), 'default_category' => 'Network activity', 'to_ids' => 1), + 'hassh-md5' => array('desc' => __("hassh is a network fingerprinting standard which can be used to identify specific Client SSH implementations. The fingerprints can be easily stored, searched and shared in the form of an MD5 fingerprint."), 'default_category' => 'Network activity', 'to_ids' => 1), + 'hasshserver-md5' => array('desc' => __("hasshServer is a network fingerprinting standard which can be used to identify specific Server SSH implementations. The fingerprints can be easily stored, searched and shared in the form of an MD5 fingerprint."), 'default_category' => 'Network activity', 'to_ids' => 1), + 'regkey' => array('desc' => __("Registry key or value"), 'default_category' => 'Persistence mechanism', 'to_ids' => 1), + 'regkey|value' => array('desc' => __("Registry value + data separated by |"), 'default_category' => 'Persistence mechanism', 'to_ids' => 1), + 'AS' => array('desc' => __('Autonomous system'), 'default_category' => 'Network activity', 'to_ids' => 0), + 'snort' => array('desc' => __('An IDS rule in Snort rule-format'), 'formdesc' => __("An IDS rule in Snort rule-format. This rule will be automatically rewritten in the NIDS exports."), 'default_category' => 'Network activity', 'to_ids' => 1), + 'bro' => array('desc' => __('An NIDS rule in the Bro rule-format'), 'formdesc' => __("An NIDS rule in the Bro rule-format."), 'default_category' => 'Network activity', 'to_ids' => 1), + 'zeek' => array('desc' => __('An NIDS rule in the Zeek rule-format'), 'formdesc' => __("An NIDS rule in the Zeek rule-format."), 'default_category' => 'Network activity', 'to_ids' => 1), + 'community-id' => array('desc' => __('a community ID flow hashing algorithm to map multiple traffic monitors into common flow id'), 'formdesc' => __("a community ID flow hashing algorithm to map multiple traffic monitors into common flow id"), 'default_category' => 'Network activity', 'to_ids' => 1), + 'pattern-in-file' => array('desc' => __('Pattern in file that identifies the malware'), 'default_category' => 'Payload installation', 'to_ids' => 1), + 'pattern-in-traffic' => array('desc' => __('Pattern in network traffic that identifies the malware'), 'default_category' => 'Network activity', 'to_ids' => 1), + 'pattern-in-memory' => array('desc' => __('Pattern in memory dump that identifies the malware'), 'default_category' => 'Payload installation', 'to_ids' => 1), + 'pattern-filename' => array('desc' => __('A pattern in the name of a file'), 'default_category' => 'Payload installation', 'to_ids' => 1), + 'pgp-public-key' => array('desc' => __('A PGP public key'), 'default_category' => 'Person', 'to_ids' => 0), + 'pgp-private-key' => array('desc' => __('A PGP private key'), 'default_category' => 'Person', 'to_ids' => 0), + 'yara' => array('desc' => __('Yara signature'), 'default_category' => 'Payload installation', 'to_ids' => 1), + 'stix2-pattern' => array('desc' => __('STIX 2 pattern'), 'default_category' => 'Payload installation', 'to_ids' => 1), + 'sigma' => array('desc' => __('Sigma - Generic Signature Format for SIEM Systems'), 'default_category' => 'Payload installation', 'to_ids' => 1), + 'gene' => array('desc' => __('GENE - Go Evtx sigNature Engine'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0), + 'kusto-query' => array('desc' => __('Kusto query - Kusto from Microsoft Azure is a service for storing and running interactive analytics over Big Data.'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0), + 'mime-type' => array('desc' => __('A media type (also MIME type and content type) is a two-part identifier for file formats and format contents transmitted on the Internet'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0), + 'identity-card-number' => array('desc' => __('Identity card number'), 'default_category' => 'Person', 'to_ids' => 0), + 'cookie' => array('desc' => __('HTTP cookie as often stored on the user web client. This can include authentication cookie or session cookie.'), 'default_category' => 'Network activity', 'to_ids' => 0), + 'vulnerability' => array('desc' => __('A reference to the vulnerability used in the exploit'), 'default_category' => 'External analysis', 'to_ids' => 0), + 'cpe' => array('desc' => __('Common Platform Enumeration - structured naming scheme for information technology systems, software, and packages.'), 'default_category' => 'External analysis', 'to_ids' => 0), + 'weakness' => array('desc'=> __('A reference to the weakness used in the exploit'), 'default_category' => 'External analysis', 'to_ids' => 0), + 'attachment' => array('desc' => __('Attachment with external information'), 'formdesc' => __("Please upload files using the Upload Attachment button."), 'default_category' => 'External analysis', 'to_ids' => 0), + 'malware-sample' => array('desc' => __('Attachment containing encrypted malware sample'), 'formdesc' => __("Please upload files using the Upload Attachment button."), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'link' => array('desc' => __('Link to an external information'), 'default_category' => 'External analysis', 'to_ids' => 0), + 'comment' => array('desc' => __('Comment or description in a human language'), 'formdesc' => __('Comment or description in a human language. This will not be correlated with other attributes'), 'default_category' => 'Other', 'to_ids' => 0), + 'text' => array('desc' => __('Name, ID or a reference'), 'default_category' => 'Other', 'to_ids' => 0), + 'hex' => array('desc' => __('A value in hexadecimal format'), 'default_category' => 'Other', 'to_ids' => 0), + 'other' => array('desc' => __('Other attribute'), 'default_category' => 'Other', 'to_ids' => 0), + 'named pipe' => array('desc' => __('Named pipe, use the format \\.\pipe\'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0), + 'mutex' => array('desc' => __('Mutex, use the format \BaseNamedObjects\'), 'default_category' => 'Artifacts dropped', 'to_ids' => 1), + 'process-state' => array('desc' => __('State of a process'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0), + 'target-user' => array('desc' => __('Attack Targets Username(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0), + 'target-email' => array('desc' => __('Attack Targets Email(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0), + 'target-machine' => array('desc' => __('Attack Targets Machine Name(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0), + 'target-org' => array('desc' => __('Attack Targets Department or Organization(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0), + 'target-location' => array('desc' => __('Attack Targets Physical Location(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0), + 'target-external' => array('desc' => __('External Target Organizations Affected by this Attack'), 'default_category' => 'Targeting data', 'to_ids' => 0), + 'btc' => array('desc' => __('Bitcoin Address'), 'default_category' => 'Financial fraud', 'to_ids' => 1), + 'dash' => array('desc' => __('Dash Address'), 'default_category' => 'Financial fraud', 'to_ids' => 1), + 'xmr' => array('desc' => __('Monero Address'), 'default_category' => 'Financial fraud', 'to_ids' => 1), + 'iban' => array('desc' => __('International Bank Account Number'), 'default_category' => 'Financial fraud', 'to_ids' => 1), + 'bic' => array('desc' => __('Bank Identifier Code Number also known as SWIFT-BIC, SWIFT code or ISO 9362 code'), 'default_category' => 'Financial fraud', 'to_ids' => 1), + 'bank-account-nr' => array('desc' => __('Bank account number without any routing number'), 'default_category' => 'Financial fraud', 'to_ids' => 1), + 'aba-rtn' => array('desc' => __('ABA routing transit number'), 'default_category' => 'Financial fraud', 'to_ids' => 1), + 'bin' => array('desc' => __('Bank Identification Number'), 'default_category' => 'Financial fraud', 'to_ids' => 1), + 'cc-number' => array('desc' => __('Credit-Card Number'), 'default_category' => 'Financial fraud', 'to_ids' => 1), + 'prtn' => array('desc' => __('Premium-Rate Telephone Number'), 'default_category' => 'Financial fraud', 'to_ids' => 1), + 'phone-number' => array('desc' => __('Telephone Number'), 'default_category' => 'Person', 'to_ids' => 0), + 'threat-actor' => array('desc' => __('A string identifying the threat actor'), 'default_category' => 'Attribution', 'to_ids' => 0), + 'campaign-name' => array('desc' => __('Associated campaign name'), 'default_category' => 'Attribution', 'to_ids' => 0), + 'campaign-id' => array('desc' => __('Associated campaign ID'), 'default_category' => 'Attribution', 'to_ids' => 0), + 'malware-type' => array('desc' => '', 'default_category' => 'Payload delivery', 'to_ids' => 0), + 'uri' => array('desc' => __('Uniform Resource Identifier'), 'default_category' => 'Network activity', 'to_ids' => 1), + 'authentihash' => array('desc' => __('Authenticode executable signature hash'), 'formdesc' => __("You are encouraged to use filename|authentihash instead. Authenticode executable signature hash, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'vhash' => array('desc' => __('A VirusTotal checksum'), 'formdesc' => __("You are encouraged to use filename|vhash instead. A checksum from VirusTotal, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'ssdeep' => array('desc' => __('A checksum in ssdeep format'), 'formdesc' => __("You are encouraged to use filename|ssdeep instead. A checksum in the SSDeep format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'imphash' => array('desc' => __('Import hash - a hash created based on the imports in the sample.'), 'formdesc' => __("You are encouraged to use filename|imphash instead. A hash created based on the imports in the sample, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'telfhash' => array('desc' => __('telfhash is symbol hash for ELF files, just like imphash is imports hash for PE files.'), 'formdesc' => __("You are encouraged to use a file object with telfash"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'pehash' => array('desc' => __('PEhash - a hash calculated based of certain pieces of a PE executable file'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'impfuzzy' => array('desc' => __('A fuzzy hash of import table of Portable Executable format'), 'formdesc' => __("You are encouraged to use filename|impfuzzy instead. A fuzzy hash created based on the imports in the sample, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'sha224' => array('desc' => __('A checksum in sha-224 format'), 'formdesc' => __("You are encouraged to use filename|sha224 instead. A checksum in sha224 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'sha384' => array('desc' => __('A checksum in sha-384 format'), 'formdesc' => __("You are encouraged to use filename|sha384 instead. A checksum in sha384 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'sha512' => array('desc' => __('A checksum in sha-512 format'), 'formdesc' => __("You are encouraged to use filename|sha512 instead. A checksum in sha512 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'sha512/224' => array('desc' => __('A checksum in the sha-512/224 format'), 'formdesc' => __("You are encouraged to use filename|sha512/224 instead. A checksum in sha512/224 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'sha512/256' => array('desc' => __('A checksum in the sha-512/256 format'), 'formdesc' => __("You are encouraged to use filename|sha512/256 instead. A checksum in sha512/256 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'sha3-224' => array('desc' => __('A checksum in sha3-224 format'), 'formdesc' => __("You are encouraged to use filename|sha3-224 instead. A checksum in sha3-224 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'sha3-256' => array('desc' => __('A checksum in sha3-256 format'), 'formdesc' => __("You are encouraged to use filename|sha3-256 instead. A checksum in sha3-256 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'sha3-384' => array('desc' => __('A checksum in sha3-384 format'), 'formdesc' => __("You are encouraged to use filename|sha3-384 instead. A checksum in sha3-384 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'sha3-512' => array('desc' => __('A checksum in sha3-512 format'), 'formdesc' => __("You are encouraged to use filename|sha3-512 instead. A checksum in sha3-512 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'tlsh' => array('desc' => __('A checksum in the Trend Micro Locality Sensitive Hash format'), 'formdesc' => __("You are encouraged to use filename|tlsh instead. A checksum in the Trend Micro Locality Sensitive Hash format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'cdhash' => array('desc' => __('An Apple Code Directory Hash, identifying a code-signed Mach-O executable file'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'filename|authentihash' => array('desc' => __('A checksum in md5 format'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'filename|vhash' => array('desc' => __('A filename and a VirusTotal hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'filename|ssdeep' => array('desc' => __('A checksum in ssdeep format'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'filename|imphash' => array('desc' => __('Import hash - a hash created based on the imports in the sample.'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'filename|impfuzzy' => array('desc' => __('Import fuzzy hash - a fuzzy hash created based on the imports in the sample.'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'filename|pehash' => array('desc' => __('A filename and a PEhash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'filename|sha224' => array('desc' => __('A filename and a sha-224 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'filename|sha384' => array('desc' => __('A filename and a sha-384 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'filename|sha512' => array('desc' => __('A filename and a sha-512 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'filename|sha512/224' => array('desc' => __('A filename and a sha-512/224 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'filename|sha512/256' => array('desc' => __('A filename and a sha-512/256 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'filename|sha3-224' => array('desc' => __('A filename and an sha3-224 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'filename|sha3-256' => array('desc' => __('A filename and an sha3-256 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'filename|sha3-384' => array('desc' => __('A filename and an sha3-384 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'filename|sha3-512' => array('desc' => __('A filename and an sha3-512 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'filename|tlsh' => array('desc' => __('A filename and a Trend Micro Locality Sensitive Hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'windows-scheduled-task' => array('desc' => __('A scheduled task in windows'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0), + 'windows-service-name' => array('desc' => __('A windows service name. This is the name used internally by windows. Not to be confused with the windows-service-displayname.'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0), + 'windows-service-displayname' => array('desc' => __('A windows service\'s displayname, not to be confused with the windows-service-name. This is the name that applications will generally display as the service\'s name in applications.'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0), + 'whois-registrant-email' => array('desc' => __('The e-mail of a domain\'s registrant, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0), + 'whois-registrant-phone' => array('desc' => __('The phone number of a domain\'s registrant, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0), + 'whois-registrant-name' => array('desc' => __('The name of a domain\'s registrant, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0), + 'whois-registrant-org' => array('desc' => __('The org of a domain\'s registrant, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0), + 'whois-registrar' => array('desc' => __('The registrar of the domain, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0), + 'whois-creation-date' => array('desc' => __('The date of domain\'s creation, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0), + // 'targeted-threat-index' => array('desc' => ''), // currently not mapped! + // 'mailslot' => array('desc' => 'MailSlot interprocess communication'), // currently not mapped! + // 'pipe' => array('desc' => 'Pipeline (for named pipes use the attribute type "named pipe")'), // currently not mapped! + // 'ssl-cert-attributes' => array('desc' => 'SSL certificate attributes'), // currently not mapped! + 'x509-fingerprint-sha1' => array('desc' => __('X509 fingerprint in SHA-1 format'), 'default_category' => 'Network activity', 'to_ids' => 1), + 'x509-fingerprint-md5' => array('desc' => __('X509 fingerprint in MD5 format'), 'default_category' => 'Network activity', 'to_ids' => 1), + 'x509-fingerprint-sha256' => array('desc' => __('X509 fingerprint in SHA-256 format'), 'default_category' => 'Network activity', 'to_ids' => 1), + 'dns-soa-email' => array('desc' => __('RFC1035 mandates that DNS zones should have a SOA (Statement Of Authority) record that contains an email address where a PoC for the domain could be contacted. This can sometimes be used for attribution/linkage between different domains even if protected by whois privacy'), 'default_category' => 'Attribution', 'to_ids' => 0), + 'size-in-bytes' => array('desc' => __('Size expressed in bytes'), 'default_category' => 'Other', 'to_ids' => 0), + 'counter' => array('desc' => __('An integer counter, generally to be used in objects'), 'default_category' => 'Other', 'to_ids' => 0), + 'datetime' => array('desc' => __('Datetime in the ISO 8601 format'), 'default_category' => 'Other', 'to_ids' => 0), + 'port' => array('desc' => __('Port number'), 'default_category' => 'Network activity', 'to_ids' => 0), + 'ip-dst|port' => array('desc' => __('IP destination and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1), + 'ip-src|port' => array('desc' => __('IP source and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1), + 'hostname|port' => array('desc' => __('Hostname and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1), + 'mac-address' => array('desc' => __('Mac address'), 'default_category' => 'Network activity', 'to_ids' => 0), + 'mac-eui-64' => array('desc' => __('Mac EUI-64 address'), 'default_category' => 'Network activity', 'to_ids' => 0), + // verify IDS flag defaults for these + 'email-dst-display-name' => array('desc' => __('Email destination display name'), 'default_category' => 'Payload delivery', 'to_ids' => 0), + 'email-src-display-name' => array('desc' => __('Email source display name'), 'default_category' => 'Payload delivery', 'to_ids' => 0), + 'email-header' => array('desc' => __('Email header'), 'default_category' => 'Payload delivery', 'to_ids' => 0), + 'email-reply-to' => array('desc' => __('Email reply to header'), 'default_category' => 'Payload delivery', 'to_ids' => 0), + 'email-x-mailer' => array('desc' => __('Email x-mailer header'), 'default_category' => 'Payload delivery', 'to_ids' => 0), + 'email-mime-boundary' => array('desc' => __('The email mime boundary separating parts in a multipart email'), 'default_category' => 'Payload delivery', 'to_ids' => 0), + 'email-thread-index' => array('desc' => __('The email thread index header'), 'default_category' => 'Payload delivery', 'to_ids' => 0), + 'email-message-id' => array('desc' => __('The email message ID'), 'default_category' => 'Payload delivery', 'to_ids' => 0), + 'github-username' => array('desc' => __('A github user name'), 'default_category' => 'Social network', 'to_ids' => 0), + 'github-repository' => array('desc' => __('A github repository'), 'default_category' => 'Social network', 'to_ids' => 0), + 'github-organisation' => array('desc' => __('A github organisation'), 'default_category' => 'Social network', 'to_ids' => 0), + 'jabber-id' => array('desc' => __('Jabber ID'), 'default_category' => 'Social network', 'to_ids' => 0), + 'twitter-id' => array('desc' => __('Twitter ID'), 'default_category' => 'Social network', 'to_ids' => 0), + 'first-name' => array('desc' => __('First name of a natural person'), 'default_category' => 'Person', 'to_ids' => 0), + 'middle-name' => array('desc' => __('Middle name of a natural person'), 'default_category' => 'Person', 'to_ids' => 0), + 'last-name' => array('desc' => __('Last name of a natural person'), 'default_category' => 'Person', 'to_ids' => 0), + 'date-of-birth' => array('desc' => __('Date of birth of a natural person (in YYYY-MM-DD format)'), 'default_category' => 'Person', 'to_ids' => 0), + 'place-of-birth' => array('desc' => __('Place of birth of a natural person'), 'default_category' => 'Person', 'to_ids' => 0), + 'gender' => array('desc' => __('The gender of a natural person (Male, Female, Other, Prefer not to say)'), 'default_category' => 'Person', 'to_ids' => 0), + 'passport-number' => array('desc' => __('The passport number of a natural person'), 'default_category' => 'Person', 'to_ids' => 0), + 'passport-country' => array('desc' => __('The country in which the passport was issued'), 'default_category' => 'Person', 'to_ids' => 0), + 'passport-expiration' => array('desc' => __('The expiration date of a passport'), 'default_category' => 'Person', 'to_ids' => 0), + 'redress-number' => array('desc' => __('The Redress Control Number is the record identifier for people who apply for redress through the DHS Travel Redress Inquiry Program (DHS TRIP). DHS TRIP is for travelers who have been repeatedly identified for additional screening and who want to file an inquiry to have erroneous information corrected in DHS systems'), 'default_category' => 'Person', 'to_ids' => 0), + 'nationality' => array('desc' => __('The nationality of a natural person'), 'default_category' => 'Person', 'to_ids' => 0), + 'visa-number' => array('desc' => __('Visa number'), 'default_category' => 'Person', 'to_ids' => 0), + 'issue-date-of-the-visa' => array('desc' => __('The date on which the visa was issued'), 'default_category' => 'Person', 'to_ids' => 0), + 'primary-residence' => array('desc' => __('The primary residence of a natural person'), 'default_category' => 'Person', 'to_ids' => 0), + 'country-of-residence' => array('desc' => __('The country of residence of a natural person'), 'default_category' => 'Person', 'to_ids' => 0), + 'special-service-request' => array('desc' => __('A Special Service Request is a function to an airline to provide a particular facility for A Passenger or passengers. '), 'default_category' => 'Person', 'to_ids' => 0), + 'frequent-flyer-number' => array('desc' => __('The frequent flyer number of a passenger'), 'default_category' => 'Person', 'to_ids' => 0), + // Do we really need remarks? Or just use comment/text for this? + //'remarks' => array('desc' => '', 'default_category' => 'Person', 'to_ids' => 0), + 'travel-details' => array('desc' => __('Travel details'), 'default_category' => 'Person', 'to_ids' => 0), + 'payment-details' => array('desc' => __('Payment details'), 'default_category' => 'Person', 'to_ids' => 0), + 'place-port-of-original-embarkation' => array('desc' => __('The orignal port of embarkation'), 'default_category' => 'Person', 'to_ids' => 0), + 'place-port-of-clearance' => array('desc' => __('The port of clearance'), 'default_category' => 'Person', 'to_ids' => 0), + 'place-port-of-onward-foreign-destination' => array('desc' => __('A Port where the passenger is transiting to'), 'default_category' => 'Person', 'to_ids' => 0), + 'passenger-name-record-locator-number' => array('desc' => __('The Passenger Name Record Locator is a key under which the reservation for a trip is stored in the system. The PNR contains, among other data, the name, flight segments and address of the passenger. It is defined by a combination of five or six letters and numbers.'), 'default_category' => 'Person', 'to_ids' => 0), + 'mobile-application-id' => array('desc' => __('The application id of a mobile application'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'chrome-extension-id' => array('desc' => __('Chrome extension id'), 'default_category' => 'Payload delivery', 'to_ids' => 1), + 'cortex' => array('desc' => __('Cortex analysis result'), 'default_category' => 'External analysis', 'to_ids' => 0), + 'boolean' => array('desc' => __('Boolean value - to be used in objects'), 'default_category' => 'Other', 'to_ids' => 0), + 'anonymised' => array('desc' => __('Anonymised value - described with the anonymisation object via a relationship'), 'formdesc' => __('Anonymised value - described with the anonymisation object via a relationship.'), 'default_category' => 'Other', 'to_ids' => 0) + // Not convinced about this. + //'url-regex' => array('desc' => '', 'default_category' => 'Person', 'to_ids' => 0), + ); + } } diff --git a/app/Model/AuthKey.php b/app/Model/AuthKey.php index 68bc698f7..f4711b6b7 100644 --- a/app/Model/AuthKey.php +++ b/app/Model/AuthKey.php @@ -42,22 +42,38 @@ class AuthKey extends AppModel $this->data['AuthKey']['authkey_end'] = substr($authkey, -4); $this->data['AuthKey']['authkey_raw'] = $authkey; $this->authkey_raw = $authkey; + + $validity = Configure::read('Security.advanced_authkeys_validity'); if (empty($this->data['AuthKey']['expiration'])) { - $this->data['AuthKey']['expiration'] = 0; + $this->data['AuthKey']['expiration'] = $validity ? strtotime("+$validity days") : 0; } else { - $this->data['AuthKey']['expiration'] = strtotime($this->data['AuthKey']['expiration']); + $expiration = is_numeric($this->data['AuthKey']['expiration']) ? + (int)$this->data['AuthKey']['expiration'] : + strtotime($this->data['AuthKey']['expiration']); + + if ($expiration === false) { + $this->invalidate('expiration', __('Expiration must be in YYYY-MM-DD format.')); + } + if ($validity && $expiration > strtotime("+$validity days")) { + $this->invalidate('expiration', __('Maximal key validity is %s days.', $validity)); + } + $this->data['AuthKey']['expiration'] = $expiration; } } return true; } + /** + * @param string $authkey + * @return array|false + */ public function getAuthUserByAuthKey($authkey) { $start = substr($authkey, 0, 4); $end = substr($authkey, -4); $existing_authkeys = $this->find('all', [ 'recursive' => -1, - 'fields' => ['authkey', 'user_id'], + 'fields' => ['id', 'authkey', 'user_id', 'expiration'], 'conditions' => [ 'OR' => [ 'expiration >' => time(), @@ -70,7 +86,12 @@ class AuthKey extends AppModel $passwordHasher = $this->getHasher(); foreach ($existing_authkeys as $existing_authkey) { if ($passwordHasher->check($authkey, $existing_authkey['AuthKey']['authkey'])) { - return $this->User->getAuthUser($existing_authkey['AuthKey']['user_id']); + $user = $this->User->getAuthUser($existing_authkey['AuthKey']['user_id']); + if ($user) { + $user['authkey_id'] = $existing_authkey['AuthKey']['id']; + $user['authkey_expiration'] = $existing_authkey['AuthKey']['expiration']; + } + return $user; } } return false; @@ -105,6 +126,67 @@ class AuthKey extends AppModel } } + /** + * @param int $id + * @return array + * @throws Exception + */ + public function getKeyUsage($id) + { + $redis = $this->setupRedisWithException(); + $data = $redis->hGetAll("misp:authkey_usage:$id"); + + $output = []; + $uniqueIps = []; + foreach ($data as $key => $count) { + list($date, $ip) = explode(':', $key); + $uniqueIps[$ip] = true; + if (isset($output[$date])) { + $output[$date] += $count; + } else { + $output[$date] = $count; + } + } + // Data from redis are not sorted + ksort($output); + + $lastUsage = $redis->get("misp:authkey_last_usage:$id"); + $lastUsage = $lastUsage === false ? null : (int)$lastUsage; + + return [$output, $lastUsage, count($uniqueIps)]; + } + + /** + * @param array $ids + * @return array + * @throws Exception + */ + public function getLastUsageForKeys(array $ids) + { + $redis = $this->setupRedisWithException(); + $keys = array_map(function($id) { + return "misp:authkey_last_usage:$id"; + }, $ids); + $lastUsages = $redis->mget($keys); + $output = []; + foreach (array_values($ids) as $i => $id) { + $output[$id] = $lastUsages[$i] === false ? null : (int)$lastUsages[$i]; + } + return $output; + } + + /** + * When key is deleted, update after `date_modified` for user that was assigned to that key, so session data + * will be realoaded and canceled. + * @see AppController::_refreshAuth + */ + public function afterDelete() + { + parent::afterDelete(); + $userId = $this->data['AuthKey']['user_id']; + $this->User->updateAll(['date_modified' => time()], ['User.id' => $userId]); + } + /** * @return AbstractPasswordHasher */ diff --git a/app/Model/CorrelationExclusion.php b/app/Model/CorrelationExclusion.php new file mode 100644 index 000000000..4c18fc0e9 --- /dev/null +++ b/app/Model/CorrelationExclusion.php @@ -0,0 +1,108 @@ + array( + 'userModel' => 'User', + 'userKey' => 'user_id', + 'change' => 'full'), + 'Containable', + ); + + public function afterSave($created, $options = array()) + { + $this->cacheValues(); + } + + public function afterDelete() + { + $this->cacheValues(); + } + + public function cacheValues() + { + try { + $redis = $this->setupRedisWithException(); + } catch (Exception $e) { + return false; + } + $redis->del($this->key); + $exclusions = $this->find('column', [ + 'fields' => ['value'] + ]); + $redis->sAddArray($this->key, $exclusions); + } + + public function cleanRouter($user) + { + if (Configure::read('MISP.background_jobs')) { + $this->Job = ClassRegistry::init('Job'); + $this->Job->create(); + $data = [ + 'worker' => 'default', + 'job_type' => 'clean_correlation_exclusions', + 'job_input' => '', + 'status' => 0, + 'retries' => 0, + 'org' => $user['Organisation']['name'], + 'message' => __('Cleaning up excluded correlations.'), + ]; + $this->Job->save($data); + $jobId = $this->Job->id; + $process_id = CakeResque::enqueue( + 'default', + 'AdminShell', + ['cleanExcludedCorrelations', $jobId], + true + ); + $this->Job->saveField('process_id', $process_id); + $message = __('Cleanup queued for background execution.'); + } else { + $this->clean(); + } + } + + public function clean($jobId = false) + { + try { + $redis = $this->setupRedisWithException(); + } catch (Exception $e) { + return false; + } + $this->Correlation = ClassRegistry::init('Correlation'); + $exclusions = $redis->sMembers($this->key); + $conditions = []; + $exclusions = array_chunk($exclusions, 100); + if ($jobId) { + $this->Job = ClassRegistry::init('Job'); + $this->Job->id = $jobId; + } + $total = count($exclusions); + foreach ($exclusions as $exclusion_chunk) { + $i = 0; + foreach ($exclusion_chunk as $exclusion) { + $i += 1; + if (!empty($exclusion)) { + if ($exclusion[0] === '%' || substr($exclusion, -1) === '%') { + $conditions['OR'][] = ['Correlation.value LIKE' => $exclusion]; + } else { + $conditions['OR']['Correlation.value'][] = $exclusion; + } + } + if (!empty($conditions)) { + $this->Correlation->deleteAll($conditions); + } + if ($i % 100 === 0) { + $this->Job->saveProgress($jobId, 'Chunk ' . $i . '/' . $total, $i * 100 / $total); + } + } + } + } +} diff --git a/app/Model/Event.php b/app/Model/Event.php index fbab11391..2596c4f82 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -10,6 +10,8 @@ App::uses('TmpFileTool', 'Tools'); * @property Attribute $Attribute * @property ShadowAttribute $ShadowAttribute * @property EventTag $EventTag + * @property SharingGroup $SharingGroup + * @property ThreatLevel $ThreatLevel */ class Event extends AppModel { @@ -61,120 +63,7 @@ class Event extends AppModel public $shortDist = array(0 => 'Organisation', 1 => 'Community', 2 => 'Connected', 3 => 'All', 4 => ' sharing Group'); - public $export_types = array( - 'json' => array( - 'extension' => '.json', - 'type' => 'JSON', - 'scope' => 'Event', - 'requiresPublished' => 0, - 'params' => array('includeAttachments' => 1, 'ignore' => 1, 'returnFormat' => 'json'), - 'description' => 'Click this to download all events and attributes that you have access to in MISP JSON format.', - ), - 'xml' => array( - 'extension' => '.xml', - 'type' => 'XML', - 'scope' => 'Event', - 'params' => array('includeAttachments' => 1, 'ignore' => 1, 'returnFormat' => 'xml'), - 'requiresPublished' => 0, - 'description' => 'Click this to download all events and attributes that you have access to in MISP XML format.', - ), - 'csv_sig' => array( - 'extension' => '.csv', - 'type' => 'CSV_Sig', - 'scope' => 'Event', - 'requiresPublished' => 1, - 'params' => array('published' => 1, 'to_ids' => 1, 'returnFormat' => 'csv'), - 'description' => 'Click this to download all attributes that are indicators and that you have access to (except file attachments) in CSV format.', - ), - 'csv_all' => array( - 'extension' => '.csv', - 'type' => 'CSV_All', - 'scope' => 'Event', - 'requiresPublished' => 0, - 'params' => array('ignore' => 1, 'returnFormat' => 'csv'), - 'description' => 'Click this to download all attributes that you have access to (except file attachments) in CSV format.', - ), - 'suricata' => array( - 'extension' => '.rules', - 'type' => 'Suricata', - 'scope' => 'Attribute', - 'requiresPublished' => 1, - 'params' => array('returnFormat' => 'suricata'), - 'description' => 'Click this to download all network related attributes that you have access to under the Suricata rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a allowedlist containing host, domain name and IP numbers to exclude from the NIDS export.', - ), - 'snort' => array( - 'extension' => '.rules', - 'type' => 'Snort', - 'scope' => 'Attribute', - 'requiresPublished' => 1, - 'params' => array('returnFormat' => 'snort'), - 'description' => 'Click this to download all network related attributes that you have access to under the Snort rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a allowedlist containing host, domain name and IP numbers to exclude from the NIDS export.', - ), - 'bro' => array( - 'extension' => '.intel', - 'type' => 'Bro', - 'scope' => 'Attribute', - 'requiresPublished' => 1, - 'params' => array('returnFormat' => 'bro'), - 'description' => 'Click this to download all network related attributes that you have access to under the Bro rule format. Only published events and attributes marked as IDS Signature are exported. Administration is able to maintain a allowedlist containing host, domain name and IP numbers to exclude from the NIDS export.', - ), - 'stix' => array( - 'extension' => '.xml', - 'type' => 'STIX', - 'scope' => 'Event', - 'requiresPublished' => 1, - 'params' => array('returnFormat' => 'stix', 'includeAttachments' => 1), - 'description' => 'Click this to download a STIX document containing the STIX version of all events and attributes that you have access to.' - ), - 'stix-json' => array( - 'extension' => '.json', - 'type' => 'STIX', - 'scope' => 'Event', - 'requiresPublished' => 1, - 'params' => array('returnFormat' => 'stix', 'includeAttachments' => 1), - 'description' => 'Click this to download a STIX document containing the STIX version of all events and attributes that you have access to.' - ), - 'stix2' => array( - 'extension' => '.json', - 'type' => 'STIX2', - 'scope' => 'Event', - 'requiresPublished' => 1, - 'params' => array('returnFormat' => 'stix2', 'includeAttachments' => 1), - 'description' => 'Click this to download a STIX2 document containing the STIX2 version of all events and attributes that you have access to.' - ), - 'rpz' => array( - 'extension' => '.txt', - 'type' => 'RPZ', - 'scope' => 'Attribute', - 'requiresPublished' => 1, - 'params' => array('returnFormat' => 'rpz'), - 'description' => 'Click this to download an RPZ Zone file generated from all ip-src/ip-dst, hostname, domain attributes. This can be useful for DNS level firewalling. Only published events and attributes marked as IDS Signature are exported.' - ), - 'text' => array( - 'extension' => '.txt', - 'type' => 'TEXT', - 'scope' => 'Attribute', - 'requiresPublished' => 1, - 'params' => array('returnFormat' => 'text', 'includeAttachments' => 1), - 'description' => 'Click on one of the buttons below to download all the attributes with the matching type. This list can be used to feed forensic software when searching for susipicious files. Only published events and attributes marked as IDS Signature are exported.' - ), - 'yara' => array( - 'extension' => '.yara', - 'type' => 'Yara', - 'scope' => 'Event', - 'requiresPublished' => 1, - 'params' => array('returnFormat' => 'yara'), - 'description' => 'Click this to download Yara rules generated from all relevant attributes.' - ), - 'yara-json' => array( - 'extension' => '.json', - 'type' => 'Yara', - 'scope' => 'Event', - 'requiresPublished' => 1, - 'params' => array('returnFormat' => 'yara-json'), - 'description' => 'Click this to download Yara rules generated from all relevant attributes. Rules are returned in a JSON format with information about origin (generated or parsed) and validity.' - ), - ); + public $export_types = []; public $validFormats = array( 'attack' => array('html', 'AttackExport', 'html'), @@ -653,10 +542,10 @@ class Event extends AppModel } } - public function attachtagsToEvents($events) + public function attachTagsToEvents(array $events) { $tagsToFetch = array(); - foreach ($events as $k => $event) { + foreach ($events as $event) { if (!empty($event['EventTag'])) { foreach ($event['EventTag'] as $et) { $tagsToFetch[$et['tag_id']] = $et['tag_id']; @@ -668,11 +557,11 @@ class Event extends AppModel 'recursive' => -1, 'order' => false )); - $tags = Set::combine($tags, '{n}.Tag.id', '{n}'); + $tags = array_column(array_column($tags, 'Tag'), null, 'id'); foreach ($events as $k => $event) { if (!empty($event['EventTag'])) { foreach ($event['EventTag'] as $k2 => $et) { - $events[$k]['EventTag'][$k2]['Tag'] = $tags[$et['tag_id']]['Tag']; + $events[$k]['EventTag'][$k2]['Tag'] = $tags[$et['tag_id']]; } } } @@ -687,7 +576,7 @@ class Event extends AppModel $sgids = array(-1); } $this->Correlation = ClassRegistry::init('Correlation'); - $eventIds = Set::extract('/Event/id', $events); + $eventIds = array_column(array_column($events, 'Event'), 'id'); $conditionsCorrelation = $this->__buildEventConditionsCorrelation($user, $eventIds, $sgids); $correlations = $this->Correlation->find('all', array( 'fields' => array('Correlation.1_event_id', 'count(distinct(Correlation.event_id)) as count'), @@ -697,14 +586,14 @@ class Event extends AppModel )); $correlations = Hash::combine($correlations, '{n}.Correlation.1_event_id', '{n}.0.count'); foreach ($events as &$event) { - $event['Event']['correlation_count'] = (isset($correlations[$event['Event']['id']])) ? $correlations[$event['Event']['id']] : 0; + $event['Event']['correlation_count'] = isset($correlations[$event['Event']['id']]) ? $correlations[$event['Event']['id']] : 0; } return $events; } public function attachSightingsCountToEvents($user, $events) { - $eventIds = Set::extract('/Event/id', $events); + $eventIds = array_column(array_column($events, 'Event'), 'id'); $this->Sighting = ClassRegistry::init('Sighting'); $sightings = $this->Sighting->find('all', array( 'fields' => array('Sighting.event_id', 'count(distinct(Sighting.id)) as count'), @@ -714,14 +603,14 @@ class Event extends AppModel )); $sightings = Hash::combine($sightings, '{n}.Sighting.event_id', '{n}.0.count'); foreach ($events as $key => $event) { - $events[$key]['Event']['sightings_count'] = (isset($sightings[$event['Event']['id']])) ? $sightings[$event['Event']['id']] : 0; + $events[$key]['Event']['sightings_count'] = isset($sightings[$event['Event']['id']]) ? $sightings[$event['Event']['id']] : 0; } return $events; } public function attachProposalsCountToEvents($user, $events) { - $eventIds = Set::extract('/Event/id', $events); + $eventIds = array_column(array_column($events, 'Event'), 'id'); $proposals = $this->ShadowAttribute->find('all', array( 'fields' => array('ShadowAttribute.event_id', 'count(distinct(ShadowAttribute.id)) as count'), 'conditions' => array('event_id' => $eventIds, 'deleted' => 0), @@ -730,14 +619,14 @@ class Event extends AppModel )); $proposals = Hash::combine($proposals, '{n}.ShadowAttribute.event_id', '{n}.0.count'); foreach ($events as $key => $event) { - $events[$key]['Event']['proposals_count'] = (isset($proposals[$event['Event']['id']])) ? $proposals[$event['Event']['id']] : 0; + $events[$key]['Event']['proposals_count'] = isset($proposals[$event['Event']['id']]) ? $proposals[$event['Event']['id']] : 0; } return $events; } public function attachDiscussionsCountToEvents($user, $events) { - $eventIds = Set::extract('/Event/id', $events); + $eventIds = array_column(array_column($events, 'Event'), 'id'); $this->Thread = ClassRegistry::init('Thread'); $threads = $this->Thread->find('list', array( 'conditions' => array('Thread.event_id' => $eventIds), @@ -842,18 +731,16 @@ class Event extends AppModel // ii. Atttibute has a distribution between 1-3 (community only, connected communities, all orgs) // iii. Attribute has a sharing group that the user is accessible to view $conditionsCorrelation = $this->__buildEventConditionsCorrelation($user, $eventId, $sgids); - $correlations = $this->Correlation->find('list', array( - 'fields' => array('Correlation.event_id', 'Correlation.event_id'), - 'conditions' => $conditionsCorrelation, - 'recursive' => 0, - 'group' => 'Correlation.event_id', - 'order' => array('Correlation.event_id DESC'))); + $relatedEventIds = $this->Correlation->find('column', array( + 'fields' => array('Correlation.event_id'), + 'conditions' => $conditionsCorrelation, + 'unique' => true, + )); - if (empty($correlations)) { + if (empty($relatedEventIds)) { return []; } - $relatedEventIds = array_values($correlations); // now look up the event data for these attributes $conditions = $this->createEventConditions($user); $conditions['AND'][] = array('Event.id' => $relatedEventIds); @@ -1087,7 +974,7 @@ class Event extends AppModel public function uploadEventToServer($event, $server, $HttpSocket = null, $scope = 'events') { $this->Server = ClassRegistry::init('Server'); - $push = $this->Server->checkVersionCompatibility($server['Server']['id'], false, $HttpSocket); + $push = $this->Server->checkVersionCompatibility($server, false, $HttpSocket); if ($scope === 'events' && empty($push['canPush'])) { return 'The remote user is not a sync user - the upload of the event has been blocked.'; } elseif ($scope === 'sightings' && empty($push['canPush']) && empty($push['canSight'])) { @@ -1187,14 +1074,18 @@ class Event extends AppModel if (is_numeric($event)) { return $event; } - $url = $server['Server']['url']; $HttpSocket = $this->setupHttpSocket($server, $HttpSocket); $request = $this->setupSyncRequest($server); if ($scope === 'sightings') { $scope .= '/bulkSaveSightings'; $urlPath = $event['Event']['uuid']; } + $url = $server['Server']['url']; $uri = $url . '/' . $scope . $this->__getLastUrlPathComponent($urlPath); + if ($scope === 'event') { + // After creating or editing event, it is not necessary to fetch full event + $uri .= '/metadata:1'; + } $data = json_encode($event); if (!empty(Configure::read('Security.sync_audit'))) { $pushLogEntry = sprintf( @@ -1466,40 +1357,32 @@ class Event extends AppModel * @param int $eventId * @param array $server * @param null|HttpSocket $HttpSocket - * @param boolean $metadataOnly, if True, we only retrieve the metadata - * without attributes and attachments which is much faster + * @param boolean $metadataOnly, if True, we only retrieve the metadata, without attributes and attachments which is much faster * @return array * @throws Exception */ - public function downloadEventFromServer($eventId, $server, $HttpSocket=null, $metadataOnly=false) + public function downloadEventFromServer($eventId, $server, HttpSocket $HttpSocket=null, $metadataOnly=false) { $url = $server['Server']['url']; $HttpSocket = $this->setupHttpSocket($server, $HttpSocket); $request = $this->setupSyncRequest($server); if ($metadataOnly) { $uri = $url . '/events/index'; - $data = ['eventid' => $eventId]; - $data = json_encode($data); + $data = json_encode(['eventid' => $eventId]); $response = $HttpSocket->post($uri, $data, $request); } else { $uri = $url . '/events/view/' . $eventId . '/deleted[]:0/deleted[]:1/excludeGalaxy:1'; if (!empty($server['Server']['internal'])) { $uri = $uri . '/excludeLocalTags:1'; } - $response = $HttpSocket->get($uri, $data = '', $request); + $response = $HttpSocket->get($uri, [], $request); } - if ($response === false) { - throw new Exception("Could not reach '$uri'."); - } else if (!$response->isOk()) { + if (!$response->isOk()) { throw new Exception("Fetching the '$uri' failed with HTTP error {$response->code}: {$response->reasonPhrase}"); } - $event = json_decode($response->body, true); - if ($event === null) { - throw new Exception('Could not parse event JSON: ' . json_last_error_msg(), json_last_error()); - } - return $event; + return $this->jsonDecode($response->body); } public function quickDelete($event) @@ -1802,11 +1685,10 @@ class Event extends AppModel { $conditions = $this->createEventConditions($user); $conditions['AND'][] = $params['conditions']; - $results = array_values($this->find('list', array( + $results = $this->find('column', array( 'conditions' => $conditions, - 'recursive' => -1, 'fields' => array('Event.id') - ))); + )); return $results; } @@ -1882,9 +1764,9 @@ class Event extends AppModel if ($list) { $params = array( 'conditions' => $conditions, - 'recursive' => -1, + 'fields' => ['Event.id'], ); - $results = array_values($this->find('list', $params)); + $results = $this->find('column', $params); } else { $params = array( 'conditions' => $conditions, @@ -2587,6 +2469,12 @@ class Event extends AppModel $event['Event']['extensionEvents'][$eventMeta['id']] = $eventMeta; $thingsToMerge = array('Attribute', 'Object', 'ShadowAttribute', 'Galaxy'); foreach ($thingsToMerge as $thingToMerge) { + if (!isset($event[$thingToMerge])) { + $event[$thingToMerge] = []; + } + if (!isset($extensionEvent[$thingToMerge])) { + $extensionEvent[$thingToMerge] = []; + } $event[$thingToMerge] = array_merge($event[$thingToMerge], $extensionEvent[$thingToMerge]); } // Merge event reports if requested @@ -2697,7 +2585,7 @@ class Event extends AppModel $existingOrg = $this->Orgc->find('first', array( 'recursive' => -1, 'conditions' => array('Orgc.name' => $org), - 'fields' => array('Orgc.name', 'Orgc.id') + 'fields' => array('Orgc.id') )); if (empty($existingOrg)) { $params['org']['OR'][$k] = -1; @@ -2714,7 +2602,7 @@ class Event extends AppModel $existingOrg = $this->Orgc->find('first', array( 'recursive' => -1, 'conditions' => array('Orgc.name' => $org), - 'fields' => array('Orgc.name', 'Orgc.id') + 'fields' => array('Orgc.id') )); if (!empty($existingOrg)) { $temp[] = $existingOrg['Orgc']['id']; @@ -3197,16 +3085,18 @@ class Event extends AppModel $userCount = count($usersWithAccess); $this->UserSetting = ClassRegistry::init('UserSetting'); foreach ($usersWithAccess as $k => $user) { - if ($this->UserSetting->checkPublishFilter($user, $event)) { - // Fetch event for user that will receive alert e-mail to respect all ACLs - $eventForUser = $this->fetchEvent($user, [ - 'eventid' => $id, - 'includeAllTags' => true, - 'includeEventCorrelations' => true, - ])[0]; + // Fetch event for user that will receive alert e-mail to respect all ACLs + $eventForUser = $this->fetchEvent($user, [ + 'eventid' => $id, + 'includeAllTags' => true, + 'includeEventCorrelations' => true, + 'noEventReports' => true, + 'noSightings' => true, + ])[0]; + if ($this->UserSetting->checkPublishFilter($user, $eventForUser)) { $body = $this->__buildAlertEmailBody($eventForUser, $user, $oldpublish); - $this->User->sendEmail(array('User' => $user), $body, $bodyNoEnc, $subject); + $this->User->sendEmail(['User' => $user], $body, $bodyNoEnc, $subject); } if ($jobId) { $this->Job->saveProgress($jobId, null, $k / $userCount * 100); @@ -4204,7 +4094,7 @@ class Event extends AppModel if (isset($data['Event']['EventReport'])) { foreach ($data['Event']['EventReport'] as $i => $report) { $nothingToChange = false; - $result = $this->EventReport->editReport($user, $report, $this->id, true, $nothingToChange); + $result = $this->EventReport->editReport($user, ['EventReport' => $report], $this->id, true, $nothingToChange); if (!empty($result)) { $validationErrors['EventReport'][] = $result; } @@ -6149,8 +6039,8 @@ class Event extends AppModel } } } + $data[$dataType . 'Tag'] = array_values($data[$dataType . 'Tag']); } - $data[$dataType . 'Tag'] = array_values($data[$dataType . 'Tag']); return $data; } @@ -6994,7 +6884,7 @@ class Event extends AppModel unset($result); unset($temp); $tmpfile->write($exportTool->footer($exportToolParams)); - return $tmpfile->finish(); + return $tmpfile; } /* @@ -7046,7 +6936,7 @@ class Event extends AppModel 'model_id' => 0, 'email' => 'SYSTEM', 'action' => 'error', - 'title' => sprintf('Event fetch potential memory exhaustion. During the fetching of events, a large event (#%s) was detected that exceeds the available PHP memory. Consider raising the PHP max_memory setting to at least %sM', $largest_event_id, ceil($largest_event/$memory_scaling_factor)), + 'title' => sprintf('Event fetch potential memory exhaustion.' . PHP_EOL . 'During the fetching of events, a large event (#%s) was detected that exceeds the available PHP memory.' . PHP_EOL . 'Consider raising the PHP max_memory setting to at least %sM', $largest_event_id, ceil($largest_event/$memory_scaling_factor)), 'change' => null, )); } diff --git a/app/Model/EventReport.php b/app/Model/EventReport.php index 30a93c438..2e508a932 100644 --- a/app/Model/EventReport.php +++ b/app/Model/EventReport.php @@ -161,6 +161,8 @@ class EventReport extends AppModel $errors[] = __('Event Report not found.'); return $errors; } + } else { + $report['EventReport']['id'] = $existingReport['EventReport']['id']; } if ($fromPull) { @@ -521,7 +523,8 @@ class EventReport extends AppModel return $errors; } - public function applySuggestions($user, $report, $contentWithSuggestions, $suggestionsMapping) { + public function applySuggestions(array $user, array $report, $contentWithSuggestions, array $suggestionsMapping) + { $errors = []; $replacedContent = $contentWithSuggestions; $success = 0; @@ -546,10 +549,10 @@ class EventReport extends AppModel return $errors; } - public function applySuggestionsInText($contentWithSuggestions, $attribute, $value) + public function applySuggestionsInText($contentWithSuggestions, array $attribute, $value) { - $textToBeReplaced = sprintf('@[suggestion](%s)', $value); - $textToInject = sprintf('@[attribute](%s)', $attribute['Attribute']['uuid']); + $textToBeReplaced = "@[suggestion]($value)"; + $textToInject = "@[attribute]({$attribute['Attribute']['uuid']})"; $replacedContent = str_replace($textToBeReplaced, $textToInject, $contentWithSuggestions); return $replacedContent; } @@ -642,25 +645,36 @@ class EventReport extends AppModel ]; } - public function transformFreeTextIntoSuggestion($content, $complexTypeToolResult) + public function transformFreeTextIntoSuggestion($content, array $complexTypeToolResult) { $replacedContent = $content; - $suggestionsMapping = []; $typeToCategoryMapping = $this->Event->Attribute->typeToCategoryMapping(); - foreach ($complexTypeToolResult as $i => $complexTypeToolEntry) { + + // Sort by original value string length, longest values first + usort($complexTypeToolResult, function ($a, $b) { + $strlenA = strlen($a['original_value']); + $strlenB = strlen($b['original_value']); + if ($strlenA === $strlenB) { + return 0; + } + return ($strlenA < $strlenB) ? 1 : -1; + }); + + $suggestionsMapping = []; + foreach ($complexTypeToolResult as $complexTypeToolEntry) { $textToBeReplaced = $complexTypeToolEntry['value']; - $textToInject = sprintf('@[suggestion](%s)', $textToBeReplaced); + $textToInject = "@[suggestion]($textToBeReplaced)"; $suggestionsMapping[$textToBeReplaced] = [ 'category' => $typeToCategoryMapping[$complexTypeToolEntry['default_type']][0], 'type' => $complexTypeToolEntry['default_type'], 'value' => $textToBeReplaced, 'to_ids' => $complexTypeToolEntry['to_ids'], ]; - $replacedContent = str_replace($textToBeReplaced, $textToInject, $replacedContent); + $replacedContent = str_replace($complexTypeToolEntry['original_value'], $textToInject, $replacedContent); } return [ 'contentWithSuggestions' => $replacedContent, - 'suggestionsMapping' => $suggestionsMapping + 'suggestionsMapping' => $suggestionsMapping, ]; } @@ -674,21 +688,17 @@ class EventReport extends AppModel return $complexTypeToolResult; } - public function getComplexTypeToolResultFromReport($content) + public function getComplexTypeToolResultWithReplacements(array $user, array $report) { App::uses('ComplexTypeTool', 'Tools'); $complexTypeTool = new ComplexTypeTool(); $this->Warninglist = ClassRegistry::init('Warninglist'); $complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists()); - $complexTypeToolResult = $complexTypeTool->checkComplexRouter($content, 'freetext'); - return $complexTypeToolResult; - } - public function getComplexTypeToolResultWithReplacements($user, $report) - { - $complexTypeToolResult = $this->getComplexTypeToolResultFromReport($report['EventReport']['content']); + $complexTypeToolResult = $complexTypeTool->checkFreeText($report['EventReport']['content']); $replacementResult = $this->transformFreeTextIntoReplacement($user, $report, $complexTypeToolResult); - $complexTypeToolResult = $this->getComplexTypeToolResultFromReport($replacementResult['contentWithReplacements']); + $complexTypeToolResult = $complexTypeTool->checkFreeText($replacementResult['contentWithReplacements']); + return [ 'complexTypeToolResult' => $complexTypeToolResult, 'replacementResult' => $replacementResult, @@ -700,6 +710,7 @@ class EventReport extends AppModel * * @param array $user * @param array $report + * @param array $options * @return array */ public function extractWithReplacements(array $user, array $report, array $options = []) @@ -713,7 +724,6 @@ class EventReport extends AppModel 'attack' => true, ]; $options = array_merge($baseOptions, $options); - $originalContent = $report['EventReport']['content']; $this->GalaxyCluster = ClassRegistry::init('GalaxyCluster'); $mitreAttackGalaxyId = $this->GalaxyCluster->Galaxy->getMitreAttackGalaxyId(); $clusterContain = ['Tag']; @@ -734,17 +744,21 @@ class EventReport extends AppModel 'contain' => $clusterContain ]); + $originalContent = $report['EventReport']['content']; + // Remove all existing event report markers + $content = preg_replace("/@\[(attribute|tag|galaxymatrix)]\([^)]*\)/", '', $originalContent); + if ($options['tags']) { $this->Tag = ClassRegistry::init('Tag'); $tags = $this->Tag->fetchUsableTags($user); - foreach ($tags as $i => $tag) { + foreach ($tags as $tag) { $tagName = $tag['Tag']['name']; - $found = $this->isValidReplacementTag($originalContent, $tagName); + $found = $this->isValidReplacementTag($content, $tagName); if ($found) { $replacedContext[$tagName][$tagName] = $tag['Tag']; } else { $tagNameUpper = strtoupper($tagName); - $found = $this->isValidReplacementTag($originalContent, $tagNameUpper); + $found = $this->isValidReplacementTag($content, $tagNameUpper); if ($found) { $replacedContext[$tagNameUpper][$tagName] = $tag['Tag']; } @@ -752,10 +766,10 @@ class EventReport extends AppModel } } - foreach ($clusters as $i => $cluster) { + foreach ($clusters as $cluster) { $cluster['GalaxyCluster']['colour'] = '#0088cc'; $tagName = $cluster['GalaxyCluster']['tag_name']; - $found = $this->isValidReplacementTag($originalContent, $tagName); + $found = $this->isValidReplacementTag($content, $tagName); if ($found) { $replacedContext[$tagName][$tagName] = $cluster['GalaxyCluster']; } @@ -765,10 +779,10 @@ class EventReport extends AppModel $replacedContext[$cluster['GalaxyCluster']['value']][$tagName] = $cluster['GalaxyCluster']; } if ($options['synonyms']) { - foreach ($cluster['GalaxyElement'] as $j => $element) { + foreach ($cluster['GalaxyElement'] as $element) { if (strlen($element['value']) >= $options['synonyms_min_characters']) { $toSearch = ' ' . $element['value'] . ' '; - $found = strpos($originalContent, $toSearch) !== false; + $found = strpos($content, $toSearch) !== false; if ($found) { $replacedContext[$element['value']][$tagName] = $cluster['GalaxyCluster']; } @@ -783,22 +797,22 @@ class EventReport extends AppModel 'conditions' => ['GalaxyCluster.galaxy_id' => $mitreAttackGalaxyId], 'contain' => $clusterContain ]); - foreach ($attackClusters as $i => $cluster) { + foreach ($attackClusters as $cluster) { $cluster['GalaxyCluster']['colour'] = '#0088cc'; $tagName = $cluster['GalaxyCluster']['tag_name']; $toSearch = ' ' . $cluster['GalaxyCluster']['value'] . ' '; - $found = strpos($originalContent, $toSearch) !== false; + $found = strpos($content, $toSearch) !== false; if ($found) { $replacedContext[$cluster['GalaxyCluster']['value']][$tagName] = $cluster['GalaxyCluster']; } else { $clusterParts = explode(' - ', $cluster['GalaxyCluster']['value'], 2); $toSearch = ' ' . $clusterParts[0] . ' '; - $found = strpos($originalContent, $toSearch) !== false; + $found = strpos($content, $toSearch) !== false; if ($found) { $replacedContext[$clusterParts[0]][$tagName] = $cluster['GalaxyCluster']; - } else { + } else if (isset($clusterParts[1])) { $toSearch = ' ' . $clusterParts[1] . ' '; - $found = strpos($originalContent, $toSearch) !== false; + $found = strpos($content, $toSearch) !== false; if ($found) { $replacedContext[$clusterParts[1]][$tagName] = $cluster['GalaxyCluster']; } @@ -810,14 +824,32 @@ class EventReport extends AppModel 'replacedContext' => $replacedContext ]; if ($options['replace']) { + // Sort by original value string length, longest values first + uksort($replacedContext, function ($a, $b) { + $strlenA = strlen($a); + $strlenB = strlen($b); + if ($strlenA === $strlenB) { + return 0; + } + return ($strlenA < $strlenB) ? 1 : -1; + }); + $content = $originalContent; + $secondPassReplace = []; + // Replace in two pass to prevent double replace + $id = 0; foreach ($replacedContext as $rawText => $replacements) { // Replace with first one until a better strategy is found reset($replacements); $replacement = key($replacements); - $textToInject = sprintf('@[tag](%s)', $replacement); - $content = str_replace($rawText, $textToInject, $content); + ++$id; + $content = str_replace($rawText, "@[mark]($id)", $content); + $secondPassReplace[$id] = "@[tag]($replacement)"; } + + $content = preg_replace_callback("/@\[mark]\(([^)]*)\)/", function ($matches) use ($secondPassReplace) { + return $secondPassReplace[$matches[1]]; + }, $content); $toReturn['contentWithReplacements'] = $content; } return $toReturn; @@ -835,7 +867,6 @@ class EventReport extends AppModel 'event_id' => $event_id, 'url' => $url ]; - $module = $this->isFetchURLModuleEnabled(); if (!empty($module)) { $result = $this->Module->queryModuleServer($modulePayload, false); if (empty($result['results'][0]['values'][0])) { @@ -853,7 +884,7 @@ class EventReport extends AppModel } /** - * findValidReplacementTag Search if tagName is in content and is not wrapped in a tag reference + * findValidReplacementTag Search if tagName is in content * * @param string $content * @param string $tagName @@ -861,25 +892,8 @@ class EventReport extends AppModel */ private function isValidReplacementTag($content, $tagName) { - $lastIndex = 0; - $allIndices = []; $toSearch = strpos($tagName, ':') === false ? ' ' . $tagName . ' ' : $tagName; - while (($lastIndex = strpos($content, $toSearch, $lastIndex)) !== false) { - $allIndices[] = $lastIndex; - $lastIndex = $lastIndex + strlen($toSearch); - } - if (empty($allIndices)) { - return false; - } else { - $wrapper = '@[tag]('; - foreach ($allIndices as $i => $index) { - $stringBeforeTag = substr($content, $index - strlen($wrapper), strlen($wrapper)); - if ($stringBeforeTag != $wrapper) { - return true; - } - } - return false; - } + return strpos($content, $toSearch) !== false; } public function attachTagsAfterReplacements($user, $replacedContext, $eventId) diff --git a/app/Model/Feed.php b/app/Model/Feed.php index a24cfdc06..e94d481ab 100644 --- a/app/Model/Feed.php +++ b/app/Model/Feed.php @@ -180,7 +180,7 @@ class Feed extends AppModel $tmpFile->write(trim($data)); unset($data); - return $tmpFile->csv(); + return $tmpFile->intoParsedCsv(); } /** @@ -364,18 +364,18 @@ class Feed extends AppModel $redisResultToAttributePosition = []; foreach ($attributes as $k => $attribute) { - if (in_array($attribute['type'], $this->Attribute->nonCorrelatingTypes)) { + if (in_array($attribute['type'], $this->Attribute->nonCorrelatingTypes, true)) { continue; // attribute type is not correlateable } if (!empty($attribute['disable_correlation'])) { continue; // attribute correlation is disabled } - if (in_array($attribute['type'], $compositeTypes)) { + if (in_array($attribute['type'], $compositeTypes, true)) { list($value1, $value2) = explode('|', $attribute['value']); $parts = [$value1]; - if (!in_array($attribute['type'], $this->Attribute->primaryOnlyCorrelatingTypes)) { + if (!in_array($attribute['type'], $this->Attribute->primaryOnlyCorrelatingTypes, true)) { $parts[] = $value2; } } else { @@ -442,12 +442,17 @@ class Feed extends AppModel if (!isset($event[$scope][$sourceId])) { $event[$scope][$sourceId] = $source[$scope]; } + $attributePosition = $redisResultToAttributePosition[$hitIds[$k]]; - $attributes[$attributePosition][$scope][] = $source[$scope]; + $alreadyAttached = isset($attributes[$attributePosition][$scope]) && + in_array($sourceId, array_column($attributes[$attributePosition][$scope], 'id')); + if (!$alreadyAttached) { + $attributes[$attributePosition][$scope][] = $source[$scope]; + } $sourceHasHit = true; } } - // Append also exact MISP feed event UUID + // Append also exact MISP feed or server event UUID // TODO: This can be optimised in future to do that in one pass if ($sourceHasHit && ($scope === 'Server' || $source[$scope]['source_format'] === 'misp')) { $pipe = $redis->multi(Redis::PIPELINE); @@ -475,7 +480,9 @@ class Feed extends AppModel $attributePosition = $eventUuidHitPosition[$sourceHitPos]; foreach ($attributes[$attributePosition][$scope] as $tempKey => $tempFeed) { if ($tempFeed['id'] == $feedId) { - $attributes[$attributePosition][$scope][$tempKey]['event_uuids'][] = $eventUuid; + if (empty($attributes[$attributePosition][$scope][$tempKey]['event_uuids']) || !in_array($eventUuid, $attributes[$attributePosition][$scope][$tempKey]['event_uuids'])) { + $attributes[$attributePosition][$scope][$tempKey]['event_uuids'][] = $eventUuid; + } break; } } @@ -598,11 +605,6 @@ class Feed extends AppModel ) ); - // Enable gzipped responses if PHP has 'gzdecode' method - if (function_exists('gzdecode')) { - $result['header']['Accept-Encoding'] = 'gzip'; - } - $commit = $this->checkMIPSCommit(); if ($commit) { $result['header']['commit'] = $commit; @@ -1759,30 +1761,22 @@ class Feed extends AppModel $request = $this->__createFeedRequest($feed['Feed']['headers']); - if ($followRedirect) { - $response = $this->getFollowRedirect($HttpSocket, $uri, $request); - } else { - $response = $HttpSocket->get($uri, array(), $request); + try { + if ($followRedirect) { + $response = $this->getFollowRedirect($HttpSocket, $uri, $request); + } else { + $response = $HttpSocket->get($uri, array(), $request); + } + } catch (Exception $e) { + throw new Exception("Fetching the '$uri' failed with exception: {$e->getMessage()}", 0, $e); } - if ($response === false) { - throw new Exception("Could not reach '$uri'."); - } else if ($response->code != 200) { // intentionally != + if ($response->code != 200) { // intentionally != throw new Exception("Fetching the '$uri' failed with HTTP error {$response->code}: {$response->reasonPhrase}"); } $data = $response->body; - $contentEncoding = $response->getHeader('Content-Encoding'); - if ($contentEncoding === 'gzip') { - $data = gzdecode($data); - if ($data === false) { - throw new Exception("Fetching the '$uri' failed, response should be gzip encoded, but gzip decoding failed."); - } - } else if ($contentEncoding) { - throw new Exception("Fetching the '$uri' failed, because remote server returns unsupported content encoding '$contentEncoding'"); - } - $contentType = $response->getHeader('Content-Type'); if ($contentType === 'application/zip') { $zipFile = new File($this->tempFileName()); diff --git a/app/Model/Galaxy.php b/app/Model/Galaxy.php index dc443c9d3..020bd291a 100644 --- a/app/Model/Galaxy.php +++ b/app/Model/Galaxy.php @@ -92,7 +92,7 @@ class Galaxy extends AppModel return $this->find('list', array('recursive' => -1, 'fields' => array('type', 'id'))); } - private function __update_prepare_template(array $cluster_package, array $galaxies): array + private function __update_prepare_template(array $cluster_package, array $galaxies) { return [ 'source' => isset($cluster_package['source']) ? $cluster_package['source'] : '', @@ -104,7 +104,7 @@ class Galaxy extends AppModel ]; } - private function __getPreExistingClusters(array $galaxies, array $cluster_package): array + private function __getPreExistingClusters(array $galaxies, array $cluster_package) { $temp = $this->GalaxyCluster->find('all', array( 'conditions' => array( @@ -120,7 +120,7 @@ class Galaxy extends AppModel return $existingClusters; } - private function __deleteOutdated(bool $force, array $cluster_package, array $existingClusters): array + private function __deleteOutdated(bool $force, array $cluster_package, array $existingClusters) { // Delete all existing outdated clusters $cluster_ids_to_delete = array(); @@ -152,7 +152,7 @@ class Galaxy extends AppModel return $cluster_package; } - private function __createClusters($cluster_package, $template): array + private function __createClusters($cluster_package, $template) { $relations = []; $elements = []; @@ -377,12 +377,23 @@ class Galaxy extends AppModel $result = $this->Tag->$connectorModel->save($toSave); if ($result) { if ($target_type !== 'tag_collection') { + $date = new DateTime(); if ($target_type === 'event') { $event = $target; + } else if ($target_type === 'attribute') { + $target['Attribute']['timestamp'] = $date->getTimestamp(); + $this->Tag->AttributeTag->Attribute->save($target); + if (!empty($target['Attribute']['object_id'])) { + $container_object = $this->Tag->AttributeTag->Attribute->Object->find('first', [ + 'recursive' => -1, + 'conditions' => ['id' => $target['Attribute']['object_id']] + ]); + $container_object['Object']['timestamp'] = $date->getTimestamp(); + $this->Tag->AttributeTag->Attribute->Object->save($container_object); + } } $this->Tag->EventTag->Event->insertLock($user, $event['Event']['id']); $event['Event']['published'] = 0; - $date = new DateTime(); $event['Event']['timestamp'] = $date->getTimestamp(); $this->Tag->EventTag->Event->save($event); } diff --git a/app/Model/GalaxyCluster.php b/app/Model/GalaxyCluster.php index 24dc55d2a..26df51ec8 100644 --- a/app/Model/GalaxyCluster.php +++ b/app/Model/GalaxyCluster.php @@ -1,5 +1,7 @@ __clusterCache[$name])) { return $this->__clusterCache[$name]; } @@ -1104,8 +1116,8 @@ class GalaxyCluster extends AppModel ); } - $tmpfile = tmpfile(); - fwrite($tmpfile, $exportTool->header($exportToolParams)); + $tmpfile = new TmpFileTool(); + $tmpfile->write($exportTool->header($exportToolParams)); $loop = false; if (empty($params['limit'])) { $memory_in_mb = $this->convert_to_memory_limit_to_mb(ini_get('memory_limit')); @@ -1115,48 +1127,32 @@ class GalaxyCluster extends AppModel $params['page'] = 1; } $this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams, $elementCounter); - fwrite($tmpfile, $exportTool->footer($exportToolParams)); - fseek($tmpfile, 0); - if (fstat($tmpfile)['size']) { - $final = fread($tmpfile, fstat($tmpfile)['size']); - } else { - $final = ''; - } - fclose($tmpfile); - return $final; + $tmpfile->write($exportTool->footer($exportToolParams)); + return $tmpfile; } - private function __iteratedFetch($user, &$params, &$loop, &$tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0) + private function __iteratedFetch($user, $params, $loop, TmpFileTool $tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0) { - $continue = true; - while ($continue) { - $temp = ''; + $elementCounter = 0; + $separator = $exportTool->separator($exportToolParams); + do { $results = $this->fetchGalaxyClusters($user, $params, $full=$params['full']); if (empty($results)) { - $loop = false; - return true; + break; // nothing found, skip rest } - if ($elementCounter !== 0 && !empty($results)) { - $temp .= $exportTool->separator($exportToolParams); + $resultCount = count($results); + $elementCounter += $resultCount; + foreach ($results as $cluster) { + $handlerResult = $exportTool->handler($cluster, $exportToolParams); + if ($handlerResult !== '') { + $tmpfile->writeWithSeparator($handlerResult, $separator); + } + } + if ($resultCount < $params['limit']) { + break; } $params['page'] += 1; - $i = 0; - foreach ($results as $cluster) { - $elementCounter++; - $handlerResult = $exportTool->handler($cluster, $exportToolParams); - $temp .= $handlerResult; - if ($handlerResult !== '') { - if ($i != count($results) -1) { - $temp .= $exportTool->separator($exportToolParams); - } - } - $i++; - } - if (!$loop) { - $continue = false; - } - fwrite($tmpfile, $temp); - } + } while ($loop); return true; } @@ -1407,9 +1403,9 @@ class GalaxyCluster extends AppModel { $clusterTagNames = []; foreach ($events as $event) { - foreach ($event['EventTag'] as $k2 => $eventTag) { + foreach ($event['EventTag'] as $eventTag) { if ($eventTag['Tag']['is_galaxy']) { - $clusterTagNames[] = strtolower($eventTag['Tag']['name']); + $clusterTagNames[strtolower($eventTag['Tag']['name'])] = true; } } } @@ -1419,7 +1415,7 @@ class GalaxyCluster extends AppModel } $options = [ - 'conditions' => ['LOWER(GalaxyCluster.tag_name)' => $clusterTagNames], + 'conditions' => ['LOWER(GalaxyCluster.tag_name)' => array_keys($clusterTagNames)], ]; if (!$fetchFullCluster) { $options['contain'] = ['Galaxy']; @@ -1564,7 +1560,7 @@ class GalaxyCluster extends AppModel { $this->Server = ClassRegistry::init('Server'); $this->Log = ClassRegistry::init('Log'); - $push = $this->Server->checkVersionCompatibility($server['Server']['id'], false, $HttpSocket); + $push = $this->Server->checkVersionCompatibility($server, false, $HttpSocket); if (empty($push['canPush']) && empty($push['canPushGalaxyCluster'])) { return __('The remote user does not have the permission to manipulate galaxies - the upload of the galaxy clusters has been blocked.'); } @@ -1837,12 +1833,12 @@ class GalaxyCluster extends AppModel * - string pull everything * - string pull updates of cluster present locally * - string pull clusters based on tags present locally - * @return void The number of pulled clusters + * @return int The number of pulled clusters */ public function pullGalaxyClusters(array $user, array $server, $technique = 'full') { $this->Server = ClassRegistry::init('Server'); - $compatible = $this->Server->checkVersionCompatibility($server['Server']['id'], $user)['supportEditOfGalaxyCluster']; + $compatible = $this->Server->checkVersionCompatibility($server, $user)['supportEditOfGalaxyCluster']; if (!$compatible) { return 0; } @@ -1875,12 +1871,11 @@ class GalaxyCluster extends AppModel $clusterIds = $this->Server->getElligibleClusterIdsFromServerForPull($server, $HttpSocket=null, $onlyUpdateLocalCluster=true, $elligibleClusters=$localClustersToUpdate); } elseif ("pull_relevant_clusters" === $technique) { // Fetch all local custom cluster tags then fetch their corresponding clusters on the remote end - $tagNames = $this->Tag->find('list', array( + $tagNames = $this->Tag->find('column', array( 'conditions' => array( 'Tag.is_custom_galaxy' => true ), 'fields' => array('Tag.name'), - 'recursive' => -1, )); $clusterUUIDs = array(); $re = '/^misp-galaxy:[^:="]+="(?[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12})"$/m'; diff --git a/app/Model/Log.php b/app/Model/Log.php index 12d5ba470..1cf7aee90 100644 --- a/app/Model/Log.php +++ b/app/Model/Log.php @@ -336,6 +336,11 @@ class Log extends AppModel $elasticSearchClient->pushDocument($logIndex, "log", $data); } + // Do not save request action logs to syslog, because they contain no information + if ($data['Log']['action'] === 'request') { + return true; + } + // write to syslogd as well if enabled if ($this->syslog === null) { if (Configure::read('Security.syslog')) { diff --git a/app/Model/MispObject.php b/app/Model/MispObject.php index daf1790e9..8d23328e4 100644 --- a/app/Model/MispObject.php +++ b/app/Model/MispObject.php @@ -81,6 +81,8 @@ class MispObject extends AppModel ) ); + private $__objectDuplicationCheckCache = []; + public function buildFilterConditions(&$params) { $conditions = []; @@ -302,32 +304,44 @@ class MispObject extends AppModel { $newObjectAttributes = array(); $existingObjectAttributes = array(); - foreach ($object['Attribute'] as $attribute) { + if (isset($object['Object']['Attribute'])) { + $attributeArray = $object['Object']['Attribute']; + } else { + $attributeArray = $object['Attribute']; + } + foreach ($attributeArray as $attribute) { + if ($attribute['type'] === 'malware-sample') { + if (strpos($attribute['value'], '|') === false && !empty($attribute['data'])) { + $attribute['value'] = $attribute['value'] . '|' . md5(base64_decode($attribute['data'])); + } + } $newObjectAttributes[] = hash( 'sha256', - $attribute['object_relation'] . $attribute['category'] . $attribute['type'] . $attribute['value'] + $attribute['object_relation'] . $attribute['category'] . $attribute['type'] . $this->data['Attribute']['value'] = $this->Attribute->modifyBeforeValidation($attribute['type'], $attribute['value']) ); } $newObjectAttributeCount = count($newObjectAttributes); - $existingObjects = $this->find('all', array( - 'recursive' => -1, - 'contain' => array( - 'Attribute' => array( - 'fields' => array('value', 'type', 'category', 'object_relation'), - 'conditions' => array('Attribute.deleted' => 0) - ) - ), - 'fields' => array('template_uuid'), - 'conditions' => array('template_uuid' => $object['Object']['template_uuid'], 'Object.deleted' => 0) - )); + if (!isset($this->__objectDuplicationCheckCache[$object['Object']['template_uuid']])) { + $this->__objectDuplicationCheckCache[$object['Object']['template_uuid']] = $this->find('all', array( + 'recursive' => -1, + 'contain' => array( + 'Attribute' => array( + 'fields' => array('value', 'type', 'category', 'object_relation'), + 'conditions' => array('Attribute.deleted' => 0) + ) + ), + 'fields' => array('template_uuid'), + 'conditions' => array('template_uuid' => $object['Object']['template_uuid'], 'Object.deleted' => 0, 'event_id' => $eventId) + )); + } $oldObjects = array(); - foreach ($existingObjects as $k => $existingObject) { + foreach ($this->__objectDuplicationCheckCache[$object['Object']['template_uuid']] as $k => $existingObject) { $temp = array(); if (!empty($existingObject['Attribute']) && $newObjectAttributeCount == count($existingObject['Attribute'])) { foreach ($existingObject['Attribute'] as $existingAttribute) { $temp[] = hash( 'sha256', - $attribute['object_relation'] . $existingAttribute['category'] . $existingAttribute['type'] . $existingAttribute['value'] + $existingAttribute['object_relation'] . $existingAttribute['category'] . $existingAttribute['type'] . $existingAttribute['value'] ); } if (empty(array_diff($temp, $newObjectAttributes))) { @@ -884,6 +898,23 @@ class MispObject extends AppModel if (!isset($object['Object'])) { $object = array('Object' => $object); } + if (!empty($object['Object']['breakOnDuplicate'])) { + $duplicate = $this->checkForDuplicateObjects($object, $eventId); + if ($duplicate) { + $log->create(); + $log->save(array( + 'org' => $user['Organisation']['name'], + 'model' => 'Object', + 'model_id' => 0, + 'email' => $user['email'], + 'action' => 'add', + 'user_id' => $user['id'], + 'title' => 'Object dropped due to it being a duplicate and breakOnDuplicate being requested for Event ' . $eventId, + 'change' => 'Duplicate object found.', + )); + return true; + } + } if (empty($log)) { $log = ClassRegistry::init('Log'); } @@ -1443,7 +1474,7 @@ class MispObject extends AppModel } $this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams, $elementCounter); $tmpfile->write($exportTool->footer($exportToolParams)); - return $tmpfile->finish(); + return $tmpfile; } private function __iteratedFetch($user, &$params, &$loop, TmpFileTool $tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0) diff --git a/app/Model/ObjectTemplate.php b/app/Model/ObjectTemplate.php index 2608c01b1..f721d48cb 100644 --- a/app/Model/ObjectTemplate.php +++ b/app/Model/ObjectTemplate.php @@ -31,6 +31,8 @@ class ObjectTemplate extends AppModel public $validate = array( ); + public $objectsDir = APP . 'files/misp-objects/objects'; + public function afterFind($results, $primary = false) { foreach ($results as $k => $result) { @@ -49,10 +51,9 @@ class ObjectTemplate extends AppModel public function update($user = false, $type = false, $force = false) { - $objectsDir = APP . 'files/misp-objects/objects'; - $directories = glob($objectsDir . '/*', GLOB_ONLYDIR); + $directories = $this->getTemplateDirectoryPaths(); foreach ($directories as $k => $dir) { - $dir = str_replace($objectsDir, '', $dir); + $dir = str_replace($this->objectsDir, '', $dir); $directories[$k] = $dir; } $updated = array(); @@ -60,10 +61,10 @@ class ObjectTemplate extends AppModel if ($type && '/' . $type != $dir) { continue; } - if (!file_exists($objectsDir . DS . $dir . DS . 'definition.json')) { + if (!file_exists($this->objectsDir . DS . $dir . DS . 'definition.json')) { continue; } - $file = new File($objectsDir . DS . $dir . DS . 'definition.json'); + $file = new File($this->objectsDir . DS . $dir . DS . 'definition.json'); $template = json_decode($file->read(), true); $file->close(); if (!isset($template['version'])) { @@ -316,4 +317,56 @@ class ObjectTemplate extends AppModel } return 1; } + + public function getRawFromDisk($uuidOrName) + { + $template = []; + if (Validation::uuid($uuidOrName)) { + foreach ($this->readTemplatesFromDisk() as $templateFromDisk) { + if ($templateFromDisk['uuid'] == $uuidOrName) { + $template = $templateFromDisk; + break; + } + } + } else { + $allTemplateNames = $this->getTemplateDirectoryPaths(false); + if (in_array($uuidOrName, $allTemplateNames)) { // ensure the path is not out of scope + $template = $this->readTemplateFromDisk($this->getFullPathFromTemplateName($uuidOrName)); + } + } + return $template; + } + + private function readTemplateFromDisk($path) + { + $file = new File($path, false); + if (!$file->exists()) { + return false; + } + $template = json_decode($file->read(), true); + $file->close(); + return $template; + } + + private function readTemplatesFromDisk() + { + foreach ($this->getTemplateDirectoryPaths() as $dirpath) { + $filepath = $dirpath . DS . 'definition.json'; + $template = $this->readTemplateFromDisk($filepath); + if (isset($template['uuid'])) { + yield $template; + } + } + } + + private function getTemplateDirectoryPaths($fullPath=true) + { + $dir = new Folder($this->objectsDir, false); + return $dir->read(true, false, $fullPath)[0]; + } + + private function getFullPathFromTemplateName($templateName) + { + return $this->objectsDir . DS . $templateName . DS . 'definition.json'; + } } diff --git a/app/Model/Organisation.php b/app/Model/Organisation.php index c6929c344..738e26bfa 100644 --- a/app/Model/Organisation.php +++ b/app/Model/Organisation.php @@ -1,6 +1,10 @@ alias]['id']; } - public function createOrgFromName($name, $user_id, $local) + /** + * @param string $name Organisation name + * @param int $userId Organisation creator + * @param bool $local True if organisation should be marked as local + * @return int Existing or newly created organisation ID + * @throws Exception + */ + public function createOrgFromName($name, $userId, $local) { - $existingOrg = $this->find('first', array( - 'recursive' => -1, - 'conditions' => array('name' => $name) - )); + $existingOrg = $this->find('first', [ + 'recursive' => -1, + 'conditions' => ['name' => $name], + 'fields' => ['id'], + ]); if (empty($existingOrg)) { $this->create(); - $organisation = array( - 'uuid' =>CakeText::uuid(), - 'name' => $name, - 'local' => $local, - 'created_by' => $user_id - ); + $organisation = [ + 'name' => $name, + 'local' => $local, + 'created_by' => $userId, + ]; $this->save($organisation); return $this->id; } @@ -472,6 +483,82 @@ class Organisation extends AppModel return $suggestedOrg; } + /** + * Hide organisation view from users if they haven't yet contributed data and Security.hide_organisation_index_from_users is enabled + * + * @see Organisation::canSee if you want to check multiple orgs + * @param array $user + * @param int $orgId + * @return bool + */ + public function canSee(array $user, $orgId) + { + if ($user['org_id'] == $orgId) { + return true; // User can see his own org. + } + if (!$user['Role']['perm_sharing_group'] && Configure::read('Security.hide_organisation_index_from_users')) { + // Check if there is event from given org that can current user see + $eventConditions = $this->Event->createEventConditions($user); + $eventConditions['AND']['Event.orgc_id'] = $orgId; + $event = $this->Event->find('first', array( + 'fields' => array('Event.id'), + 'recursive' => -1, + 'conditions' => $eventConditions, + )); + if (empty($event)) { + $proposalConditions = $this->Event->ShadowAttribute->buildConditions($user); + $proposalConditions['AND']['ShadowAttribute.org_id'] = $orgId; + $proposal = $this->Event->ShadowAttribute->find('first', array( + 'fields' => array('ShadowAttribute.id'), + 'recursive' => -1, + 'conditions' => $proposalConditions, + 'contain' => ['Event', 'Attribute'], + )); + if (empty($proposal)) { + return false; + } + } + } + return true; + } + + /** + * Create conditions for fetching orgs based on user permission. + * @see Organisation::canSee if you want to check just one org + * @param array $user + * @return array|array[] + */ + public function createConditions(array $user) + { + if (!$user['Role']['perm_sharing_group'] && Configure::read('Security.hide_organisation_index_from_users')) { + $allowedOrgs = [$user['org_id']]; + + $eventConditions = $this->Event->createEventConditions($user); + $orgsWithEvent = $this->Event->find('column', [ + 'fields' => ['Event.orgc_id'], + 'conditions' => $eventConditions, + 'unique' => true, + ]); + $allowedOrgs = array_merge($allowedOrgs, $orgsWithEvent); + + $proposalConditions = $this->Event->ShadowAttribute->buildConditions($user); + // Do not check orgs that we already can see + $proposalConditions['AND'][]['NOT'] = ['ShadowAttribute.org_id' => $allowedOrgs]; + $orgsWithProposal = $this->Event->ShadowAttribute->find('column', [ + 'fields' => ['ShadowAttribute.org_id'], + 'conditions' => $proposalConditions, + 'contain' => ['Event', 'Attribute'], + 'unique' => true, + 'order' => false, + ]); + + $allowedOrgs = array_merge($allowedOrgs, $orgsWithProposal); + return ['AND' => ['id' => $allowedOrgs]]; + } + + return []; + } + private function getCountryGalaxyCluster() { static $list; @@ -507,10 +594,9 @@ class Organisation extends AppModel */ public function getCountries() { - $countries = ['International']; - foreach ($this->getCountryGalaxyCluster() as $country) { - $countries[] = $country['description']; - } + $countries = array_column($this->getCountryGalaxyCluster(), 'description'); + sort($countries); + array_unshift($countries, 'Internation'); return $countries; } } diff --git a/app/Model/Role.php b/app/Model/Role.php index 6600f9a16..c43e0ab3c 100644 --- a/app/Model/Role.php +++ b/app/Model/Role.php @@ -1,8 +1,12 @@ array( 'rule' => array('valueNotEmpty'), @@ -232,6 +236,18 @@ class Role extends AppModel return true; } + public function afterSave($created, $options = array()) + { + // After role change, update `date_modified` field for all user with this role to apply this change to already + // logged users. + if (!$created && !empty($this->data)) { + $roleId = $this->data['Role']['id']; + $this->User->updateAll(['date_modified' => time()], ['role_id' => $roleId]); + } + + parent::afterSave($created, $options); + } + public function afterFind($results, $primary = false) { foreach ($results as $key => $val) { diff --git a/app/Model/Server.php b/app/Model/Server.php index ed59b5bd3..16e1d6cae 100644 --- a/app/Model/Server.php +++ b/app/Model/Server.php @@ -2,6 +2,9 @@ App::uses('AppModel', 'Model'); App::uses('GpgTool', 'Tools'); +/** + * @property-read array $serverSettings + */ class Server extends AppModel { const SETTING_CRITICAL = 0, @@ -180,2349 +183,6 @@ class Server extends AppModel 'header' => __('Managing the background workers') ) ); - - $this->serverSettings = array( - 'MISP' => array( - 'branch' => 1, - 'baseurl' => array( - 'level' => 0, - 'description' => __('The base url of the application (in the format https://www.mymispinstance.com or https://myserver.com/misp). Several features depend on this setting being correctly set to function.'), - 'value' => '', - 'errorMessage' => __('The currenty set baseurl does not match the URL through which you have accessed the page. Disregard this if you are accessing the page via an alternate URL (for example via IP address).'), - 'test' => 'testBaseURL', - 'type' => 'string', - ), - 'external_baseurl' => array( - 'level' => 0, - 'description' => __('The base url of the application (in the format https://www.mymispinstance.com) as visible externally/by other MISPs. MISP will encode this URL in sharing groups when including itself. If this value is not set, the baseurl is used as a fallback.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testURL', - 'type' => 'string', - ), - 'live' => array( - 'level' => 0, - 'description' => __('Unless set to true, the instance will only be accessible by site admins.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testLive', - 'type' => 'boolean', - ), - 'language' => array( - 'level' => 0, - 'description' => __('Select the language MISP should use. The default is english.'), - 'value' => 'eng', - 'errorMessage' => '', - 'test' => 'testLanguage', - 'type' => 'string', - 'optionsSource' => 'AvailableLanguages', - 'afterHook' => 'cleanCacheFiles' - ), - 'default_attribute_memory_coefficient' => array( - 'level' => 1, - 'description' => __('This values controls the internal fetcher\'s memory envelope when it comes to attributes. The number provided is the amount of attributes that can be loaded for each MB of PHP memory available in one shot. Consider lowering this number if your instance has a lot of attribute tags / attribute galaxies attached.'), - 'value' => 80, - 'errorMessage' => '', - 'test' => 'testForNumeric', - 'type' => 'numeric', - 'null' => true - ), - 'default_event_memory_divisor' => array( - 'level' => 1, - 'description' => __('This value controls the divisor for attribute weighting when it comes to loading full events. Meaning that it will load coefficient / divisor number of attributes per MB of memory available. Consider raising this number if you have a lot of correlations or highly contextualised events (large number of event level galaxies/tags).'), - 'value' => 3, - 'errorMessage' => '', - 'test' => 'testForNumeric', - 'type' => 'numeric', - 'null' => true - ), - 'enable_advanced_correlations' => array( - 'level' => 0, - 'description' => __('Enable some performance heavy correlations (currently CIDR correlation)'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'server_settings_skip_backup_rotate' => array( - 'level' => 1, - 'description' => __('Enable this setting to directly save the config.php file without first creating a temporary file and moving it to avoid concurency issues. Generally not recommended, but useful when for example other tools modify/maintain the config.php file.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'python_bin' => array( - 'level' => 1, - 'description' => __('It is highly recommended to install all the python dependencies in a virtualenv. The recommended location is: %s/venv', ROOT), - 'value' => false, - 'errorMessage' => '', - 'null' => false, - 'test' => 'testForBinExec', - 'beforeHook' => 'beforeHookBinExec', - 'type' => 'string', - 'cli_only' => 1 - ), - 'ca_path' => array( - 'level' => 1, - 'description' => __('MISP will default to the bundled mozilla certificate bundle shipped with the framework, which is rather stale. If you wish to use an alternate bundle, just set this setting using the path to the bundle to use. This setting can only be modified via the CLI.'), - 'value' => APP . 'Lib/cakephp/lib/Cake/Config/cacert.pem', - 'errorMessage' => '', - 'null' => true, - 'test' => 'testForCABundle', - 'type' => 'string', - 'cli_only' => 1 - ), - 'disable_auto_logout' => array( - 'level' => 1, - 'description' => __('In some cases, a heavily used MISP instance can generate unwanted blackhole errors due to a high number of requests hitting the server. Disable the auto logout functionality to ease the burden on the system.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'ssdeep_correlation_threshold' => array( - 'level' => 1, - 'description' => __('Set the ssdeep score at which to consider two ssdeep hashes as correlating [1-100]'), - 'value' => 40, - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'numeric' - ), - 'max_correlations_per_event' => array( - 'level' => 1, - 'description' => __('Sets the maximum number of correlations that can be fetched with a single event. For extreme edge cases this can prevent memory issues. The default value is 5k.'), - 'value' => 5000, - 'errorMessage' => '', - 'test' => 'testForNumeric', - 'type' => 'numeric', - 'null' => true - ), - 'maintenance_message' => array( - 'level' => 2, - 'description' => __('The message that users will see if the instance is not live.'), - 'value' => 'Great things are happening! MISP is undergoing maintenance, but will return shortly. You can contact the administration at $email.', - 'errorMessage' => __('If this is not set the default value will be used.'), - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'name' => array( - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'version' => array( - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'disable_cached_exports' => array( - 'level' => 1, - 'description' => __('Cached exports can take up a considerable amount of space and can be disabled instance wide using this setting. Disabling the cached exports is not recommended as it\'s a valuable feature, however, if your server is having free space issues it might make sense to take this step.'), - 'value' => false, - 'null' => true, - 'errorMessage' => '', - 'test' => 'testDisableCache', - 'type' => 'boolean', - 'afterHook' => 'disableCacheAfterHook', - ), - 'disable_threat_level' => array( - 'level' => 1, - 'description' => __('Disable displaying / modifications to the threat level altogether on the instance (deprecated field).'), - 'value' => false, - 'null' => true, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'header' => array( - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'footermidleft' => array( - 'level' => 2, - 'description' => __('Footer text prepending the "Powered by MISP" text.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'footermidright' => array( - 'level' => 2, - 'description' => __('Footer text following the "Powered by MISP" text.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'footerpart1' => array( - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'footerpart2' => array( - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'footer' => array( - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'footerversion' => array( - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'footer_logo' => array( - 'level' => 2 , - 'description' => __('If set, this setting allows you to display a logo on the right side of the footer. Upload it as a custom image in the file management tool.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForCustomImage', - 'type' => 'string', - ), - 'home_logo' => array( - 'level' => 2 , - 'description' => __('If set, this setting allows you to display a logo as the home icon. Upload it as a custom image in the file management tool.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForCustomImage', - 'type' => 'string', - ), - 'main_logo' => array( - 'level' => 2 , - 'description' => __('If set, the image specified here will replace the main MISP logo on the login screen. Upload it as a custom image in the file management tool.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForCustomImage', - 'type' => 'string', - ), - 'org' => array( - 'level' => 1, - 'description' => __('The organisation tag of the hosting organisation. This is used in the e-mail subjects.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'host_org_id' => array( - 'level' => 0, - 'description' => __('The hosting organisation of this instance. If this is not selected then replication instances cannot be added.'), - 'value' => '0', - 'errorMessage' => '', - 'test' => 'testLocalOrg', - 'type' => 'numeric', - 'optionsSource' => 'LocalOrgs', - ), - 'uuid' => array( - 'level' => 0, - 'description' => __('The MISP instance UUID. This UUID is used to identify this instance.'), - 'value' => '0', - 'errorMessage' => __('No valid UUID set'), - 'test' => 'testUuid', - 'type' => 'string' - ), - 'logo' => array( - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'showorg' => array( - 'level' => 0, - 'description' => __('Setting this setting to \'false\' will hide all organisation names / logos.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ), - 'threatlevel_in_email_subject' => array( - 'level' => 2, - 'description' => __('Put the event threat level in the notification E-mail subject.'), - 'value' => true, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ), - 'email_subject_TLP_string' => array( - 'level' => 2, - 'description' => __('This is the TLP string for e-mails when email_subject_tag is not found.'), - 'value' => 'tlp:amber', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'email_subject_tag' => array( - 'level' => 2, - 'description' => __('If this tag is set on an event it\'s value will be sent in the E-mail subject. If the tag is not set the email_subject_TLP_string will be used.'), - 'value' => 'tlp', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'email_subject_include_tag_name' => array( - 'level' => 2, - 'description' => __('Include in name of the email_subject_tag in the subject. When false only the tag value is used.'), - 'value' => true, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ), - 'taxii_sync' => array( - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'taxii_client_path' => array( - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'background_jobs' => array( - 'level' => 1, - 'description' => __('Enables the use of MISP\'s background processing.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testBoolTrue', - 'type' => 'boolean', - ), - 'attachments_dir' => array( - 'level' => 2, - 'description' => __('Directory where attachments are stored. MISP will NOT migrate the existing data if you change this setting. The only safe way to change this setting is in config.php, when MISP is not running, and after having moved/copied the existing data to the new location. This directory must already exist and be writable and readable by the MISP application.'), - 'value' => APP . '/files', # GUI display purpose only. - 'errorMessage' => '', - 'null' => false, - 'test' => 'testForWritableDir', - 'type' => 'string', - 'cli_only' => 1 - ), - 'cached_attachments' => array( - 'level' => 1, - 'description' => __('Allow the XML caches to include the encoded attachments.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ), - 'download_attachments_on_load' => array( - 'level' => 2, - 'description' => __('Always download attachments when loaded by a user in a browser'), - 'value' => true, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ), - 'osuser' => array( - 'level' => 0, - 'description' => __('The Unix user MISP (php) is running as'), - 'value' => 'www-data', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'email' => array( - 'level' => 0, - 'description' => __('The e-mail address that MISP should use for all notifications'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'disable_emailing' => array( - 'level' => 0, - 'description' => __('You can disable all e-mailing using this setting. When enabled, no outgoing e-mails will be sent by MISP.'), - 'value' => false, - 'errorMessage' => '', - 'null' => true, - 'test' => 'testDisableEmail', - 'type' => 'boolean', - ), - 'contact' => array( - 'level' => 1, - 'description' => __('The e-mail address that MISP should include as a contact address for the instance\'s support team.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'dns' => array( - 'level' => 3, - 'description' => __('This setting is deprecated and can be safely removed.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'cveurl' => array( - 'level' => 1, - 'description' => __('Turn Vulnerability type attributes into links linking to the provided CVE lookup'), - 'value' => 'http://cve.circl.lu/cve/', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'cweurl' => array( - 'level' => 1, - 'description' => __('Turn Weakness type attributes into links linking to the provided CWE lookup'), - 'value' => 'http://cve.circl.lu/cwe/', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'disablerestalert' => array( - 'level' => 1, - 'description' => __('This setting controls whether notification e-mails will be sent when an event is created via the REST interface. It might be a good idea to disable this setting when first setting up a link to another instance to avoid spamming your users during the initial pull. Quick recap: True = Emails are NOT sent, False = Emails are sent on events published via sync / REST.'), - 'value' => true, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ), - 'extended_alert_subject' => array( - 'level' => 1, - 'description' => __('enabling this flag will allow the event description to be transmitted in the alert e-mail\'s subject. Be aware that this is not encrypted by GnuPG, so only enable it if you accept that part of the event description will be sent out in clear-text.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'default_event_distribution' => array( - 'level' => 0, - 'description' => __('The default distribution setting for events (0-3).'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'options' => array('0' => 'Your organisation only', '1' => 'This community only', '2' => 'Connected communities', '3' => 'All communities'), - ), - 'default_attribute_distribution' => array( - 'level' => 0, - 'description' => __('The default distribution setting for attributes, set it to \'event\' if you would like the attributes to default to the event distribution level. (0-3 or "event")'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'options' => array('0' => 'Your organisation only', '1' => 'This community only', '2' => 'Connected communities', '3' => 'All communities', 'event' => 'Inherit from event'), - ), - 'default_event_threat_level' => array( - 'level' => 1, - 'description' => __('The default threat level setting when creating events.'), - 'value' => 4, - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'options' => array('1' => 'High', '2' => 'Medium', '3' => 'Low', '4' => 'undefined'), - ), - 'default_event_tag_collection' => array( - 'level' => 0, - 'description' => __('The tag collection to be applied to all events created manually.'), - 'value' => 0, - 'errorMessage' => '', - 'test' => 'testTagCollections', - 'type' => 'numeric', - 'optionsSource' => 'TagCollections', - ), - 'default_publish_alert' => array( - 'level' => 0, - 'description' => __('The default setting for publish alerts when creating users.'), - 'value' => true, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'tagging' => array( - 'level' => 1, - 'description' => __('Enable the tagging feature of MISP. This is highly recommended.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ), - 'full_tags_on_event_index' => array( - 'level' => 2, - 'description' => __('Show the full tag names on the event index.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'options' => array(0 => 'Minimal tags', 1 => 'Full tags', 2 => 'Shortened tags'), - ), - 'welcome_text_top' => array( - 'level' => 2, - 'description' => __('Used on the login page, before the MISP logo'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'welcome_text_bottom' => array( - 'level' => 2, - 'description' => __('Used on the login page, after the MISP logo'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'welcome_logo' => array( - 'level' => 2, - 'description' => __('Used on the login page, to the left of the MISP logo, upload it as a custom image in the file management tool.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForCustomImage', - 'type' => 'string', - ), - 'welcome_logo2' => array( - 'level' => 2, - 'description' => __('Used on the login page, to the right of the MISP logo, upload it as a custom image in the file management tool.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForCustomImage', - 'type' => 'string', - ), - 'title_text' => array( - 'level' => 2, - 'description' => __('Used in the page title, after the name of the page'), - 'value' => 'MISP', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'take_ownership_xml_import' => array( - 'level' => 2, - 'description' => __('Allows users to take ownership of an event uploaded via the "Add MISP XML" button. This allows spoofing the creator of a manually imported event, also breaking possibly breaking the original intended releasability. Synchronising with an instance that has a different creator for the same event can lead to unwanted consequences.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ), - 'terms_download' => array( - 'level' => 2, - 'description' => __('Choose whether the terms and conditions should be displayed inline (false) or offered as a download (true)'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'terms_file' => array( - 'level' => 2, - 'description' => __('The filename of the terms and conditions file. Make sure that the file is located in your MISP/app/files/terms directory'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForTermsFile', - 'type' => 'string' - ), - 'showorgalternate' => array( - 'level' => 2, - 'description' => __('True enables the alternate org fields for the event index (source org and member org) instead of the traditional way of showing only an org field. This allows users to see if an event was uploaded by a member organisation on their MISP instance, or if it originated on an interconnected instance.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'unpublishedprivate' => array( - 'level' => 2, - 'description' => __('True will deny access to unpublished events to users outside the organization of the submitter except site admins.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'newUserText' => array( - 'level' => 1, - 'bigField' => true, - 'description' => __('The message sent to the user after account creation (has to be sent manually from the administration interface). Use \\n for line-breaks. The following variables will be automatically replaced in the text: $password = a new temporary password that MISP generates, $username = the user\'s e-mail address, $misp = the url of this instance, $org = the organisation that the instance belongs to, as set in MISP.org, $contact = the e-mail address used to contact the support team, as set in MISP.contact. For example, "the password for $username is $password" would appear to a user with the e-mail address user@misp.org as "the password for user@misp.org is hNamJae81".'), - 'value' => 'Dear new MISP user,\n\nWe would hereby like to welcome you to the $org MISP community.\n\n Use the credentials below to log into MISP at $misp, where you will be prompted to manually change your password to something of your own choice.\n\nUsername: $username\nPassword: $password\n\nIf you have any questions, don\'t hesitate to contact us at: $contact.\n\nBest regards,\nYour $org MISP support team', - 'errorMessage' => '', - 'test' => 'testPasswordResetText', - 'type' => 'string' - ), - 'passwordResetText' => array( - 'level' => 1, - 'bigField' => true, - 'description' => __('The message sent to the users when a password reset is triggered. Use \\n for line-breaks. The following variables will be automatically replaced in the text: $password = a new temporary password that MISP generates, $username = the user\'s e-mail address, $misp = the url of this instance, $contact = the e-mail address used to contact the support team, as set in MISP.contact. For example, "the password for $username is $password" would appear to a user with the e-mail address user@misp.org as "the password for user@misp.org is hNamJae81".'), - 'value' => 'Dear MISP user,\n\nA password reset has been triggered for your account. Use the below provided temporary password to log into MISP at $misp, where you will be prompted to manually change your password to something of your own choice.\n\nUsername: $username\nYour temporary password: $password\n\nIf you have any questions, don\'t hesitate to contact us at: $contact.\n\nBest regards,\nYour $org MISP support team', - 'errorMessage' => '', - 'test' => 'testPasswordResetText', - 'type' => 'string' - ), - 'enableEventBlocklisting' => array( - 'level' => 1, - 'description' => __('Since version 2.3.107 you can start blocklisting event UUIDs to prevent them from being pushed to your instance. This functionality will also happen silently whenever an event is deleted, preventing a deleted event from being pushed back from another instance.'), - 'value' => true, - 'type' => 'boolean', - 'test' => 'testBool' - ), - 'enableOrgBlocklisting' => array( - 'level' => 1, - 'description' => __('Blocklisting organisation UUIDs to prevent the creation of any event created by the blocklisted organisation.'), - 'value' => true, - 'type' => 'boolean', - 'test' => 'testBool' - ), - 'log_client_ip' => array( - 'level' => 1, - 'description' => __('If enabled, all log entries will include the IP address of the user.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'beforeHook' => 'ipLogBeforeHook' - ), - 'log_client_ip_header' => array( - 'level' => 1, - 'description' => __('If log_client_ip is enabled, you can customize which header field contains the client\'s IP address. This is generally used when you have a reverse proxy infront of your MISP instance.'), - 'value' => 'REMOTE_ADDR', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true, - ), - 'log_auth' => array( - 'level' => 1, - 'description' => __('If enabled, MISP will log all successful authentications using API keys. The requested URLs are also logged.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ), - 'log_skip_db_logs_completely' => array( - 'level' => 0, - 'description' => __('This functionality allows you to completely disable any logs from being saved in your SQL backend. This is HIGHLY advised against, you lose all the functionalities provided by the audit log subsystem along with the event history (as these are built based on the logs on the fly). Only enable this if you understand and accept the associated risks.'), - 'value' => false, - 'errorMessage' => __('Logging has now been disabled - your audit logs will not capture failed authentication attempts, your event history logs are not being populated and no system maintenance messages are being logged.'), - 'test' => 'testBoolFalse', - 'type' => 'boolean', - 'null' => true - ), - 'log_paranoid' => array( - 'level' => 0, - 'description' => __('If this functionality is enabled all page requests will be logged. Keep in mind this is extremely verbose and will become a burden to your database.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBoolFalse', - 'type' => 'boolean', - 'null' => true - ), - 'log_paranoid_skip_db' => array( - 'level' => 0, - 'description' => __('You can decide to skip the logging of the paranoid logs to the database.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testParanoidSkipDb', - 'type' => 'boolean', - 'null' => true - ), - 'log_paranoid_include_post_body' => array( - 'level' => 0, - 'description' => __('If paranoid logging is enabled, include the POST body in the entries.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'log_user_ips' => array( - 'level' => 0, - 'description' => __('Log user IPs on each request. 30 day retention for lookups by IP to get the last authenticated user ID for the given IP, whilst on the reverse, indefinitely stores all associated IPs for a user ID.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'delegation' => array( - 'level' => 1, - 'description' => __('This feature allows users to create org only events and ask another organisation to take ownership of the event. This allows organisations to remain anonymous by asking a partner to publish an event for them.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'showCorrelationsOnIndex' => array( - 'level' => 1, - 'description' => __('When enabled, the number of correlations visible to the currently logged in user will be visible on the event index UI. This comes at a performance cost but can be very useful to see correlating events at a glance.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'showProposalsCountOnIndex' => array( - 'level' => 1, - 'description' => __('When enabled, the number of proposals for the events are shown on the index.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'showSightingsCountOnIndex' => array( - 'level' => 1, - 'description' => __('When enabled, the aggregate number of attribute sightings within the event becomes visible to the currently logged in user on the event index UI.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'showDiscussionsCountOnIndex' => array( - 'level' => 1, - 'description' => __('When enabled, the aggregate number of discussion posts for the event becomes visible to the currently logged in user on the event index UI.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'disableUserSelfManagement' => array( - 'level' => 1, - 'description' => __('When enabled only Org and Site admins can edit a user\'s profile.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => false, - ), - 'disable_user_login_change' => array( - 'level' => self::SETTING_RECOMMENDED, - 'description' => __('When enabled only Site admins can change user email. This should be enabled if you manage user logins by external system.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => false, - ), - 'disable_user_password_change' => array( - 'level' => self::SETTING_RECOMMENDED, - 'description' => __('When enabled only Site admins can change user password. This should be enabled if you manage user passwords by external system.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => false, - ), - 'disable_user_add' => array( - 'level' => self::SETTING_RECOMMENDED, - 'description' => __('When enabled, Org Admins could not add new users. This should be enabled if you manage users by external system.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => false, - ), - 'block_event_alert' => array( - 'level' => 1, - 'description' => __('Enable this setting to start blocking alert e-mails for events with a certain tag. Define the tag in MISP.block_event_alert_tag.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => false, - ), - 'block_event_alert_tag' => array( - 'level' => 1, - 'description' => __('If the MISP.block_event_alert setting is set, alert e-mails for events tagged with the tag defined by this setting will be blocked.'), - 'value' => 'no-alerts="true"', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => false, - ), - 'org_alert_threshold' => array( - 'level' => 1, - 'description' => __('Set a value to limit the number of email alerts that events can generate per creator organisation (for example, if an organisation pushes out 2000 events in one shot, only alert on the first 20).'), - 'value' => 0, - 'errorMessage' => '', - 'test' => 'testForNumeric', - 'type' => 'numeric', - 'null' => true, - ), - 'block_old_event_alert' => array( - 'level' => 1, - 'description' => __('Enable this setting to start blocking alert e-mails for old events. The exact timing of what constitutes an old event is defined by MISP.block_old_event_alert_age.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => false, - ), - 'block_old_event_alert_age' => array( - 'level' => 1, - 'description' => __('If the MISP.block_old_event_alert setting is set, this setting will control how old an event can be for it to be alerted on. The "timestamp" field of the event is used. Expected format: integer, in days'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testForNumeric', - 'type' => 'numeric', - 'null' => false, - ), - 'block_old_event_alert_by_date' => array( - 'level' => 1, - 'description' => __('If the MISP.block_old_event_alert setting is set, this setting will control the threshold for the event.date field, indicating how old an event can be for it to be alerted on. The "date" field of the event is used. Expected format: integer, in days'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testForNumeric', - 'type' => 'numeric', - 'null' => false, - ), - 'tmpdir' => array( - 'level' => 1, - 'description' => __('Please indicate the temp directory you wish to use for certain functionalities in MISP. By default this is set to /tmp and will be used among others to store certain temporary files extracted from imports during the import process.'), - 'value' => '/tmp', - 'errorMessage' => '', - 'test' => 'testForPath', - 'type' => 'string', - 'null' => true, - 'cli_only' => 1 - ), - 'custom_css' => array( - 'level' => 2, - 'description' => __('If you would like to customise the css, simply drop a css file in the /var/www/MISP/app/webroot/css directory and enter the name here.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForStyleFile', - 'type' => 'string', - 'null' => true, - ), - 'proposals_block_attributes' => array( - 'level' => 0, - 'description' => __('Enable this setting to allow blocking attributes from to_ids sensitive exports if a proposal has been made to it to remove the IDS flag or to remove the attribute altogether. This is a powerful tool to deal with false-positives efficiently.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => false, - ), - 'incoming_tags_disabled_by_default' => array( - 'level' => 1, - 'description' => __('Enable this settings if new tags synced / added via incoming events from any source should not be selectable by users by default.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => false - ), - 'completely_disable_correlation' => array( - 'level' => 0, - 'description' => __('*WARNING* This setting will completely disable the correlation on this instance and remove any existing saved correlations. Enabling this will trigger a full recorrelation of all data which is an extremely long and costly procedure. Only enable this if you know what you\'re doing.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBoolFalse', - 'type' => 'boolean', - 'null' => true, - 'afterHook' => 'correlationAfterHook', - ), - 'allow_disabling_correlation' => array( - 'level' => 0, - 'description' => __('*WARNING* This setting will give event creators the possibility to disable the correlation of individual events / attributes that they have created.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBoolFalse', - 'type' => 'boolean', - 'null' => true - ), - 'redis_host' => array( - 'level' => 0, - 'description' => __('The host running the redis server to be used for generic MISP tasks such as caching. This is not to be confused by the redis server used by the background processing.'), - 'value' => '127.0.0.1', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'redis_port' => array( - 'level' => 0, - 'description' => __('The port used by the redis server to be used for generic MISP tasks such as caching. This is not to be confused by the redis server used by the background processing.'), - 'value' => 6379, - 'errorMessage' => '', - 'test' => 'testForNumeric', - 'type' => 'numeric' - ), - 'redis_database' => array( - 'level' => 0, - 'description' => __('The database on the redis server to be used for generic MISP tasks. If you run more than one MISP instance, please make sure to use a different database on each instance.'), - 'value' => 13, - 'errorMessage' => '', - 'test' => 'testForNumeric', - 'type' => 'numeric' - ), - 'redis_password' => array( - 'level' => 0, - 'description' => __('The password on the redis server (if any) to be used for generic MISP tasks.'), - 'value' => '', - 'errorMessage' => '', - 'test' => null, - 'type' => 'string', - 'redacted' => true - ), - 'event_view_filter_fields' => array( - 'level' => 2, - 'description' => __('Specify which fields to filter on when you search on the event view. Default values are : "id, uuid, value, comment, type, category, Tag.name"'), - 'value' => 'id, uuid, value, comment, type, category, Tag.name', - 'errorMessage' => '', - 'test' => null, - 'type' => 'string', - ), - 'manage_workers' => array( - 'level' => 2, - 'description' => __('Set this to false if you would like to disable MISP managing its own worker processes (for example, if you are managing the workers with a systemd unit).'), - 'value' => true, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'deadlock_avoidance' => array( - 'level' => 1, - 'description' => __('Only enable this if you have some tools using MISP with extreme high concurency. General performance will be lower as normal as certain transactional queries are avoided in favour of shorter table locks.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'updateTimeThreshold' => array( - 'level' => 1, - 'description' => __('Sets the minimum time before being able to re-trigger an update if the previous one failed. (safe guard to avoid starting the same update multiple time)'), - 'value' => '7200', - 'test' => 'testForNumeric', - 'type' => 'numeric', - 'null' => true - ), - 'attribute_filters_block_only' => array( - 'level' => 1, - 'description' => __('This is a performance tweak to change the behaviour of restSearch to use attribute filters solely for blocking. This means that a lookup on the event scope with for example the type field set will be ignored unless it\'s used to strip unwanted attributes from the results. If left disabled, passing [ip-src, ip-dst] for example will return any event with at least one ip-src or ip-dst attribute. This is generally not considered to be too useful and is a heavy burden on the database.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'attachment_scan_module' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Name of enrichment module that will be used for attachment malware scanning. This module must return av-signature or sb-signature object.'), - 'value' => '', - 'errorMessage' => '', - 'type' => 'string', - 'null' => true, - ], - 'attachment_scan_hash_only' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Send to attachment scan module just file hash. This can be useful if module sends attachment to remote service and you don\'t want to leak real data.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true, - ], - 'attachment_scan_timeout' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('How long to wait for scan results in seconds.'), - 'value' => 30, - 'errorMessage' => '', - 'test' => 'testForPositiveInteger', - 'type' => 'numeric', - 'null' => true, - ] - ), - 'GnuPG' => array( - 'branch' => 1, - 'binary' => array( - 'level' => 2, - 'description' => __('The location of the GnuPG executable. If you would like to use a different GnuPG executable than /usr/bin/gpg, you can set it here. If the default is fine, just keep the setting suggested by MISP.'), - 'value' => '/usr/bin/gpg', - 'errorMessage' => '', - 'test' => 'testForGPGBinary', - 'type' => 'string', - 'cli_only' => 1 - ), - 'onlyencrypted' => array( - 'level' => 0, - 'description' => __('Allow (false) unencrypted e-mails to be sent to users that don\'t have a GnuPG key.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ), - 'bodyonlyencrypted' => array( - 'level' => 2, - 'description' => __('Allow (false) the body of unencrypted e-mails to contain details about the event.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ), - 'sign' => array( - 'level' => 2, - 'description' => __('Enable the signing of GnuPG emails. By default, GnuPG emails are signed'), - 'value' => true, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ), - 'email' => array( - 'level' => 0, - 'description' => __('The e-mail address that the instance\'s GnuPG key is tied to.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'password' => array( - 'level' => 1, - 'description' => __('The password (if it is set) of the GnuPG key of the instance.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'redacted' => true - ), - 'homedir' => array( - 'level' => 0, - 'description' => __('The location of the GnuPG homedir.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'obscure_subject' => array( - 'level' => 2, - 'description' => __('When enabled, subject in signed and encrypted e-mails will not send in unencrypted form.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ) - ), - 'SMIME' => array( - 'branch' => 1, - 'enabled' => array( - 'level' => 2, - 'description' => __('Enable SMIME encryption. The encryption posture of the GnuPG.onlyencrypted and GnuPG.bodyonlyencrypted settings are inherited if SMIME is enabled.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ), - 'email' => array( - 'level' => 2, - 'description' => __('The e-mail address that the instance\'s SMIME key is tied to.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'cert_public_sign' => array( - 'level' => 2, - 'description' => __('The location of the public half of the signing certificate.'), - 'value' => '/var/www/MISP/.smime/email@address.com.pem', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'key_sign' => array( - 'level' => 2, - 'description' => __('The location of the private half of the signing certificate.'), - 'value' => '/var/www/MISP/.smime/email@address.com.key', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'password' => array( - 'level' => 2, - 'description' => __('The password (if it is set) of the SMIME key of the instance.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'redacted' => true - ), - ), - 'Proxy' => array( - 'branch' => 1, - 'host' => array( - 'level' => 2, - 'description' => __('The hostname of an HTTP proxy for outgoing sync requests. Leave empty to not use a proxy.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'port' => array( - 'level' => 2, - 'description' => __('The TCP port for the HTTP proxy.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForNumeric', - 'type' => 'numeric', - ), - 'method' => array( - 'level' => 2, - 'description' => __('The authentication method for the HTTP proxy. Currently supported are Basic or Digest. Leave empty for no proxy authentication.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'user' => array( - 'level' => 2, - 'description' => __('The authentication username for the HTTP proxy.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'password' => array( - 'level' => 2, - 'description' => __('The authentication password for the HTTP proxy.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - ), - 'Security' => array( - 'branch' => 1, - 'disable_form_security' => array( - 'level' => 0, - 'description' => __('Disabling this setting will remove all form tampering protection. Do not set this setting pretty much ever. You were warned.'), - 'value' => false, - 'errorMessage' => 'This setting leaves your users open to CSRF attacks. Do not please consider disabling this setting.', - 'test' => 'testBoolFalse', - 'type' => 'boolean', - 'null' => true - ), - 'salt' => array( - 'level' => 0, - 'description' => __('The salt used for the hashed passwords. You cannot reset this from the GUI, only manually from the settings.php file. Keep in mind, this will invalidate all passwords in the database.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testSalt', - 'type' => 'string', - 'editable' => false, - 'redacted' => true - ), - 'advanced_authkeys' => array( - 'level' => 0, - 'description' => __('Advanced authkeys will allow each user to create and manage a set of authkeys for themselves, each with individual expirations and comments. API keys are stored in a hashed state and can no longer be recovered from MISP. Users will be prompted to note down their key when creating a new authkey. You can generate a new set of API keys for all users on demand in the diagnostics page, or by triggering %s.', sprintf('%s', $this->baseurl, __('the advanced upgrade'))), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ), - 'auth_enforced' => [ - 'level' => self::SETTING_OPTIONAL, - 'description' => __('This optional can be enabled if external auth provider is used and when set to true, it will disable default form authentication.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ], - 'rest_client_enable_arbitrary_urls' => array( - 'level' => 0, - 'description' => __('Enable this setting if you wish for users to be able to query any arbitrary URL via the rest client. Keep in mind that queries are executed by the MISP server, so internal IPs in your MISP\'s network may be reachable.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'rest_client_baseurl' => array( - 'level' => 1, - 'description' => __('If left empty, the baseurl of your MISP is used. However, in some instances (such as port-forwarded VM installations) this will not work. You can override the baseurl with a url through which your MISP can reach itself (typically https://127.0.0.1 would work).'), - 'value' => false, - 'errorMessage' => '', - 'test' => null, - 'type' => 'string' - ), - 'syslog' => array( - 'level' => 0, - 'description' => __('Enable this setting to pass all audit log entries directly to syslog. Keep in mind, this is verbose and will include user, organisation, event data.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'syslog_to_stderr' => array( - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Write syslog messages also to standard error output.'), - 'value' => true, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'syslog_ident' => array( - 'level' => self::SETTING_OPTIONAL, - 'description' => __('Syslog message identifier.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ), - 'do_not_log_authkeys' => array( - 'level' => 0, - 'description' => __('If enabled, any authkey will be replaced by asterisks in Audit log.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'disable_browser_cache' => array( - 'level' => 0, - 'description' => __('If enabled, HTTP headers that block browser cache will be send. Static files (like images or JavaScripts) will still be cached, but not generated pages.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true, - ), - 'check_sec_fetch_site_header' => [ - 'level' => 0, - 'description' => __('If enabled, any POST, PUT or AJAX request will be allow just when Sec-Fetch-Site header is not defined or contains "same-origin".'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true, - ], - 'email_otp_enabled' => array( - 'level'=> 2, - 'description' => __('Enable two step authentication with a OTP sent by email. Requires e-mailing to be enabled. Warning: You cannot use it in combination with external authentication plugins.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'beforeHook' => 'otpBeforeHook', - 'type' => 'boolean', - 'null' => true - ), - 'email_otp_length' => array ( - 'level' => 2, - 'description' => __('Define the length of the OTP code sent by email'), - 'value' => '6', - 'errorMessage' => '', - 'type' => 'numeric', - 'test' => 'testForNumeric', - 'null' => true, - ), - 'email_otp_validity' => array ( - 'level' => 2, - 'description' => __('Define the validity (in minutes) of the OTP code sent by email'), - 'value' => '5', - 'errorMessage' => '', - 'type' => 'numeric', - 'test' => 'testForNumeric', - 'null' => true, - ), - 'email_otp_text' => array( - 'level' => 2, - 'bigField' => true, - 'description' => __('The message sent to the user when a new OTP is requested. Use \\n for line-breaks. The following variables will be automatically replaced in the text: $otp = the new OTP generated by MISP, $username = the user\'s e-mail address, $org the Organisation managing the instance, $misp = the url of this instance, $contact = the e-mail address used to contact the support team (as set in MISP.contact), $ip the IP used to complete the first step of the login and $validity the validity time in minutes.'), - 'value' => 'Dear MISP user,\n\nYou have attempted to login to MISP ($misp) from $ip with username $username.\n\n Use the following OTP to log into MISP: $otp\n This code is valid for the next $validity minutes.\n\nIf you have any questions, don\'t hesitate to contact us at: $contact.\n\nBest regards,\nYour $org MISP support team', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true, - ), - 'email_otp_exceptions' => array( - 'level' => 2, - 'bigField' => true, - 'description' => __('A comma separated list of emails for which the OTP is disabled. Note that if you remove someone from this list, the OTP will only be asked at next login.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true, - ), - 'allow_self_registration' => array( - 'level' => 1, - 'description' => __('Enabling this setting will allow users to have access to the pre-auth registration form. This will create an inbox entry for administrators to review.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'self_registration_message' => array( - 'level' => 1, - 'bigField' => true, - 'description' => __('The message sent shown to anyone trying to self-register.'), - 'value' => 'If you would like to send us a registration request, please fill out the form below. Make sure you fill out as much information as possible in order to ease the task of the administrators.', - 'errorMessage' => '', - 'test' => false, - 'type' => 'string' - ), - 'password_policy_length' => array( - 'level' => 2, - 'description' => __('Password length requirement. If it is not set or it is set to 0, then the default value is assumed (12).'), - 'value' => '12', - 'errorMessage' => '', - 'test' => 'testPasswordLength', - 'type' => 'numeric', - ), - 'password_policy_complexity' => array( - 'level' => 2, - 'description' => __('Password complexity requirement. Leave it empty for the default setting (3 out of 4, with either a digit or a special char) or enter your own regex. Keep in mind that the length is checked in another key. Default (simple 3 out of 4 or minimum 16 characters): /^((?=.*\d)|(?=.*\W+))(?![\n])(?=.*[A-Z])(?=.*[a-z]).*$|.{16,}/'), - 'value' => '/^((?=.*\d)|(?=.*\W+))(?![\n])(?=.*[A-Z])(?=.*[a-z]).*$|.{16,}/', - 'errorMessage' => '', - 'test' => 'testPasswordRegex', - 'type' => 'string', - ), - 'require_password_confirmation' => array( - 'level' => 1, - 'description' => __('Enabling this setting will require users to submit their current password on any edits to their profile (including a triggered password change). For administrators, the confirmation will be required when changing the profile of any user. Could potentially mitigate an attacker trying to change a compromised user\'s password in order to establish persistance, however, enabling this feature will be highly annoying to users.'), - 'value' => true, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'sanitise_attribute_on_delete' => array( - 'level' => 1, - 'description' => __('Enabling this setting will sanitise the contents of an attribute on a soft delete'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'hide_organisation_index_from_users' => array( - 'level' => 1, - 'description' => __('Enabling this setting will block the organisation index from being visible to anyone besides site administrators on the current instance. Keep in mind that users can still see organisations that produce data via events, proposals, event history log entries, etc.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'disable_local_feed_access' => array( - 'level' => 0, - 'description' => __('Disabling this setting will allow the creation/modification of local feeds (as opposed to network feeds). Enabling this setting will restrict feed sources to be network based only. When disabled, keep in mind that a malicious site administrator could get access to any arbitrary file on the system that the apache user has access to. Make sure that proper safe-guards are in place. This setting can only be modified via the CLI.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true, - 'cli_only' => 1 - ), - 'allow_unsafe_apikey_named_param' => array( - 'level' => 0, - 'description' => __('Allows passing the API key via the named url parameter "apikey" - highly recommended not to enable this, but if you have some dodgy legacy tools that cannot pass the authorization header it can work as a workaround. Again, only use this as a last resort.'), - 'value' => false, - 'errorMessage' => __('You have enabled the passing of API keys via URL parameters. This is highly recommended against, do you really want to reveal APIkeys in your logs?...'), - 'test' => 'testBoolFalse', - 'type' => 'boolean', - 'null' => true - ), - 'allow_cors' => array( - 'level' => 1, - 'description' => __('Allow cross-origin requests to this instance, matching origins given in Security.cors_origins. Set to false to totally disable'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'cors_origins' => array( - 'level' => 1, - 'description' => __('Set the origins from which MISP will allow cross-origin requests. Useful for external integration. Comma seperate if you need more than one.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ), - 'sync_audit' => array( - 'level' => 1, - 'description' => __('Enable this setting to create verbose logs of synced event data for debugging reasons. Logs are saved in your MISP directory\'s app/files/scripts/tmp/ directory.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBoolFalse', - 'type' => 'boolean', - 'null' => true - ), - 'user_monitoring_enabled' => array( - 'level' => 1, - 'description' => __('Enables the functionality to monitor users - thereby enabling all logging functionalities for a single user. This functionality is intrusive and potentially heavy on the system - use it with care.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ) - ), - 'SecureAuth' => array( - 'branch' => 1, - 'amount' => array( - 'level' => 0, - 'description' => __('The number of tries a user can try to login and fail before the bruteforce protection kicks in.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForNumeric', - 'type' => 'string', - ), - 'expire' => array( - 'level' => 0, - 'description' => __('The duration (in seconds) of how long the user will be locked out when the allowed number of login attempts are exhausted.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForNumeric', - 'type' => 'string', - ), - ), - 'Session' => array( - 'branch' => 1, - 'autoRegenerate' => array( - 'level' => 0, - 'description' => __('Set to true to automatically regenerate sessions after x number of requests. This might lead to the user getting de-authenticated and is frustrating in general, so only enable it if you really need to regenerate sessions. (Not recommended)'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBoolFalse', - 'type' => 'boolean', - ), - 'checkAgent' => array( - 'level' => 0, - 'description' => __('Set to true to check for the user agent string in each request. This can lead to occasional logouts (not recommended).'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBoolFalse', - 'type' => 'boolean', - ), - 'defaults' => array( - 'level' => 0, - 'description' => __('The session type used by MISP. The default setting is php, which will use the session settings configured in php.ini for the session data (supported options: php, database). The recommended option is php and setting your PHP up to use redis sessions via your php.ini. Just add \'session.save_handler = redis\' and "session.save_path = \'tcp://localhost:6379\'" (replace the latter with your redis connection) to '), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForSessionDefaults', - 'type' => 'string', - 'options' => array('php' => 'php', 'database' => 'database', 'cake' => 'cake', 'cache' => 'cache'), - ), - 'timeout' => array( - 'level' => 0, - 'description' => __('The timeout duration of sessions (in MINUTES). 0 does not mean infinite for the PHP session handler, instead sessions will invalidate immediately.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForNumeric', - 'type' => 'string' - ), - 'cookieTimeout' => array( - 'level' => 0, - 'description' => __('The expiration of the cookie (in MINUTES). The session timeout gets refreshed frequently, however the cookies do not. Generally it is recommended to have a much higher cookie_timeout than timeout.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForCookieTimeout', - 'type' => 'numeric' - ) - ), - 'Plugin' => array( - 'branch' => 1, - 'RPZ_policy' => array( - 'level' => 2, - 'description' => __('The default policy action for the values added to the RPZ.'), - 'value' => 1, - 'errorMessage' => '', - 'test' => 'testForRPZBehaviour', - 'type' => 'numeric', - 'options' => array(0 => 'DROP', 1 => 'NXDOMAIN', 2 => 'NODATA', 3 => 'Local-Data', 4 => 'PASSTHRU', 5 => 'TCP-only' ), - ), - 'RPZ_walled_garden' => array( - 'level' => 2, - 'description' => __('The default walled garden used by the RPZ export if the Local-Data policy setting is picked for the export.'), - 'value' => '127.0.0.1', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'RPZ_serial' => array( - 'level' => 2, - 'description' => __('The serial in the SOA portion of the zone file. (numeric, best practice is yyyymmddrr where rr is the two digit sub-revision of the file. $date will automatically get converted to the current yyyymmdd, so $date00 is a valid setting). Setting it to $time will give you an unixtime-based serial (good then you need more than 99 revisions per day).'), - 'value' => '$date00', - 'errorMessage' => '', - 'test' => 'testForRPZSerial', - 'type' => 'string', - ), - 'RPZ_refresh' => array( - 'level' => 2, - 'description' => __('The refresh specified in the SOA portion of the zone file. (in seconds, or shorthand duration such as 15m)'), - 'value' => '2h', - 'errorMessage' => '', - 'test' => 'testForRPZDuration', - 'type' => 'string', - ), - 'RPZ_retry' => array( - 'level' => 2, - 'description' => __('The retry specified in the SOA portion of the zone file. (in seconds, or shorthand duration such as 15m)'), - 'value' => '30m', - 'errorMessage' => '', - 'test' => 'testForRPZDuration', - 'type' => 'string', - ), - 'RPZ_expiry' => array( - 'level' => 2, - 'description' => __('The expiry specified in the SOA portion of the zone file. (in seconds, or shorthand duration such as 15m)'), - 'value' => '30d', - 'errorMessage' => '', - 'test' => 'testForRPZDuration', - 'type' => 'string', - ), - 'RPZ_minimum_ttl' => array( - 'level' => 2, - 'description' => __('The minimum TTL specified in the SOA portion of the zone file. (in seconds, or shorthand duration such as 15m)'), - 'value' => '1h', - 'errorMessage' => '', - 'test' => 'testForRPZDuration', - 'type' => 'string', - ), - 'RPZ_ttl' => array( - 'level' => 2, - 'description' => __('The TTL of the zone file. (in seconds, or shorthand duration such as 15m)'), - 'value' => '1w', - 'errorMessage' => '', - 'test' => 'testForRPZDuration', - 'type' => 'string', - ), - 'RPZ_ns' => array( - 'level' => 2, - 'description' => __('Nameserver'), - 'value' => 'localhost.', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'RPZ_ns_alt' => array( - 'level' => 2, - 'description' => __('Alternate nameserver'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'RPZ_email' => array( - 'level' => 2, - 'description' => __('The e-mail address specified in the SOA portion of the zone file.'), - 'value' => 'root.localhost', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'Kafka_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the Kafka pub feature of MISP. Make sure that you install the requirements for the plugin to work. Refer to the installation instructions for more information.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ), - 'Kafka_brokers' => array( - 'level' => 2, - 'description' => __('A comma separated list of Kafka bootstrap brokers'), - 'value' => 'kafka:9092', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'Kafka_rdkafka_config' => array( - 'level' => 2, - 'description' => __('A path to an ini file with configuration options to be passed to rdkafka. Section headers in the ini file will be ignored.'), - 'value' => '/etc/rdkafka.ini', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - ), - 'Kafka_include_attachments' => array( - 'level' => 2, - 'description' => __('Enable this setting to include the base64 encoded payloads of malware-samples/attachments in the output.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'Kafka_event_notifications_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the publishing of any event creations/edits/deletions.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'Kafka_event_notifications_topic' => array( - 'level' => 2, - 'description' => __('Topic for publishing event creations/edits/deletions.'), - 'value' => 'misp_event', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'Kafka_event_publish_notifications_enable' => array( - 'level' => 2, - 'description' => __('If enabled it will publish to Kafka the event at the time that the event gets published in MISP. Event actions (creation or edit) will not be published to Kafka.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'Kafka_event_publish_notifications_topic' => array( - 'level' => 2, - 'description' => __('Topic for publishing event information on publish.'), - 'value' => 'misp_event_publish', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'Kafka_object_notifications_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the publishing of any object creations/edits/deletions.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'Kafka_object_notifications_topic' => array( - 'level' => 2, - 'description' => __('Topic for publishing object creations/edits/deletions.'), - 'value' => 'misp_object', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'Kafka_object_reference_notifications_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the publishing of any object reference creations/deletions.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'Kafka_object_reference_notifications_topic' => array( - 'level' => 2, - 'description' => __('Topic for publishing object reference creations/deletions.'), - 'value' => 'misp_object_reference', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'Kafka_attribute_notifications_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the publishing of any attribute creations/edits/soft deletions.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'Kafka_attribute_notifications_topic' => array( - 'level' => 2, - 'description' => __('Topic for publishing attribute creations/edits/soft deletions.'), - 'value' => 'misp_attribute', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'Kafka_shadow_attribute_notifications_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the publishing of any proposal creations/edits/deletions.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'Kafka_shadow_attribute_notifications_topic' => array( - 'level' => 2, - 'description' => __('Topic for publishing proposal creations/edits/deletions.'), - 'value' => 'misp_shadow_attribute', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'Kafka_tag_notifications_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the publishing of any tag creations/edits/deletions as well as tags being attached to / detached from various MISP elements.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'Kafka_tag_notifications_topic' => array( - 'level' => 2, - 'description' => __('Topic for publishing tag creations/edits/deletions as well as tags being attached to / detached from various MISP elements.'), - 'value' => 'misp_tag', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'Kafka_sighting_notifications_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the publishing of new sightings.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'Kafka_sighting_notifications_topic' => array( - 'level' => 2, - 'description' => __('Topic for publishing sightings.'), - 'value' => 'misp_sighting', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'Kafka_user_notifications_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the publishing of new/modified users.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'Kafka_user_notifications_topic' => array( - 'level' => 2, - 'description' => __('Topic for publishing new/modified users.'), - 'value' => 'misp_user', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'Kafka_organisation_notifications_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the publishing of new/modified organisations.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'Kafka_organisation_notifications_topic' => array( - 'level' => 2, - 'description' => __('Topic for publishing new/modified organisations.'), - 'value' => 'misp_organisation', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'Kafka_audit_notifications_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the publishing of log entries. Keep in mind, this can get pretty verbose depending on your logging settings.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'Kafka_audit_notifications_topic' => array( - 'level' => 2, - 'description' => __('Topic for publishing log entries.'), - 'value' => 'misp_audit', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'ZeroMQ_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the pub/sub feature of MISP. Make sure that you install the requirements for the plugin to work. Refer to the installation instructions for more information.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'afterHook' => 'zmqAfterHook', - ), - 'ZeroMQ_port' => array( - 'level' => 2, - 'description' => __('The port that the pub/sub feature will use.'), - 'value' => 50000, - 'errorMessage' => '', - 'test' => 'testForZMQPortNumber', - 'type' => 'numeric', - 'afterHook' => 'zmqAfterHook', - ), - 'ZeroMQ_username' => array( - 'level' => 2, - 'description' => __('The username that client need to use to connect to ZeroMQ.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'afterHook' => 'zmqAfterHook', - ), - 'ZeroMQ_password' => array( - 'level' => 2, - 'description' => __('The password that client need to use to connect to ZeroMQ.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'afterHook' => 'zmqAfterHook', - ), - 'ZeroMQ_redis_host' => array( - 'level' => 2, - 'description' => __('Location of the Redis db used by MISP and the Python PUB script to queue data to be published.'), - 'value' => 'localhost', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'afterHook' => 'zmqAfterHook', - ), - 'ZeroMQ_redis_port' => array( - 'level' => 2, - 'description' => __('The port that Redis is listening on.'), - 'value' => 6379, - 'errorMessage' => '', - 'test' => 'testForPortNumber', - 'type' => 'numeric', - 'afterHook' => 'zmqAfterHook', - ), - 'ZeroMQ_redis_password' => array( - 'level' => 2, - 'description' => __('The password, if set for Redis.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'afterHook' => 'zmqAfterHook', - ), - 'ZeroMQ_redis_database' => array( - 'level' => 2, - 'description' => __('The database to be used for queuing messages for the pub/sub functionality.'), - 'value' => 1, - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'afterHook' => 'zmqAfterHook', - ), - 'ZeroMQ_redis_namespace' => array( - 'level' => 2, - 'description' => __('The namespace to be used for queuing messages for the pub/sub functionality.'), - 'value' => 'mispq', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'afterHook' => 'zmqAfterHook', - ), - 'ZeroMQ_include_attachments' => array( - 'level' => 2, - 'description' => __('Enable this setting to include the base64 encoded payloads of malware-samples/attachments in the output.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'ZeroMQ_event_notifications_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the publishing of any event creations/edits/deletions.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'ZeroMQ_object_notifications_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the publishing of any object creations/edits/deletions.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'ZeroMQ_object_reference_notifications_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the publishing of any object reference creations/deletions.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'ZeroMQ_attribute_notifications_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the publishing of any attribute creations/edits/soft deletions.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'ZeroMQ_tag_notifications_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the publishing of any tag creations/edits/deletions as well as tags being attached to / detached from various MISP elements.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'ZeroMQ_sighting_notifications_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the publishing of new sightings to the ZMQ pubsub feed.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'ZeroMQ_user_notifications_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the publishing of new/modified users to the ZMQ pubsub feed.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'ZeroMQ_organisation_notifications_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the publishing of new/modified organisations to the ZMQ pubsub feed.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'ZeroMQ_audit_notifications_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables the publishing of log entries to the ZMQ pubsub feed. Keep in mind, this can get pretty verbose depending on your logging settings.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'ElasticSearch_logging_enable' => array( - 'level' => 2, - 'description' => __('Enabled logging to an ElasticSearch instance'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'ElasticSearch_connection_string' => array( - 'level' => 2, - 'description' => __('The URL(s) at which to access ElasticSearch - comma separate if you want to have more than one.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'ElasticSearch_log_index' => array( - 'level' => 2, - 'description' => __('The index in which to place logs'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'S3_enable' => array( - 'level' => 2, - 'description' => __('Enables or disables uploading of malware samples to S3 rather than to disk (WARNING: Get permission from amazon first!)'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'S3_bucket_name' => array( - 'level' => 2, - 'description' => __('Bucket name to upload to'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'S3_region' => array( - 'level' => 2, - 'description' => __('Region in which your S3 bucket resides'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'S3_aws_access_key' => array( - 'level' => 2, - 'description' => __('AWS key to use when uploading samples (WARNING: It\' highly recommended that you use EC2 IAM roles if at all possible)'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'S3_aws_secret_key' => array( - 'level' => 2, - 'description' => __('AWS secret key to use when uploading samples'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'Sightings_policy' => array( - 'level' => 1, - 'description' => __('This setting defines who will have access to seeing the reported sightings. The default setting is the event owner alone (in addition to everyone seeing their own contribution) with the other options being Sighting reporters (meaning the event owner and anyone that provided sighting data about the event) and Everyone (meaning anyone that has access to seeing the event / attribute).'), - 'value' => 0, - 'errorMessage' => '', - 'test' => 'testForSightingVisibility', - 'type' => 'numeric', - 'options' => array(0 => 'Event Owner', 1 => 'Sighting reporters', 2 => 'Everyone'), - ), - 'Sightings_anonymise' => array( - 'level' => 1, - 'description' => __('Enabling the anonymisation of sightings will simply aggregate all sightings instead of showing the organisations that have reported a sighting. Users will be able to tell the number of sightings their organisation has submitted and the number of sightings for other organisations'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - ), - 'Sightings_anonymise_as' => array( - 'level' => 1, - 'description' => __('When pushing sightings to another server, report all sightings from this instance as this organisation. This effectively hides all sightings from this instance behind a single organisation to the outside world. Sightings pulled from this instance follow the Sightings_policy above.'), - 'value' => '0', - 'errorMessage' => '', - 'test' => 'testLocalOrg', - 'type' => 'numeric', - 'optionsSource' => 'LocalOrgs', - ), - 'Sightings_range' => array( - 'level' => 1, - 'description' => __('Set the range in which sightings will be taken into account when generating graphs. For example a sighting with a sighted_date of 7 years ago might not be relevant anymore. Setting given in number of days, default is 365 days'), - 'value' => 365, - 'errorMessage' => '', - 'test' => 'testForNumeric', - 'type' => 'numeric' - ), - 'Sightings_sighting_db_enable' => array( - 'level' => 1, - 'description' => __('Enable SightingDB integration.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'CustomAuth_enable' => array( - 'level' => 2, - 'description' => __('Enable this functionality if you would like to handle the authentication via an external tool and authenticate with MISP using a custom header.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true, - 'beforeHook' => 'customAuthBeforeHook' - ), - 'CustomAuth_header' => array( - 'level' => 2, - 'description' => __('Set the header that MISP should look for here. If left empty it will default to the Authorization header.'), - 'value' => 'Authorization', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ), - 'CustomAuth_use_header_namespace' => array( - 'level' => 2, - 'description' => __('Use a header namespace for the auth header - default setting is enabled'), - 'value' => true, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'CustomAuth_header_namespace' => array( - 'level' => 2, - 'description' => __('The default header namespace for the auth header - default setting is HTTP_'), - 'value' => 'HTTP_', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ), - 'CustomAuth_required' => array( - 'level' => 2, - 'description' => __('If this setting is enabled then the only way to authenticate will be using the custom header. Alternatively, you can run in mixed mode that will log users in via the header if found, otherwise users will be redirected to the normal login page.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'CustomAuth_only_allow_source' => array( - 'level' => 2, - 'description' => __('If you are using an external tool to authenticate with MISP and would like to only allow the tool\'s url as a valid point of entry then set this field. '), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ), - 'CustomAuth_name' => array( - 'level' => 2, - 'description' => __('The name of the authentication method, this is cosmetic only and will be shown on the user creation page and logs.'), - 'value' => 'External authentication', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ), - 'CustomAuth_disable_logout' => array( - 'level' => 2, - 'description' => __('Disable the logout button for users authenticate with the external auth mechanism.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'Enrichment_services_enable' => array( - 'level' => 0, - 'description' => __('Enable/disable the enrichment services'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'Enrichment_timeout' => array( - 'level' => 1, - 'description' => __('Set a timeout for the enrichment services'), - 'value' => 10, - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'numeric' - ), - 'Import_services_enable' => array( - 'level' => 0, - 'description' => __('Enable/disable the import services'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'Import_timeout' => array( - 'level' => 1, - 'description' => __('Set a timeout for the import services'), - 'value' => 10, - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'numeric' - ), - 'Import_services_url' => array( - 'level' => 1, - 'description' => __('The url used to access the import services. By default, it is accessible at http://127.0.0.1:6666'), - 'value' => 'http://127.0.0.1', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'Import_services_port' => array( - 'level' => 1, - 'description' => __('The port used to access the import services. By default, it is accessible at 127.0.0.1:6666'), - 'value' => '6666', - 'errorMessage' => '', - 'test' => 'testForPortNumber', - 'type' => 'numeric' - ), - 'Export_services_url' => array( - 'level' => 1, - 'description' => __('The url used to access the export services. By default, it is accessible at http://127.0.0.1:6666'), - 'value' => 'http://127.0.0.1', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'Export_services_port' => array( - 'level' => 1, - 'description' => __('The port used to access the export services. By default, it is accessible at 127.0.0.1:6666'), - 'value' => '6666', - 'errorMessage' => '', - 'test' => 'testForPortNumber', - 'type' => 'numeric' - ), - 'Export_services_enable' => array( - 'level' => 0, - 'description' => __('Enable/disable the export services'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'Export_timeout' => array( - 'level' => 1, - 'description' => __('Set a timeout for the export services'), - 'value' => 10, - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'numeric' - ), - 'Enrichment_hover_enable' => array( - 'level' => 0, - 'description' => __('Enable/disable the hover over information retrieved from the enrichment modules'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'Enrichment_hover_popover_only' => array( - 'level' => 0, - 'description' => __('When enabled, users have to click on the magnifier icon to show the enrichment'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'Enrichment_hover_timeout' => array( - 'level' => 1, - 'description' => __('Set a timeout for the hover services'), - 'value' => 5, - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'numeric' - ), - 'Enrichment_services_url' => array( - 'level' => 1, - 'description' => __('The url used to access the enrichment services. By default, it is accessible at http://127.0.0.1:6666'), - 'value' => 'http://127.0.0.1', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'Enrichment_services_port' => array( - 'level' => 1, - 'description' => __('The port used to access the enrichment services. By default, it is accessible at 127.0.0.1:6666'), - 'value' => 6666, - 'errorMessage' => '', - 'test' => 'testForPortNumber', - 'type' => 'numeric' - ), - 'Cortex_services_url' => array( - 'level' => 1, - 'description' => __('The url used to access Cortex. By default, it is accessible at http://cortex-url'), - 'value' => 'http://127.0.0.1', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string' - ), - 'Cortex_services_port' => array( - 'level' => 1, - 'description' => __('The port used to access Cortex. By default, this is port 9000'), - 'value' => 9000, - 'errorMessage' => '', - 'test' => 'testForPortNumber', - 'type' => 'numeric' - ), - 'Cortex_services_enable' => array( - 'level' => 0, - 'description' => __('Enable/disable the Cortex services'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean' - ), - 'Cortex_authkey' => array( - 'level' => 1, - 'description' => __('Set an authentication key to be passed to Cortex'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ), - 'Cortex_timeout' => array( - 'level' => 1, - 'description' => __('Set a timeout for the Cortex services'), - 'value' => 120, - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'numeric' - ), - 'Cortex_ssl_verify_peer' => array( - 'level' => 1, - 'description' => __('Set to false to disable SSL verification. This is not recommended.'), - 'value' => true, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'Cortex_ssl_verify_host' => array( - 'level' => 1, - 'description' => __('Set to false if you wish to ignore hostname match errors when validating certificates.'), - 'value' => true, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'Cortex_ssl_allow_self_signed' => array( - 'level' => 1, - 'description' => __('Set to true to enable self-signed certificates to be accepted. This requires Cortex_ssl_verify_peer to be enabled.'), - 'value' => false, - 'errorMessage' => '', - 'test' => 'testBool', - 'type' => 'boolean', - 'null' => true - ), - 'Cortex_ssl_cafile' => array( - 'level' => 1, - 'description' => __('Set to the absolute path of the Certificate Authority file that you wish to use for verifying SSL certificates.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ), - 'CustomAuth_custom_password_reset' => array( - 'level' => 2, - 'description' => __('Provide your custom authentication users with an external URL to the authentication system to reset their passwords.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ), - 'CustomAuth_custom_logout' => array( - 'level' => 2, - 'description' => __('Provide a custom logout URL for your users that will log them out using the authentication system you use.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testForEmpty', - 'type' => 'string', - 'null' => true - ) - ), - 'debug' => array( - 'level' => 0, - 'description' => __('The debug level of the instance, always use 0 for production instances.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testDebug', - 'type' => 'numeric', - 'options' => array(0 => 'Debug off', 1 => 'Debug on', 2 => 'Debug + SQL dump'), - ), - 'site_admin_debug' => array( - 'level' => 0, - 'description' => __('The debug level of the instance for site admins. This feature allows site admins to run debug mode on a live instance without exposing it to other users. The most verbose option of debug and site_admin_debug is used for site admins.'), - 'value' => '', - 'errorMessage' => '', - 'test' => 'testDebugAdmin', - 'type' => 'boolean', - 'null' => true - ), - ); } private $__settingTabMergeRules = array( @@ -2533,14 +193,8 @@ class Server extends AppModel 'Session' => 'Security' ); - public $validEventIndexFilters = array('searchall', 'searchpublished', 'searchorg', 'searchtag', 'searcheventid', 'searchdate', 'searcheventinfo', 'searchthreatlevel', 'searchdistribution', 'searchanalysis', 'searchattribute'); - public function isOwnedByOrg($serverid, $org) - { - return $this->field('id', array('id' => $serverid, 'org' => $org)) === $serverid; - } - public function beforeSave($options = array()) { $this->data['Server']['url'] = rtrim($this->data['Server']['url'], '/'); @@ -2577,11 +231,10 @@ class Server extends AppModel } elseif ("update" === $technique) { $eventIds = $this->getEventIdsFromServer($server, false, null, true, 'events', $force); $eventModel = ClassRegistry::init('Event'); - $local_event_ids = $eventModel->find('list', array( - 'fields' => array('uuid'), - 'recursive' => -1, + $localEventUuids = $eventModel->find('column', array( + 'fields' => array('Event.uuid'), )); - return array_intersect($eventIds, $local_event_ids); + return array_intersect($eventIds, $localEventUuids); } elseif (is_numeric($technique)) { return array(intval($technique)); } elseif (Validation::uuid($technique)) { @@ -2723,7 +376,11 @@ class Server extends AppModel private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, $eventModel, $server, $user, $jobId, $force = false) { // check if the event already exist (using the uuid) - $existingEvent = $eventModel->find('first', array('conditions' => array('Event.uuid' => $event['Event']['uuid']))); + $existingEvent = $eventModel->find('first', [ + 'conditions' => ['Event.uuid' => $event['Event']['uuid']], + 'recursive' => -1, + 'fields' => ['id', 'locked'], + ]); $passAlong = $server['Server']['id']; if (!$existingEvent) { // add data for newly imported events @@ -2793,7 +450,7 @@ class Server extends AppModel } else { $email = $user['email']; } - $server['Server']['version'] = $this->getRemoteVersion($id); + $server['Server']['version'] = $this->getRemoteVersion($server); $pulledClusters = 0; if (!empty($server['Server']['pull_galaxy_clusters'])) { $this->GalaxyCluster = ClassRegistry::init('GalaxyCluster'); @@ -3116,11 +773,11 @@ class Server extends AppModel } else { if (Configure::read('MISP.enableEventBlocklisting') !== false) { $this->EventBlocklist = ClassRegistry::init('EventBlocklist'); - $blocklistHits = $this->EventBlocklist->find('list', array( - 'recursive' => -1, + $blocklistHits = $this->EventBlocklist->find('column', array( 'conditions' => array('EventBlocklist.event_uuid' => array_column($eventArray, 'uuid')), - 'fields' => array('EventBlocklist.event_uuid', 'EventBlocklist.event_uuid'), + 'fields' => array('EventBlocklist.event_uuid'), )); + $blocklistHits = array_flip($blocklistHits); foreach ($eventArray as $k => $event) { if (isset($blocklistHits[$event['uuid']])) { unset($eventArray[$k]); @@ -3130,11 +787,11 @@ class Server extends AppModel if (Configure::read('MISP.enableOrgBlocklisting') !== false) { $this->OrgBlocklist = ClassRegistry::init('OrgBlocklist'); - $blocklistHits = $this->OrgBlocklist->find('list', array( - 'recursive' => -1, + $blocklistHits = $this->OrgBlocklist->find('column', array( 'conditions' => array('OrgBlocklist.org_uuid' => array_unique(array_column($eventArray, 'orgc_uuid'))), - 'fields' => array('OrgBlocklist.org_uuid', 'OrgBlocklist.org_uuid'), + 'fields' => array('OrgBlocklist.org_uuid'), )); + $blocklistHits = array_flip($blocklistHits); foreach ($eventArray as $k => $event) { if (isset($blocklistHits[$event['orgc_uuid']])) { unset($eventArray[$k]); @@ -3156,6 +813,55 @@ class Server extends AppModel return $eventUuids; } + public function serverEventsOverlap() + { + $servers = $this->find('all', [ + 'conditions' => ['Server.pull' => 1], + 'order' => ['Server.id ASC'], + 'recursive' => -1, + ]); + + if (count($servers) < 2) { + return [$servers, []]; + } + + $serverUuids = []; + foreach ($servers as &$server) { + try { + $uuids = $this->getEventIdsFromServer($server, true, null, true); + $serverUuids[$server['Server']['id']] = array_flip($uuids); + $server['Server']['events_count'] = count($uuids); + } catch (Exception $e) { + $this->logException("Could not get event UUIDs for server {$server['Server']['id']}", $e); + } + } + unset($server); + + $compared = []; + foreach ($servers as $server) { + if (!isset($serverUuids[$server['Server']['id']])) { + continue; + } + + foreach ($servers as $server2) { + if ($server['Server']['id'] == $server2['Server']['id']) { + continue; + } + if (!isset($serverUuids[$server2['Server']['id']])) { + continue; + } + + $intersect = count(array_intersect_key($serverUuids[$server['Server']['id']], $serverUuids[$server2['Server']['id']])); + $percentage = round(100 * $intersect / $server['Server']['events_count']); + $compared[$server['Server']['id']][$server2['Server']['id']] = [ + 'percentage' => $percentage, + 'events' => $intersect, + ]; + } + } + return [$servers, $compared]; + } + /** * @param int $id Server ID * @param string|int $technique Can be 'full', 'incremental' or event ID @@ -3176,7 +882,7 @@ class Server extends AppModel } $this->Event = ClassRegistry::init('Event'); $url = $server['Server']['url']; - $push = $this->checkVersionCompatibility($id, $user); + $push = $this->checkVersionCompatibility($server, $user); if (is_array($push) && !$push['canPush'] && !$push['canSight']) { $push = 'Remote instance is outdated or no permission to push.'; } @@ -3555,7 +1261,9 @@ class Server extends AppModel $setting['value'] = 0; $setting['test'] = 'testLocalOrg'; $setting['type'] = 'numeric'; - $setting['optionsSource'] = 'LocalOrgs'; + $setting['optionsSource'] = function () { + return $this->loadLocalOrganisations(); + }; } else { $setting['test'] = 'testForEmpty'; $setting['type'] = 'string'; @@ -3712,6 +1420,16 @@ class Server extends AppModel return $options; } + private function loadLocalOrganisations() + { + $localOrgs = $this->Organisation->find('list', array( + 'conditions' => array('local' => 1), + 'recursive' => -1, + 'fields' => array('Organisation.id', 'Organisation.name') + )); + return array_replace(array(0 => __('No organisation selected.')), $localOrgs); + } + public function testTagCollections($value) { $tag_collections = $this->loadTagCollections(); @@ -3834,6 +1552,9 @@ class Server extends AppModel if (substr($value, 0, 7) === "phar://") { return 'Phar protocol not allowed.'; } + if (substr($value, 0, 5) === "s3://") { + return true; + } if (!is_dir($value)) { return 'Not a valid directory.'; } @@ -3924,7 +1645,7 @@ class Server extends AppModel if ($this->testForEmpty($value) !== true) { return $this->testForEmpty($value); } - $regex = "/^(?https?):\/\/(?([\w,\.]+))(?::(?[0-9]+))?(?\/[a-z0-9_\-\.]+)?$/i"; + $regex = "/^(?https?):\/\/(?([\w,\-,\.]+))(?::(?[0-9]+))?(?\/[a-z0-9_\-\.]+)?$/i"; if ( !preg_match($regex, $value, $matches) || strtolower($matches['proto']) != strtolower($this->getProto()) || @@ -4670,7 +2391,7 @@ class Server extends AppModel /** * @param array $server - * @return int + * @return array * @throws JsonException */ public function runPOSTtest(array $server) @@ -4685,12 +2406,13 @@ class Server extends AppModel try { $response = $HttpSocket->post($uri, json_encode(array('testString' => $testFile)), $request); + $contentEncoding = $response->getHeader('Content-Encoding'); $rawBody = $response->body; $response = $this->jsonDecode($rawBody); } catch (Exception $e) { $title = 'Error: POST connection test failed. Reason: ' . $e->getMessage(); $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', $server['Server']['id'], $title); - return 8; + return ['status' => 8]; } if (!isset($response['body']['testString']) || $response['body']['testString'] !== $testFile) { if (!empty($repsonse['body']['testString'])) { @@ -4703,28 +2425,27 @@ class Server extends AppModel $title = 'Error: POST connection test failed due to the message body not containing the expected data. Response: ' . PHP_EOL . PHP_EOL . $responseString; $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', $server['Server']['id'], $title); - return 9; + return ['status' => 9, 'content-encoding' => $contentEncoding]; } $headers = array('Accept', 'Content-type'); foreach ($headers as $header) { - if (!isset($response['headers'][$header]) || $response['headers'][$header] != 'application/json') { + if (!isset($response['headers'][$header]) || $response['headers'][$header] !== 'application/json') { $responseHeader = isset($response['headers'][$header]) ? $response['headers'][$header] : 'Header was not set.'; - $title = 'Error: POST connection test failed due to a header not matching the expected value. Expected: "application/json", received "' . $responseHeader . '"'; + $title = 'Error: POST connection test failed due to a header ' . $header . ' not matching the expected value. Expected: "application/json", received "' . $responseHeader . '"'; $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', $server['Server']['id'], $title); - return 10; + return ['status' => 10, 'content-encoding' => $contentEncoding]; } } - return 1; + return ['status' => 1, 'content-encoding' => $contentEncoding]; } - public function checkVersionCompatibility($id, $user = array(), $HttpSocket = false) + public function checkVersionCompatibility(array $server, $user = array(), $HttpSocket = false) { // for event publishing when we don't have a user. if (empty($user)) { $user = array('Organisation' => array('name' => 'SYSTEM'), 'email' => 'SYSTEM', 'id' => 0); } $localVersion = $this->checkMISPVersion(); - $server = $this->find('first', array('conditions' => array('Server.id' => $id))); $HttpSocket = $this->setupHttpSocket($server, $HttpSocket); $request = $this->setupSyncRequest($server); $uri = $server['Server']['url'] . '/servers/getVersion'; @@ -4739,12 +2460,12 @@ class Server extends AppModel if (isset($response->code)) { $title = 'Error: Connection to the server has failed.' . (isset($response->code) ? ' Returned response code: ' . $response->code : ''); } else { - $title = 'Error: Connection to the server has failed. The returned exception\'s error message was: ' . $e->getMessage(); + $title = 'Error: Connection to the server has failed. The returned exception\'s error message was: ' . $error; } $this->Log->save(array( 'org' => $user['Organisation']['name'], 'model' => 'Server', - 'model_id' => $id, + 'model_id' => $server['Server']['id'], 'email' => $user['email'], 'action' => 'error', 'user_id' => $user['id'], @@ -4752,7 +2473,7 @@ class Server extends AppModel )); return $title; } - $remoteVersion = json_decode($response->body, true); + $remoteVersion = $this->jsonDecode($response->body); $canPush = isset($remoteVersion['perm_sync']) ? $remoteVersion['perm_sync'] : false; $canSight = isset($remoteVersion['perm_sighting']) ? $remoteVersion['perm_sighting'] : false; $supportEditOfGalaxyCluster = isset($remoteVersion['perm_galaxy_editor']); @@ -4765,7 +2486,7 @@ class Server extends AppModel $this->Log->save(array( 'org' => $user['Organisation']['name'], 'model' => 'Server', - 'model_id' => $id, + 'model_id' => $server['Server']['id'], 'email' => $user['email'], 'action' => 'error', 'user_id' => $user['id'], @@ -4777,16 +2498,16 @@ class Server extends AppModel $success = false; $issueLevel = "warning"; if ($localVersion['major'] > $remoteVersion[0]) { - $response = "Sync to Server ('" . $id . "') aborted. The remote instance's MISP version is behind by a major version."; + $response = "Sync to Server ('{$server['Server']['id']}') aborted. The remote instance's MISP version is behind by a major version."; } if ($response === false && $localVersion['major'] < $remoteVersion[0]) { - $response = "Sync to Server ('" . $id . "') aborted. The remote instance is at least a full major version ahead - make sure you update your MISP instance!"; + $response = "Sync to Server ('{$server['Server']['id']}') aborted. The remote instance is at least a full major version ahead - make sure you update your MISP instance!"; } if ($response === false && $localVersion['minor'] > $remoteVersion[1]) { - $response = "Sync to Server ('" . $id . "') aborted. The remote instance's MISP version is behind by a minor version."; + $response = "Sync to Server ('{$server['Server']['id']}') aborted. The remote instance's MISP version is behind by a minor version."; } if ($response === false && $localVersion['minor'] < $remoteVersion[1]) { - $response = "Sync to Server ('" . $id . "') aborted. The remote instance is at least a full minor version ahead - make sure you update your MISP instance!"; + $response = "Sync to Server ('{$server['Server']['id']}') aborted. The remote instance is at least a full minor version ahead - make sure you update your MISP instance!"; } // if we haven't set a message yet, we're good to go. We are only behind by a hotfix version @@ -4796,13 +2517,13 @@ class Server extends AppModel $issueLevel = "error"; } if ($response === false && $localVersion['hotfix'] > $remoteVersion[2]) { - $response = "Sync to Server ('" . $id . "') initiated, but the remote instance is a few hotfixes behind."; + $response = "Sync to Server ('{$server['Server']['id']}') initiated, but the remote instance is a few hotfixes behind."; } if ($response === false && $localVersion['hotfix'] < $remoteVersion[2]) { - $response = "Sync to Server ('" . $id . "') initiated, but the remote instance is a few hotfixes ahead. Make sure you keep your instance up to date!"; + $response = "Sync to Server ('{$server['Server']['id']}') initiated, but the remote instance is a few hotfixes ahead. Make sure you keep your instance up to date!"; } if (empty($response) && $remoteVersion[2] < 111) { - $response = "Sync to Server ('" . $id . "') initiated, but version 2.4.111 is required in order to be able to pull proposals from the remote side."; + $response = "Sync to Server ('{$server['Server']['id']}') initiated, but version 2.4.111 is required in order to be able to pull proposals from the remote side."; } if ($response !== false) { @@ -4811,14 +2532,22 @@ class Server extends AppModel $this->Log->save(array( 'org' => $user['Organisation']['name'], 'model' => 'Server', - 'model_id' => $id, + 'model_id' => $server['Server']['id'], 'email' => $user['email'], 'action' => $issueLevel, 'user_id' => $user['id'], 'title' => ucfirst($issueLevel) . ': ' . $response, )); } - return array('success' => $success, 'response' => $response, 'canPush' => $canPush, 'canSight' => $canSight, 'canEditGalaxyCluster' => $canEditGalaxyCluster, 'supportEditOfGalaxyCluster' => $supportEditOfGalaxyCluster, 'version' => $remoteVersion); + return [ + 'success' => $success, + 'response' => $response, + 'canPush' => $canPush, + 'canSight' => $canSight, + 'canEditGalaxyCluster' => $canEditGalaxyCluster, + 'supportEditOfGalaxyCluster' => $supportEditOfGalaxyCluster, + 'version' => $remoteVersion, + ]; } public function isJson($string) @@ -4913,6 +2642,7 @@ class Server extends AppModel public function dbSchemaDiagnostic() { + $this->AdminSetting = ClassRegistry::init('AdminSetting'); $actualDbVersion = $this->AdminSetting->find('first', array( 'conditions' => array('setting' => 'db_version') ))['AdminSetting']['value']; @@ -4994,7 +2724,7 @@ class Server extends AppModel $field['column_name'], $field['expected']['data_type'], $length !== null ? sprintf('(%d)', $length) : '', - strpos($field['expected']['column_type'], 'unsigned') != -1 ? 'UNSIGNED' : '', + strpos($field['expected']['column_type'], 'unsigned') !== false ? 'UNSIGNED' : '', isset($field['expected']['column_default']) ? 'DEFAULT "' . $field['expected']['column_default'] . '"' : '', $field['expected']['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL', empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name'], @@ -5008,7 +2738,7 @@ class Server extends AppModel $field['column_name'], $field['expected']['data_type'], $length !== null ? sprintf('(%d)', $length) : '', - strpos($field['expected']['column_type'], 'unsigned') != -1 ? 'UNSIGNED' : '', + strpos($field['expected']['column_type'], 'unsigned') !== false ? 'UNSIGNED' : '', isset($field['expected']['column_default']) ? 'DEFAULT "' . $field['expected']['column_default'] . '"' : '', $field['expected']['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL', empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name'], @@ -5033,7 +2763,7 @@ class Server extends AppModel $expectedField['column_name'], $expectedField['data_type'], $length !== null ? sprintf('(%d)', $length) : '', - strpos($expectedField['column_type'], 'unsigned') != -1 ? 'UNSIGNED' : '', + strpos($expectedField['column_type'], 'unsigned') !== false ? 'UNSIGNED' : '', isset($expectedField['column_default']) ? 'DEFAULT "' . $expectedField['column_default'] . '"' : '', $expectedField['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL', empty($expectedField['collation_name']) ? '' : 'COLLATE ' . $expectedField['collation_name'], @@ -5705,6 +3435,32 @@ class Server extends AppModel return $settings; } + /** + * Return PHP setting in basic unit (bytes). + * @param string $setting + * @return string|int|null + */ + public function getIniSetting($setting) + { + $value = ini_get($setting); + if ($value === '') { + return null; + } + + switch ($setting) { + case 'memory_limit': + case 'upload_max_filesize': + case 'post_max_size': + return (int)preg_replace_callback('/(-?\d+)(.?)/', function ($m) { + return $m[1] * pow(1024, strpos('BKMG', $m[2])); + }, strtoupper($value)); + case 'max_execution_time': + return (int)$value; + default: + return $value; + } + } + public function killWorker($pid, $user) { if (!is_numeric($pid)) { @@ -5814,49 +3570,30 @@ class Server extends AppModel )); } - /* returns the version string of a connected instance - * error codes: - * 1: received non json response - * 2: no route to host - * 3: empty result set - */ - public function getRemoteVersion($id) + private function getRemoteVersion(array $server) { - $server = $this->find('first', array( - 'conditions' => array('Server.id' => $id), - )); - if (empty($server)) { - return 2; - } - App::uses('SyncTool', 'Tools'); - $syncTool = new SyncTool(); - $HttpSocket = $syncTool->setupHttpSocket($server); + $HttpSocket = $this->setupHttpSocket($server); $request = $this->setupSyncRequest($server); $response = $HttpSocket->get($server['Server']['url'] . '/servers/getVersion', $data = '', $request); if ($response->code == 200) { - try { - $data = json_decode($response->body, true); - } catch (Exception $e) { - return 1; - } + $data = $this->jsonDecode($response->body); if (isset($data['version']) && !empty($data['version'])) { return $data['version']; } else { - return 3; + throw new Exception("Invalid response from remote server: version field missing"); } } - return 2; } /** * Returns an array with the events - * @param int $id + * @param array $server * @param $user - not used * @param array $passedArgs * @return array * @throws Exception */ - public function previewIndex($id, $user, array $passedArgs) + public function previewIndex(array $server, $user, array $passedArgs) { $validArgs = array_merge(array('sort', 'direction', 'page', 'limit'), $this->validEventIndexFilters); $urlParams = ''; @@ -5867,7 +3604,7 @@ class Server extends AppModel } $relativeUri = '/events/index' . $urlParams; - list($events, $response) = $this->serverGetRequest($id, $relativeUri); + list($events, $response) = $this->serverGetRequest($server, $relativeUri); $totalCount = $response->getHeader('X-Result-Count') ?: 0; foreach ($events as $k => $event) { @@ -5888,15 +3625,15 @@ class Server extends AppModel /** * Returns an array with the event. - * @param int $serverId + * @param array $server * @param int $eventId * @return array * @throws Exception */ - public function previewEvent($serverId, $eventId) + public function previewEvent(array $server, $eventId) { $relativeUri = '/events/' . $eventId; - list($event) = $this->serverGetRequest($serverId, $relativeUri); + list($event) = $this->serverGetRequest($server, $relativeUri); if (!isset($event['Event']['Orgc'])) { $event['Event']['Orgc']['name'] = $event['Event']['orgc']; @@ -5955,6 +3692,11 @@ class Server extends AppModel return $validServers; } + /** + * Check installed PHP extensions and their versions. + * @return array + * @throws JsonException + */ public function extensionDiagnostics() { try { @@ -5972,10 +3714,11 @@ class Server extends AppModel } } } catch (Exception $e) { + $this->logException('Could not load extensions from composer.json', $e, LOG_NOTICE); $extensions = ['redis' => '', 'gd' => '', 'ssdeep' => '', 'zip' => '', 'intl' => '']; // Default extensions } - $results = array(); + $results = ['cli' => false]; foreach ($extensions as $extension => $reason) { $results['extensions'][$extension] = [ 'web_version' => phpversion($extension), @@ -5986,14 +3729,14 @@ class Server extends AppModel 'info' => $reason === true ? null : $reason, ]; } - if (!is_readable(APP . '/files/scripts/selftest.php')) { - $results['cli'] = false; - } else { + if (is_readable(APP . '/files/scripts/selftest.php')) { $execResult = exec('php ' . APP . '/files/scripts/selftest.php ' . escapeshellarg(json_encode(array_keys($extensions)))); - $execResult = $this->jsonDecode($execResult); - $results['cli']['phpversion'] = $execResult['phpversion']; - foreach ($execResult['extensions'] as $extension => $loaded) { - $results['extensions'][$extension]['cli_version'] = $loaded; + if (!empty($execResult)) { + $execResult = $this->jsonDecode($execResult); + $results['cli']['phpversion'] = $execResult['phpversion']; + foreach ($execResult['extensions'] as $extension => $loaded) { + $results['extensions'][$extension]['cli_version'] = $loaded; + } } } @@ -6210,7 +3953,7 @@ class Server extends AppModel return implode('\n', $result); } - public function update(array $status, &$raw = array()) + public function update(array $status, &$raw = [], array $settings = []) { $final = ''; $workingDirectoryPrefix = 'cd $(git rev-parse --show-toplevel) && '; @@ -6229,7 +3972,22 @@ class Server extends AppModel ); $final .= implode("\n", $output) . "\n\n"; } - $command1 = $workingDirectoryPrefix . 'git pull origin ' . $status['branch'] . ' 2>&1'; + if (!empty($settings['branch'])) { + $branchname = false; + preg_match('/^[a-z0-9\_]+/i', $settings['branch'], $branchname); + if (!empty($branchname)) { + $branchname = $branchname[0]; + $checkout_command = $workingDirectoryPrefix . 'git checkout ' . escapeshellarg($branchname) . ' 2>&1'; + exec($checkout_command, $output, $returnCode); + $raw[] = array( + 'input' => $checkout_command, + 'output' => $output, + 'status' => $returnCode, + ); + $status = $this->getCurrentGitStatus(); + } + } + $command1 = $workingDirectoryPrefix . 'git pull origin ' . escapeshellarg($status['branch']) . ' 2>&1'; $command2 = $workingDirectoryPrefix . 'git submodule update --init --recursive 2>&1'; $final .= $command1 . "\n\n"; $returnCode = false; @@ -6536,48 +4294,34 @@ class Server extends AppModel } /** - * @param int $serverId + * @param array $server * @param string $relativeUri * @param HttpSocket|null $HttpSocket * @return array * @throws Exception */ - private function serverGetRequest($serverId, $relativeUri, HttpSocket $HttpSocket = null) + private function serverGetRequest(array $server, $relativeUri, HttpSocket $HttpSocket = null) { - $server = $this->find('first', array( - 'conditions' => array('Server.id' => $serverId), - )); - if ($server === null) { - throw new Exception(__("Server with ID '$serverId' not found.")); - } - - if (!$HttpSocket) { - $HttpSocket = $this->setupHttpSocket($server); - } + $HttpSocket = $this->setupHttpSocket($server, $HttpSocket); $request = $this->setupSyncRequest($server); $uri = $server['Server']['url'] . $relativeUri; $response = $HttpSocket->get($uri, array(), $request); - if ($response === false) { - throw new Exception(__("Could not reach '$uri'.")); - } else if ($response->code == 404) { // intentional != - throw new NotFoundException(__("Fetching the '$uri' failed with HTTP error 404: Not Found")); + if ($response->code == 404) { // intentional != + throw new NotFoundException(__("Fetching the '%s' failed with HTTP error 404: Not Found", $uri)); } else if ($response->code == 405) { // intentional != $responseText = json_decode($response->body, true); if ($responseText !== null) { - throw new Exception(sprintf(__("Fetching the '$uri' failed with HTTP error %s: %s"), $response->code, $responseText['message'])); + throw new Exception(__("Fetching the '%s' failed with HTTP error %s: %s", $uri, $response->code, $responseText['message'])); } } if ($response->code != 200) { // intentional != - throw new Exception(sprintf(__("Fetching the '$uri' failed with HTTP error %s: %s"), $response->code, $response->reasonPhrase)); + throw new Exception(__("Fetching the '%s' failed with HTTP error %s: %s", $uri, $response->code, $response->reasonPhrase)); } - $data = json_decode($response->body, true); - if ($data === null) { - throw new Exception(__('Could not parse JSON: ') . json_last_error_msg(), json_last_error()); - } + $data = $this->jsonDecode($response->body); return array($data, $response); } @@ -6596,7 +4340,7 @@ class Server extends AppModel } catch (Exception $e) { $this->Log = ClassRegistry::init('Log'); $this->Log->create(); - $message = __('Could not reset fetch remote user account.'); + $message = __('Could not fetch remote user account.'); $this->Log->save(array( 'org' => 'SYSTEM', 'model' => 'Server', @@ -6609,14 +4353,18 @@ class Server extends AppModel return $message; } if ($response->isOk()) { - $user = json_decode($response->body, true); + $user = $this->jsonDecode($response->body); if (!empty($user['User'])) { - $result = array( - 'Email' => $user['User']['email'], - 'Role name' => isset($user['Role']['name']) ? $user['Role']['name'] : 'Unknown, outdated instance', - 'Sync flag' => isset($user['Role']['perm_sync']) ? ($user['Role']['perm_sync'] ? 1 : 0) : 'Unknown, outdated instance' - ); - return $result; + $results = [ + __('User') => $user['User']['email'], + __('Role name') => isset($user['Role']['name']) ? $user['Role']['name'] : __('Unknown, outdated instance'), + __('Sync flag') => isset($user['Role']['perm_sync']) ? ($user['Role']['perm_sync'] ? __('Yes') : __('No')) : __('Unknown, outdated instance'), + ]; + if (isset($response->headers['X-Auth-Key-Expiration'])) { + $date = new DateTime($response->headers['X-Auth-Key-Expiration']); + $results[__('Auth key expiration')] = $date->format('Y-m-d H:i:s'); + } + return $results; } else { return __('No user object received in response.'); } @@ -6624,4 +4372,2421 @@ class Server extends AppModel return $response->code; } } + + public function __get($name) + { + if ($name === 'serverSettings') { + $this->serverSettings = $this->generateServerSettings(); + return $this->serverSettings; + } + return parent::__get($name); + } + + /** + * Generate just when required + * @return array[] + */ + private function generateServerSettings() + { + return array( + 'MISP' => array( + 'branch' => 1, + 'baseurl' => array( + 'level' => 0, + 'description' => __('The base url of the application (in the format https://www.mymispinstance.com or https://myserver.com/misp). Several features depend on this setting being correctly set to function.'), + 'value' => '', + 'errorMessage' => __('The currently set baseurl does not match the URL through which you have accessed the page. Disregard this if you are accessing the page via an alternate URL (for example via IP address).'), + 'test' => 'testBaseURL', + 'type' => 'string', + ), + 'external_baseurl' => array( + 'level' => 0, + 'description' => __('The base url of the application (in the format https://www.mymispinstance.com) as visible externally/by other MISPs. MISP will encode this URL in sharing groups when including itself. If this value is not set, the baseurl is used as a fallback.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testURL', + 'type' => 'string', + ), + 'live' => array( + 'level' => 0, + 'description' => __('Unless set to true, the instance will only be accessible by site admins.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testLive', + 'type' => 'boolean', + ), + 'language' => array( + 'level' => 0, + 'description' => __('Select the language MISP should use. The default is english.'), + 'value' => 'eng', + 'errorMessage' => '', + 'test' => 'testLanguage', + 'type' => 'string', + 'optionsSource' => function () { + return $this->loadAvailableLanguages(); + }, + 'afterHook' => 'cleanCacheFiles' + ), + 'default_attribute_memory_coefficient' => array( + 'level' => 1, + 'description' => __('This values controls the internal fetcher\'s memory envelope when it comes to attributes. The number provided is the amount of attributes that can be loaded for each MB of PHP memory available in one shot. Consider lowering this number if your instance has a lot of attribute tags / attribute galaxies attached.'), + 'value' => 80, + 'errorMessage' => '', + 'test' => 'testForNumeric', + 'type' => 'numeric', + 'null' => true + ), + 'default_event_memory_divisor' => array( + 'level' => 1, + 'description' => __('This value controls the divisor for attribute weighting when it comes to loading full events. Meaning that it will load coefficient / divisor number of attributes per MB of memory available. Consider raising this number if you have a lot of correlations or highly contextualised events (large number of event level galaxies/tags).'), + 'value' => 3, + 'errorMessage' => '', + 'test' => 'testForNumeric', + 'type' => 'numeric', + 'null' => true + ), + 'enable_advanced_correlations' => array( + 'level' => 0, + 'description' => __('Enable some performance heavy correlations (currently CIDR correlation)'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'server_settings_skip_backup_rotate' => array( + 'level' => 1, + 'description' => __('Enable this setting to directly save the config.php file without first creating a temporary file and moving it to avoid concurency issues. Generally not recommended, but useful when for example other tools modify/maintain the config.php file.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'python_bin' => array( + 'level' => 1, + 'description' => __('It is highly recommended to install all the python dependencies in a virtualenv. The recommended location is: %s/venv', ROOT), + 'value' => false, + 'errorMessage' => '', + 'null' => false, + 'test' => 'testForBinExec', + 'beforeHook' => 'beforeHookBinExec', + 'type' => 'string', + 'cli_only' => 1 + ), + 'ca_path' => array( + 'level' => 1, + 'description' => __('MISP will default to the bundled mozilla certificate bundle shipped with the framework, which is rather stale. If you wish to use an alternate bundle, just set this setting using the path to the bundle to use. This setting can only be modified via the CLI.'), + 'value' => APP . 'Lib/cakephp/lib/Cake/Config/cacert.pem', + 'errorMessage' => '', + 'null' => true, + 'test' => 'testForCABundle', + 'type' => 'string', + 'cli_only' => 1 + ), + 'disable_auto_logout' => array( + 'level' => 1, + 'description' => __('In some cases, a heavily used MISP instance can generate unwanted blackhole errors due to a high number of requests hitting the server. Disable the auto logout functionality to ease the burden on the system.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'ssdeep_correlation_threshold' => array( + 'level' => 1, + 'description' => __('Set the ssdeep score at which to consider two ssdeep hashes as correlating [1-100]'), + 'value' => 40, + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'numeric' + ), + 'max_correlations_per_event' => array( + 'level' => 1, + 'description' => __('Sets the maximum number of correlations that can be fetched with a single event. For extreme edge cases this can prevent memory issues. The default value is 5k.'), + 'value' => 5000, + 'errorMessage' => '', + 'test' => 'testForNumeric', + 'type' => 'numeric', + 'null' => true + ), + 'maintenance_message' => array( + 'level' => 2, + 'description' => __('The message that users will see if the instance is not live.'), + 'value' => 'Great things are happening! MISP is undergoing maintenance, but will return shortly. You can contact the administration at $email.', + 'errorMessage' => __('If this is not set the default value will be used.'), + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'name' => array( + 'level' => 3, + 'description' => __('This setting is deprecated and can be safely removed.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'version' => array( + 'level' => 3, + 'description' => __('This setting is deprecated and can be safely removed.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'disable_cached_exports' => array( + 'level' => 1, + 'description' => __('Cached exports can take up a considerable amount of space and can be disabled instance wide using this setting. Disabling the cached exports is not recommended as it\'s a valuable feature, however, if your server is having free space issues it might make sense to take this step.'), + 'value' => false, + 'null' => true, + 'errorMessage' => '', + 'test' => 'testDisableCache', + 'type' => 'boolean', + 'afterHook' => 'disableCacheAfterHook', + ), + 'disable_threat_level' => array( + 'level' => 1, + 'description' => __('Disable displaying / modifications to the threat level altogether on the instance (deprecated field).'), + 'value' => false, + 'null' => true, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'header' => array( + 'level' => 3, + 'description' => __('This setting is deprecated and can be safely removed.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'footermidleft' => array( + 'level' => 2, + 'description' => __('Footer text prepending the "Powered by MISP" text.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'footermidright' => array( + 'level' => 2, + 'description' => __('Footer text following the "Powered by MISP" text.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'footerpart1' => array( + 'level' => 3, + 'description' => __('This setting is deprecated and can be safely removed.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'footerpart2' => array( + 'level' => 3, + 'description' => __('This setting is deprecated and can be safely removed.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'footer' => array( + 'level' => 3, + 'description' => __('This setting is deprecated and can be safely removed.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'footerversion' => array( + 'level' => 3, + 'description' => __('This setting is deprecated and can be safely removed.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'footer_logo' => array( + 'level' => 2 , + 'description' => __('If set, this setting allows you to display a logo on the right side of the footer. Upload it as a custom image in the file management tool.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForCustomImage', + 'type' => 'string', + ), + 'home_logo' => array( + 'level' => 2 , + 'description' => __('If set, this setting allows you to display a logo as the home icon. Upload it as a custom image in the file management tool.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForCustomImage', + 'type' => 'string', + ), + 'main_logo' => array( + 'level' => 2 , + 'description' => __('If set, the image specified here will replace the main MISP logo on the login screen. Upload it as a custom image in the file management tool.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForCustomImage', + 'type' => 'string', + ), + 'org' => array( + 'level' => 1, + 'description' => __('The organisation tag of the hosting organisation. This is used in the e-mail subjects.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'host_org_id' => array( + 'level' => 0, + 'description' => __('The hosting organisation of this instance. If this is not selected then replication instances cannot be added.'), + 'value' => '0', + 'errorMessage' => '', + 'test' => 'testLocalOrg', + 'type' => 'numeric', + 'optionsSource' => function () { + return $this->loadLocalOrganisations(); + }, + ), + 'uuid' => array( + 'level' => 0, + 'description' => __('The MISP instance UUID. This UUID is used to identify this instance.'), + 'value' => '0', + 'errorMessage' => __('No valid UUID set'), + 'test' => 'testUuid', + 'type' => 'string' + ), + 'logo' => array( + 'level' => 3, + 'description' => __('This setting is deprecated and can be safely removed.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'showorg' => array( + 'level' => 0, + 'description' => __('Setting this setting to \'false\' will hide all organisation names / logos.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + ), + 'threatlevel_in_email_subject' => array( + 'level' => 2, + 'description' => __('Put the event threat level in the notification E-mail subject.'), + 'value' => true, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + ), + 'email_subject_TLP_string' => array( + 'level' => 2, + 'description' => __('This is the TLP string for e-mails when email_subject_tag is not found.'), + 'value' => 'tlp:amber', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'email_subject_tag' => array( + 'level' => 2, + 'description' => __('If this tag is set on an event it\'s value will be sent in the E-mail subject. If the tag is not set the email_subject_TLP_string will be used.'), + 'value' => 'tlp', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'email_subject_include_tag_name' => array( + 'level' => 2, + 'description' => __('Include in name of the email_subject_tag in the subject. When false only the tag value is used.'), + 'value' => true, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + ), + 'taxii_sync' => array( + 'level' => 3, + 'description' => __('This setting is deprecated and can be safely removed.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'taxii_client_path' => array( + 'level' => 3, + 'description' => __('This setting is deprecated and can be safely removed.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'background_jobs' => array( + 'level' => 1, + 'description' => __('Enables the use of MISP\'s background processing.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testBoolTrue', + 'type' => 'boolean', + ), + 'attachments_dir' => array( + 'level' => 2, + 'description' => __('Directory where attachments are stored. MISP will NOT migrate the existing data if you change this setting. The only safe way to change this setting is in config.php, when MISP is not running, and after having moved/copied the existing data to the new location. This directory must already exist and be writable and readable by the MISP application.'), + 'value' => APP . '/files', # GUI display purpose only. + 'errorMessage' => '', + 'null' => false, + 'test' => 'testForWritableDir', + 'type' => 'string', + 'cli_only' => 1 + ), + 'cached_attachments' => array( + 'level' => 1, + 'description' => __('Allow the XML caches to include the encoded attachments.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + ), + 'download_attachments_on_load' => array( + 'level' => 2, + 'description' => __('Always download attachments when loaded by a user in a browser'), + 'value' => true, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + ), + 'osuser' => array( + 'level' => 0, + 'description' => __('The Unix user MISP (php) is running as'), + 'value' => 'www-data', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'email' => array( + 'level' => 0, + 'description' => __('The e-mail address that MISP should use for all notifications'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'disable_emailing' => array( + 'level' => 0, + 'description' => __('You can disable all e-mailing using this setting. When enabled, no outgoing e-mails will be sent by MISP.'), + 'value' => false, + 'errorMessage' => '', + 'null' => true, + 'test' => 'testDisableEmail', + 'type' => 'boolean', + ), + 'contact' => array( + 'level' => 1, + 'description' => __('The e-mail address that MISP should include as a contact address for the instance\'s support team.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'dns' => array( + 'level' => 3, + 'description' => __('This setting is deprecated and can be safely removed.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'cveurl' => array( + 'level' => 1, + 'description' => __('Turn Vulnerability type attributes into links linking to the provided CVE lookup'), + 'value' => 'https://cve.circl.lu/cve/', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'cweurl' => array( + 'level' => 1, + 'description' => __('Turn Weakness type attributes into links linking to the provided CWE lookup'), + 'value' => 'https://cve.circl.lu/cwe/', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'disablerestalert' => array( + 'level' => 1, + 'description' => __('This setting controls whether notification e-mails will be sent when an event is created via the REST interface. It might be a good idea to disable this setting when first setting up a link to another instance to avoid spamming your users during the initial pull. Quick recap: True = Emails are NOT sent, False = Emails are sent on events published via sync / REST.'), + 'value' => true, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + ), + 'extended_alert_subject' => array( + 'level' => 1, + 'description' => __('enabling this flag will allow the event description to be transmitted in the alert e-mail\'s subject. Be aware that this is not encrypted by GnuPG, so only enable it if you accept that part of the event description will be sent out in clear-text.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'default_event_distribution' => array( + 'level' => 0, + 'description' => __('The default distribution setting for events (0-3).'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'options' => array('0' => __('Your organisation only'), '1' => __('This community only'), '2' => __('Connected communities'), '3' => __('All communities')), + ), + 'default_attribute_distribution' => array( + 'level' => 0, + 'description' => __('The default distribution setting for attributes, set it to \'event\' if you would like the attributes to default to the event distribution level. (0-3 or "event")'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'options' => array('0' => __('Your organisation only'), '1' => __('This community only'), '2' => __('Connected communities'), '3' => __('All communities'), 'event' => __('Inherit from event')), + ), + 'default_event_threat_level' => array( + 'level' => 1, + 'description' => __('The default threat level setting when creating events.'), + 'value' => 4, + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'options' => array('1' => 'High', '2' => 'Medium', '3' => 'Low', '4' => 'undefined'), + ), + 'default_event_tag_collection' => array( + 'level' => 0, + 'description' => __('The tag collection to be applied to all events created manually.'), + 'value' => 0, + 'errorMessage' => '', + 'test' => 'testTagCollections', + 'type' => 'numeric', + 'optionsSource' => function () { + return $this->loadTagCollections(); + } + ), + 'default_publish_alert' => array( + 'level' => 0, + 'description' => __('The default setting for publish alerts when creating users.'), + 'value' => true, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'tagging' => array( + 'level' => 1, + 'description' => __('Enable the tagging feature of MISP. This is highly recommended.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + ), + 'full_tags_on_event_index' => array( + 'level' => 2, + 'description' => __('Show the full tag names on the event index.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'options' => array(0 => 'Minimal tags', 1 => 'Full tags', 2 => 'Shortened tags'), + ), + 'welcome_text_top' => array( + 'level' => 2, + 'description' => __('Used on the login page, before the MISP logo'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'welcome_text_bottom' => array( + 'level' => 2, + 'description' => __('Used on the login page, after the MISP logo'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'welcome_logo' => array( + 'level' => 2, + 'description' => __('Used on the login page, to the left of the MISP logo, upload it as a custom image in the file management tool.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForCustomImage', + 'type' => 'string', + ), + 'welcome_logo2' => array( + 'level' => 2, + 'description' => __('Used on the login page, to the right of the MISP logo, upload it as a custom image in the file management tool.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForCustomImage', + 'type' => 'string', + ), + 'title_text' => array( + 'level' => 2, + 'description' => __('Used in the page title, after the name of the page'), + 'value' => 'MISP', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'take_ownership_xml_import' => array( + 'level' => 2, + 'description' => __('Allows users to take ownership of an event uploaded via the "Add MISP XML" button. This allows spoofing the creator of a manually imported event, also breaking possibly breaking the original intended releasability. Synchronising with an instance that has a different creator for the same event can lead to unwanted consequences.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + ), + 'terms_download' => array( + 'level' => 2, + 'description' => __('Choose whether the terms and conditions should be displayed inline (false) or offered as a download (true)'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'terms_file' => array( + 'level' => 2, + 'description' => __('The filename of the terms and conditions file. Make sure that the file is located in your MISP/app/files/terms directory'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForTermsFile', + 'type' => 'string' + ), + 'showorgalternate' => array( + 'level' => 2, + 'description' => __('True enables the alternate org fields for the event index (source org and member org) instead of the traditional way of showing only an org field. This allows users to see if an event was uploaded by a member organisation on their MISP instance, or if it originated on an interconnected instance.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'unpublishedprivate' => array( + 'level' => 2, + 'description' => __('True will deny access to unpublished events to users outside the organization of the submitter except site admins.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'newUserText' => array( + 'level' => 1, + 'bigField' => true, + 'description' => __('The message sent to the user after account creation (has to be sent manually from the administration interface). Use \\n for line-breaks. The following variables will be automatically replaced in the text: $password = a new temporary password that MISP generates, $username = the user\'s e-mail address, $misp = the url of this instance, $org = the organisation that the instance belongs to, as set in MISP.org, $contact = the e-mail address used to contact the support team, as set in MISP.contact. For example, "the password for $username is $password" would appear to a user with the e-mail address user@misp.org as "the password for user@misp.org is hNamJae81".'), + 'value' => 'Dear new MISP user,\n\nWe would hereby like to welcome you to the $org MISP community.\n\n Use the credentials below to log into MISP at $misp, where you will be prompted to manually change your password to something of your own choice.\n\nUsername: $username\nPassword: $password\n\nIf you have any questions, don\'t hesitate to contact us at: $contact.\n\nBest regards,\nYour $org MISP support team', + 'errorMessage' => '', + 'test' => 'testPasswordResetText', + 'type' => 'string' + ), + 'passwordResetText' => array( + 'level' => 1, + 'bigField' => true, + 'description' => __('The message sent to the users when a password reset is triggered. Use \\n for line-breaks. The following variables will be automatically replaced in the text: $password = a new temporary password that MISP generates, $username = the user\'s e-mail address, $misp = the url of this instance, $contact = the e-mail address used to contact the support team, as set in MISP.contact. For example, "the password for $username is $password" would appear to a user with the e-mail address user@misp.org as "the password for user@misp.org is hNamJae81".'), + 'value' => 'Dear MISP user,\n\nA password reset has been triggered for your account. Use the below provided temporary password to log into MISP at $misp, where you will be prompted to manually change your password to something of your own choice.\n\nUsername: $username\nYour temporary password: $password\n\nIf you have any questions, don\'t hesitate to contact us at: $contact.\n\nBest regards,\nYour $org MISP support team', + 'errorMessage' => '', + 'test' => 'testPasswordResetText', + 'type' => 'string' + ), + 'enableEventBlocklisting' => array( + 'level' => 1, + 'description' => __('Since version 2.3.107 you can start blocklisting event UUIDs to prevent them from being pushed to your instance. This functionality will also happen silently whenever an event is deleted, preventing a deleted event from being pushed back from another instance.'), + 'value' => true, + 'type' => 'boolean', + 'test' => 'testBool' + ), + 'enableOrgBlocklisting' => array( + 'level' => 1, + 'description' => __('Blocklisting organisation UUIDs to prevent the creation of any event created by the blocklisted organisation.'), + 'value' => true, + 'type' => 'boolean', + 'test' => 'testBool' + ), + 'log_client_ip' => array( + 'level' => 1, + 'description' => __('If enabled, all log entries will include the IP address of the user.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'beforeHook' => 'ipLogBeforeHook' + ), + 'log_client_ip_header' => array( + 'level' => 1, + 'description' => __('If log_client_ip is enabled, you can customize which header field contains the client\'s IP address. This is generally used when you have a reverse proxy infront of your MISP instance.'), + 'value' => 'REMOTE_ADDR', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'null' => true, + ), + 'log_auth' => array( + 'level' => 1, + 'description' => __('If enabled, MISP will log all successful authentications using API keys. The requested URLs are also logged.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + ), + 'log_skip_db_logs_completely' => array( + 'level' => 0, + 'description' => __('This functionality allows you to completely disable any logs from being saved in your SQL backend. This is HIGHLY advised against, you lose all the functionalities provided by the audit log subsystem along with the event history (as these are built based on the logs on the fly). Only enable this if you understand and accept the associated risks.'), + 'value' => false, + 'errorMessage' => __('Logging has now been disabled - your audit logs will not capture failed authentication attempts, your event history logs are not being populated and no system maintenance messages are being logged.'), + 'test' => 'testBoolFalse', + 'type' => 'boolean', + 'null' => true + ), + 'log_paranoid' => array( + 'level' => 0, + 'description' => __('If this functionality is enabled all page requests will be logged. Keep in mind this is extremely verbose and will become a burden to your database.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBoolFalse', + 'type' => 'boolean', + 'null' => true + ), + 'log_paranoid_skip_db' => array( + 'level' => 0, + 'description' => __('You can decide to skip the logging of the paranoid logs to the database.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testParanoidSkipDb', + 'type' => 'boolean', + 'null' => true + ), + 'log_paranoid_include_post_body' => array( + 'level' => 0, + 'description' => __('If paranoid logging is enabled, include the POST body in the entries.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'log_user_ips' => array( + 'level' => 0, + 'description' => __('Log user IPs on each request. 30 day retention for lookups by IP to get the last authenticated user ID for the given IP, whilst on the reverse, indefinitely stores all associated IPs for a user ID.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'log_user_ips_authkeys' => [ + 'level' => self::SETTING_RECOMMENDED, + 'description' => __('Log user IP and key usage on each API request. All logs for given keys are deleted after one year when this key is not used.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ], + 'delegation' => array( + 'level' => 1, + 'description' => __('This feature allows users to create org only events and ask another organisation to take ownership of the event. This allows organisations to remain anonymous by asking a partner to publish an event for them.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'showCorrelationsOnIndex' => array( + 'level' => 1, + 'description' => __('When enabled, the number of correlations visible to the currently logged in user will be visible on the event index UI. This comes at a performance cost but can be very useful to see correlating events at a glance.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'showProposalsCountOnIndex' => array( + 'level' => 1, + 'description' => __('When enabled, the number of proposals for the events are shown on the index.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'showSightingsCountOnIndex' => array( + 'level' => 1, + 'description' => __('When enabled, the aggregate number of attribute sightings within the event becomes visible to the currently logged in user on the event index UI.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'showDiscussionsCountOnIndex' => array( + 'level' => 1, + 'description' => __('When enabled, the aggregate number of discussion posts for the event becomes visible to the currently logged in user on the event index UI.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'disableUserSelfManagement' => array( + 'level' => 1, + 'description' => __('When enabled only Org and Site admins can edit a user\'s profile.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => false, + ), + 'disable_user_login_change' => array( + 'level' => self::SETTING_RECOMMENDED, + 'description' => __('When enabled only Site admins can change user email. This should be enabled if you manage user logins by external system.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => false, + ), + 'disable_user_password_change' => array( + 'level' => self::SETTING_RECOMMENDED, + 'description' => __('When enabled only Site admins can change user password. This should be enabled if you manage user passwords by external system.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => false, + ), + 'disable_user_add' => array( + 'level' => self::SETTING_RECOMMENDED, + 'description' => __('When enabled, Org Admins could not add new users. This should be enabled if you manage users by external system.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => false, + ), + 'block_event_alert' => array( + 'level' => 1, + 'description' => __('Enable this setting to start blocking alert e-mails for events with a certain tag. Define the tag in MISP.block_event_alert_tag.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => false, + ), + 'block_event_alert_tag' => array( + 'level' => 1, + 'description' => __('If the MISP.block_event_alert setting is set, alert e-mails for events tagged with the tag defined by this setting will be blocked.'), + 'value' => 'no-alerts="true"', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'null' => false, + ), + 'org_alert_threshold' => array( + 'level' => 1, + 'description' => __('Set a value to limit the number of email alerts that events can generate per creator organisation (for example, if an organisation pushes out 2000 events in one shot, only alert on the first 20).'), + 'value' => 0, + 'errorMessage' => '', + 'test' => 'testForNumeric', + 'type' => 'numeric', + 'null' => true, + ), + 'block_old_event_alert' => array( + 'level' => 1, + 'description' => __('Enable this setting to start blocking alert e-mails for old events. The exact timing of what constitutes an old event is defined by MISP.block_old_event_alert_age.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => false, + ), + 'block_old_event_alert_age' => array( + 'level' => 1, + 'description' => __('If the MISP.block_old_event_alert setting is set, this setting will control how old an event can be for it to be alerted on. The "timestamp" field of the event is used. Expected format: integer, in days'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testForNumeric', + 'type' => 'numeric', + 'null' => false, + ), + 'block_old_event_alert_by_date' => array( + 'level' => 1, + 'description' => __('If the MISP.block_old_event_alert setting is set, this setting will control the threshold for the event.date field, indicating how old an event can be for it to be alerted on. The "date" field of the event is used. Expected format: integer, in days'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testForNumeric', + 'type' => 'numeric', + 'null' => false, + ), + 'tmpdir' => array( + 'level' => 1, + 'description' => __('Please indicate the temp directory you wish to use for certain functionalities in MISP. By default this is set to /tmp and will be used among others to store certain temporary files extracted from imports during the import process.'), + 'value' => '/tmp', + 'errorMessage' => '', + 'test' => 'testForPath', + 'type' => 'string', + 'null' => true, + 'cli_only' => 1 + ), + 'custom_css' => array( + 'level' => 2, + 'description' => __('If you would like to customise the css, simply drop a css file in the /var/www/MISP/app/webroot/css directory and enter the name here.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForStyleFile', + 'type' => 'string', + 'null' => true, + ), + 'proposals_block_attributes' => array( + 'level' => 0, + 'description' => __('Enable this setting to allow blocking attributes from to_ids sensitive exports if a proposal has been made to it to remove the IDS flag or to remove the attribute altogether. This is a powerful tool to deal with false-positives efficiently.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => false, + ), + 'incoming_tags_disabled_by_default' => array( + 'level' => 1, + 'description' => __('Enable this settings if new tags synced / added via incoming events from any source should not be selectable by users by default.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => false + ), + 'completely_disable_correlation' => array( + 'level' => 0, + 'description' => __('*WARNING* This setting will completely disable the correlation on this instance and remove any existing saved correlations. Enabling this will trigger a full recorrelation of all data which is an extremely long and costly procedure. Only enable this if you know what you\'re doing.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBoolFalse', + 'type' => 'boolean', + 'null' => true, + 'afterHook' => 'correlationAfterHook', + ), + 'allow_disabling_correlation' => array( + 'level' => 0, + 'description' => __('*WARNING* This setting will give event creators the possibility to disable the correlation of individual events / attributes that they have created.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBoolFalse', + 'type' => 'boolean', + 'null' => true + ), + 'redis_host' => array( + 'level' => 0, + 'description' => __('The host running the redis server to be used for generic MISP tasks such as caching. This is not to be confused by the redis server used by the background processing.'), + 'value' => '127.0.0.1', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'redis_port' => array( + 'level' => 0, + 'description' => __('The port used by the redis server to be used for generic MISP tasks such as caching. This is not to be confused by the redis server used by the background processing.'), + 'value' => 6379, + 'errorMessage' => '', + 'test' => 'testForNumeric', + 'type' => 'numeric' + ), + 'redis_database' => array( + 'level' => 0, + 'description' => __('The database on the redis server to be used for generic MISP tasks. If you run more than one MISP instance, please make sure to use a different database on each instance.'), + 'value' => 13, + 'errorMessage' => '', + 'test' => 'testForNumeric', + 'type' => 'numeric' + ), + 'redis_password' => array( + 'level' => 0, + 'description' => __('The password on the redis server (if any) to be used for generic MISP tasks.'), + 'value' => '', + 'errorMessage' => '', + 'test' => null, + 'type' => 'string', + 'redacted' => true + ), + 'event_view_filter_fields' => array( + 'level' => 2, + 'description' => __('Specify which fields to filter on when you search on the event view. Default values are : "id, uuid, value, comment, type, category, Tag.name"'), + 'value' => 'id, uuid, value, comment, type, category, Tag.name', + 'errorMessage' => '', + 'test' => null, + 'type' => 'string', + ), + 'manage_workers' => array( + 'level' => 2, + 'description' => __('Set this to false if you would like to disable MISP managing its own worker processes (for example, if you are managing the workers with a systemd unit).'), + 'value' => true, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'deadlock_avoidance' => array( + 'level' => 1, + 'description' => __('Only enable this if you have some tools using MISP with extreme high concurency. General performance will be lower as normal as certain transactional queries are avoided in favour of shorter table locks.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'updateTimeThreshold' => array( + 'level' => 1, + 'description' => __('Sets the minimum time before being able to re-trigger an update if the previous one failed. (safe guard to avoid starting the same update multiple time)'), + 'value' => '7200', + 'test' => 'testForNumeric', + 'type' => 'numeric', + 'null' => true + ), + 'attribute_filters_block_only' => array( + 'level' => 1, + 'description' => __('This is a performance tweak to change the behaviour of restSearch to use attribute filters solely for blocking. This means that a lookup on the event scope with for example the type field set will be ignored unless it\'s used to strip unwanted attributes from the results. If left disabled, passing [ip-src, ip-dst] for example will return any event with at least one ip-src or ip-dst attribute. This is generally not considered to be too useful and is a heavy burden on the database.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'attachment_scan_module' => [ + 'level' => self::SETTING_OPTIONAL, + 'description' => __('Name of enrichment module that will be used for attachment malware scanning. This module must return av-signature or sb-signature object.'), + 'value' => '', + 'errorMessage' => '', + 'type' => 'string', + 'null' => true, + ], + 'attachment_scan_hash_only' => [ + 'level' => self::SETTING_OPTIONAL, + 'description' => __('Send to attachment scan module just file hash. This can be useful if module sends attachment to remote service and you don\'t want to leak real data.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true, + ], + 'attachment_scan_timeout' => [ + 'level' => self::SETTING_OPTIONAL, + 'description' => __('How long to wait for scan results in seconds.'), + 'value' => 30, + 'errorMessage' => '', + 'test' => 'testForPositiveInteger', + 'type' => 'numeric', + 'null' => true, + ], + ), + 'GnuPG' => array( + 'branch' => 1, + 'binary' => array( + 'level' => 2, + 'description' => __('The location of the GnuPG executable. If you would like to use a different GnuPG executable than /usr/bin/gpg, you can set it here. If the default is fine, just keep the setting suggested by MISP.'), + 'value' => '/usr/bin/gpg', + 'errorMessage' => '', + 'test' => 'testForGPGBinary', + 'type' => 'string', + 'cli_only' => 1 + ), + 'onlyencrypted' => array( + 'level' => 0, + 'description' => __('Allow (false) unencrypted e-mails to be sent to users that don\'t have a GnuPG key.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + ), + 'bodyonlyencrypted' => array( + 'level' => 2, + 'description' => __('Allow (false) the body of unencrypted e-mails to contain details about the event.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + ), + 'sign' => array( + 'level' => 2, + 'description' => __('Enable the signing of GnuPG emails. By default, GnuPG emails are signed'), + 'value' => true, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + ), + 'email' => array( + 'level' => 0, + 'description' => __('The e-mail address that the instance\'s GnuPG key is tied to.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'password' => array( + 'level' => 1, + 'description' => __('The password (if it is set) of the GnuPG key of the instance.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'redacted' => true + ), + 'homedir' => array( + 'level' => 0, + 'description' => __('The location of the GnuPG homedir.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'obscure_subject' => array( + 'level' => 2, + 'description' => __('When enabled, subject in signed and encrypted e-mails will not send in unencrypted form.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + ) + ), + 'SMIME' => array( + 'branch' => 1, + 'enabled' => array( + 'level' => 2, + 'description' => __('Enable S/MIME encryption. The encryption posture of the GnuPG.onlyencrypted and GnuPG.bodyonlyencrypted settings are inherited if S/MIME is enabled.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + ), + 'email' => array( + 'level' => 2, + 'description' => __('The e-mail address that the instance\'s S/MIME key is tied to.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'cert_public_sign' => array( + 'level' => 2, + 'description' => __('The location of the public half of the signing certificate.'), + 'value' => '/var/www/MISP/.smime/email@address.com.pem', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'key_sign' => array( + 'level' => 2, + 'description' => __('The location of the private half of the signing certificate.'), + 'value' => '/var/www/MISP/.smime/email@address.com.key', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'password' => array( + 'level' => 2, + 'description' => __('The password (if it is set) of the S/MIME key of the instance.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'redacted' => true + ), + ), + 'Proxy' => array( + 'branch' => 1, + 'host' => array( + 'level' => 2, + 'description' => __('The hostname of an HTTP proxy for outgoing sync requests. Leave empty to not use a proxy.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'port' => array( + 'level' => 2, + 'description' => __('The TCP port for the HTTP proxy.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForNumeric', + 'type' => 'numeric', + ), + 'method' => array( + 'level' => 2, + 'description' => __('The authentication method for the HTTP proxy. Currently supported are Basic or Digest. Leave empty for no proxy authentication.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'user' => array( + 'level' => 2, + 'description' => __('The authentication username for the HTTP proxy.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'password' => array( + 'level' => 2, + 'description' => __('The authentication password for the HTTP proxy.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + ), + 'Security' => array( + 'branch' => 1, + 'disable_form_security' => array( + 'level' => 0, + 'description' => __('Disabling this setting will remove all form tampering protection. Do not set this setting pretty much ever. You were warned.'), + 'value' => false, + 'errorMessage' => 'This setting leaves your users open to CSRF attacks. Do not please consider disabling this setting.', + 'test' => 'testBoolFalse', + 'type' => 'boolean', + 'null' => true + ), + 'salt' => array( + 'level' => 0, + 'description' => __('The salt used for the hashed passwords. You cannot reset this from the GUI, only manually from the settings.php file. Keep in mind, this will invalidate all passwords in the database.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testSalt', + 'type' => 'string', + 'editable' => false, + 'redacted' => true + ), + 'advanced_authkeys' => array( + 'level' => 0, + 'description' => __('Advanced authkeys will allow each user to create and manage a set of authkeys for themselves, each with individual expirations and comments. API keys are stored in a hashed state and can no longer be recovered from MISP. Users will be prompted to note down their key when creating a new authkey. You can generate a new set of API keys for all users on demand in the diagnostics page, or by triggering %s.', sprintf('%s', $this->baseurl, __('the advanced upgrade'))), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + ), + 'advanced_authkeys_validity' => [ + 'level' => self::SETTING_OPTIONAL, + 'description' => __('Maximal key lifetime in days. Use can limit that validity even more. Just newly created keys will be affected. When not set, key validity is not limited.'), + 'value' => '', + 'errorMessage' => '', + 'type' => 'numeric', + 'test' => 'testForNumeric', + 'null' => true, + ], + 'authkey_keep_session' => [ + 'level' => self::SETTING_OPTIONAL, + 'description' => __('When enabled, session is kept between API requests.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true, + ], + 'auth_enforced' => [ + 'level' => self::SETTING_OPTIONAL, + 'description' => __('This optional can be enabled if external auth provider is used and when set to true, it will disable default form authentication.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + ], + 'rest_client_enable_arbitrary_urls' => array( + 'level' => 0, + 'description' => __('Enable this setting if you wish for users to be able to query any arbitrary URL via the rest client. Keep in mind that queries are executed by the MISP server, so internal IPs in your MISP\'s network may be reachable.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'rest_client_baseurl' => array( + 'level' => 1, + 'description' => __('If left empty, the baseurl of your MISP is used. However, in some instances (such as port-forwarded VM installations) this will not work. You can override the baseurl with a url through which your MISP can reach itself (typically https://127.0.0.1 would work).'), + 'value' => false, + 'errorMessage' => '', + 'test' => null, + 'type' => 'string' + ), + 'syslog' => array( + 'level' => 0, + 'description' => __('Enable this setting to pass all audit log entries directly to syslog. Keep in mind, this is verbose and will include user, organisation, event data.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'syslog_to_stderr' => array( + 'level' => self::SETTING_OPTIONAL, + 'description' => __('Write syslog messages also to standard error output.'), + 'value' => true, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'syslog_ident' => array( + 'level' => self::SETTING_OPTIONAL, + 'description' => __('Syslog message identifier.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'null' => true + ), + 'do_not_log_authkeys' => array( + 'level' => 0, + 'description' => __('If enabled, any authkey will be replaced by asterisks in Audit log.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'disable_browser_cache' => array( + 'level' => 0, + 'description' => __('If enabled, HTTP headers that block browser cache will be send. Static files (like images or JavaScripts) will still be cached, but not generated pages.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true, + ), + 'check_sec_fetch_site_header' => [ + 'level' => 0, + 'description' => __('If enabled, any POST, PUT or AJAX request will be allow just when Sec-Fetch-Site header is not defined or contains "same-origin".'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true, + ], + 'email_otp_enabled' => array( + 'level'=> 2, + 'description' => __('Enable two step authentication with a OTP sent by email. Requires e-mailing to be enabled. Warning: You cannot use it in combination with external authentication plugins.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'beforeHook' => 'otpBeforeHook', + 'type' => 'boolean', + 'null' => true + ), + 'email_otp_length' => array ( + 'level' => 2, + 'description' => __('Define the length of the OTP code sent by email'), + 'value' => '6', + 'errorMessage' => '', + 'type' => 'numeric', + 'test' => 'testForNumeric', + 'null' => true, + ), + 'email_otp_validity' => array ( + 'level' => 2, + 'description' => __('Define the validity (in minutes) of the OTP code sent by email'), + 'value' => '5', + 'errorMessage' => '', + 'type' => 'numeric', + 'test' => 'testForNumeric', + 'null' => true, + ), + 'email_otp_text' => array( + 'level' => 2, + 'bigField' => true, + 'description' => __('The message sent to the user when a new OTP is requested. Use \\n for line-breaks. The following variables will be automatically replaced in the text: $otp = the new OTP generated by MISP, $username = the user\'s e-mail address, $org the Organisation managing the instance, $misp = the url of this instance, $contact = the e-mail address used to contact the support team (as set in MISP.contact), $ip the IP used to complete the first step of the login and $validity the validity time in minutes.'), + 'value' => 'Dear MISP user,\n\nYou have attempted to login to MISP ($misp) from $ip with username $username.\n\n Use the following OTP to log into MISP: $otp\n This code is valid for the next $validity minutes.\n\nIf you have any questions, don\'t hesitate to contact us at: $contact.\n\nBest regards,\nYour $org MISP support team', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'null' => true, + ), + 'email_otp_exceptions' => array( + 'level' => 2, + 'bigField' => true, + 'description' => __('A comma separated list of emails for which the OTP is disabled. Note that if you remove someone from this list, the OTP will only be asked at next login.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'null' => true, + ), + 'allow_self_registration' => array( + 'level' => 1, + 'description' => __('Enabling this setting will allow users to have access to the pre-auth registration form. This will create an inbox entry for administrators to review.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'self_registration_message' => array( + 'level' => 1, + 'bigField' => true, + 'description' => __('The message sent shown to anyone trying to self-register.'), + 'value' => 'If you would like to send us a registration request, please fill out the form below. Make sure you fill out as much information as possible in order to ease the task of the administrators.', + 'errorMessage' => '', + 'test' => false, + 'type' => 'string' + ), + 'password_policy_length' => array( + 'level' => 2, + 'description' => __('Password length requirement. If it is not set or it is set to 0, then the default value is assumed (12).'), + 'value' => '12', + 'errorMessage' => '', + 'test' => 'testPasswordLength', + 'type' => 'numeric', + ), + 'password_policy_complexity' => array( + 'level' => 2, + 'description' => __('Password complexity requirement. Leave it empty for the default setting (3 out of 4, with either a digit or a special char) or enter your own regex. Keep in mind that the length is checked in another key. Default (simple 3 out of 4 or minimum 16 characters): /^((?=.*\d)|(?=.*\W+))(?![\n])(?=.*[A-Z])(?=.*[a-z]).*$|.{16,}/'), + 'value' => '/^((?=.*\d)|(?=.*\W+))(?![\n])(?=.*[A-Z])(?=.*[a-z]).*$|.{16,}/', + 'errorMessage' => '', + 'test' => 'testPasswordRegex', + 'type' => 'string', + ), + 'require_password_confirmation' => array( + 'level' => 1, + 'description' => __('Enabling this setting will require users to submit their current password on any edits to their profile (including a triggered password change). For administrators, the confirmation will be required when changing the profile of any user. Could potentially mitigate an attacker trying to change a compromised user\'s password in order to establish persistance, however, enabling this feature will be highly annoying to users.'), + 'value' => true, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'sanitise_attribute_on_delete' => array( + 'level' => 1, + 'description' => __('Enabling this setting will sanitise the contents of an attribute on a soft delete'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'hide_organisation_index_from_users' => array( + 'level' => 1, + 'description' => __('Enabling this setting will block the organisation index from being visible to anyone besides site administrators on the current instance. Keep in mind that users can still see organisations that produce data via events, proposals, event history log entries, etc.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'hide_organisations_in_sharing_groups' => [ + 'level' => self::SETTING_RECOMMENDED, + 'description' => __('Enabling this setting will block the organisation list from being visible in sharing group besides user with sharing group permission.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ], + 'disable_local_feed_access' => array( + 'level' => 0, + 'description' => __('Disabling this setting will allow the creation/modification of local feeds (as opposed to network feeds). Enabling this setting will restrict feed sources to be network based only. When disabled, keep in mind that a malicious site administrator could get access to any arbitrary file on the system that the apache user has access to. Make sure that proper safe-guards are in place. This setting can only be modified via the CLI.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true, + 'cli_only' => 1 + ), + 'allow_unsafe_apikey_named_param' => array( + 'level' => 0, + 'description' => __('Allows passing the API key via the named url parameter "apikey" - highly recommended not to enable this, but if you have some dodgy legacy tools that cannot pass the authorization header it can work as a workaround. Again, only use this as a last resort.'), + 'value' => false, + 'errorMessage' => __('You have enabled the passing of API keys via URL parameters. This is highly recommended against, do you really want to reveal APIkeys in your logs?...'), + 'test' => 'testBoolFalse', + 'type' => 'boolean', + 'null' => true + ), + 'allow_cors' => array( + 'level' => 1, + 'description' => __('Allow cross-origin requests to this instance, matching origins given in Security.cors_origins. Set to false to totally disable'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'cors_origins' => array( + 'level' => 1, + 'description' => __('Set the origins from which MISP will allow cross-origin requests. Useful for external integration. Comma seperate if you need more than one.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'null' => true + ), + 'sync_audit' => array( + 'level' => 1, + 'description' => __('Enable this setting to create verbose logs of synced event data for debugging reasons. Logs are saved in your MISP directory\'s app/files/scripts/tmp/ directory.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBoolFalse', + 'type' => 'boolean', + 'null' => true + ), + 'user_monitoring_enabled' => array( + 'level' => 1, + 'description' => __('Enables the functionality to monitor users - thereby enabling all logging functionalities for a single user. This functionality is intrusive and potentially heavy on the system - use it with care.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'username_in_response_header' => [ + 'level' => self::SETTING_OPTIONAL, + 'description' => __('When enabled, logged in username will be included in X-Username HTTP response header. This is useful for request logging on webserver/proxy side.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ] + ), + 'SecureAuth' => array( + 'branch' => 1, + 'amount' => array( + 'level' => 0, + 'description' => __('The number of tries a user can try to login and fail before the bruteforce protection kicks in.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForNumeric', + 'type' => 'string', + ), + 'expire' => array( + 'level' => 0, + 'description' => __('The duration (in seconds) of how long the user will be locked out when the allowed number of login attempts are exhausted.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForNumeric', + 'type' => 'string', + ), + ), + 'Session' => array( + 'branch' => 1, + 'autoRegenerate' => array( + 'level' => 0, + 'description' => __('Set to true to automatically regenerate sessions after x number of requests. This might lead to the user getting de-authenticated and is frustrating in general, so only enable it if you really need to regenerate sessions. (Not recommended)'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBoolFalse', + 'type' => 'boolean', + ), + 'checkAgent' => array( + 'level' => 0, + 'description' => __('Set to true to check for the user agent string in each request. This can lead to occasional logouts (not recommended).'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBoolFalse', + 'type' => 'boolean', + ), + 'defaults' => array( + 'level' => 0, + 'description' => __('The session type used by MISP. The default setting is php, which will use the session settings configured in php.ini for the session data (supported options: php, database). The recommended option is php and setting your PHP up to use redis sessions via your php.ini. Just add \'session.save_handler = redis\' and "session.save_path = \'tcp://localhost:6379\'" (replace the latter with your redis connection) to '), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForSessionDefaults', + 'type' => 'string', + 'options' => array('php' => 'php', 'database' => 'database', 'cake' => 'cake', 'cache' => 'cache'), + ), + 'timeout' => array( + 'level' => 0, + 'description' => __('The timeout duration of sessions (in MINUTES). 0 does not mean infinite for the PHP session handler, instead sessions will invalidate immediately.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForNumeric', + 'type' => 'string' + ), + 'cookieTimeout' => array( + 'level' => 0, + 'description' => __('The expiration of the cookie (in MINUTES). The session timeout gets refreshed frequently, however the cookies do not. Generally it is recommended to have a much higher cookie_timeout than timeout.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForCookieTimeout', + 'type' => 'numeric' + ) + ), + 'Plugin' => array( + 'branch' => 1, + 'RPZ_policy' => array( + 'level' => 2, + 'description' => __('The default policy action for the values added to the RPZ.'), + 'value' => 1, + 'errorMessage' => '', + 'test' => 'testForRPZBehaviour', + 'type' => 'numeric', + 'options' => array(0 => 'DROP', 1 => 'NXDOMAIN', 2 => 'NODATA', 3 => 'Local-Data', 4 => 'PASSTHRU', 5 => 'TCP-only' ), + ), + 'RPZ_walled_garden' => array( + 'level' => 2, + 'description' => __('The default walled garden used by the RPZ export if the Local-Data policy setting is picked for the export.'), + 'value' => '127.0.0.1', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'RPZ_serial' => array( + 'level' => 2, + 'description' => __('The serial in the SOA portion of the zone file. (numeric, best practice is yyyymmddrr where rr is the two digit sub-revision of the file. $date will automatically get converted to the current yyyymmdd, so $date00 is a valid setting). Setting it to $time will give you an unixtime-based serial (good then you need more than 99 revisions per day).'), + 'value' => '$date00', + 'errorMessage' => '', + 'test' => 'testForRPZSerial', + 'type' => 'string', + ), + 'RPZ_refresh' => array( + 'level' => 2, + 'description' => __('The refresh specified in the SOA portion of the zone file. (in seconds, or shorthand duration such as 15m)'), + 'value' => '2h', + 'errorMessage' => '', + 'test' => 'testForRPZDuration', + 'type' => 'string', + ), + 'RPZ_retry' => array( + 'level' => 2, + 'description' => __('The retry specified in the SOA portion of the zone file. (in seconds, or shorthand duration such as 15m)'), + 'value' => '30m', + 'errorMessage' => '', + 'test' => 'testForRPZDuration', + 'type' => 'string', + ), + 'RPZ_expiry' => array( + 'level' => 2, + 'description' => __('The expiry specified in the SOA portion of the zone file. (in seconds, or shorthand duration such as 15m)'), + 'value' => '30d', + 'errorMessage' => '', + 'test' => 'testForRPZDuration', + 'type' => 'string', + ), + 'RPZ_minimum_ttl' => array( + 'level' => 2, + 'description' => __('The minimum TTL specified in the SOA portion of the zone file. (in seconds, or shorthand duration such as 15m)'), + 'value' => '1h', + 'errorMessage' => '', + 'test' => 'testForRPZDuration', + 'type' => 'string', + ), + 'RPZ_ttl' => array( + 'level' => 2, + 'description' => __('The TTL of the zone file. (in seconds, or shorthand duration such as 15m)'), + 'value' => '1w', + 'errorMessage' => '', + 'test' => 'testForRPZDuration', + 'type' => 'string', + ), + 'RPZ_ns' => array( + 'level' => 2, + 'description' => __('Nameserver'), + 'value' => 'localhost.', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'RPZ_ns_alt' => array( + 'level' => 2, + 'description' => __('Alternate nameserver'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'RPZ_email' => array( + 'level' => 2, + 'description' => __('The e-mail address specified in the SOA portion of the zone file.'), + 'value' => 'root.localhost', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'Kafka_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the Kafka pub feature of MISP. Make sure that you install the requirements for the plugin to work. Refer to the installation instructions for more information.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + ), + 'Kafka_brokers' => array( + 'level' => 2, + 'description' => __('A comma separated list of Kafka bootstrap brokers'), + 'value' => 'kafka:9092', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'Kafka_rdkafka_config' => array( + 'level' => 2, + 'description' => __('A path to an ini file with configuration options to be passed to rdkafka. Section headers in the ini file will be ignored.'), + 'value' => '/etc/rdkafka.ini', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + ), + 'Kafka_include_attachments' => array( + 'level' => 2, + 'description' => __('Enable this setting to include the base64 encoded payloads of malware-samples/attachments in the output.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Kafka_event_notifications_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the publishing of any event creations/edits/deletions.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Kafka_event_notifications_topic' => array( + 'level' => 2, + 'description' => __('Topic for publishing event creations/edits/deletions.'), + 'value' => 'misp_event', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'Kafka_event_publish_notifications_enable' => array( + 'level' => 2, + 'description' => __('If enabled it will publish to Kafka the event at the time that the event gets published in MISP. Event actions (creation or edit) will not be published to Kafka.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Kafka_event_publish_notifications_topic' => array( + 'level' => 2, + 'description' => __('Topic for publishing event information on publish.'), + 'value' => 'misp_event_publish', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'Kafka_object_notifications_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the publishing of any object creations/edits/deletions.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Kafka_object_notifications_topic' => array( + 'level' => 2, + 'description' => __('Topic for publishing object creations/edits/deletions.'), + 'value' => 'misp_object', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'Kafka_object_reference_notifications_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the publishing of any object reference creations/deletions.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Kafka_object_reference_notifications_topic' => array( + 'level' => 2, + 'description' => __('Topic for publishing object reference creations/deletions.'), + 'value' => 'misp_object_reference', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'Kafka_attribute_notifications_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the publishing of any attribute creations/edits/soft deletions.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Kafka_attribute_notifications_topic' => array( + 'level' => 2, + 'description' => __('Topic for publishing attribute creations/edits/soft deletions.'), + 'value' => 'misp_attribute', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'Kafka_shadow_attribute_notifications_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the publishing of any proposal creations/edits/deletions.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Kafka_shadow_attribute_notifications_topic' => array( + 'level' => 2, + 'description' => __('Topic for publishing proposal creations/edits/deletions.'), + 'value' => 'misp_shadow_attribute', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'Kafka_tag_notifications_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the publishing of any tag creations/edits/deletions as well as tags being attached to / detached from various MISP elements.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Kafka_tag_notifications_topic' => array( + 'level' => 2, + 'description' => __('Topic for publishing tag creations/edits/deletions as well as tags being attached to / detached from various MISP elements.'), + 'value' => 'misp_tag', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'Kafka_sighting_notifications_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the publishing of new sightings.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Kafka_sighting_notifications_topic' => array( + 'level' => 2, + 'description' => __('Topic for publishing sightings.'), + 'value' => 'misp_sighting', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'Kafka_user_notifications_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the publishing of new/modified users.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Kafka_user_notifications_topic' => array( + 'level' => 2, + 'description' => __('Topic for publishing new/modified users.'), + 'value' => 'misp_user', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'Kafka_organisation_notifications_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the publishing of new/modified organisations.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Kafka_organisation_notifications_topic' => array( + 'level' => 2, + 'description' => __('Topic for publishing new/modified organisations.'), + 'value' => 'misp_organisation', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'Kafka_audit_notifications_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the publishing of log entries. Keep in mind, this can get pretty verbose depending on your logging settings.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Kafka_audit_notifications_topic' => array( + 'level' => 2, + 'description' => __('Topic for publishing log entries.'), + 'value' => 'misp_audit', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'ZeroMQ_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the pub/sub feature of MISP. Make sure that you install the requirements for the plugin to work. Refer to the installation instructions for more information.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'afterHook' => 'zmqAfterHook', + ), + 'ZeroMQ_port' => array( + 'level' => 2, + 'description' => __('The port that the pub/sub feature will use.'), + 'value' => 50000, + 'errorMessage' => '', + 'test' => 'testForZMQPortNumber', + 'type' => 'numeric', + 'afterHook' => 'zmqAfterHook', + ), + 'ZeroMQ_username' => array( + 'level' => 2, + 'description' => __('The username that client need to use to connect to ZeroMQ.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'afterHook' => 'zmqAfterHook', + ), + 'ZeroMQ_password' => array( + 'level' => 2, + 'description' => __('The password that client need to use to connect to ZeroMQ.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'afterHook' => 'zmqAfterHook', + ), + 'ZeroMQ_redis_host' => array( + 'level' => 2, + 'description' => __('Location of the Redis db used by MISP and the Python PUB script to queue data to be published.'), + 'value' => 'localhost', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'afterHook' => 'zmqAfterHook', + ), + 'ZeroMQ_redis_port' => array( + 'level' => 2, + 'description' => __('The port that Redis is listening on.'), + 'value' => 6379, + 'errorMessage' => '', + 'test' => 'testForPortNumber', + 'type' => 'numeric', + 'afterHook' => 'zmqAfterHook', + ), + 'ZeroMQ_redis_password' => array( + 'level' => 2, + 'description' => __('The password, if set for Redis.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'afterHook' => 'zmqAfterHook', + ), + 'ZeroMQ_redis_database' => array( + 'level' => 2, + 'description' => __('The database to be used for queuing messages for the pub/sub functionality.'), + 'value' => 1, + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'afterHook' => 'zmqAfterHook', + ), + 'ZeroMQ_redis_namespace' => array( + 'level' => 2, + 'description' => __('The namespace to be used for queuing messages for the pub/sub functionality.'), + 'value' => 'mispq', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'afterHook' => 'zmqAfterHook', + ), + 'ZeroMQ_include_attachments' => array( + 'level' => 2, + 'description' => __('Enable this setting to include the base64 encoded payloads of malware-samples/attachments in the output.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'ZeroMQ_event_notifications_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the publishing of any event creations/edits/deletions.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'ZeroMQ_object_notifications_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the publishing of any object creations/edits/deletions.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'ZeroMQ_object_reference_notifications_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the publishing of any object reference creations/deletions.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'ZeroMQ_attribute_notifications_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the publishing of any attribute creations/edits/soft deletions.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'ZeroMQ_tag_notifications_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the publishing of any tag creations/edits/deletions as well as tags being attached to / detached from various MISP elements.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'ZeroMQ_sighting_notifications_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the publishing of new sightings to the ZMQ pubsub feed.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'ZeroMQ_user_notifications_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the publishing of new/modified users to the ZMQ pubsub feed.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'ZeroMQ_organisation_notifications_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the publishing of new/modified organisations to the ZMQ pubsub feed.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'ZeroMQ_audit_notifications_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables the publishing of log entries to the ZMQ pubsub feed. Keep in mind, this can get pretty verbose depending on your logging settings.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'ElasticSearch_logging_enable' => array( + 'level' => 2, + 'description' => __('Enabled logging to an ElasticSearch instance'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'ElasticSearch_connection_string' => array( + 'level' => 2, + 'description' => __('The URL(s) at which to access ElasticSearch - comma separate if you want to have more than one.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'ElasticSearch_log_index' => array( + 'level' => 2, + 'description' => __('The index in which to place logs'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'S3_enable' => array( + 'level' => 2, + 'description' => __('Enables or disables uploading of malware samples to S3 rather than to disk (WARNING: Get permission from amazon first!)'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'S3_bucket_name' => array( + 'level' => 2, + 'description' => __('Bucket name to upload to'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'S3_region' => array( + 'level' => 2, + 'description' => __('Region in which your S3 bucket resides'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'S3_aws_access_key' => array( + 'level' => 2, + 'description' => __('AWS key to use when uploading samples (WARNING: It\' highly recommended that you use EC2 IAM roles if at all possible)'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'S3_aws_secret_key' => array( + 'level' => 2, + 'description' => __('AWS secret key to use when uploading samples'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'Sightings_policy' => array( + 'level' => 1, + 'description' => __('This setting defines who will have access to seeing the reported sightings. The default setting is the event owner organisation alone (in addition to everyone seeing their own contribution) with the other options being Sighting reporters (meaning the event owner and any organisation that provided sighting data about the event) and Everyone (meaning anyone that has access to seeing the event / attribute).'), + 'value' => 0, + 'errorMessage' => '', + 'test' => 'testForSightingVisibility', + 'type' => 'numeric', + 'options' => array( + 0 => __('Event Owner Organisation'), + 1 => __('Sighting reporters'), + 2 => __('Everyone'), + 3 => __('Event Owner + host org sightings'), + ), + ), + 'Sightings_anonymise' => array( + 'level' => 1, + 'description' => __('Enabling the anonymisation of sightings will simply aggregate all sightings instead of showing the organisations that have reported a sighting. Users will be able to tell the number of sightings their organisation has submitted and the number of sightings for other organisations'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + ), + 'Sightings_anonymise_as' => array( + 'level' => 1, + 'description' => __('When pushing sightings to another server, report all sightings from this instance as this organisation. This effectively hides all sightings from this instance behind a single organisation to the outside world. Sightings pulled from this instance follow the Sightings_policy above.'), + 'value' => '0', + 'errorMessage' => '', + 'test' => 'testLocalOrg', + 'type' => 'numeric', + 'optionsSource' => function () { + return $this->loadLocalOrganisations(); + }, + ), + 'Sightings_range' => array( + 'level' => 1, + 'description' => __('Set the range in which sightings will be taken into account when generating graphs. For example a sighting with a sighted_date of 7 years ago might not be relevant anymore. Setting given in number of days, default is 365 days'), + 'value' => 365, + 'errorMessage' => '', + 'test' => 'testForNumeric', + 'type' => 'numeric' + ), + 'Sightings_sighting_db_enable' => array( + 'level' => 1, + 'description' => __('Enable SightingDB integration.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'CustomAuth_enable' => array( + 'level' => 2, + 'description' => __('Enable this functionality if you would like to handle the authentication via an external tool and authenticate with MISP using a custom header.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true, + 'beforeHook' => 'customAuthBeforeHook' + ), + 'CustomAuth_header' => array( + 'level' => 2, + 'description' => __('Set the header that MISP should look for here. If left empty it will default to the Authorization header.'), + 'value' => 'Authorization', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'null' => true + ), + 'CustomAuth_use_header_namespace' => array( + 'level' => 2, + 'description' => __('Use a header namespace for the auth header - default setting is enabled'), + 'value' => true, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'CustomAuth_header_namespace' => array( + 'level' => 2, + 'description' => __('The default header namespace for the auth header - default setting is HTTP_'), + 'value' => 'HTTP_', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'null' => true + ), + 'CustomAuth_required' => array( + 'level' => 2, + 'description' => __('If this setting is enabled then the only way to authenticate will be using the custom header. Alternatively, you can run in mixed mode that will log users in via the header if found, otherwise users will be redirected to the normal login page.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'CustomAuth_only_allow_source' => array( + 'level' => 2, + 'description' => __('If you are using an external tool to authenticate with MISP and would like to only allow the tool\'s url as a valid point of entry then set this field. '), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'null' => true + ), + 'CustomAuth_name' => array( + 'level' => 2, + 'description' => __('The name of the authentication method, this is cosmetic only and will be shown on the user creation page and logs.'), + 'value' => 'External authentication', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'null' => true + ), + 'CustomAuth_disable_logout' => array( + 'level' => 2, + 'description' => __('Disable the logout button for users authenticate with the external auth mechanism.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Enrichment_services_enable' => array( + 'level' => 0, + 'description' => __('Enable/disable the enrichment services'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Enrichment_timeout' => array( + 'level' => 1, + 'description' => __('Set a timeout for the enrichment services'), + 'value' => 10, + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'numeric' + ), + 'Import_services_enable' => array( + 'level' => 0, + 'description' => __('Enable/disable the import services'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Import_timeout' => array( + 'level' => 1, + 'description' => __('Set a timeout for the import services'), + 'value' => 10, + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'numeric' + ), + 'Import_services_url' => array( + 'level' => 1, + 'description' => __('The url used to access the import services. By default, it is accessible at http://127.0.0.1:6666'), + 'value' => 'http://127.0.0.1', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'Import_services_port' => array( + 'level' => 1, + 'description' => __('The port used to access the import services. By default, it is accessible at 127.0.0.1:6666'), + 'value' => '6666', + 'errorMessage' => '', + 'test' => 'testForPortNumber', + 'type' => 'numeric' + ), + 'Export_services_url' => array( + 'level' => 1, + 'description' => __('The url used to access the export services. By default, it is accessible at http://127.0.0.1:6666'), + 'value' => 'http://127.0.0.1', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'Export_services_port' => array( + 'level' => 1, + 'description' => __('The port used to access the export services. By default, it is accessible at 127.0.0.1:6666'), + 'value' => '6666', + 'errorMessage' => '', + 'test' => 'testForPortNumber', + 'type' => 'numeric' + ), + 'Export_services_enable' => array( + 'level' => 0, + 'description' => __('Enable/disable the export services'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Export_timeout' => array( + 'level' => 1, + 'description' => __('Set a timeout for the export services'), + 'value' => 10, + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'numeric' + ), + 'Enrichment_hover_enable' => array( + 'level' => 0, + 'description' => __('Enable/disable the hover over information retrieved from the enrichment modules'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Enrichment_hover_popover_only' => array( + 'level' => 0, + 'description' => __('When enabled, users have to click on the magnifier icon to show the enrichment'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Enrichment_hover_timeout' => array( + 'level' => 1, + 'description' => __('Set a timeout for the hover services'), + 'value' => 5, + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'numeric' + ), + 'Enrichment_services_url' => array( + 'level' => 1, + 'description' => __('The url used to access the enrichment services. By default, it is accessible at http://127.0.0.1:6666'), + 'value' => 'http://127.0.0.1', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'Enrichment_services_port' => array( + 'level' => 1, + 'description' => __('The port used to access the enrichment services. By default, it is accessible at 127.0.0.1:6666'), + 'value' => 6666, + 'errorMessage' => '', + 'test' => 'testForPortNumber', + 'type' => 'numeric' + ), + 'Cortex_services_url' => array( + 'level' => 1, + 'description' => __('The url used to access Cortex. By default, it is accessible at http://cortex-url'), + 'value' => 'http://127.0.0.1', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string' + ), + 'Cortex_services_port' => array( + 'level' => 1, + 'description' => __('The port used to access Cortex. By default, this is port 9000'), + 'value' => 9000, + 'errorMessage' => '', + 'test' => 'testForPortNumber', + 'type' => 'numeric' + ), + 'Cortex_services_enable' => array( + 'level' => 0, + 'description' => __('Enable/disable the Cortex services'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean' + ), + 'Cortex_authkey' => array( + 'level' => 1, + 'description' => __('Set an authentication key to be passed to Cortex'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'null' => true + ), + 'Cortex_timeout' => array( + 'level' => 1, + 'description' => __('Set a timeout for the Cortex services'), + 'value' => 120, + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'numeric' + ), + 'Cortex_ssl_verify_peer' => array( + 'level' => 1, + 'description' => __('Set to false to disable SSL verification. This is not recommended.'), + 'value' => true, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'Cortex_ssl_verify_host' => array( + 'level' => 1, + 'description' => __('Set to false if you wish to ignore hostname match errors when validating certificates.'), + 'value' => true, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'Cortex_ssl_allow_self_signed' => array( + 'level' => 1, + 'description' => __('Set to true to enable self-signed certificates to be accepted. This requires Cortex_ssl_verify_peer to be enabled.'), + 'value' => false, + 'errorMessage' => '', + 'test' => 'testBool', + 'type' => 'boolean', + 'null' => true + ), + 'Cortex_ssl_cafile' => array( + 'level' => 1, + 'description' => __('Set to the absolute path of the Certificate Authority file that you wish to use for verifying SSL certificates.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'null' => true + ), + 'CustomAuth_custom_password_reset' => array( + 'level' => 2, + 'description' => __('Provide your custom authentication users with an external URL to the authentication system to reset their passwords.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'null' => true + ), + 'CustomAuth_custom_logout' => array( + 'level' => 2, + 'description' => __('Provide a custom logout URL for your users that will log them out using the authentication system you use.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testForEmpty', + 'type' => 'string', + 'null' => true + ) + ), + 'debug' => array( + 'level' => 0, + 'description' => __('The debug level of the instance, always use 0 for production instances.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testDebug', + 'type' => 'numeric', + 'options' => array(0 => 'Debug off', 1 => 'Debug on', 2 => 'Debug + SQL dump'), + ), + 'site_admin_debug' => array( + 'level' => 0, + 'description' => __('The debug level of the instance for site admins. This feature allows site admins to run debug mode on a live instance without exposing it to other users. The most verbose option of debug and site_admin_debug is used for site admins.'), + 'value' => '', + 'errorMessage' => '', + 'test' => 'testDebugAdmin', + 'type' => 'boolean', + 'null' => true + ), + ); + } } diff --git a/app/Model/ShadowAttribute.php b/app/Model/ShadowAttribute.php index 993fd08d8..26ca56687 100644 --- a/app/Model/ShadowAttribute.php +++ b/app/Model/ShadowAttribute.php @@ -8,6 +8,8 @@ App::uses('ComplexTypeTool', 'Tools'); /** * @property Event $Event * @property Attribute $Attribute + * @property-read array $typeDefinitions + * @property-read array $categoryDefinitions */ class ShadowAttribute extends AppModel { @@ -73,9 +75,6 @@ class ShadowAttribute extends AppModel 'attachment' ); - // definitions of categories - public $categoryDefinitions; - public $order = array("ShadowAttribute.event_id" => "DESC", "ShadowAttribute.type" => "ASC"); public $validate = array( @@ -149,11 +148,22 @@ class ShadowAttribute extends AppModel ) ); - public function __construct($id = false, $table = null, $ds = null) + public function __isset($name) { - parent::__construct($id, $table, $ds); - $this->categoryDefinitions = $this->Attribute->categoryDefinitions; - $this->typeDefinitions = $this->Attribute->typeDefinitions; + if ($name === 'typeDefinitions' || $name === 'categoryDefinitions') { + return true; + } + return parent::__isset($name); + } + + public function __get($name) + { + if ($name === 'categoryDefinitions') { + return $this->Attribute->categoryDefinitions; + } else if ($name === 'typeDefinitions') { + return $this->Attribute->typeDefinitions; + } + return parent::__get($name); } // The Associations below have been created with all possible keys, those that are not needed can be removed @@ -354,7 +364,7 @@ class ShadowAttribute extends AppModel { $category = $this->data['ShadowAttribute']['category']; if (isset($this->categoryDefinitions[$category]['types'])) { - return in_array($fields['type'], $this->categoryDefinitions[$category]['types']); + return in_array($fields['type'], $this->categoryDefinitions[$category]['types'], true); } return false; } @@ -481,13 +491,13 @@ class ShadowAttribute extends AppModel */ public function getEventContributors($eventId) { - $orgs = $this->find('all', array( - 'fields' => array('DISTINCT(ShadowAttribute.org_id)'), + $orgIds = $this->find('column', array( + 'fields' => array('ShadowAttribute.org_id'), 'conditions' => array('event_id' => $eventId), - 'recursive' => -1, + 'unique' => true, 'order' => false )); - if (empty($orgs)) { + if (empty($orgIds)) { return []; } @@ -495,8 +505,8 @@ class ShadowAttribute extends AppModel return $this->Organisation->find('list', array( 'recursive' => -1, 'fields' => array('id', 'name'), - 'conditions' => array('Organisation.id' => Hash::extract($orgs, "{n}.ShadowAttribute.org_id"))) - ); + 'conditions' => array('Organisation.id' => $orgIds) + )); } /** @@ -809,4 +819,49 @@ class ShadowAttribute extends AppModel )); } } + + public function saveAttachment($shadowAttribute, $path_suffix='') + { + $result = $this->loadAttachmentTool()->saveShadow($shadowAttribute['event_id'], $shadowAttribute['id'], $shadowAttribute['data'], $path_suffix); + if ($result) { + $this->loadAttachmentScan()->backgroundScan(AttachmentScan::TYPE_SHADOW_ATTRIBUTE, $shadowAttribute); + } + return $result; + } + + /** + * @param array $shadowAttribute + * @param bool $thumbnail + * @param int $maxWidth - When $thumbnail is true + * @param int $maxHeight - When $thumbnail is true + * @return string + * @throws Exception + */ + public function getPictureData(array $shadowAttribute, $thumbnail=false, $maxWidth=200, $maxHeight=200) + { + if ($thumbnail && extension_loaded('gd')) { + if ($maxWidth == 200 && $maxHeight == 200) { + // Return thumbnail directly if already exists + try { + return $this->getAttachment($shadowAttribute['ShadowAttribute'], $path_suffix = '_thumbnail'); + } catch (NotFoundException $e) { + // pass + } + } + + // Thumbnail doesn't exists, we need to generate it + $imageData = $this->getAttachment($shadowAttribute['ShadowAttribute']); + $imageData = $this->Attribute->resizeImage($imageData, $maxWidth, $maxHeight); + + // Save just when requested default thumbnail size + if ($maxWidth == 200 && $maxHeight == 200) { + $shadowAttribute['ShadowAttribute']['data'] = $imageData; + $this->saveAttachment($shadowAttribute['ShadowAttribute'], $path_suffix='_thumbnail'); + } + } else { + $imageData = $this->getAttachment($shadowAttribute['ShadowAttribute']); + } + + return $imageData; + } } diff --git a/app/Model/SharingGroup.php b/app/Model/SharingGroup.php index e3e60ee58..7245fb9d1 100644 --- a/app/Model/SharingGroup.php +++ b/app/Model/SharingGroup.php @@ -111,26 +111,20 @@ class SharingGroup extends AppModel return false; } - public function fetchAllAuthorisedForServer($server) - { - $sgs = $this->SharingGroupOrg->fetchAllAuthorised($server['RemoteOrg']['id']); - $sgs = array_merge($sgs, $this->SharingGroupServer->fetchAllSGsForServer($server['Server']['id'])); - return $sgs; - } - /** * Returns a list of all sharing groups that the user is allowed to see. * Scope can be: * - full: Entire SG object with all organisations and servers attached - * - simplified: Just imporant fields from SG, organisations and servers - * - name: array in ID => name key => value format - * - uuid - * - false: array with all IDs + * - simplified: Just important fields from SG, organisations and servers + * - sharing_group: specific scope that fetch just necessary information for generating distribution graph + * - name: array in ID => name format + * - uuid: array in ID => uuid format + * - false: array with all sharing group IDs * * @param array $user * @param string|false $scope - * @param bool $active - * @param int|false $id + * @param bool $active If true, return only active sharing groups + * @param int|array|false $id * @return array */ public function fetchAllAuthorised(array $user, $scope = false, $active = false, $id = false) @@ -144,11 +138,10 @@ class SharingGroup extends AppModel } if ($user['Role']['perm_site_admin']) { - $ids = array_values($this->find('list', array( - 'recursive' => -1, + $ids = $this->find('column', array( 'fields' => array('id'), 'conditions' => $conditions - ))); + )); } else { $ids = array_unique(array_merge( $this->SharingGroupServer->fetchAllAuthorised(), @@ -200,6 +193,23 @@ class SharingGroup extends AppModel 'order' => 'SharingGroup.name ASC' )); return $this->appendOrgsAndServers($sgs, $fieldsOrg, $fieldsServer); + } elseif ($scope === 'distribution_graph') { + // Specific scope that fetch just necessary information for distribution graph + // @see DistributionGraphTool + $canSeeOrgs = $user['Role']['perm_sharing_group'] || !Configure::read('Security.hide_organisations_in_sharing_groups'); + $sgs = $this->find('all', array( + 'contain' => $canSeeOrgs ? ['SharingGroupOrg' => ['org_id']] : [], + 'conditions' => $conditions, + 'fields' => ['SharingGroup.id', 'SharingGroup.name', 'SharingGroup.org_id'], + 'order' => 'SharingGroup.name ASC' + )); + if ($canSeeOrgs) { + return $this->appendOrgsAndServers($sgs, ['id', 'name'], []); + } + foreach ($sgs as &$sg) { + $sg['SharingGroupOrg'] = []; + } + return $sgs; } elseif ($scope === 'name') { $sgs = $this->find('list', array( 'recursive' => -1, @@ -230,8 +240,10 @@ class SharingGroup extends AppModel { $orgsToFetch = []; $serverToFetch = []; - foreach($sharingGroups as $sg) { - $orgsToFetch[$sg['SharingGroup']['org_id']] = true; + foreach ($sharingGroups as $sg) { + if (isset($sg['SharingGroup']['org_id'])) { + $orgsToFetch[$sg['SharingGroup']['org_id']] = true; + } if (isset($sg['SharingGroupOrg'])) { foreach ($sg['SharingGroupOrg'] as $sgo) { $orgsToFetch[$sgo['org_id']] = true; @@ -272,7 +284,7 @@ class SharingGroup extends AppModel } foreach ($sharingGroups as &$sg) { - if (isset($orgsById[$sg['SharingGroup']['org_id']])) { + if (isset($sg['SharingGroup']['org_id']) && isset($orgsById[$sg['SharingGroup']['org_id']])) { $sg['Organisation'] = $orgsById[$sg['SharingGroup']['org_id']]; } @@ -426,23 +438,23 @@ class SharingGroup extends AppModel return $this->__sgAuthorisationCache['access'][boolval($adminCheck)][$id]; } if (Validation::uuid($id)) { - $sgid = $this->SharingGroup->find('first', array( + $sgid = $this->find('first', array( 'conditions' => array('SharingGroup.uuid' => $id), 'recursive' => -1, 'fields' => array('SharingGroup.id') )); if (empty($sgid)) { - throw new MethodNotAllowedException('Invalid sharing group.'); + return false; } $id = $sgid['SharingGroup']['id']; + } else { + if (!$this->exists($id)) { + return false; + } } if (!isset($user['id'])) { throw new MethodNotAllowedException('Invalid user.'); } - $this->id = $id; - if (!$this->exists()) { - return false; - } if (($adminCheck && $user['Role']['perm_site_admin']) || $this->SharingGroupServer->checkIfAuthorised($id) || $this->SharingGroupOrg->checkIfAuthorised($id, $user['org_id'])) { $this->__sgAuthorisationCache['access'][boolval($adminCheck)][$id] = true; return true; @@ -451,24 +463,28 @@ class SharingGroup extends AppModel return false; } - public function checkIfOwner($user, $id) + /** + * @param array $user + * @param string|int $id Sharing group ID or UUID + * @return bool False if sharing group doesn't exists or user org is not sharing group owner + */ + public function checkIfOwner(array $user, $id) { if (!isset($user['id'])) { throw new MethodNotAllowedException('Invalid user.'); } - $this->id = $id; - if (!$this->exists()) { + $sg = $this->find('first', array( + 'conditions' => Validation::uuid($id) ? ['SharingGroup.uuid' => $id] : ['SharingGroup.id' => $id], + 'recursive' => -1, + 'fields' => array('org_id'), + )); + if (empty($sg)) { return false; } if ($user['Role']['perm_site_admin']) { return true; } - $sg = $this->find('first', array( - 'conditions' => array('SharingGroup.id' => $id), - 'recursive' => -1, - 'fields' => array('id', 'org_id'), - )); - return ($sg['SharingGroup']['org_id'] == $user['org_id']); + return $sg['SharingGroup']['org_id'] == $user['org_id']; } // Get all organisation ids that can see a SG @@ -684,7 +700,7 @@ class SharingGroup extends AppModel if ($force) { $sgids = $existingSG['SharingGroup']['id']; $editedSG = $existingSG['SharingGroup']; - $attributes = array('name', 'releasability', 'description', 'created', 'modified', 'active'); + $attributes = ['name', 'releasability', 'description', 'created', 'modified', 'active', 'roaming']; foreach ($attributes as $a) { if (isset($sg[$a])) { $editedSG[$a] = $sg[$a]; diff --git a/app/Model/SharingGroupOrg.php b/app/Model/SharingGroupOrg.php index afbd5a395..5580fc501 100644 --- a/app/Model/SharingGroupOrg.php +++ b/app/Model/SharingGroupOrg.php @@ -78,12 +78,11 @@ class SharingGroupOrg extends AppModel */ public function fetchAllAuthorised($org_id) { - $sgs = $this->find('list', array( + $sgs = $this->find('column', array( 'conditions' => array('org_id' => $org_id), - 'recursive' => -1, - 'fields' => array('sharing_group_id'), + 'fields' => array('SharingGroupOrg.sharing_group_id'), )); - return array_values($sgs); + return $sgs; } // pass a sharing group ID and an organisation ID, returns true if it has a matching attached organisation object diff --git a/app/Model/SharingGroupServer.php b/app/Model/SharingGroupServer.php index e7135ebfd..f96ed5e60 100644 --- a/app/Model/SharingGroupServer.php +++ b/app/Model/SharingGroupServer.php @@ -87,12 +87,11 @@ class SharingGroupServer extends AppModel // This basically lists all SGs that allow everyone on the instance to see events tagged with it public function fetchAllAuthorised() { - $sgs = $this->find('list', array( + $sgs = $this->find('column', array( 'conditions' => array('all_orgs' => 1, 'server_id' => 0), - 'recursive' => -1, - 'fields' => array('sharing_group_id'), + 'fields' => array('SharingGroupServer.sharing_group_id'), )); - return array_values($sgs); + return $sgs; } // pass a sharing group ID, returns true if it has an attached server object with "all_orgs" ticked @@ -108,20 +107,4 @@ class SharingGroupServer extends AppModel } return false; } - - public function fetchAllSGsForServer($server_id) - { - $sgs = $this->find('all', array( - 'recursive' => -1, - 'conditions' => array('server_id' => $server_id) - )); - if (empty($sgs)) { - return array(); - } - $sgids = array(); - foreach ($sgs as $temp) { - $sgids[] = $temp[$this->alias]['id']; - } - return $sgids; - } } diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php index c19c77de3..03c833c55 100644 --- a/app/Model/Sighting.php +++ b/app/Model/Sighting.php @@ -14,7 +14,8 @@ class Sighting extends AppModel // Possible values of `Plugin.Sightings_policy` setting const SIGHTING_POLICY_EVENT_OWNER = 0, SIGHTING_POLICY_SIGHTING_REPORTER = 1, - SIGHTING_POLICY_EVERYONE = 2; + SIGHTING_POLICY_EVERYONE = 2, + SIGHTING_POLICY_HOST_ORG = 3; // the same as SIGHTING_POLICY_EVENT_OWNER, but also sightings from host org are visible private $orgCache = []; @@ -124,23 +125,54 @@ class Sighting extends AppModel } } - public function captureSighting($sighting, $attribute_id, $event_id, $user) + /** + * @param array $sightings + * @param int $attributeId + * @param int $eventId + * @param array $user + * @return bool + */ + public function captureSightings(array $sightings, $attributeId, $eventId, array $user) { - $org_id = 0; - if (!empty($sighting['Organisation'])) { - $org_id = $this->Organisation->captureOrg($sighting['Organisation'], $user); - } - if (isset($sighting['id'])) { + // Since sightings are immutable (it is not possible to change it from web interface), we can check + // if sighting with given uuid already exists and skip them + $existingSighting = $this->existing($sightings); + + // Fetch existing organisations in bulk + $existingOrganisations = $this->existingOrganisations($sightings); + + $toSave = []; + foreach ($sightings as $sighting) { + if (isset($existingSighting[$sighting['uuid']])) { + continue; // already exists, skip + } + + $orgId = 0; + if (isset($sighting['Organisation'])) { + if (isset($existingOrganisations[$sighting['Organisation']['uuid']])) { + $orgId = $existingOrganisations[$sighting['Organisation']['uuid']]; + } else { + $orgId = $this->Organisation->captureOrg($sighting['Organisation'], $user); + } + } unset($sighting['id']); + + $sighting['org_id'] = $orgId; + $sighting['event_id'] = $eventId; + $sighting['attribute_id'] = $attributeId; + $toSave[] = $sighting; } - $sighting['org_id'] = $org_id; - $sighting['event_id'] = $event_id; - $sighting['attribute_id'] = $attribute_id; - $this->create(); - return $this->save($sighting); + + return $this->saveMany($toSave); } - public function getSighting($id, $user) + /** + * @param int $id + * @param array $user + * @param bool $withEvent + * @return array + */ + public function getSighting($id, array $user, $withEvent = true) { $sighting = $this->find('first', array( 'recursive' => -1, @@ -149,10 +181,7 @@ class Sighting extends AppModel 'fields' => array('Attribute.value', 'Attribute.id', 'Attribute.uuid', 'Attribute.type', 'Attribute.category', 'Attribute.to_ids') ), 'Event' => array( - 'fields' => array('Event.id', 'Event.uuid', 'Event.orgc_id', 'Event.org_id', 'Event.info'), - 'Orgc' => array( - 'fields' => array('Orgc.name') - ) + 'fields' => $withEvent ? ['Event.id', 'Event.uuid', 'Event.orgc_id', 'Event.org_id', 'Event.info'] : ['Event.org_id'], ) ), 'conditions' => array('Sighting.id' => $id) @@ -161,11 +190,7 @@ class Sighting extends AppModel return array(); } - if (!isset($event)) { - $event = array('Event' => $sighting['Event']); - } - - $ownEvent = $user['Role']['perm_site_admin'] || $event['Event']['org_id'] == $user['org_id']; + $ownEvent = $user['Role']['perm_site_admin'] || $sighting['Event']['org_id'] == $user['org_id']; if (!$ownEvent) { $sightingPolicy = $this->sightingsPolicy(); // if sighting policy == 0 then return false if the sighting doesn't belong to the user @@ -180,27 +205,30 @@ class Sighting extends AppModel return array(); } } - } - $anonymise = Configure::read('Plugin.Sightings_anonymise'); - if ($anonymise) { - if ($sighting['Sighting']['org_id'] != $user['org_id']) { - unset($sighting['Sighting']['org_id']); - unset($sighting['Organisation']); + else if ($sightingPolicy === self::SIGHTING_POLICY_HOST_ORG) { + if ($sighting['Sighting']['org_id'] != $user['org_id'] || $sighting['Sighting']['org_id'] != Configure::read('MISP.host_org_id')) { + return array(); + } } } - // rearrange it to match the event format of fetchevent - if (isset($sighting['Organisation'])) { - $sighting['Sighting']['Organisation'] = $sighting['Organisation']; - unset($sighting['Organisation']); + + // Put event organisation name from cache + if ($withEvent) { + $sighting['Event']['Orgc']['name'] = $this->getOrganisationById($sighting['Event']['orgc_id'])['name']; } + + $anonymise = Configure::read('Plugin.Sightings_anonymise'); + if ($anonymise && $sighting['Sighting']['org_id'] != $user['org_id']) { + unset($sighting['Sighting']['org_id']); + } + // rearrange it to match the event format of fetchevent $result = array( 'Sighting' => $sighting['Sighting'] ); - $result['Sighting']['Event'] = $sighting['Event']; - $result['Sighting']['Attribute'] = $sighting['Attribute']; - if (!empty($sighting['Organisation'])) { - $result['Sighting']['Organisation'] = $sighting['Organisation']; + if ($withEvent) { + $result['Sighting']['Event'] = $sighting['Event']; } + $result['Sighting']['Attribute'] = $sighting['Attribute']; return $result; } @@ -223,6 +251,8 @@ class Sighting extends AppModel $sightingsPolicy = $this->sightingsPolicy(); if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER) { $conditions['Sighting.org_id'] = $user['org_id']; + } else if ($sightingsPolicy === self::SIGHTING_POLICY_HOST_ORG) { + $conditions['Sighting.org_id'] = [$user['org_id'], Configure::read('MISP.host_org_id')]; } // TODO: Currently, we dont support `SIGHTING_POLICY_SIGHTING_REPORTER` for tags $sparklineData = []; @@ -231,7 +261,7 @@ class Sighting extends AppModel $objectElement = ucfirst($context) . 'Tag'; foreach ($sightings as $sighting) { $tagId = $sighting[$objectElement]['tag_id']; - $date = $sighting['Sighting']['date_sighting']; + $date = $sighting['Sighting']['date']; $count = (int)$sighting['Sighting']['sighting_count']; if (isset($sparklineData[$tagId][$date]['sighting'])) { @@ -269,6 +299,8 @@ class Sighting extends AppModel if (!$this->isReporter($attribute['Event']['id'], $user['org_id'])) { continue; // skip attribute } + } else if ($sightingsPolicy === self::SIGHTING_POLICY_HOST_ORG) { + $attributeConditions['Sighting.org_id'] = [$user['org_id'], Configure::read('MISP.host_org_id')]; } } $conditions['OR'][] = $attributeConditions; @@ -290,19 +322,21 @@ class Sighting extends AppModel return ['data' => [], 'csv' => []]; } - $sightingPolicy = $this->sightingsPolicy(); + $sightingsPolicy = $this->sightingsPolicy(); $conditions = []; foreach ($events as $event) { $eventCondition = ['Sighting.event_id' => $event['Event']['id']]; $ownEvent = $user['Role']['perm_site_admin'] || $event['Event']['org_id'] == $user['org_id']; if (!$ownEvent) { - if ($sightingPolicy === self::SIGHTING_POLICY_EVENT_OWNER) { + if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER) { $eventCondition['Sighting.org_id'] = $user['org_id']; - } else if ($sightingPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) { + } else if ($sightingsPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) { if (!$this->isReporter($event['Event']['id'], $user['org_id'])) { continue; } + } else if ($sightingsPolicy === self::SIGHTING_POLICY_HOST_ORG) { + $eventCondition['Sighting.org_id'] = [$user['org_id'], Configure::read('MISP.host_org_id')]; } } $conditions['OR'][] = $eventCondition; @@ -324,18 +358,18 @@ class Sighting extends AppModel } // Returns date in `Y-m-d` format - $this->virtualFields['date_sighting'] = $this->dateVirtualColumn(); + $this->virtualFields['date'] = $this->dateVirtualColumn(); $this->virtualFields['sighting_count'] = 'COUNT(id)'; $this->virtualFields['last_timestamp'] = 'MAX(date_sighting)'; $groupedSightings = $this->find('all', array( 'conditions' => $conditions, - 'fields' => ['org_id', 'attribute_id', 'type', 'date_sighting', 'last_timestamp', 'sighting_count'], + 'fields' => ['org_id', 'attribute_id', 'type', 'date', 'last_timestamp', 'sighting_count'], 'recursive' => -1, - 'group' => ['org_id', 'attribute_id', 'type', 'date_sighting'], - 'order' => ['date_sighting'], // from oldest + 'group' => ['org_id', 'attribute_id', 'type', 'date'], + 'order' => ['date'], // from oldest )); unset( - $this->virtualFields['date_sighting'], + $this->virtualFields['date'], $this->virtualFields['sighting_count'], $this->virtualFields['last_timestamp'] ); @@ -361,17 +395,17 @@ class Sighting extends AppModel ] ]); // Returns date in `Y-m-d` format - $this->virtualFields['date_sighting'] = $this->dateVirtualColumn(); + $this->virtualFields['date'] = $this->dateVirtualColumn(); $this->virtualFields['sighting_count'] = 'COUNT(Sighting.id)'; - $sightings = $this->find('all', array( + $sightings = $this->find('all', [ 'recursive' => -1, 'contain' => [ucfirst($context) . 'Tag'], 'conditions' => $conditions, - 'fields' => [ucfirst($context) . 'Tag.tag_id', 'date_sighting', 'sighting_count'], - 'group' => [ucfirst($context) . 'Tag.id', 'date_sighting'], - 'order' => ['date_sighting'], // from oldest - )); - unset($this->virtualFields['date_sighting'], $this->virtualFields['sighting_count']); + 'fields' => [ucfirst($context) . 'Tag.tag_id', 'date', 'sighting_count'], + 'group' => [ucfirst($context) . 'Tag.tag_id', 'date'], + 'order' => ['date'], // from oldest + ]); + unset($this->virtualFields['date'], $this->virtualFields['sighting_count']); return $sightings; } @@ -389,7 +423,7 @@ class Sighting extends AppModel $type = $this->type[$sighting['type']]; $orgName = isset($sighting['Organisation']['name']) ? $sighting['Organisation']['name'] : __('Others'); $count = (int)$sighting['sighting_count']; - $inRange = strtotime($sighting['date_sighting']) >= $range; + $inRange = strtotime($sighting['date']) >= $range; foreach ([$sighting['attribute_id'], 'all'] as $needle) { if (!isset($sightingsData[$needle][$type])) { @@ -407,10 +441,10 @@ class Sighting extends AppModel } if ($inRange) { - if (isset($sparklineData[$needle][$sighting['date_sighting']][$type])) { - $sparklineData[$needle][$sighting['date_sighting']][$type] += $count; + if (isset($sparklineData[$needle][$sighting['date']][$type])) { + $sparklineData[$needle][$sighting['date']][$type] += $count; } else { - $sparklineData[$needle][$sighting['date_sighting']][$type] = $count; + $sparklineData[$needle][$sighting['date']][$type] = $count; } } } @@ -517,13 +551,15 @@ class Sighting extends AppModel $ownEvent = $user['Role']['perm_site_admin'] || $event['Event']['org_id'] == $user['org_id']; if (!$ownEvent) { - $sightingPolicy = $this->sightingsPolicy(); - if ($sightingPolicy === self::SIGHTING_POLICY_EVENT_OWNER) { + $sightingsPolicy = $this->sightingsPolicy(); + if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER) { $conditions['Sighting.org_id'] = $user['org_id']; - } elseif ($sightingPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) { + } elseif ($sightingsPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) { if (!$this->isReporter($event['Event']['id'], $user['org_id'])) { return array(); } + } else if ($sightingsPolicy === self::SIGHTING_POLICY_HOST_ORG) { + $conditions['Sighting.org_id'] = [$user['org_id'], Configure::read('MISP.host_org_id')]; } } if ($extraConditions !== false) { @@ -659,6 +695,10 @@ class Sighting extends AppModel return $result; } + /** + * @return bool + * @deprecated + */ public function addUuids() { $sightings = $this->find('all', array( @@ -743,10 +783,14 @@ class Sighting extends AppModel return $sightings; // site admin can see all sightings, do not limit him } $sightingsPolicy = $this->sightingsPolicy(); - if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER) { + if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER || $sightingsPolicy === self::SIGHTING_POLICY_HOST_ORG) { $userOrgId = $user['org_id']; + $allowedOrgs = [$userOrgId]; + if ($sightingsPolicy === self::SIGHTING_POLICY_HOST_ORG) { + $allowedOrgs[] = Configure::read('MISP.host_org_id'); + } foreach ($sightings as $k => $sighting) { - if ($eventOwnerOrgIdList[$sighting['Sighting']['event_id']] !== $userOrgId && $sighting['Sighting']['org_id'] !== $userOrgId) { + if ($eventOwnerOrgIdList[$sighting['Sighting']['event_id']] !== $userOrgId && !in_array($sighting['Sighting']['org_id'], $allowedOrgs)) { unset($sightings[$k]); } } @@ -789,9 +833,9 @@ class Sighting extends AppModel if (isset($filters['context']) && !in_array($filters['context'], $allowedContext, true)) { throw new MethodNotAllowedException(__('Invalid context.')); } - // ensure that an id is provided if context is set - if (!empty($filters['context']) && !isset($filters['id'])) { - throw new MethodNotAllowedException(__('An id must be provided if the context is set.')); + // ensure that an id or uuid is provided if context is set + if (!empty($filters['context']) && !(isset($filters['id']) || isset($filters['uuid'])) ) { + throw new MethodNotAllowedException(__('An ID or UUID must be provided if the context is set.')); } if (!isset($this->validFormats[$returnFormat][1])) { @@ -811,7 +855,9 @@ class Sighting extends AppModel } else { $timeCondition = '30d'; } - $conditions = $this->Attribute->setTimestampConditions($timeCondition, array(), $scope = 'Sighting.date_sighting'); + + $contain = []; + $conditions = $this->Attribute->setTimestampConditions($timeCondition, [], $scope = 'Sighting.date_sighting'); if (isset($filters['type'])) { $conditions['Sighting.type'] = $filters['type']; @@ -824,7 +870,11 @@ class Sighting extends AppModel } foreach ($filters['org_id'] as $k => $org_id) { if (Validation::uuid($org_id)) { - $org = $this->Organisation->find('first', array('conditions' => array('Organisation.uuid' => $org_id), 'recursive' => -1, 'fields' => array('Organisation.id'))); + $org = $this->Organisation->find('first', array( + 'conditions' => array('Organisation.uuid' => $org_id), + 'recursive' => -1, + 'fields' => array('Organisation.id'), + )); if (empty($org)) { $filters['org_id'][$k] = -1; } else { @@ -847,56 +897,37 @@ class Sighting extends AppModel } } - // fetch sightings matching the query - $sightings = $this->find('list', array( - 'recursive' => -1, - 'conditions' => $conditions, - 'fields' => array('id'), - )); - $sightings = array_values($sightings); - - $filters['requested_attributes'] = array('id', 'attribute_id', 'event_id', 'org_id', 'date_sighting', 'uuid', 'source', 'type'); - - // apply ACL and sighting policies - $allowedSightings = array(); - $additional_attribute_added = false; - $additional_event_added = false; - foreach ($sightings as $sid) { - $sight = $this->getSighting($sid, $user); - if (!empty($sight)) { - $sight['Sighting']['value'] = $sight['Sighting']['Attribute']['value']; - // by default, do not include event and attribute - if (!isset($filters['includeAttribute']) || !$filters['includeAttribute']) { - unset($sight["Sighting"]["Attribute"]); - } else if (!$additional_attribute_added) { - $filters['requested_attributes'] = array_merge($filters['requested_attributes'], array('attribute_uuid', 'attribute_type', 'attribute_category', 'attribute_to_ids', 'attribute_value')); - $additional_attribute_added = true; - } - - if (!isset($filters['includeEvent']) || !$filters['includeEvent']) { - unset($sight["Sighting"]["Event"]); - } else if (!$additional_event_added) { - $filters['requested_attributes'] = array_merge($filters['requested_attributes'], array('event_uuid', 'event_orgc_id', 'event_org_id', 'event_info', 'event_Orgc_name')); - $additional_event_added = true; - } - if (!empty($sight)) { - array_push($allowedSightings, $sight); - } + if (!empty($filters['uuid'])) { + if ($filters['context'] === 'attribute') { + $conditions['Attribute.uuid'] = $filters['uuid']; + $contain[] = 'Attribute'; + } elseif ($filters['context'] === 'event') { + $conditions['Event.uuid'] = $filters['uuid']; + $contain[] = 'Event'; } } - $params = array( - 'conditions' => array(), //result already filtered - ); + // fetch sightings matching the query + $sightingIds = $this->find('column', [ + 'conditions' => $conditions, + 'fields' => ['Sighting.id'], + 'contain' => $contain, + ]); - if (!isset($this->validFormats[$returnFormat])) { - // this is where the new code path for the export modules will go - throw new NotFoundException('Invalid export format.'); + $includeAttribute = isset($filters['includeAttribute']) && $filters['includeAttribute']; + $includeEvent = isset($filters['includeEvent']) && $filters['includeEvent']; + $requestedAttributes = ['id', 'attribute_id', 'event_id', 'org_id', 'date_sighting', 'uuid', 'source', 'type']; + if ($includeAttribute) { + $requestedAttributes = array_merge($requestedAttributes, ['attribute_uuid', 'attribute_type', 'attribute_category', 'attribute_to_ids', 'attribute_value']); } + if ($includeEvent) { + $requestedAttributes = array_merge($requestedAttributes, ['event_uuid', 'event_orgc_id', 'event_org_id', 'event_info', 'event_Orgc_name']); + } + $filters['requested_attributes'] = $requestedAttributes; $exportToolParams = array( 'user' => $user, - 'params' => $params, + 'params' => ['conditions' => []], //result already filtered 'returnFormat' => $returnFormat, 'scope' => 'Sighting', 'filters' => $filters @@ -904,57 +935,110 @@ class Sighting extends AppModel $tmpfile = new TmpFileTool(); $tmpfile->write($exportTool->header($exportToolParams)); + $separator = $exportTool->separator($exportToolParams); - $temp = ''; - $i = 0; - foreach ($allowedSightings as $sighting) { - $temp .= $exportTool->handler($sighting, $exportToolParams); - if ($temp !== '') { - if ($i != count($allowedSightings) -1) { - $temp .= $exportTool->separator($exportToolParams); + foreach ($sightingIds as $sightingId) { + // apply ACL and sighting policies + $sighting = $this->getSighting($sightingId, $user, $includeEvent); + if (!empty($sighting)) { + $sighting['Sighting']['value'] = $sighting['Sighting']['Attribute']['value']; + if (!$includeAttribute) { + unset($sighting['Sighting']['Attribute']); } + $tmpfile->writeWithSeparator($exportTool->handler($sighting, $exportToolParams), $separator); } - $i++; } - $tmpfile->write($temp); + $tmpfile->write($exportTool->footer($exportToolParams)); - return $tmpfile->finish(); + return $tmpfile; } /** * @param int|string $eventId Event ID or UUID * @param array $sightings * @param array $user - * @param null $passAlong - * @return int|string Number of saved sightings or error message as string + * @param int|null $passAlong Server ID + * @return int Number of saved sightings + * @throws Exception */ - public function bulkSaveSightings($eventId, $sightings, $user, $passAlong = null) + public function bulkSaveSightings($eventId, array $sightings, array $user, $passAlong = null) { $event = $this->Event->fetchSimpleEvent($user, $eventId); if (empty($event)) { - return 'Event not found or not accessible by this user.'; + throw new NotFoundException('Event not found or not accessible by this user.'); } - $saved = 0; + + // Since sightings are immutable (it is not possible to change it from web interface), we can check + // if sighting with given uuid already exists and skip them + $existingSighting = $this->existing($sightings); + + // Fetch existing organisations in bulk + $existingOrganisations = $this->existingOrganisations($sightings); + + // Fetch attributes IDs and event IDs + $attributesToTransform = $this->Attribute->fetchAttributesSimple($user, [ + 'conditions' => ['Attribute.uuid' => array_unique(array_column($sightings, 'attribute_uuid'))], + 'fields' => ['Attribute.id', 'Attribute.uuid', 'Attribute.event_id'], + ]); + $attributes = []; + foreach ($attributesToTransform as $attribute) { + $attributes[$attribute['Attribute']['uuid']] = [$attribute['Attribute']['id'], $attribute['Attribute']['event_id']]; + } + + $toSave = []; foreach ($sightings as $s) { + if (isset($existingSighting[$s['uuid']])) { + continue; // sighting already exists + } + if (!isset($attributes[$s['attribute_uuid']])) { + continue; // attribute doesn't exists or user don't have permission to access it + } + list($attributeId, $eventId) = $attributes[$s['attribute_uuid']]; + + if ($s['type'] === '2') { + // remove existing expiration by the same org if it exists + $this->deleteAll(array( + 'Sighting.org_id' => $user['org_id'], + 'Sighting.type' => 2, + 'Sighting.attribute_id' => $attributeId, + )); + } + $saveOnBehalfOf = false; if ($user['Role']['perm_sync']) { if (isset($s['org_id'])) { if ($s['org_id'] != 0 && !empty($s['Organisation'])) { - $saveOnBehalfOf = $this->Event->Orgc->captureOrg($s['Organisation'], $user); + if (isset($existingOrganisations[$s['Organisation']['uuid']])) { + $saveOnBehalfOf = $existingOrganisations[$s['Organisation']['uuid']]; + } else { + $saveOnBehalfOf = $this->Organisation->captureOrg($s['Organisation'], $user); + } } else { $saveOnBehalfOf = 0; } } } - $result = $this->saveSightings($s['attribute_uuid'], false, $s['date_sighting'], $user, $s['type'], $s['source'], $s['uuid'], false, $saveOnBehalfOf); - if (is_numeric($result)) { - $saved += $result; - } + + $toSave[] = [ + 'attribute_id' => $attributeId, + 'event_id' => $eventId, + 'org_id' => $saveOnBehalfOf === false ? $user['org_id'] : $saveOnBehalfOf, + 'date_sighting' => $s['date_sighting'], + 'type' => $s['type'], + 'source' => $s['source'], + 'uuid' => $s['uuid'], + ]; } - if ($saved > 0) { + if (empty($toSave)) { + return 0; + } + + if ($this->saveMany($toSave)) { $this->Event->publishRouter($event['Event']['id'], $passAlong, $user, 'sightings'); + return count($toSave); + } else { + return 0; } - return $saved; } public function pullSightings($user, $server) @@ -1005,6 +1089,36 @@ class Sighting extends AppModel return strtotime("-$rangeInDays days"); } + /** + * @param array $sightings + * @return array Existing sightings UUID in key + */ + private function existing(array $sightings) + { + $existingSighting = $this->find('column', [ + 'fields' => ['Sighting.uuid'], + 'conditions' => ['uuid' => array_column($sightings, 'uuid')], + ]); + // Move UUID to array key + return array_flip($existingSighting); + } + + /** + * @param array $sightings + * @return array Organisation UUID => Organisation ID + */ + private function existingOrganisations(array $sightings) + { + $organisations = array_column($sightings, 'Organisation'); + if (empty($organisations)) { + return []; + } + return $this->Organisation->find('list', [ + 'fields' => ['Organisation.uuid', 'Organisation.id'], + 'conditions' => ['Organisation.uuid' => array_unique(array_column($organisations, 'uuid'))], + ]); + } + /** * Sighting reporters setting * If the event has any sightings for the user's org, then the user is a sighting reporter for the event too. diff --git a/app/Model/Tag.php b/app/Model/Tag.php index dafd2349a..7cbe0e934 100644 --- a/app/Model/Tag.php +++ b/app/Model/Tag.php @@ -91,6 +91,9 @@ class Tag extends AppModel if (!isset($this->data['Tag']['exportable'])) { $this->data['Tag']['exportable'] = 1; } + if (isset($this->data['Tag']['name']) && strlen($this->data['Tag']['name']) >= 255) { + $this->data['Tag']['name'] = substr($this->data['Tag']['name'], 0, 255); + } $this->data['Tag']['is_galaxy'] = preg_match($this->reGalaxy, $this->data['Tag']['name']); $this->data['Tag']['is_custom_galaxy'] = preg_match($this->reCustomGalaxy, $this->data['Tag']['name']); return true; @@ -174,8 +177,8 @@ class Tag extends AppModel { $conditions = array(); if (!$user['Role']['perm_site_admin']) { - $conditions['Tag.org_id'] = array(0, $user['User']['org_id']); - $conditions['Tag.user_id'] = array(0, $user['User']['id']); + $conditions['Tag.org_id'] = array(0, $user['org_id']); + $conditions['Tag.user_id'] = array(0, $user['id']); $conditions['Tag.hide_tag'] = 0; } return $this->find('all', array('conditions' => $conditions, 'recursive' => -1)); @@ -463,13 +466,12 @@ class Tag extends AppModel public function getTagsByName($tag_names, $containTagConnectors = true) { - $contain = array('EventTag', 'AttributeTag'); $tag_params = array( - 'recursive' => -1, - 'conditions' => array('name' => $tag_names) + 'recursive' => -1, + 'conditions' => array('name' => $tag_names) ); if ($containTagConnectors) { - $tag_params['contain'] = $contain; + $tag_params['contain'] = array('EventTag', 'AttributeTag'); } $tags_temp = $this->find('all', $tag_params); $tags = array(); @@ -479,16 +481,19 @@ class Tag extends AppModel return $tags; } + /** + * @param string $namespace + * @param bool $containTagConnectors + * @return array Uppercase tag name in key + */ public function getTagsForNamespace($namespace, $containTagConnectors = true) { - - $contain = array('EventTag', 'AttributeTag'); $tag_params = array( - 'recursive' => -1, - 'conditions' => array('UPPER(name) LIKE' => strtoupper($namespace) . '%'), + 'recursive' => -1, + 'conditions' => array('LOWER(name) LIKE' => strtolower($namespace) . '%'), ); if ($containTagConnectors) { - $tag_params['contain'] = $contain; + $tag_params['contain'] = array('EventTag', 'AttributeTag'); } $tags_temp = $this->find('all', $tag_params); $tags = array(); @@ -511,13 +516,11 @@ class Tag extends AppModel } $id = $tag['Tag']['id']; } - $event_ids = $this->EventTag->find('list', array( - 'recursive' => -1, + $event_ids = $this->EventTag->find('column', array( 'conditions' => array('EventTag.tag_id' => $id), - 'fields' => array('EventTag.event_id', 'EventTag.event_id'), - 'order' => array('EventTag.event_id') + 'fields' => array('EventTag.event_id'), )); - $params = array('conditions' => array('Event.id' => array_values($event_ids))); + $params = array('conditions' => array('Event.id' => $event_ids)); $events = $this->EventTag->Event->fetchSimpleEvents($user, $params, true); foreach ($events as $k => $event) { $event['Event']['Orgc'] = $event['Orgc']; diff --git a/app/Model/Taxonomy.php b/app/Model/Taxonomy.php index bd03e90a3..f726db4df 100644 --- a/app/Model/Taxonomy.php +++ b/app/Model/Taxonomy.php @@ -1,6 +1,9 @@ exists()) { continue; } - $vocab = json_decode($file->read(), true); - $file->close(); - if ($vocab === null) { + try { + $vocab = $this->jsonDecode($file->read()); + } catch (Exception $e) { $updated['fails'][] = array('namespace' => $dir, 'fail' => "File machinetag.json is not valid JSON."); continue; } @@ -91,17 +94,55 @@ class Taxonomy extends AppModel return $updated; } - private function __updateVocab($vocab, $current, $skipUpdateFields = array()) + /** + * @param array $vocab + * @return int Taxonomy ID + * @throws Exception + */ + public function import(array $vocab) + { + foreach (['namespace', 'description', 'predicates'] as $requiredField) { + if (!isset($vocab[$requiredField])) { + throw new Exception("Required field '$requiredField' not provided."); + } + } + if (!is_array($vocab['predicates'])) { + throw new Exception("Field 'predicates' must be array."); + } + if (isset($vocab['values']) && !is_array($vocab['values'])) { + throw new Exception("Field 'values' must be array."); + } + if (!isset($vocab['version'])) { + $vocab['version'] = 1; + } + $current = $this->find('first', array( + 'conditions' => array('namespace' => $vocab['namespace']), + 'recursive' => -1, + 'fields' => array('version', 'enabled', 'namespace') + )); + $result = $this->__updateVocab($vocab, $current); + if (is_array($result)) { + throw new Exception('Could not save taxonomy because of validation errors: ' . json_encode($result)); + } + return (int)$result; + } + + private function __updateVocab(array $vocab, $current, array $skipUpdateFields = []) { $enabled = 0; - $taxonomy = array(); if (!empty($current)) { if ($current['Taxonomy']['enabled']) { $enabled = 1; } - $this->deleteAll(array('Taxonomy.namespace' => $current['Taxonomy']['namespace'])); + $this->deleteAll(['Taxonomy.namespace' => $current['Taxonomy']['namespace']]); } - $taxonomy['Taxonomy'] = array('namespace' => $vocab['namespace'], 'description' => $vocab['description'], 'version' => $vocab['version'], 'exclusive' => !empty($vocab['exclusive']), 'enabled' => $enabled); + $taxonomy = ['Taxonomy' => [ + 'namespace' => $vocab['namespace'], + 'description' => $vocab['description'], + 'version' => $vocab['version'], + 'exclusive' => !empty($vocab['exclusive']), + 'enabled' => $enabled, + ]]; $predicateLookup = array(); foreach ($vocab['predicates'] as $k => $predicate) { $taxonomy['Taxonomy']['TaxonomyPredicate'][$k] = $predicate; @@ -109,14 +150,15 @@ class Taxonomy extends AppModel } if (!empty($vocab['values'])) { foreach ($vocab['values'] as $value) { - if (empty($taxonomy['Taxonomy']['TaxonomyPredicate'][$predicateLookup[$value['predicate']]]['TaxonomyEntry'])) { - $taxonomy['Taxonomy']['TaxonomyPredicate'][$predicateLookup[$value['predicate']]]['TaxonomyEntry'] = $value['entry']; + $predicatePosition = $predicateLookup[$value['predicate']]; + if (empty($taxonomy['Taxonomy']['TaxonomyPredicate'][$predicatePosition]['TaxonomyEntry'])) { + $taxonomy['Taxonomy']['TaxonomyPredicate'][$predicatePosition]['TaxonomyEntry'] = $value['entry']; } else { - $taxonomy['Taxonomy']['TaxonomyPredicate'][$predicateLookup[$value['predicate']]]['TaxonomyEntry'] = array_merge($taxonomy['Taxonomy']['TaxonomyPredicate'][$predicateLookup[$value['predicate']]]['TaxonomyEntry'], $value['entry']); + $taxonomy['Taxonomy']['TaxonomyPredicate'][$predicatePosition]['TaxonomyEntry'] = array_merge($taxonomy['Taxonomy']['TaxonomyPredicate'][$predicatePosition]['TaxonomyEntry'], $value['entry']); } } } - $result = $this->saveAssociated($taxonomy, array('deep' => true)); + $result = $this->saveAssociated($taxonomy, ['deep' => true]); if ($result) { $this->__updateTags($this->id, $skipUpdateFields); return $this->id; @@ -175,7 +217,7 @@ class Taxonomy extends AppModel if ($filter) { $namespaceLength = strlen($taxonomy['Taxonomy']['namespace']); foreach ($entries as $k => $entry) { - if (strpos(substr(strtoupper($entry['tag']), $namespaceLength), strtoupper($filter)) === false) { + if (strpos(substr(mb_strtolower($entry['tag']), $namespaceLength), mb_strtolower($filter)) === false) { unset($entries[$k]); } } @@ -189,8 +231,7 @@ class Taxonomy extends AppModel public function getAllTaxonomyTags($inverse = false, $user = false, $full = false) { $this->Tag = ClassRegistry::init('Tag'); - $taxonomyIdList = $this->find('list', array('fields' => array('id'))); - $taxonomyIdList = array_keys($taxonomyIdList); + $taxonomyIdList = $this->find('column', array('fields' => array('Taxonomy.id'))); $allTaxonomyTags = array(); foreach ($taxonomyIdList as $taxonomy) { $allTaxonomyTags = array_merge($allTaxonomyTags, array_keys($this->getTaxonomyTags($taxonomy, true))); @@ -284,8 +325,8 @@ class Taxonomy extends AppModel if (empty($taxonomy)) { return false; } - $tag_names = Hash::extract($taxonomy, 'entries.{n}.tag'); - $tags = $this->Tag->getTagsByName($tag_names, false); + $tagNames = array_column($taxonomy['entries'], 'tag'); + $tags = $this->Tag->getTagsByName($tagNames, false); if (isset($taxonomy['entries'])) { foreach ($taxonomy['entries'] as $key => $temp) { if (isset($tags[strtoupper($temp['tag'])])) { @@ -307,13 +348,12 @@ class Taxonomy extends AppModel private function __updateTags($id, $skipUpdateFields = array()) { - $this->Tag = ClassRegistry::init('Tag'); App::uses('ColourPaletteTool', 'Tools'); $paletteTool = new ColourPaletteTool(); $taxonomy = $this->__getTaxonomy($id, array('full' => true)); $colours = $paletteTool->generatePaletteFromString($taxonomy['Taxonomy']['namespace'], count($taxonomy['entries'])); $this->Tag = ClassRegistry::init('Tag'); - $tags = $this->Tag->getTagsForNamespace($taxonomy['Taxonomy']['namespace']); + $tags = $this->Tag->getTagsForNamespace($taxonomy['Taxonomy']['namespace'], false); foreach ($taxonomy['entries'] as $k => $entry) { if (isset($tags[strtoupper($entry['tag'])])) { $temp = $tags[strtoupper($entry['tag'])]; @@ -501,12 +541,9 @@ class Taxonomy extends AppModel return $taxonomies; } - public function getTaxonomyForTag($tagName, $metaOnly = false, $fullTaxonomy = False) + public function getTaxonomyForTag($tagName, $metaOnly = false, $fullTaxonomy = false) { - if (preg_match('/^[^:="]+:[^:="]+="[^:="]+"$/i', $tagName)) { - $temp = explode(':', $tagName); - $pieces = array_merge(array($temp[0]), explode('=', $temp[1])); - $pieces[2] = trim($pieces[2], '"'); + if (preg_match('/^([^:="]+):([^:="]+)="([^:="]+)"$/i', $tagName, $matches)) { $contain = array( 'TaxonomyPredicate' => array( 'TaxonomyEntry' => array() @@ -514,15 +551,15 @@ class Taxonomy extends AppModel ); if (!$fullTaxonomy) { $contain['TaxonomyPredicate']['conditions'] = array( - 'LOWER(TaxonomyPredicate.value)' => strtolower($pieces[1]) + 'LOWER(TaxonomyPredicate.value)' => mb_strtolower($matches[2]), ); $contain['TaxonomyPredicate']['TaxonomyEntry']['conditions'] = array( - 'LOWER(TaxonomyEntry.value)' => strtolower($pieces[2]) + 'LOWER(TaxonomyEntry.value)' => mb_strtolower($matches[3]), ); } $taxonomy = $this->find('first', array( 'recursive' => -1, - 'conditions' => array('LOWER(Taxonomy.namespace)' => strtolower($pieces[0])), + 'conditions' => array('LOWER(Taxonomy.namespace)' => mb_strtolower($matches[1])), 'contain' => $contain )); if ($metaOnly && !empty($taxonomy)) { @@ -534,12 +571,12 @@ class Taxonomy extends AppModel $contain = array('TaxonomyPredicate' => array()); if (!$fullTaxonomy) { $contain['TaxonomyPredicate']['conditions'] = array( - 'LOWER(TaxonomyPredicate.value)' => strtolower($pieces[1]) + 'LOWER(TaxonomyPredicate.value)' => mb_strtolower($pieces[1]) ); } $taxonomy = $this->find('first', array( 'recursive' => -1, - 'conditions' => array('LOWER(Taxonomy.namespace)' => strtolower($pieces[0])), + 'conditions' => array('LOWER(Taxonomy.namespace)' => mb_strtolower($pieces[0])), 'contain' => $contain )); if ($metaOnly && !empty($taxonomy)) { diff --git a/app/Model/TaxonomyPredicate.php b/app/Model/TaxonomyPredicate.php index fb025d696..412021138 100644 --- a/app/Model/TaxonomyPredicate.php +++ b/app/Model/TaxonomyPredicate.php @@ -1,6 +1,9 @@ getUserById($id); - if (empty($user)) { - return $user; + if (empty($id)) { + throw new InvalidArgumentException('Invalid user ID.'); } - return $this->rearrangeToAuthForm($user); + $conditions = ['User.id' => $id]; + return $this->getAuthUserByConditions($conditions); } // get the current user and rearrange it to be in the same format as in the auth component - public function getAuthUserByAuthkey($id) + public function getAuthUserByAuthkey($authkey) { - $conditions = array('User.authkey' => $id); - $user = $this->find('first', array('conditions' => $conditions, 'recursive' => -1,'contain' => array('Organisation', 'Role', 'Server'))); - if (empty($user)) { - return $user; + if (empty($authkey)) { + throw new InvalidArgumentException('Invalid user auth key.'); } - return $this->rearrangeToAuthForm($user); + $conditions = array('User.authkey' => $authkey); + return $this->getAuthUserByConditions($conditions); } public function getAuthUserByExternalAuth($auth_key) { + if (empty($auth_key)) { + throw new InvalidArgumentException('Invalid user external auth key.'); + } $conditions = array( 'User.external_auth_key' => $auth_key, 'User.external_auth_required' => true ); - $user = $this->find('first', array( + return $this->getAuthUserByConditions($conditions); + } + + /** + * @param array $conditions + * @return array|null + */ + private function getAuthUserByConditions(array $conditions) + { + $user = $this->find('first', [ 'conditions' => $conditions, 'recursive' => -1, - 'contain' => array( + 'contain' => [ 'Organisation', 'Role', - 'Server' - ) - )); + 'Server', + ], + ]); if (empty($user)) { return $user; } @@ -696,9 +707,6 @@ class User extends AppModel $user['User']['Role'] = $user['Role']; $user['User']['Organisation'] = $user['Organisation']; $user['User']['Server'] = $user['Server']; - if (isset($user['UserSetting'])) { - $user['User']['UserSetting'] = $user['UserSetting']; - } return $user['User']; } @@ -880,38 +888,6 @@ class User extends AppModel return $fields; } - public function getMembersCount($org_id = false) - { - // for Organizations List - $conditions = array(); - $findType = 'all'; - if ($org_id !== false) { - $findType = 'first'; - $conditions = array('User.org_id' => $org_id); - } - $fields = array('org_id', 'COUNT(User.id) AS num_members'); - $params = array( - 'fields' => $fields, - 'recursive' => -1, - 'group' => array('org_id'), - 'order' => array('org_id'), - 'conditions' => $conditions - ); - $orgs = $this->find($findType, $params); - if (empty($orgs)) { - return 0; - } - if ($org_id !== false) { - return $orgs[0]['num_members']; - } else { - $usersPerOrg = []; - foreach ($orgs as $key => $value) { - $usersPerOrg[$value['User']['org_id']] = $value[0]['num_members']; - } - return $usersPerOrg; - } - } - public function findAdminsResponsibleForUser($user) { $admin = $this->find('first', array( @@ -967,7 +943,7 @@ class User extends AppModel if ($result) { $this->id = $user['User']['id']; $this->saveField('password', $password); - $this->saveField('change_pw', '1'); + $this->updateField($user['User'], 'change_pw', 1); if ($simpleReturn) { return true; } else { @@ -983,10 +959,9 @@ class User extends AppModel public function getOrgAdminsForOrg($org_id, $excludeUserId = false) { - $adminRoles = $this->Role->find('list', array( - 'recursive' => -1, + $adminRoles = $this->Role->find('column', array( 'conditions' => array('perm_admin' => 1), - 'fields' => array('Role.id', 'Role.id') + 'fields' => array('Role.id') )); $conditions = array( 'User.org_id' => $org_id, @@ -1142,7 +1117,7 @@ class User extends AppModel if (empty(Configure::read('Security.advanced_authkeys'))) { $oldKey = $this->data['User']['authkey']; $newkey = $this->generateAuthKey(); - $this->saveField('authkey', $newkey); + $this->updateField($updatedUser['User'], 'authkey', $newkey); $this->extralog( $user, 'reset_auth_key', @@ -1212,6 +1187,27 @@ class User extends AppModel $syslog->write('notice', "$description -- $action" . (empty($fieldResult) ? '' : ' -- ' . $result['Log']['change'])); } + /** + * @return array|null + * @throws Exception + */ + public function getGpgPublicKey() + { + $email = Configure::read('GnuPG.email'); + if (!$email) { + throw new Exception("Configuration option 'GnuPG.email' is not set, public key cannot be exported."); + } + + $cryptGpg = $this->initializeGpg(); + $fingerprint = $cryptGpg->getFingerprint($email); + if (!$fingerprint) { + return null; + } + + $publicKey = $cryptGpg->exportPublicKey($fingerprint); + return array($fingerprint, $publicKey); + } + public function getOrgActivity($orgId, $params=array()) { $conditions = array(); @@ -1265,24 +1261,6 @@ class User extends AppModel return $data; } - /* - * Set the monitoring flag in Configure for the current user - * Reads the state from redis - */ - public function setMonitoring($user) - { - if ( - !empty(Configure::read('Security.user_monitoring_enabled')) - ) { - $redis = $this->setupRedis(); - if (!empty($redis->sismember('misp:monitored_users', $user['id']))) { - Configure::write('Security.monitored', 1); - return true; - } - } - Configure::write('Security.monitored', 0); - } - public function registerUser($added_by, $registration, $org_id, $role_id) { $user = array( 'email' => $registration['data']['email'], @@ -1382,7 +1360,7 @@ class User extends AppModel $name => $value, ], true, ['id', $name, 'date_modified']); if (!$success) { - throw new RuntimeException("Could not save field `$name` with value `$value` for user `{$user['id']}`."); + throw new RuntimeException("Could not save setting $name for user {$user['id']}."); } } diff --git a/app/Model/UserSetting.php b/app/Model/UserSetting.php index 8dc179d84..4c3ef2a15 100644 --- a/app/Model/UserSetting.php +++ b/app/Model/UserSetting.php @@ -201,55 +201,50 @@ class UserSetting extends AppModel public function getDefaulRestSearchParameters($user) { - $setting = $this->find('first', array( - 'recursive' => -1, - 'conditions' => array( - 'UserSetting.user_id' => $user['id'], - 'UserSetting.setting' => 'default_restsearch_parameters' - ) - )); - $parameters = array(); - if (!empty($setting)) { - $parameters = $setting['UserSetting']['value']; - } - return $parameters; + return $this->getValueForUser($user['id'], 'default_restsearch_parameters') ?: []; } public function getTagNumericalValueOverride($userId) { - $setting = $this->find('first', array( - 'recursive' => -1, - 'conditions' => array( - 'UserSetting.user_id' => $userId, - 'UserSetting.setting' => 'tag_numerical_value_override' - ) - )); - $parameters = array(); - if (!empty($setting)) { - $parameters = $setting['UserSetting']['value']; - } - return $parameters; + return $this->getValueForUser($userId, 'tag_numerical_value_override') ?: []; } - /* - * Check whether the event is something the user is interested (to be alerted on) - * + /** + * @param int $userId + * @param string $setting + * @return mixed|null */ - public function checkPublishFilter($user, $event) + public function getValueForUser($userId, $setting) + { + $output = $this->find('first', array( + 'recursive' => -1, + 'fields' => ['value'], + 'conditions' => array( + 'UserSetting.user_id' => $userId, + 'UserSetting.setting' => $setting, + ) + )); + if ($output) { + return $output['UserSetting']['value']; + } + return null; + } + + /** + * Check whether the event is something the user is interested (to be alerted on) + * @param $user + * @param $event + * @return bool + */ + public function checkPublishFilter(array $user, array $event) { - $rule = $this->find('first', array( - 'recursive' => -1, - 'conditions' => array( - 'UserSetting.user_id' => $user['id'], - 'UserSetting.setting' => 'publish_alert_filter' - ) - )); + $rule = $this->getValueForUser($user['id'], 'publish_alert_filter'); // We should return true if no setting has been configured, or there's a setting with an empty value - if (empty($rule) || empty($rule['UserSetting']['value'])) { + if (empty($rule)) { return true; } // recursively evaluate the boolean tree to true/false and return the value - $result = $this->__recursiveConvert($rule['UserSetting']['value'], $event); + $result = $this->__recursiveConvert($rule, $event); if (isset($result[0])) { return $result[0]; } else { @@ -257,7 +252,7 @@ class UserSetting extends AppModel } } - /* + /** * Convert a complex rule set recursively * takes as params a rule branch and an event to check against * evaluate whether the rule set evaluates as true/false @@ -285,9 +280,9 @@ class UserSetting extends AppModel } } } - $toReturn []= $temp; + $toReturn[] = $temp; } else { - $toReturn []= $this->__checkEvent($k, $v, $event); + $toReturn[] = $this->__checkEvent($k, $v, $event); } } return $toReturn; @@ -304,6 +299,7 @@ class UserSetting extends AppModel * - Tag.name (checks against both event and attribute tags) * - Orgc.uuid * - Orgc.name + * - ThreatLevel.name * Values passed can be used for direct string comparisons or alternatively * as substring matches by encapsulating the string in a pair of "%" characters * Each rule can take a list of values @@ -333,6 +329,8 @@ class UserSetting extends AppModel Hash::extract($event, 'Object.{n}.Attribute.{n}.AttributeTag.{n}.Tag.name'), Hash::extract($event, 'EventTag.{n}.Tag.name') ); + } else if ($rule === 'ThreatLevel.name') { + $values = [$event['ThreatLevel']['name']]; } if (!empty($values)) { foreach ($values as $extracted_value) { @@ -406,18 +404,14 @@ class UserSetting extends AppModel return true; } + /** + * @param int $user_id + * @param string $setting + * @return array|mixed + * @deprecated + */ public function getSetting($user_id, $setting) { - $setting = $this->find('first', array( - 'recursive' => -1, - 'conditions' => array( - 'UserSetting.user_id' => $user_id, - 'UserSetting.setting' => $setting - ) - )); - if (empty($setting)) { - return array(); - } - return $setting['UserSetting']['value']; + return $this->getValueForUser($user_id, $setting) ?: []; } } diff --git a/app/Model/Warninglist.php b/app/Model/Warninglist.php index ab98bbb5a..115a31887 100644 --- a/app/Model/Warninglist.php +++ b/app/Model/Warninglist.php @@ -1,5 +1,6 @@ multi(Redis::PIPELINE); foreach ($saveToCache as $attributeKey => $json) { - $redis->setex($attributeKey, 3600, $json); // cache for one hour + $redis->setex($attributeKey, 8 * 3600, $json); // cache for eight hour } $pipe->exec(); } @@ -304,8 +305,7 @@ class Warninglist extends AppModel if ($id && $warninglist['Warninglist']['id'] != $id) { continue; } - $entries = $this->WarninglistEntry->find('list', array( - 'recursive' => -1, + $entries = $this->WarninglistEntry->find('column', array( 'conditions' => array('warninglist_id' => $warninglist['Warninglist']['id']), 'fields' => array('value') )); @@ -390,49 +390,15 @@ class Warninglist extends AppModel if ($redis !== false && $redis->exists('misp:warninglist_entries_cache:' . $id)) { return $redis->sMembers('misp:warninglist_entries_cache:' . $id); } else { - $entries = array_values($this->WarninglistEntry->find('list', array( - 'recursive' => -1, + $entries = $this->WarninglistEntry->find('column', array( 'conditions' => array('warninglist_id' => $id), - 'fields' => array('value') - ))); + 'fields' => array('WarninglistEntry.value') + )); $this->cacheWarninglistEntries($entries, $id); return $entries; } } - /** - * Filter out invalid IPv4 or IPv4 CIDR and append maximum netmaks if no netmask is given. - * @param array $inputValues - * @return array - */ - private function filterCidrList($inputValues) - { - $outputValues = []; - foreach ($inputValues as $v) { - $v = strtolower($v); - $parts = explode('/', $v, 2); - if (filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) { - $maximumNetmask = 32; - } else if (filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) { - $maximumNetmask = 128; - } else { - // IP address part of CIDR is invalid - continue; - } - - if (!isset($parts[1])) { - // If CIDR doesnt contains '/', we will consider CIDR as /32 for IPv4 or /128 for IPv6 - $v = "$v/$maximumNetmask"; - } else if ($parts[1] > $maximumNetmask || $parts[1] < 0) { - // Netmask part of CIDR is invalid - continue; - } - - $outputValues[$v] = true; - } - return $outputValues; - } - /** * For 'hostname', 'string' and 'cidr' warninglist type, values are just in keys to save memory. * @@ -461,7 +427,7 @@ class Warninglist extends AppModel } $values = $output; } else if ($warninglist['Warninglist']['type'] === 'cidr') { - $values = $this->filterCidrList($values); + $values = new CidrTool($values); } $this->entriesCache[$id] = $values; @@ -482,7 +448,7 @@ class Warninglist extends AppModel if ($object['to_ids'] || $this->showForAll) { foreach ($warninglists as $list) { - if (in_array('ALL', $list['types']) || in_array($object['type'], $list['types'])) { + if (in_array('ALL', $list['types'], true) || in_array($object['type'], $list['types'], true)) { $result = $this->__checkValue($this->getFilteredEntries($list), $object['value'], $object['type'], $list['Warninglist']['type']); if ($result !== false) { $object['warnings'][] = array( @@ -499,7 +465,7 @@ class Warninglist extends AppModel } /** - * @param array $listValues + * @param array|CidrTool $listValues * @param string $value * @param string $type * @param string $listType @@ -514,7 +480,7 @@ class Warninglist extends AppModel } foreach ($value as $v) { if ($listType === 'cidr') { - $result = $this->__evalCidrList($listValues, $v); + $result = $listValues->contains($v); } elseif ($listType === 'string') { $result = $this->__evalString($listValues, $v); } elseif ($listType === 'substring') { @@ -538,75 +504,6 @@ class Warninglist extends AppModel return $this->__checkValue($listValues, $value, '', $type) !== false; } - private function __evalCidrList($listValues, $value) - { - $valueMask = null; - if (strpos($value, '/') !== false) { - list($value, $valueMask) = explode('/', $value); - } - - $match = false; - if (filter_var($value, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) { - // This code converts IP address to all possible CIDRs that can contains given IP address - // and then check if given hash table contains that CIDR. - $ip = ip2long($value); - // Start from 1, because doesn't make sense to check 0.0.0.0/0 match - for ($bits = 1; $bits <= 32; $bits++) { - $mask = -1 << (32 - $bits); - $needle = long2ip($ip & $mask) . "/$bits"; - if (isset($listValues[$needle])) { - $match = $needle; - break; - } - } - - } elseif (filter_var($value, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) { - foreach ($listValues as $lv => $foo) { - if (strpos($lv, ':') !== false) { // Filter out IPv4 CIDR, IPv6 CIDR must contain colon - if ($this->__ipv6InCidr($value, $lv)) { - $match = $lv; - break; - } - } - } - } - - if ($match && $valueMask) { - $matchMask = explode('/', $match)[1]; - if ($valueMask < $matchMask) { - return false; - } - } - - return $match; - } - - /** - * Using solution from https://github.com/symfony/symfony/blob/master/src/Symfony/Component/HttpFoundation/IpUtils.php - * - * @param string $ip - * @param string $cidr - * @return bool - */ - private function __ipv6InCidr($ip, $cidr) - { - list($address, $netmask) = explode('/', $cidr); - - $bytesAddr = unpack('n*', inet_pton($address)); - $bytesTest = unpack('n*', inet_pton($ip)); - - for ($i = 1, $ceil = ceil($netmask / 16); $i <= $ceil; ++$i) { - $left = $netmask - 16 * ($i - 1); - $left = ($left <= 16) ? $left : 16; - $mask = ~(0xffff >> $left) & 0xffff; - if (($bytesAddr[$i] & $mask) != ($bytesTest[$i] & $mask)) { - return false; - } - } - - return true; - } - /** * Check for exact match. * @@ -677,14 +574,13 @@ class Warninglist extends AppModel */ public function fetchTLDLists() { - $tldLists = $this->find('list', array( + $tldLists = $this->find('column', array( 'conditions' => array('Warninglist.name' => $this->__tlds), - 'recursive' => -1, - 'fields' => array('Warninglist.id', 'Warninglist.id') + 'fields' => array('Warninglist.id') )); $tlds = array(); if (!empty($tldLists)) { - $tlds = $this->WarninglistEntry->find('list', array( + $tlds = $this->WarninglistEntry->find('column', array( 'conditions' => array('WarninglistEntry.warninglist_id' => $tldLists), 'fields' => array('WarninglistEntry.value') )); @@ -692,7 +588,7 @@ class Warninglist extends AppModel $tlds[$key] = strtolower($value); } } - if (!in_array('onion', $tlds)) { + if (!in_array('onion', $tlds, true)) { $tlds[] = 'onion'; } return $tlds; @@ -710,7 +606,7 @@ class Warninglist extends AppModel } foreach ($warninglists as $warninglist) { - if (in_array('ALL', $warninglist['types']) || in_array($attribute['type'], $warninglist['types'])) { + if (in_array('ALL', $warninglist['types'], true) || in_array($attribute['type'], $warninglist['types'], true)) { $result = $this->__checkValue($this->getFilteredEntries($warninglist), $attribute['value'], $attribute['type'], $warninglist['Warninglist']['type']); if ($result !== false) { return false; diff --git a/app/Plugin/ShibbAuth/Controller/Component/Auth/ApacheShibbAuthenticate.php b/app/Plugin/ShibbAuth/Controller/Component/Auth/ApacheShibbAuthenticate.php index 44e14c20e..fa830d337 100644 --- a/app/Plugin/ShibbAuth/Controller/Component/Auth/ApacheShibbAuthenticate.php +++ b/app/Plugin/ShibbAuth/Controller/Component/Auth/ApacheShibbAuthenticate.php @@ -145,6 +145,7 @@ class ApacheShibbAuthenticate extends BaseAuthenticate * @param string $org * @param array $user * @return int + * @throws Exception */ private function checkOrganization($org, $user) { @@ -166,7 +167,7 @@ class ApacheShibbAuthenticate extends BaseAuthenticate if ($user) { $orgUserId = $user['id']; } - $orgId = $orgModel->createOrgFromName($org, $orgUserId, 0); // Created with local set to 0 by default + $orgId = $orgModel->createOrgFromName($org, $orgUserId, true); CakeLog::info("User organisation `$org` created with ID $orgId."); } else { $orgId = $orgAux['Organisation']['id']; @@ -192,13 +193,13 @@ class ApacheShibbAuthenticate extends BaseAuthenticate foreach ($groupList as $group) { // TODO: Can be optimized inverting the search group and using only array_key_exists if (array_key_exists($group, $groupRoleMatching)) { //In case there is an group not defined in the config.php file - CakeLog::write('info', "User group ${group} found."); + CakeLog::write('info', "User group $group found."); $roleVal = $groupRoleMatching[$group]; if ($roleVal <= $roleId || $roleId == -1) { $roleId = $roleVal; $roleChanged = true; } - CakeLog::write('info', "User role ${roleId} assigned."); + CakeLog::write('info', "User role $roleId assigned."); } } return array($roleChanged, $roleId); @@ -217,7 +218,8 @@ class ApacheShibbAuthenticate extends BaseAuthenticate private function updateUserRole($roleChanged, array $user, $roleId, User $userModel) { if ($roleChanged && $user['role_id'] != $roleId) { - CakeLog::write('warning', "User role changed from ${user['role_id']} to $roleId."); + $message = "User role changed from ${user['role_id']} to $roleId for user ${user['email']} (${user['id']})."; + CakeLog::write('warning', $message); $userModel->updateField($user, 'role_id', $roleId); } return $user; @@ -233,7 +235,8 @@ class ApacheShibbAuthenticate extends BaseAuthenticate private function updateUserOrg($orgId, array $user, User $userModel) { if ($user['org_id'] != $orgId) { - CakeLog::write('warning', "User organisation $orgId changed."); + $message = "User organisation changed from ${user['org_id']} to $orgId for user ${user['email']} (${user['id']})."; + CakeLog::write('warning', $message); $user['org_id'] = $orgId; // Different role either increase or decrease permissions $userModel->updateField($user, 'org_id', $orgId); } diff --git a/app/Plugin/ShibbAuth/README.md b/app/Plugin/ShibbAuth/README.md index 9e4d20920..bfcc1ce5b 100644 --- a/app/Plugin/ShibbAuth/README.md +++ b/app/Plugin/ShibbAuth/README.md @@ -68,7 +68,7 @@ Edit your MISP apache configuration by adding the below (location depends on you ```Apache SetHandler shib - + ``` Enable the plugin at bootstrap.php: @@ -144,7 +144,7 @@ If used with Apache as webserver it might be useful to make a distinction to fil If you want the logout button to work for killing your session, you can use the CustomAuth plugin to configure a custom logout url, by default the url should be https://<host>/Shibboleth.sso/Logout. This leads to a local logout. If you want to also trigger a logout at the identity provider, you can use the return mechanism. In this case you will need to change the allowed redirects. Your logout url will look like https://<host>/Shibboleth.sso/Logout?return=https:///Logout. Edit your shibboleth configuration (often at /etc/shibboleth/shibboleth2.xml) as necessary. Relevant shibboleth documentation can be found at https://wiki.shibboleth.net/confluence/display/SP3/Logout and https://wiki.shibboleth.net/confluence/display/SP3/Sessions. ```xml ``` diff --git a/app/Test/CidrToolTest.php b/app/Test/CidrToolTest.php new file mode 100644 index 000000000..2ea58bc29 --- /dev/null +++ b/app/Test/CidrToolTest.php @@ -0,0 +1,41 @@ +assertFalse($cidrTool->contains('1.2.3.4')); + } + + public function testIpv4Fullmask(): void + { + $cidrTool = new CidrTool(['1.2.3.4/32']); + $this->assertEquals('1.2.3.4/32', $cidrTool->contains('1.2.3.4')); + } + + public function testIpv4WithoutNetmask(): void + { + $cidrTool = new CidrTool(['1.2.3.4']); + $this->assertEquals('1.2.3.4/32', $cidrTool->contains('1.2.3.4')); + } + + public function testIpv4(): void + { + $cidrTool = new CidrTool(['10.0.0.0/8', '8.0.0.0/8', '9.0.0.0/8']); + $this->assertEquals('8.0.0.0/8', $cidrTool->contains('8.8.8.8')); + $this->assertFalse($cidrTool->contains('::1')); + $this->assertFalse($cidrTool->contains('7.1.2.3')); + } + + public function testIpv6(): void + { + $cidrTool = new CidrTool(['2001:0db8:1234::/48']); + $this->assertEquals('2001:db8:1234::/48', $cidrTool->contains('2001:0db8:1234:0000:0000:0000:0000:0000')); + $this->assertEquals('2001:db8:1234::/48', $cidrTool->contains('2001:0db8:1234:ffff:ffff:ffff:ffff:ffff')); + $this->assertFalse($cidrTool->contains('2002:0db8:1234:ffff:ffff:ffff:ffff:ffff')); + } +} diff --git a/app/View/Allowedlists/admin_index.ctp b/app/View/Allowedlists/admin_index.ctp index 6dd754af2..1943b30bd 100644 --- a/app/View/Allowedlists/admin_index.ctp +++ b/app/View/Allowedlists/admin_index.ctp @@ -4,13 +4,6 @@ '; + if (empty($ajax)) { + echo $this->element('/genericElements/SideMenu/side_menu', $menuData); + } +?> + diff --git a/app/View/Dashboards/list_templates.ctp b/app/View/Dashboards/list_templates.ctp index 3a772c962..edd37d9cf 100644 --- a/app/View/Dashboards/list_templates.ctp +++ b/app/View/Dashboards/list_templates.ctp @@ -91,12 +91,3 @@ )); echo ''; echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'dashboard', 'menuItem' => 'dashboardTemplateIndex')); -?> - diff --git a/app/View/Dashboards/widget_loader.ctp b/app/View/Dashboards/widget_loader.ctp index d2e65d58b..0eda51ac0 100644 --- a/app/View/Dashboards/widget_loader.ctp +++ b/app/View/Dashboards/widget_loader.ctp @@ -8,7 +8,7 @@ ?> -Js->writeBuffer(); -?> diff --git a/app/View/Elements/Users/userIndexTable.ctp b/app/View/Elements/Users/userIndexTable.ctp index c76934add..519241cbe 100644 --- a/app/View/Elements/Users/userIndexTable.ctp +++ b/app/View/Elements/Users/userIndexTable.ctp @@ -4,10 +4,12 @@ Paginator->sort('org_ci', __('Org'));?> Paginator->sort('role_id', __('Role'));?> Paginator->sort('email');?> + Paginator->sort('authkey');?> - Paginator->sort('autoalert');?> - Paginator->sort('contactalert');?> - Paginator->sort('gpgkey', __('PGP key'));?> + + Paginator->sort('autoalert', __('Event alert'));?> + Paginator->sort('contactalert', __('Contact alert'));?> + Paginator->sort('gpgkey', __('PGP Key'));?> Paginator->sort('certif_public', 'S/MIME');?> @@ -40,9 +42,11 @@   + ****************************************  + @@ -85,7 +89,7 @@ - + Html->link('', array('admin' => true, 'action' => 'edit', $user['User']['id']), array('class' => 'fa fa-edit', 'title' => __('Edit'), 'aria-label' => __('Edit'))); echo $this->Form->postLink('', array('admin' => true, 'action' => 'delete', $user['User']['id']), array('class' => 'fa fa-trash', 'title' => __('Delete'), 'aria-label' => __('Delete')), __('Are you sure you want to delete # %s? It is highly recommended to never delete users but to disable them instead.', $user['User']['id'])); diff --git a/app/View/Elements/ajaxTags.ctp b/app/View/Elements/ajaxTags.ctp index 05711dfa1..0199fb099 100644 --- a/app/View/Elements/ajaxTags.ctp +++ b/app/View/Elements/ajaxTags.ctp @@ -76,7 +76,7 @@ ); if (!empty($tag['Tag']['id'])) { $span_tag = sprintf( - '%s', + '%s', sprintf( '%s%s%s', $baseurl, @@ -86,6 +86,7 @@ $aStyle, $aClass, $aText, + h($tag['Tag']['id']), isset($aTextModified) ? $aTextModified : $aText ); } else { diff --git a/app/View/Elements/eventattribute.ctp b/app/View/Elements/eventattribute.ctp index bc605e21c..0871cbedd 100644 --- a/app/View/Elements/eventattribute.ctp +++ b/app/View/Elements/eventattribute.ctp @@ -36,14 +36,12 @@ $url = array_merge(array('controller' => 'events', 'action' => 'viewEventAttributes', $event['Event']['id']), $params); $this->Paginator->options(array( 'url' => $url, - 'update' => '#attributes_div', - 'evalScripts' => true, - 'before' => '$(".loading").show()', - 'complete' => '$(".loading").hide()', + 'data-paginator' => '#attributes_div', )); - echo $this->Paginator->prev('« ' . __('previous'), array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'prev disabled', 'escape' => false, 'disabledTag' => 'span')); - echo $this->Paginator->numbers(array('modulus' => 60, 'separator' => '', 'tag' => 'li', 'currentClass' => 'red', 'currentTag' => 'span')); - echo $this->Paginator->next(__('next') . ' »', array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'next disabled', 'escape' => false, 'disabledTag' => 'span')); + $paginatorLinks = $this->Paginator->prev('« ' . __('previous'), array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'prev disabled', 'escape' => false, 'disabledTag' => 'span')); + $paginatorLinks .= $this->Paginator->numbers(array('modulus' => 60, 'separator' => '', 'tag' => 'li', 'currentClass' => 'red', 'currentTag' => 'span')); + $paginatorLinks .= $this->Paginator->next(__('next') . ' »', array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'next disabled', 'escape' => false, 'disabledTag' => 'span')); + echo $paginatorLinks; ?>
  • + Form->create('Sighting', ['id' => 'SightingForm', 'url' => $baseurl . '/sightings/add/', 'style' => 'display:none;']); + echo $this->Form->input('id', ['label' => false, 'type' => 'number']); + echo $this->Form->input('type', ['label' => false]); + echo $this->Form->end(); + ?>
    @@ -201,18 +206,7 @@ attributes or the appropriate distribution level. If you think there is a mistak