MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch '2.4' of github.com:MISP/MISP into pr-5856

pull/5925/head
mokaddem 2020-05-18 09:22:46 +02:00
parent a0476bda3d c1e2a5479a
commit 944b613a19
No known key found for this signature in database
GPG Key ID: 164C473F627A06FA
87 changed files with 10450 additions and 3971 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.github/PULL_REQUEST_TEMPLATE.md vendored
View File

@ -2,7 +2,7 @@
* One Pull Request per fix/feature/change/...
* Keep the amount of commits per PR as small as possible: if for any reason, you need to fix your commit after the pull request, please squash the changes in one single commit (or tell us why not)
* Always make sure it is mergeable in the default branch (as of today 2016-06-03: branch 2.4)
* Always make sure it is mergeable in the default branch (as of today 2020-05-05: branch 2.4)
* Please make sure Travis CI works on this request, or update the test cases if needed
* Any major changes adding a functionality should be disabled by default in the config
@ -16,8 +16,3 @@ If it fixes an existing issue, please use github syntax: `#<IssueID>`
- [ ] Does it require a DB change?
- [ ] Are you using it in production?
- [ ] Does it require a change in the API (PyMISP for example)?
#### Release Type:
- [ ] Major
- [ ] Minor
- [X] Patch

3
.gitmodules vendored
View File

@ -40,3 +40,6 @@
[submodule "app/files/misp-decaying-models"]
path = app/files/misp-decaying-models
url = https://github.com/MISP/misp-decaying-models.git
[submodule "app/files/scripts/misp-opendata"]
path = app/files/scripts/misp-opendata
url = https://github.com/MISP/misp-opendata

3
.travis.yml
View File

@ -76,6 +76,7 @@ install:
- sudo cp travis/database.php app/Config/database.php
- sudo cp app/Config/core.default.php app/Config/core.php
- sudo cp app/Config/config.default.php app/Config/config.php
- sudo cp travis/email.php app/Config/email.php
# Ensure the perms
- sudo chown -R $USER:www-data `pwd`/app/Config
- sudo chmod -R 770 `pwd`/app/Config
@ -107,6 +108,8 @@ install:
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_port" 6379'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_database" 13'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_password" ""'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg"'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.password" "travistest"'
- sudo -E su $USER -c 'app/Console/cake Admin updateGalaxies'
- sudo -E su $USER -c 'app/Console/cake Admin updateTaxonomies'
- sudo -E su $USER -c 'app/Console/cake Admin updateWarningLists'

120
INSTALL/INSTALL.sh
View File

@ -377,18 +377,19 @@ EOF
checkInstaller () {
# Workaround: shasum is not available on RHEL, only checking sha512
if [[ $FLAVOUR == "rhel" ]] || [[ $FLAVOUR == "centos" ]]; then
INSTsum=$(sha512sum ${0} | cut -f1 -d\ )
/usr/bin/wget --no-cache -q -O /tmp/INSTALL.sh.sha512 https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/INSTALL.sh.sha512
INSTsum=$(sha512sum ${0} | cut -f1 -d\ )
/usr/bin/wget --no-cache -q -O /tmp/INSTALL.sh.sha512 https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/INSTALL.sh.sha512
chsum=$(cat /tmp/INSTALL.sh.sha512)
if [[ "${chsum}" == "${INSTsum}" ]]; then
echo "SHA512 matches"
else
echo "SHA512: ${chsum} does not match the installer sum of: ${INSTsum}"
# exit 1 # uncomment when/if PR is merged
fi
if [[ "${chsum}" == "${INSTsum}" ]]; then
echo "SHA512 matches"
else
echo "SHA512: ${chsum} does not match the installer sum of: ${INSTsum}"
# exit 1 # uncomment when/if PR is merged
fi
else
# TODO: Implement $FLAVOUR checks and install depending on the platform we are on
if [[ $(which shasum > /dev/null 2>&1 ; echo $?) != 0 ]]; then
sudo apt update
sudo apt install libdigest-sha-perl -qyy
fi
# SHAsums to be computed, not the -- notatiation is for ease of use with rhash
@ -672,8 +673,7 @@ setBaseURL () {
CONN=$(ip -br -o -4 a |grep UP |head -1 |tr -d "UP")
IFACE=`echo $CONN |awk {'print $1'}`
IP=`echo $CONN |awk {'print $2'}| cut -f1 -d/`
# TODO: Consider "QEMU"
if [[ "$(checkManufacturer)" != "innotek GmbH" ]] && [[ "$(checkManufacturer)" != "VMware, Inc." ]]; then
if [[ "$(checkManufacturer)" != "innotek GmbH" ]] && [[ "$(checkManufacturer)" != "VMware, Inc." ]] && [[ "$(checkManufacturer)" != "QEMU" ]]; then
debug "We guess that this is a physical machine and cannot possibly guess what the MISP_BASEURL might be."
if [[ "$UNATTENDED" != "1" ]]; then
echo "You can now enter your own MISP_BASEURL, if you wish to NOT do that, the MISP_BASEURL will be empty, which will work, but ideally you configure it afterwards."
@ -701,12 +701,20 @@ setBaseURL () {
MISP_BASEURL="https://misp.local"
# Webserver configuration
FQDN='misp.local'
else
elif [[ "$(checkManufacturer)" == "innotek GmbH" ]]; then
MISP_BASEURL='https://localhost:8443'
IP=$(ip addr show | awk '$1 == "inet" {gsub(/\/.*$/, "", $2); print $2}' |grep -v "127.0.0.1" |tail -1)
sudo iptables -t nat -A OUTPUT -p tcp --dport 8443 -j DNAT --to ${IP}:443
# Webserver configuration
FQDN='localhost.localdomain'
elif [[ "$(checkManufacturer)" == "VMware, Inc." ]]; then
MISP_BASEURL='""'
# Webserver configuration
FQDN='misp.local'
else
MISP_BASEURL='""'
# Webserver configuration
FQDN='misp.local'
fi
}
@ -1331,8 +1339,6 @@ installCore () {
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
cd $PATH_TO_MISP/app/files/scripts/python-maec
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
# FIXME: Remove once stix-fixed
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
# install STIX2.0 library to support STIX 2.0 export:
cd ${PATH_TO_MISP}/cti-python-stix2
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
@ -1340,6 +1346,23 @@ installCore () {
# install PyMISP
cd ${PATH_TO_MISP}/PyMISP
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
# FIXME: Remove libfaup etc once the egg has the library baked-in
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
cd /tmp
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
cd gtcaca
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
# install pydeep
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/pydeep.git
@ -1651,15 +1674,32 @@ WantedBy=multi-user.target" | sudo tee /etc/systemd/system/misp-workers.service
# Main MISP Modules install function
mispmodules () {
cd /usr/local/src/
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
## TODO: checkUsrLocalSrc in main doc
debug "Cloning misp-modules"
$SUDO_CMD git clone https://github.com/MISP/misp-modules.git
cd misp-modules
$SUDO_CMD git clone git://github.com/stricaud/gtcaca.git
$SUDO_CMD git clone git://github.com/stricaud/faup.git
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
# Install gtcaca
cd gtcaca
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
# Install faup
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
cd ../../misp-modules
# some misp-modules dependencies
sudo apt install libpq5 libjpeg-dev tesseract-ocr libpoppler-cpp-dev imagemagick libopencv-dev zbar-tools libzbar0 libzbar-dev libfuzzy-dev -y
# If you build an egg, the user you build it as need write permissions in the CWD
sudo chgrp $WWW_USER .
sudo chmod g+w .
sudo chmod og+w .
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I -r REQUIREMENTS
sudo chgrp staff .
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I .
@ -1809,8 +1849,8 @@ mail2misp () {
cd /usr/local/src/
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
$SUDO_CMD git clone https://github.com/MISP/mail_to_misp.git
$SUDO_CMD git clone git://github.com/stricaud/faup.git faup
$SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
sudo chown -R ${MISP_USER}:${MISP_USER} faup mail_to_misp gtcaca
cd gtcaca
$SUDO_CMD mkdir -p build
@ -2018,9 +2058,6 @@ installCoreRHEL () {
cd $PATH_TO_MISP/app/files/scripts/mixbox
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
# FIXME: Remove once stix-fixed
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
# install STIX2.0 library to support STIX 2.0 export:
cd $PATH_TO_MISP/cti-python-stix2
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
@ -2072,6 +2109,25 @@ installCoreRHEL () {
cd $PATH_TO_MISP/PyMISP
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U .
# FIXME: Remove libfaup etc once the egg has the library baked-in
# BROKEN: This needs to be tested on RHEL/CentOS
##sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
cd /tmp
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
cd gtcaca
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
# Enable dependencies detection in the diagnostics page
# This allows MISP to detect GnuPG, the Python modules' versions and to read the PHP settings.
# The LD_LIBRARY_PATH setting is needed for rh-git218 to work
@ -2651,21 +2707,21 @@ installSupported () {
if [[ "$1" =~ ^PHP= ]]; then
PHP_VER=$(echo $1 |cut -f2 -d=)
if [[ "$PHP_VER" == "7.2" ]]; then
if [[ "$PHP_VER" == 7.2 ]]; then
# Install PHP 7.2 Dependencies - functionLocation('INSTALL.ubuntu1804.md')
[[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp72
elif [[ "$PHP_VER" == "7.3" ]]; then
elif [[ "$PHP_VER" == 7.3 ]]; then
# Install PHP 7.4 Dependencies - functionLocation('INSTALL.ubuntu2004.md')
[[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp74
elif [[ "$PHP_VER" == "7.4" ]]; then
# Install PHP 7.3 Dependencies - functionLocation('generic/supportFunctions.md')
[[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp73
elif [[ "$PHP_VER" == "7.0" ]]; then
elif [[ "$PHP_VER" == 7.4 ]]; then
# Install PHP 7.3 Dependencies - functionLocation('generic/supportFunctions.md')
[[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp74
elif [[ "$PHP_VER" == 7.0 ]]; then
# Install PHP 7.0 Dependencies - functionLocation('generic/supportFunctions.md')
[[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp70
fi
else
# Install PHP 7.2 Dependencies - functionLocation('INSTALL.ubuntu1804.md')
# Install PHP 7.2 Dependencies by dangerous default - functionLocation('INSTALL.ubuntu1804.md')
[[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp72
fi
progress 4
@ -3173,6 +3229,7 @@ x86_64-fedora-30
x86_64-debian-stretch
x86_64-debian-buster
x86_64-ubuntu-bionic
x86_64-ubuntu-focal
x86_64-kali-2019.1
x86_64-kali-2019.2
x86_64-kali-2019.3
@ -3187,6 +3244,7 @@ armv7l-debian-jessie
armv7l-debian-stretch
armv7l-debian-buster
armv7l-ubuntu-bionic
armv7l-ubuntu-focal
"
# Check if we actually support this configuration
@ -3208,12 +3266,18 @@ if [ "${FLAVOUR}" == "ubuntu" ]; then
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
installSupported && exit || exit
fi
if [ "${RELEASE}" == "20.04" ]; then
echo "Install on Ubuntu 20.04 LTS fully supported."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
installSupported PHP="7.4" && exit || exit
fi
if [ "${RELEASE}" == "18.10" ]; then
echo "Install on Ubuntu 18.10 partially supported, bye."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
installSupported && exit || exit
fi
if [ "${RELEASE}" == "19.04" ]; then
echo "Install on Ubuntu 19.04 under development."
echo "Install on Ubuntu 19.04 partially supported bye."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
installSupported && exit || exit
exit 1

6
INSTALL/INSTALL.sh.sfv
View File

@ -1,5 +1,5 @@
; Generated by RHash v1.3.8 on 2020-04-27 at 19:39.56
; Generated by RHash v1.3.9 on 2020-05-13 at 12:47.53
; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
;
; 130366 19:39.56 2020-04-27 INSTALL.sh
INSTALL.sh 8AFDDFA23C1154790947FA1C09DBC7599614F48D 1DC92AFF146065ECB85D5C5C211E252D5C54D6F86A61C29DDED37A9ECA4540E4 3F2C936AEE2773A29DD02477E463402945576DEC32C3003BFF3132B783E2C93D14EA99E8FADFEEE655C746D4C634BE08 DBBF519D97372DDD5337C1F58252BFE9A86B0C8ABB95A375420D8D543C5239D71C24287A75FAFC389DBFDC657B1B4080530D0D3CB44D5F94152F2ABBB9B23174
; 132625 12:47.53 2020-05-13 INSTALL.sh
INSTALL.sh 39D54E983C460248BD68CEE97BA24736DDF9A479 016DDE3CF5F0199C66A23B72F8F6D5951972C2B9CF1B5AE5159D650695442D8A D245A7EB12AEC9BCEC2C391C9B7023F6C8AE3713324FC410DC42A9A9620C13E4BC9CA456F62A40AE17F864D021492750 BE6ACAE2DFD1BA7FD29ED7F73098EE54552DB91EBEE3B58E239D54D4AD7AFDD3FDB212689D550B990A253A17A37E2E8BA3C4C41C129ECBD78E0EACB97F0085D5

2
INSTALL/INSTALL.sh.sha1
View File

@ -1 +1 @@
8afddfa23c1154790947fa1c09dbc7599614f48d INSTALL.sh
39d54e983c460248bd68cee97ba24736ddf9a479 INSTALL.sh

2
INSTALL/INSTALL.sh.sha256
View File

@ -1 +1 @@
1dc92aff146065ecb85d5c5c211e252d5c54d6f86a61c29dded37a9eca4540e4 INSTALL.sh
016dde3cf5f0199c66a23b72f8f6d5951972c2b9cf1b5ae5159d650695442d8a INSTALL.sh

2
INSTALL/INSTALL.sh.sha384
View File

@ -1 +1 @@
3f2c936aee2773a29dd02477e463402945576dec32c3003bff3132b783e2c93d14ea99e8fadfeee655c746d4c634be08 INSTALL.sh
d245a7eb12aec9bcec2c391c9b7023f6c8ae3713324fc410dc42a9a9620c13e4bc9ca456f62a40ae17f864d021492750 INSTALL.sh

2
INSTALL/INSTALL.sh.sha512
View File

@ -1 +1 @@
dbbf519d97372ddd5337c1f58252bfe9a86b0c8abb95a375420d8d543c5239d71c24287a75fafc389dbfdc657b1b4080530d0d3cb44d5f94152f2abbb9b23174 INSTALL.sh
be6acae2dfd1ba7fd29ed7f73098ee54552db91ebee3b58e239d54d4ad7afdd3fdb212689d550b990a253a17a37e2e8ba3c4c41c129ecbd78e0eacb97f0085d5 INSTALL.sh

24
INSTALL/INSTALL.tpl.sh
View File

@ -278,21 +278,21 @@ installSupported () {
if [[ "$1" =~ ^PHP= ]]; then
PHP_VER=$(echo $1 |cut -f2 -d=)
if [[ "$PHP_VER" == "7.2" ]]; then
if [[ "$PHP_VER" == 7.2 ]]; then
# Install PHP 7.2 Dependencies - functionLocation('INSTALL.ubuntu1804.md')
[[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp72
elif [[ "$PHP_VER" == "7.3" ]]; then
elif [[ "$PHP_VER" == 7.3 ]]; then
# Install PHP 7.4 Dependencies - functionLocation('INSTALL.ubuntu2004.md')
[[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp74
elif [[ "$PHP_VER" == "7.4" ]]; then
# Install PHP 7.3 Dependencies - functionLocation('generic/supportFunctions.md')
[[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp73
elif [[ "$PHP_VER" == "7.0" ]]; then
elif [[ "$PHP_VER" == 7.4 ]]; then
# Install PHP 7.3 Dependencies - functionLocation('generic/supportFunctions.md')
[[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp74
elif [[ "$PHP_VER" == 7.0 ]]; then
# Install PHP 7.0 Dependencies - functionLocation('generic/supportFunctions.md')
[[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp70
fi
else
# Install PHP 7.2 Dependencies - functionLocation('INSTALL.ubuntu1804.md')
# Install PHP 7.2 Dependencies by dangerous default - functionLocation('INSTALL.ubuntu1804.md')
[[ -n $CORE ]] || [[ -n $ALL ]] && installDepsPhp72
fi
progress 4
@ -800,6 +800,7 @@ x86_64-fedora-30
x86_64-debian-stretch
x86_64-debian-buster
x86_64-ubuntu-bionic
x86_64-ubuntu-focal
x86_64-kali-2019.1
x86_64-kali-2019.2
x86_64-kali-2019.3
@ -814,6 +815,7 @@ armv7l-debian-jessie
armv7l-debian-stretch
armv7l-debian-buster
armv7l-ubuntu-bionic
armv7l-ubuntu-focal
"
# Check if we actually support this configuration
@ -835,12 +837,18 @@ if [ "${FLAVOUR}" == "ubuntu" ]; then
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
installSupported && exit || exit
fi
if [ "${RELEASE}" == "20.04" ]; then
echo "Install on Ubuntu 20.04 LTS fully supported."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
installSupported PHP="7.4" && exit || exit
fi
if [ "${RELEASE}" == "18.10" ]; then
echo "Install on Ubuntu 18.10 partially supported, bye."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
installSupported && exit || exit
fi
if [ "${RELEASE}" == "19.04" ]; then
echo "Install on Ubuntu 19.04 under development."
echo "Install on Ubuntu 19.04 partially supported bye."
echo "Please report bugs/issues here: https://github.com/MISP/MISP/issues"
installSupported && exit || exit
exit 1

38
INSTALL/MYSQL.sql
View File

@ -916,7 +916,7 @@ CREATE TABLE IF NOT EXISTS `shadow_attribute_correlations` (
-- Table structure for table `sharing_group_orgs`
--
CREATE TABLE `sharing_group_orgs` (
CREATE TABLE IF NOT EXISTS `sharing_group_orgs` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`sharing_group_id` int(11) NOT NULL,
`org_id` int(11) NOT NULL,
@ -932,7 +932,7 @@ CREATE TABLE `sharing_group_orgs` (
-- Table structure for table `sharing_group_servers`
--
CREATE TABLE `sharing_group_servers` (
CREATE TABLE IF NOT EXISTS `sharing_group_servers` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`sharing_group_id` int(11) NOT NULL,
`server_id` int(11) NOT NULL,
@ -948,7 +948,7 @@ CREATE TABLE `sharing_group_servers` (
-- Table structure for table `sharing_groups`
--
CREATE TABLE `sharing_groups` (
CREATE TABLE IF NOT EXISTS `sharing_groups` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`name` varchar(255) CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL,
`releasability` text CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL,
@ -1355,14 +1355,14 @@ CREATE TABLE IF NOT EXISTS `whitelist` (
-- Default values for initial installation
--
INSERT INTO `admin_settings` (`id`, `setting`, `value`) VALUES
INSERT IGNORE INTO `admin_settings` (`id`, `setting`, `value`) VALUES
(1, 'db_version', '40');
INSERT INTO `feeds` (`id`, `provider`, `name`, `url`, `distribution`, `default`, `enabled`) VALUES
INSERT IGNORE INTO `feeds` (`id`, `provider`, `name`, `url`, `distribution`, `default`, `enabled`) VALUES
(1, 'CIRCL', 'CIRCL OSINT Feed', 'https://www.circl.lu/doc/misp/feed-osint', 3, 1, 0),
(2, 'Botvrij.eu', 'The Botvrij.eu Data', 'https://www.botvrij.eu/data/feed-osint', 3, 1, 0);
INSERT INTO `regexp` (`id`, `regexp`, `replacement`, `type`) VALUES
INSERT IGNORE INTO `regexp` (`id`, `regexp`, `replacement`, `type`) VALUES
(1, '/.:.ProgramData./i', '%ALLUSERSPROFILE%\\\\', 'ALL'),
(2, '/.:.Documents and Settings.All Users./i', '%ALLUSERSPROFILE%\\\\', 'ALL'),
(3, '/.:.Program Files.Common Files./i', '%COMMONPROGRAMFILES%\\\\', 'ALL'),
@ -1407,22 +1407,22 @@ INSERT INTO `feeds` (`id`, `provider`, `name`, `url`, `distribution`, `default`,
-- 7. Read Only - read
--
INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`)
INSERT IGNORE INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`)
VALUES (1, 'admin', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0);
INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`)
INSERT IGNORE INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`)
VALUES (2, 'Org Admin', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0);
INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`)
INSERT IGNORE INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`)
VALUES (3, 'User', NOW(), NOW(), 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1);
INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`)
INSERT IGNORE INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`)
VALUES (4, 'Publisher', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0);
INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`)
INSERT IGNORE INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`)
VALUES (5, 'Sync user', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0);
INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`)
INSERT IGNORE INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`)
VALUES (6, 'Read Only', NOW(), NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
-- --------------------------------------------------------
@ -1431,7 +1431,7 @@ VALUES (6, 'Read Only', NOW(), NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
-- Initial threat levels
--
INSERT INTO `threat_levels` (`id`, `name`, `description`, `form_description`)
INSERT IGNORE INTO `threat_levels` (`id`, `name`, `description`, `form_description`)
VALUES
(1, 'High', '*high* means sophisticated APT malware or 0-day attack', 'Sophisticated APT malware or 0-day attack'),
(2, 'Medium', '*medium* means APT malware', 'APT malware'),
@ -1444,13 +1444,13 @@ VALUES
-- Default templates
--
INSERT INTO `templates` (`id`, `name`, `description`, `org`, `share`) VALUES
INSERT IGNORE INTO `templates` (`id`, `name`, `description`, `org`, `share`) VALUES
(1, 'Phishing E-mail', 'Create a MISP event about a Phishing E-mail.', 'MISP', 1),
(2, 'Phishing E-mail with malicious attachment', 'A MISP event based on Spear-phishing containing a malicious attachment. This event can include anything from the description of the e-mail itself, the malicious attachment and its description as well as the results of the analysis done on the malicious f', 'MISP', 1),
(3, 'Malware Report', 'This is a template for a generic malware report. ', 'MISP', 1),
(4, 'Indicator List', 'A simple template for indicator lists.', 'MISP', 1);
INSERT INTO `template_elements` (`id`, `template_id`, `position`, `element_definition`) VALUES
INSERT IGNORE INTO `template_elements` (`id`, `template_id`, `position`, `element_definition`) VALUES
(1, 1, 2, 'attribute'),
(2, 1, 3, 'attribute'),
(3, 1, 1, 'text'),
@ -1497,7 +1497,7 @@ INSERT INTO `template_elements` (`id`, `template_id`, `position`, `element_defin
(46, 4, 2, 'attribute'),
(47, 4, 3, 'attribute');
INSERT INTO `template_element_attributes` (`id`, `template_element_id`, `name`, `description`, `to_ids`, `category`, `complex`, `type`, `mandatory`, `batch`) VALUES
INSERT IGNORE INTO `template_element_attributes` (`id`, `template_element_id`, `name`, `description`, `to_ids`, `category`, `complex`, `type`, `mandatory`, `batch`) VALUES
(1, 1, 'From address', 'The source address from which the e-mail was sent.', 1, 'Payload delivery', 0, 'email-src', 1, 1),
(2, 2, 'Malicious url', 'The malicious url in the e-mail body.', 1, 'Payload delivery', 0, 'url', 1, 1),
(3, 4, 'E-mail subject', 'The subject line of the e-mail.', 0, 'Payload delivery', 0, 'email-subject', 1, 0),
@ -1529,13 +1529,13 @@ INSERT INTO `template_element_attributes` (`id`, `template_element_id`, `name`,
(29, 46, 'Network Indicators', 'Paste any combination of IP addresses, hostnames, domains or URL', 1, 'Network activity', 1, 'CnC', 0, 1),
(30, 47, 'File Indicators', 'Paste any file hashes that you have (MD5, SHA1, SHA256) or filenames below. You can also add filename and hash pairs by using the following syntax for each applicable column: filename|hash ', 1, 'Payload installation', 1, 'File', 0, 1);
INSERT INTO `template_element_files` (`id`, `template_element_id`, `name`, `description`, `category`, `malware`, `mandatory`, `batch`) VALUES
INSERT IGNORE INTO `template_element_files` (`id`, `template_element_id`, `name`, `description`, `category`, `malware`, `mandatory`, `batch`) VALUES
(1, 14, 'Malicious Attachment', 'The file (or files) that was (were) attached to the e-mail itself.', 'Payload delivery', 1, 0, 1),
(2, 21, 'Payload installation', 'Payload installation detected during the analysis', 'Payload installation', 1, 0, 1),
(3, 30, 'Malware sample', 'The sample that the report is based on', 'Payload delivery', 1, 0, 0),
(4, 40, 'Artifacts dropped (Sample)', 'Upload any files that were dropped during the analysis.', 'Artifacts dropped', 1, 0, 1);
INSERT INTO `template_element_texts` (`id`, `name`, `template_element_id`, `text`) VALUES
INSERT IGNORE INTO `template_element_texts` (`id`, `name`, `template_element_id`, `text`) VALUES
(1, 'Required fields', 3, 'The fields below are mandatory.'),
(2, 'Optional information', 5, 'All of the fields below are optional, please fill out anything that''s applicable.'),
(4, 'Required Fields', 11, 'The following fields are mandatory'),
@ -1548,6 +1548,6 @@ INSERT INTO `template_element_texts` (`id`, `name`, `template_element_id`, `text
(11, 'Persistence mechanism', 41, 'The following fields allow you to describe the persistence mechanism used by the malware'),
(12, 'Indicators', 45, 'Just paste your list of indicators based on type into the appropriate field. All of the fields are optional, so inputting a list of IP addresses into the Network indicator field for example is sufficient to complete this template.');
INSERT INTO `org_blacklists` (`org_uuid`, `created`, `org_name`, `comment`) VALUES
INSERT IGNORE INTO `org_blacklists` (`org_uuid`, `created`, `org_name`, `comment`) VALUES
('58d38339-7b24-4386-b4b4-4c0f950d210f', NOW(), 'Setec Astrononomy', 'default example'),
('58d38326-eda8-443a-9fa8-4e12950d210f', NOW(), 'Acme Finance', 'default example');

2
PyMISP

@ -1 +1 @@
Subproject commit 0faa75824f4dbac2b14919bb17e9d0fef79026d7
Subproject commit 7178d3a8a04e5de74e60fbb96c9c670ca527a0be

11
SECURITY.md Normal file
View File

@ -0,0 +1,11 @@
## Reporting security vulnerabilities for MISP or related MISP project repositories
Reporting security vulnerabilities is of great importance for us, as MISP is used in multiple critical infrastructures.
In the case of a security vulnerability report, we ask the reporter to send it directly to [CIRCL](https://www.circl.lu/contact/), if possible encrypted with the following GnuPG key: **CA57 2205 C002 4E06 BA70 BE89 EAAD CFFC 22BD 4CD5**. We usually fix reported and confirmed security vulnerabilities in less than 48 hours, followed by a software release containing the fixes within the following days.
If you report security vulnerabilities, do not forget to **tell us if and how you want to be acknowledged** and if you already requested CVE(s). Otherwise, we will request the CVE(s) directly.
As one of the critical user-bases of MISP consists of the CSIRT community, it is our duty to clearly state which bug could be abused and have a security impact on a MISP instance. CVE assignment is performed even for minor bugs suspected of having a security impact. This allows every user with MISP instances set up in their environments to understand which bugs could impact their security.
We firmly believe that, even though unfortunately it is often not regarded as common practice in our industry, being as transparent as possible about vulnerabilities, no matter how minor, is of crucial importance. At MISP Project, we care about the security of our users and prefer to have a high number of published CVEs rather than sweeping some of them under the rug.

2
VERSION.json
View File

@ -1 +1 @@
{"major":2, "minor":4, "hotfix":124}
{"major":2, "minor":4, "hotfix":125}

6
app/Console/Command/ServerShell.php
View File

@ -70,9 +70,13 @@ class ServerShell extends AppShell
$this->Job->save($data);
$jobId = $this->Job->id;
}
$force = false;
if (!empty($this->args[4]) && $this->args[4] === 'force') {
$force = true;
}
$this->Server->id = $serverId;
$server = $this->Server->read(null, $serverId);
$result = $this->Server->pull($user, $serverId, $technique, $server, $jobId);
$result = $this->Server->pull($user, $serverId, $technique, $server, $jobId, $force);
$this->Job->id = $jobId;
$this->Job->save(array(
'id' => $jobId,

19
app/Controller/AppController.php
View File

@ -46,8 +46,8 @@ class AppController extends Controller
public $helpers = array('Utility', 'OrgImg', 'FontAwesome', 'UserName', 'DataPathCollector');
private $__queryVersion = '104';
public $pyMispVersion = '2.4.123';
private $__queryVersion = '106';
public $pyMispVersion = '2.4.125';
public $phpmin = '7.2';
public $phprec = '7.4';
public $pythonmin = '3.6';
@ -57,6 +57,8 @@ class AppController extends Controller
public $baseurl = '';
public $sql_dump = false;
private $isRest = null;
// Used for _isAutomation(), a check that returns true if the controller & action combo matches an action that is a non-xml and non-json automation method
// This is used to allow authentication via headers for methods not covered by _isRest() - as that only checks for JSON and XML formats
public $automationArray = array(
@ -361,7 +363,11 @@ class AppController extends Controller
}
}
} else {
if ($this->params['controller'] !== 'users' || !in_array($this->params['action'], array('login', 'register'))) {
$pre_auth_actions = array('login', 'register');
if (!empty(Configure::read('Security.email_otp_enabled'))) {
$pre_auth_actions[] = 'email_otp';
}
if ($this->params['controller'] !== 'users' || !in_array($this->params['action'], $pre_auth_actions)) {
if (!$this->request->is('ajax')) {
$this->Session->write('pre_login_requested_url', $this->here);
}
@ -680,6 +686,11 @@ class AppController extends Controller
protected function _isRest()
{
// This method is surprisingly slow and called many times for one request, so it make sense to cache the result.
if ($this->isRest !== null) {
return $this->isRest;
}
$api = $this->__isApiFunction($this->request->params['controller'], $this->request->params['action']);
if (isset($this->RequestHandler) && ($api || $this->RequestHandler->isXml() || $this->_isJson() || $this->_isCsv())) {
if ($this->_isJson()) {
@ -687,8 +698,10 @@ class AppController extends Controller
throw new MethodNotAllowedException('Invalid JSON input. Make sure that the JSON input is a correctly formatted JSON string. This request has been blocked to avoid an unfiltered request.');
}
}
$this->isRest = true;
return true;
} else {
$this->isRest = false;
return false;
}
}

135
app/Controller/AttributesController.php
View File

@ -3,6 +3,9 @@ App::uses('AppController', 'Controller');
App::uses('Folder', 'Utility');
App::uses('File', 'Utility');
/**
* @property Attribute $Attribute
*/
class AttributesController extends AppController
{
public $components = array('Security', 'RequestHandler', 'Cidr');
@ -148,6 +151,12 @@ class AttributesController extends AppController
if (!isset($this->request->data['Attribute'])) {
$this->request->data = array('Attribute' => $this->request->data);
}
if (isset($this->request->data['Attribute']['distribution']) && $this->request->data['Attribute']['distribution'] == 4) {
$sg = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1, $this->request->data['Attribute']['sharing_group_id']);
if (empty($sg)) {
throw new MethodNotAllowedException(__('Invalid Sharing Group or not authorised.'));
}
}
//
// multiple attributes in batch import
//
@ -410,22 +419,19 @@ class AttributesController extends AppController
public function add_attachment($eventId = null)
{
if ($this->request->is('post')) {
$hashes = array('md5' => 'malware-sample', 'sha1' => 'filename|sha1', 'sha256' => 'filename|sha256');
$this->loadModel('Event');
$this->Event->id = $this->request->data['Attribute']['event_id'];
$this->Event->recursive = -1;
$event = $this->Event->read();
$this->Attribute->Event->id = $this->request->data['Attribute']['event_id'];
$this->Attribute->Event->recursive = -1;
$event = $this->Attribute->Event->read();
if (empty($event)) {
throw new NotFoundException(__('Invalid Event.'));
}
if (!$this->_isSiteAdmin() && ($this->Event->data['Event']['orgc_id'] != $this->_checkOrg() || !$this->userRole['perm_modify'])) {
if (!$this->_isSiteAdmin() && ($this->Attribute->Event->data['Event']['orgc_id'] != $this->_checkOrg() || !$this->userRole['perm_modify'])) {
throw new UnauthorizedException(__('You do not have permission to do that.'));
}
$partialFails = array();
$fails = array();
$success = 0;
foreach ($this->request->data['Attribute']['values'] as $k => $value) {
foreach ($this->request->data['Attribute']['values'] as $value) {
// Check if there were problems with the file upload
// only keep the last part of the filename, this should prevent directory attacks
$filename = basename($value['name']);
@ -449,11 +455,6 @@ class AttributesController extends AppController
$filename,
$tmpfile
);
if ($result) {
$success++;
} else {
$fails[] = $filename;
}
} else {
$result = $this->Attribute->simpleAddMalwareSample(
$eventId,
@ -461,15 +462,16 @@ class AttributesController extends AppController
$filename,
$tmpfile
);
if ($result) {
$success++;
} else {
$fails[] = $filename;
}
}
if ($result) {
$success++;
} else {
$fails[] = $filename;
}
if (!empty($result)) {
foreach ($result['Object'] as $object) {
$this->loadModel('MispObject');
$object['distribution'] = $this->request->data['Attribute']['distribution'];
if (!empty($this->request->data['sharing_group_id'])) {
$object['sharing_group_id'] = $this->request->data['Attribute']['sharing_group_id'];
@ -477,11 +479,11 @@ class AttributesController extends AppController
foreach ($object['Attribute'] as $ka => $attribute) {
$object['Attribute'][$ka]['distribution'] = 5;
}
$this->MispObject->captureObject(array('Object' => $object), $eventId, $this->Auth->user());
$this->Attribute->Object->captureObject(array('Object' => $object), $eventId, $this->Auth->user());
}
if (!empty($result['ObjectReference'])) {
foreach ($result['ObjectReference'] as $reference) {
$this->MispObject->ObjectReference->smartSave($reference, $eventId);
$this->Attribute->Object->ObjectReference->smartSave($reference, $eventId);
}
}
}
@ -508,20 +510,17 @@ class AttributesController extends AppController
}
}
}
$message = 'The attachment(s) have been uploaded.';
if (!empty($partialFails)) {
$message .= ' Some of the attributes however could not be created.';
}
$message = __('The attachment(s) have been uploaded.');
if (!empty($fails)) {
$message = 'Some of the attachments failed to upload. The failed files were: ' . implode(', ', $fails) . ' - This can be caused by the attachments already existing in the event.';
$message = __('Some of the attachments failed to upload. The failed files were: %s - This can be caused by the attachments already existing in the event.', implode(', ', $fails));
}
if (empty($success)) {
if (empty($fails)) {
$message = 'The attachment(s) could not be saved. please contact your administrator.';
$message = __('The attachment(s) could not be saved. Please contact your administrator.');
}
} else {
$this->Event->id = $this->request->data['Attribute']['event_id'];
$this->Event->saveField('published', 0);
$this->Attribute->Event->id = $this->request->data['Attribute']['event_id'];
$this->Attribute->Event->saveField('published', 0);
}
if (empty($success) && !empty($fails)) {
$this->Flash->error($message);
@ -536,57 +535,43 @@ class AttributesController extends AppController
// set the event_id in the form
$this->request->data['Attribute']['event_id'] = $eventId;
}
$event = $this->Attribute->Event->findById($eventId);
if (empty($event)) {
throw new NotFoundException(__('Invalid Event.'));
}
if (!$this->_isRest()) {
$this->Attribute->Event->insertLock($this->Auth->user(), $eventId);
}
// combobox for categories
$categories = array_keys($this->Attribute->categoryDefinitions);
// just get them with attachments..
// Filter categories that contains attachment type
$selectedCategories = array();
foreach ($categories as $category) {
$types = $this->Attribute->categoryDefinitions[$category]['types'];
$alreadySet = false;
foreach ($types as $type) {
if ($this->Attribute->typeIsAttachment($type) && !$alreadySet) {
// add to the whole..
foreach ($this->Attribute->categoryDefinitions as $category => $values) {
foreach ($values['types'] as $type) {
if ($this->Attribute->typeIsAttachment($type)) {
$selectedCategories[] = $category;
$alreadySet = true;
continue;
continue 2;
}
}
}
$categories = $this->_arrayToValuesIndexArray($selectedCategories);
$this->set('categories', $categories);
$this->set('attrDescriptions', $this->Attribute->fieldDescriptions);
$this->set('typeDefinitions', $this->Attribute->typeDefinitions);
$this->set('categoryDefinitions', $this->Attribute->categoryDefinitions);
$this->set('zippedDefinitions', $this->Attribute->zippedDefinitions);
$this->set('uploadDefinitions', $this->Attribute->uploadDefinitions);
$this->set('advancedExtractionAvailable', $this->Attribute->isAdvancedExtractionAvailable());
// combobox for distribution
$this->loadModel('Event');
$this->set('distributionLevels', $this->Event->Attribute->distributionLevels);
foreach ($this->Attribute->categoryDefinitions as $key => $value) {
$info['category'][$key] = array('key' => $key, 'desc' => isset($value['formdesc'])? $value['formdesc'] : $value['desc']);
}
foreach ($this->Event->Attribute->distributionLevels as $key => $value) {
$info['distribution'][$key] = array('key' => $value, 'desc' => $this->Attribute->distributionDescriptions[$key]['formdesc']);
}
$this->set('info', $info);
$this->set('distributionLevels', $this->Attribute->distributionLevels);
$this->set('info', $this->__getInfo());
$this->loadModel('SharingGroup');
$sgs = $this->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1);
$this->set('sharingGroups', $sgs);
$events = $this->Event->findById($eventId);
if (empty($events)) {
throw new NotFoundException(__('Invalid Event.'));
}
$this->set('currentDist', $events['Event']['distribution']);
$this->set('published', $events['Event']['published']);
$this->set('currentDist', $event['Event']['distribution']);
$this->set('published', $event['Event']['published']);
}
@ -852,6 +837,12 @@ class AttributesController extends AppController
if (!isset($this->request->data['Attribute'])) {
$this->request->data = array('Attribute' => $this->request->data);
}
if (isset($this->request->data['Attribute']['distribution']) && $this->request->data['Attribute']['distribution'] == 4) {
$sg = $this->Attribute->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1, $this->request->data['Attribute']['sharing_group_id']);
if (empty($sg)) {
throw new MethodNotAllowedException(__('Invalid Sharing Group or not authorised.'));
}
}
$existingAttribute = $this->Attribute->findByUuid($this->Attribute->data['Attribute']['uuid']);
// check if the attribute has a timestamp already set (from a previous instance that is trying to edit via synchronisation)
// check which attribute is newer
@ -3250,4 +3241,28 @@ class AttributesController extends AppController
return $this->RestResponse->viewData($final, $responseType, false, true, 'search.' . $type . '.' . $responseType);
}
}
private function __getInfo()
{
$info = array('category' => array(), 'type' => array(), 'distribution' => array());
foreach ($this->Attribute->categoryDefinitions as $key => $value) {
$info['category'][$key] = array(
'key' => $key,
'desc' => isset($value['formdesc']) ? $value['formdesc'] : $value['desc']
);
}
foreach ($this->Attribute->typeDefinitions as $key => $value) {
$info['type'][$key] = array(
'key' => $key,
'desc' => isset($value['formdesc']) ? $value['formdesc'] : $value['desc']
);
}
foreach ($this->Attribute->distributionLevels as $key => $value) {
$info['distribution'][$key] = array(
'key' => $value,
'desc' => $this->Attribute->distributionDescriptions[$key]['formdesc']
);
}
return $info;
}
}

32
app/Controller/Component/ACLComponent.php
View File

@ -285,21 +285,22 @@ class ACLComponent extends Component
'view' => array('*')
),
'objects' => array(
'add' => array('perm_add'),
'addValueField' => array('perm_add'),
'delete' => array('perm_add'),
'edit' => array('perm_add'),
'get_row' => array('perm_add'),
'orphanedObjectDiagnostics' => array(),
'editField' => array('perm_add'),
'fetchEditForm' => array('perm_add'),
'fetchViewValue' => array('*'),
'quickAddAttributeForm' => array('perm_add'),
'quickFetchTemplateWithValidObjectAttributes' => array('perm_add'),
'proposeObjectsFromAttributes' => array('*'),
'groupAttributesIntoObject' => array('perm_add'),
'revise_object' => array('perm_add'),
'view' => array('*'),
'add' => array('perm_add'),
'addValueField' => array('perm_add'),
'delete' => array('perm_add'),
'edit' => array('perm_add'),
'get_row' => array('perm_add'),
'orphanedObjectDiagnostics' => array(),
'editField' => array('perm_add'),
'fetchEditForm' => array('perm_add'),
'fetchViewValue' => array('*'),
'quickAddAttributeForm' => array('perm_add'),
'quickFetchTemplateWithValidObjectAttributes' => array('perm_add'),
'restSearch' => array('*'),
'proposeObjectsFromAttributes' => array('*'),
'groupAttributesIntoObject' => array('perm_add'),
'revise_object' => array('perm_add'),
'view' => array('*'),
),
'objectReferences' => array(
'add' => array('perm_add'),
@ -569,6 +570,7 @@ class ACLComponent extends Component
'discardRegistrations' => array('perm_site_admin'),
'downloadTerms' => array('*'),
'edit' => array('*'),
'email_otp' => array('*'),
'searchGpgKey' => array('*'),
'fetchGpgKey' => array('*'),
'histogram' => array('*'),

8
app/Controller/Component/RestResponseComponent.php
View File

@ -72,7 +72,7 @@ class RestResponseComponent extends Component
'restSearch' => array(
'description' => "Search MISP using a list of filter parameters and return the data in the selected format. The search is available on an event and an attribute level, just select the scope via the URL (/events/restSearch vs /attributes/restSearch). Besides the parameters listed, other, format specific ones can be passed along (for example: requested_attributes and includeContext for the CSV export). This API allows pagination via the page and limit parameters.",
'mandatory' => array('returnFormat'),
'optional' => array('page', 'limit', 'value', 'type', 'category', 'org', 'tag', 'tags', 'searchall', 'date', 'last', 'eventid', 'withAttachments', 'metadata', 'uuid', 'published', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'sgReferenceOnly', 'eventinfo', 'excludeLocalTags'),
'optional' => array('page', 'limit', 'value', 'type', 'category', 'org', 'tag', 'tags', 'searchall', 'date', 'last', 'eventid', 'withAttachments', 'metadata', 'uuid', 'published', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'sgReferenceOnly', 'eventinfo', 'excludeLocalTags', 'threat_level_id'),
'params' => array()
)
),
@ -1080,7 +1080,7 @@ class RestResponseComponent extends Component
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' ),
'help' => __('Will not return Attributes, shadow attribute and objects')
'help' => __('Will only return the metadata of the given query scope, contained data is omitted.')
),
'minimal' => array(
'input' => 'radio',
@ -1456,13 +1456,13 @@ class RestResponseComponent extends Component
'input' => 'select',
'type' => 'integer',
'operators' => ['equal', 'not_equal'],
'values' => array( 1 => 'Hight', 2 => 'Medium', 3 => 'Low', 4 => 'Undefined')
'values' => array( 1 => 'High', 2 => 'Medium', 3 => 'Low', 4 => 'Undefined')
),
'threatlevel' => array(
'input' => 'select',
'type' => 'integer',
'operators' => ['equal', 'not_equal'],
'values' => array( 1 => 'Hight', 2 => 'Medium', 3 => 'Low', 4 => 'Undefined')
'values' => array( 1 => 'High', 2 => 'Medium', 3 => 'Low', 4 => 'Undefined')
),
'time' => array(
'input' => 'text',

5
app/Controller/Component/RestSearchComponent.php
View File

@ -10,13 +10,14 @@ class RestSearchComponent extends Component
'published', 'timestamp','enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags',
'includeProposals', 'returnFormat', 'limit', 'page', 'requested_attributes', 'includeContext', 'headerless',
'includeWarninglistHits', 'attackGalaxy', 'object_relation', 'includeSightings', 'includeCorrelations', 'includeDecayScore',
'decayingModel', 'excludeDecayed', 'modelOverrides', 'includeFullModel', 'score', 'attribute_timestamp', 'first_seen', 'last_seen'
'decayingModel', 'excludeDecayed', 'modelOverrides', 'includeFullModel', 'score', 'attribute_timestamp', 'first_seen', 'last_seen',
'threat_level_id'
),
'Event' => array(
'returnFormat', 'value', 'type', 'category', 'org', 'tags', 'searchall', 'from', 'to', 'last', 'eventid', 'withAttachments',
'metadata', 'uuid', 'publish_timestamp', 'timestamp', 'published', 'enforceWarninglist', 'sgReferenceOnly',
'limit', 'page', 'requested_attributes', 'includeContext', 'headerless', 'includeWarninglistHits', 'attackGalaxy', 'to_ids', 'deleted',
'excludeLocalTags', 'date', 'includeSightingdb', 'tag', 'object_relation'
'excludeLocalTags', 'date', 'includeSightingdb', 'tag', 'object_relation', 'threat_level_id'
),
'Object' => array(
'returnFormat', 'value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp',

93
app/Controller/EventsController.php
View File

@ -1403,12 +1403,8 @@ class EventsController extends AppController
$this->set($alias, $currentModel->{$variable});
}
}
$cluster_names = $this->GalaxyCluster->find('list', array('fields' => array('GalaxyCluster.tag_name'), 'group' => array('GalaxyCluster.tag_name', 'GalaxyCluster.id')));
foreach ($event['EventTag'] as $k => $eventTag) {
if (in_array($eventTag['Tag']['name'], $cluster_names)) {
unset($event['EventTag'][$k]);
}
}
$this->Event->removeGalaxyClusterTags($event);
$tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($event['EventTag']);
foreach ($tagConflicts['global'] as $tagConflict) {
@ -1430,11 +1426,9 @@ class EventsController extends AppController
}
$modDate = date("Y-m-d", $attribute['timestamp']);
$modificationMap[$modDate] = empty($modificationMap[$modDate])? 1 : $modificationMap[date("Y-m-d", $attribute['timestamp'])] + 1;
foreach ($attribute['AttributeTag'] as $k2 => $attributeTag) {
if (in_array($attributeTag['Tag']['name'], $cluster_names)) {
unset($event['Attribute'][$k]['AttributeTag'][$k2]);
}
}
$this->Event->Attribute->removeGalaxyClusterTags($event['Attribute'][$k]);
$tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']);
foreach ($tagConflicts['global'] as $tagConflict) {
$warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy'];
@ -1463,11 +1457,9 @@ class EventsController extends AppController
}
$modDate = date("Y-m-d", $attribute['timestamp']);
$modificationMap[$modDate] = empty($modificationMap[$modDate])? 1 : $modificationMap[date("Y-m-d", $attribute['timestamp'])] + 1;
foreach ($attribute['AttributeTag'] as $k3 => $attributeTag) {
if (in_array($attributeTag['Tag']['name'], $cluster_names)) {
unset($event['Object'][$k]['Attribute'][$k2]['AttributeTag'][$k3]);
}
}
$this->Event->Attribute->removeGalaxyClusterTags($event['Object'][$k]['Attribute'][$k2]);
$tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']);
foreach ($tagConflicts['global'] as $tagConflict) {
$warningTagConflicts[$tagConflict['taxonomy']['Taxonomy']['namespace']] = $tagConflict['taxonomy'];
@ -1659,7 +1651,7 @@ class EventsController extends AppController
if (!empty($this->params['named']['excludeGalaxy'])) {
$conditions['excludeGalaxy'] = 1;
}
if (!empty($this->params['named']['extended'])) {
if (!empty($this->params['named']['extended']) || !empty($this->request->data['extended'])) {
$conditions['extended'] = 1;
$this->set('extended', 1);
} else {
@ -2280,18 +2272,34 @@ class EventsController extends AppController
}
}
foreach ($resultArray as $key => $result) {
if ($has_pipe = strpos($result['default_type'], '|') !== false || $result['default_type'] === 'malware-sample') {
$pieces = explode('|', $result['value']);
$or = array('Attribute.value1' => $pieces,
'Attribute.value2' => $pieces);
} else {
$or = array('Attribute.value1' => $result['value'], 'Attribute.value2' => $result['value']);
}
$options = array(
'conditions' => array('OR' => array('Attribute.value1' => $result['value'], 'Attribute.value2' => $result['value'])),
'fields' => array('Attribute.type', 'Attribute.category', 'Attribute.value', 'Attribute.comment'),
'order' => false
'conditions' => array('OR' => $or),
'fields' => array('Attribute.type', 'Attribute.category', 'Attribute.value', 'Attribute.comment'),
'order' => false
);
$resultArray[$key]['related'] = $this->Event->Attribute->fetchAttributes($this->Auth->user(), $options);
}
// combobox for distribution
$distributions = $this->Event->Attribute->distributionLevels;
$sgs = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1);
if (empty($sgs)) {
unset($distributions[4]);
}
$this->set('event', array('Event' => array('id' => $target_id)));
$this->set('resultArray', $resultArray);
$this->set('typeList', array_keys($this->Event->Attribute->typeDefinitions));
$this->set('defaultCategories', $this->Event->Attribute->defaultCategories);
$this->set('typeCategoryMapping', $typeCategoryMapping);
$this->set('distributions', $distributions);
$this->set('sgs', $sgs);
$this->set('title', 'Merge Results');
$this->set('importComment', 'Merged from event ' . $source_id);
$this->render('resolved_attributes');
@ -3715,8 +3723,15 @@ class EventsController extends AppController
}
}
foreach ($resultArray as $key => $result) {
if ($has_pipe = strpos($result['default_type'], '|') !== false || $result['default_type'] === 'malware-sample') {
$pieces = explode('|', $result['value']);
$or = array('Attribute.value1' => $pieces,
'Attribute.value2' => $pieces);
} else {
$or = array('Attribute.value1' => $result['value'], 'Attribute.value2' => $result['value']);
}
$options = array(
'conditions' => array('OR' => array('Attribute.value1' => $result['value'], 'Attribute.value2' => $result['value'])),
'conditions' => array('OR' => $or),
'fields' => array('Attribute.type', 'Attribute.category', 'Attribute.value', 'Attribute.comment'),
'order' => false,
'flatten' => 1
@ -3819,7 +3834,7 @@ class EventsController extends AppController
$this->Event->insertLock($this->Auth->user(), $id);
$attributes = json_decode($this->request->data['Attribute']['JsonObject'], true);
$default_comment = $this->request->data['Attribute']['default_comment'];
$force = $this->request->data['Attribute']['force'];
$force = $this->_isSiteAdmin() && $this->request->data['Attribute']['force'];
$flashMessage = $this->Event->processFreeTextDataRouter($this->Auth->user(), $attributes, $id, $default_comment, $force);
$this->Flash->info($flashMessage);
$this->redirect(array('controller' => 'events', 'action' => 'view', $id));
@ -4514,16 +4529,20 @@ class EventsController extends AppController
if (!in_array($type, $validTools)) {
throw new MethodNotAllowedException('Invalid type.');
}
App::uses('EventTimelineTool', 'Tools');
$grapher = new EventTimelineTool();
$data = $this->request->is('post') ? $this->request->data : array();
$dataFiltering = array_key_exists('filtering', $data) ? $data['filtering'] : array();
$scope = isset($data['scope']) ? $data['scope'] : 'seen';
$extended = isset($this->params['named']['extended']) ? 1 : 0;
$grapher->construct($this->Event, $this->Auth->user(), $dataFiltering, $extended);
$json = $grapher->get_timeline($id);
if ($scope == 'seen') {
$json = $grapher->get_timeline($id);
} elseif ($scope == 'sightings') {
$json = $grapher->get_sighting_timeline($id);
}
array_walk_recursive($json, function (&$item, $key) {
if (!mb_detect_encoding($item, 'utf-8', true)) {
@ -4983,10 +5002,17 @@ class EventsController extends AppController
}
}
foreach ($resultArray as $key => $result) {
if ($has_pipe = strpos($result['default_type'], '|') !== false || $result['default_type'] === 'malware-sample') {
$pieces = explode('|', $result['value']);
$or = array('Attribute.value1' => $pieces,
'Attribute.value2' => $pieces);
} else {
$or = array('Attribute.value1' => $result['value'], 'Attribute.value2' => $result['value']);
}
$options = array(
'conditions' => array('OR' => array('Attribute.value1' => $result['value'], 'Attribute.value2' => $result['value'])),
'fields' => array('Attribute.type', 'Attribute.category', 'Attribute.value', 'Attribute.comment'),
'order' => false
'conditions' => array('OR' => $or),
'fields' => array('Attribute.type', 'Attribute.category', 'Attribute.value', 'Attribute.comment'),
'order' => false
);
$resultArray[$key]['related'] = $this->Event->Attribute->fetchAttributes($this->Auth->user(), $options);
if (isset($result['data'])) {
@ -5166,10 +5192,17 @@ class EventsController extends AppController
}
}
foreach ($resultArray as $key => $result) {
if ($has_pipe = strpos($result['default_type'], '|') !== false || $result['default_type'] === 'malware-sample') {
$pieces = explode('|', $result['value']);
$or = array('Attribute.value1' => $pieces,
'Attribute.value2' => $pieces);
} else {
$or = array('Attribute.value1' => $result['value'], 'Attribute.value2' => $result['value']);
}
$options = array(
'conditions' => array('OR' => array('Attribute.value1' => $result['value'], 'Attribute.value2' => $result['value'])),
'fields' => array('Attribute.type', 'Attribute.category', 'Attribute.value', 'Attribute.comment'),
'order' => false
'conditions' => array('OR' => $or),
'fields' => array('Attribute.type', 'Attribute.category', 'Attribute.value', 'Attribute.comment'),
'order' => false
);
$resultArray[$key]['related'] = $this->Event->Attribute->fetchAttributes($this->Auth->user(), $options);
}

6
app/Controller/FeedsController.php
View File

@ -364,7 +364,11 @@ class FeedsController extends AppController
}
}
if (!isset($this->request->data['Feed']['settings'])) {
$this->request->data['Feed']['settings'] = array();
if (!empty($this->Feed->data['Feed']['settings'])) {
$this->request->data['Feed']['settings'] = $this->Feed->data['Feed']['settings'];
} else {
$this->request->data['Feed']['settings'] = array();
}
} else {
if (!empty($this->request->data['Feed']['settings']['common']['excluderegex']) && !$this->__checkRegex($this->request->data['Feed']['settings']['common']['excluderegex'])) {
$this->Flash->error('Invalid exclude regex. Make sure it\'s a delimited PCRE regex pattern.');

7
app/Controller/ServersController.php
View File

@ -992,13 +992,15 @@ class ServersController extends AppController
if ($tab == 'diagnostics' || $tab == 'download' || $this->_isRest()) {
$php_ini = php_ini_loaded_file();
$this->set('php_ini', $php_ini);
$advanced_attachments = shell_exec($this->Server->getPythonVersion() . ' ' . APP . 'files/scripts/generate_file_objects.py -c');
$malwareTool = new MalwareTool();
try {
$advanced_attachments = json_decode($advanced_attachments, true);
$advanced_attachments = $malwareTool->checkAdvancedExtractionStatus($this->Server->getPythonVersion());
} catch (Exception $e) {
$this->log($e->getMessage(), LOG_NOTICE);
$advanced_attachments = false;
}
$this->set('advanced_attachments', $advanced_attachments);
// check if the current version of MISP is outdated or not
$version = $this->__checkVersion();
@ -1622,6 +1624,7 @@ class ServersController extends AppController
$result = $pubSubTool->statusCheck();
if (!empty($result)) {
$this->set('events', $result['publishCount']);
$this->set('messages', $result['messageCount']);
$this->set('time', date('Y/m/d H:i:s', $result['timestamp']));
$this->set('time2', date('Y/m/d H:i:s', $result['timestampSettings']));
}

171
app/Controller/UsersController.php
View File

@ -31,6 +31,9 @@ class UsersController extends AppController
// what pages are allowed for non-logged-in users
$allowedActions = array('login', 'logout');
if(!empty(Configure::read('Security.email_otp_enabled'))) {
$allowedActions[] = 'email_otp';
}
if (!empty(Configure::read('Security.allow_self_registration'))) {
$allowedActions[] = 'register';
}
@ -1116,33 +1119,15 @@ class UsersController extends AppController
$this->Auth->constructAuthenticate();
}
}
if ($this->request->is('post') && Configure::read('Security.email_otp_enabled')) {
$user = $this->Auth->identify($this->request, $this->response);
if ($user) {
$this->Session->write('email_otp_user', $user);
return $this->redirect('email_otp');
}
}
if ($this->Auth->login()) {
$this->User->extralog($this->Auth->user(), "login");
$this->User->Behaviors->disable('SysLogLogable.SysLogLogable');
$this->User->id = $this->Auth->user('id');
$user = $this->User->find('first', array(
'conditions' => array(
'User.id' => $this->Auth->user('id')
),
'recursive' => -1
));
$lastUserLogin = $user['User']['last_login'];
unset($user['User']['password']);
$user['User']['action'] = 'login';
$user['User']['last_login'] = $this->Auth->user('current_login');
$user['User']['current_login'] = time();
$this->User->save($user['User'], true, array('id', 'last_login', 'current_login'));
if (empty($this->Auth->authenticate['Form']['passwordHasher']) && !empty($passwordToSave)) {
$this->User->saveField('password', $passwordToSave);
}
$this->User->Behaviors->enable('SysLogLogable.SysLogLogable');
if ($lastUserLogin) {
$readableDatetime = (new DateTime())->setTimestamp($lastUserLogin)->format('D, d M y H:i:s O'); // RFC822
$this->Flash->info(sprintf('Welcome! Last login was on %s', $readableDatetime));
}
// no state changes are ever done via GET requests, so it is safe to return to the original page:
$this->redirect($this->Auth->redirectUrl());
// $this->redirect(array('controller' => 'events', 'action' => 'index'));
$this->_postlogin();
} else {
$dataSourceConfig = ConnectionManager::getDataSource('default')->config;
$dataSource = $dataSourceConfig['datasource'];
@ -1224,6 +1209,35 @@ class UsersController extends AppController
}
}
private function _postlogin()
{
$this->User->extralog($this->Auth->user(), "login");
$this->User->Behaviors->disable('SysLogLogable.SysLogLogable');
$this->User->id = $this->Auth->user('id');
$user = $this->User->find('first', array(
'conditions' => array(
'User.id' => $this->Auth->user('id')
),
'recursive' => -1
));
$lastUserLogin = $user['User']['last_login'];
unset($user['User']['password']);
$user['User']['action'] = 'login';
$user['User']['last_login'] = $this->Auth->user('current_login');
$user['User']['current_login'] = time();
$this->User->save($user['User'], true, array('id', 'last_login', 'current_login'));
if (empty($this->Auth->authenticate['Form']['passwordHasher']) && !empty($passwordToSave)) {
$this->User->saveField('password', $passwordToSave);
}
$this->User->Behaviors->enable('SysLogLogable.SysLogLogable');
if ($lastUserLogin) {
$readableDatetime = (new DateTime())->setTimestamp($lastUserLogin)->format('D, d M y H:i:s O'); // RFC822
$this->Flash->info(__('Welcome! Last login was on %s', $readableDatetime));
}
// no state changes are ever done via GET requests, so it is safe to return to the original page:
$this->redirect($this->Auth->redirectUrl());
}
public function routeafterlogin()
{
// Events list
@ -1656,6 +1670,90 @@ class UsersController extends AppController
}
}
public function email_otp()
{
$user = $this->Session->read('email_otp_user');
if(empty($user)) {
$this->redirect('login');
}
$redis = $this->User->setupRedis();
$user_id = $user['id'];
if ($this->request->is('post') && isset($this->request->data['User']['otp'])) {
$stored_otp = $redis->get('misp:otp:'.$user_id);
if (!empty($stored_otp) && $this->request->data['User']['otp'] == $stored_otp) {
// we invalidate the previously generated OTP
$redis->delete('misp:otp:'.$user_id);
// We login the user with CakePHP
$this->Auth->login($user);
$this->_postlogin();
} else {
$this->Flash->error(__("The OTP is incorrect or has expired"));
}
} else {
// GET Request
// We check for exceptions
$exception_list = Configure::read('Security.email_otp_exceptions');
if (!empty($exception_list)) {
$exceptions = explode(",", $exception_list);
foreach ($exceptions as &$exception) {
if ($user['email'] == trim($exception)) {
// We login the user with CakePHP
$this->Auth->login($user);
$this->_postlogin();
}
}
}
$this->loadModel('Server');
// Generating the OTP
$digits = !empty(Configure::read('Security.email_otp_length')) ? Configure::read('Security.email_otp_length') : $this->Server->serverSettings['Security']['email_otp_length']['value'];
$otp = "";
for ($i=0; $i<$digits; $i++) {
$otp.= random_int(0,9);
}
// We use Redis to cache the OTP
$redis->set('misp:otp:'.$user_id, $otp);
$validity = !empty(Configure::read('Security.email_otp_validity')) ? Configure::read('Security.email_otp_validity') : $this->Server->serverSettings['Security']['email_otp_validity']['value'];
$redis->expire('misp:otp:'.$user_id, (int) $validity * 60);
// Email construction
$body = !empty(Configure::read('Security.email_otp_text')) ? Configure::read('Security.email_otp_text') : $this->Server->serverSettings['Security']['email_otp_text']['value'];
$body = str_replace('$misp', Configure::read('MISP.baseurl'), $body);
$body = str_replace('$org', Configure::read('MISP.org'), $body);
$body = str_replace('$contact', Configure::read('MISP.contact'), $body);
$body = str_replace('$validity', $validity, $body);
$body = str_replace('$otp', $otp, $body);
$body = str_replace('$ip', $this->__getClientIP(), $body);
$body = str_replace('$username', $user['email'], $body);
$result = $this->User->sendEmail(array('User' => $user), $body, false, "[MISP] Email OTP");
if ( $result ) {
$this->Flash->success(__("An email containing a OTP has been sent."));
} else {
$this->Flash->error(__("The email couldn't be sent, please reach out to your administrator."));
}
}
}
/**
* Helper function to determine the IP of a client (proxy aware)
*/
private function __getClientIP() {
$x_forwarded = filter_input(INPUT_SERVER, 'HTTP_X_FORWARDED_FOR', FILTER_SANITIZE_STRING);
$client_ip = filter_input(INPUT_SERVER, 'HTTP_CLIENT_IP', FILTER_SANITIZE_STRING);
if (!empty($x_forwarded)) {
$x_forwarded = explode(",", $x_forwarded);
return $x_forwarded[0];
} elseif(!empty($client_ip)){
return $_client_ip;
} else {
return filter_input(INPUT_SERVER, 'REMOTE_ADDR', FILTER_SANITIZE_STRING);
}
}
// shows some statistics about the instance
public function statistics($page = 'data')
{
@ -1742,6 +1840,7 @@ class UsersController extends AppController
$stats['user_count_pgp'] = $this->User->find('count', array('recursive' => -1, 'conditions' => array('User.gpgkey !=' => '')));
$stats['org_count'] = count($orgs);
$stats['local_org_count'] = count($local_orgs);
$stats['contributing_org_count'] = $this->User->Event->find('count', array('recursive' => -1, 'group' => array('Event.orgc_id')));
$stats['average_user_per_org'] = round($stats['user_count'] / $stats['local_org_count'], 1);
$this->loadModel('Thread');
@ -2275,7 +2374,7 @@ class UsersController extends AppController
$requestObject['message'] = '';
}
if (empty($requestObject['email'])) {
throw new InvalidArgumentException(__('We require at least the email field to be filled.'));
throw new BadRequestException(__('We require at least the email field to be filled.'));
}
$this->loadModel('Inbox');
$this->Inbox->create();
@ -2374,8 +2473,13 @@ class UsersController extends AppController
$id = $this->params['named']['id'];
}
$this->loadModel('Inbox');
if (Validation::uuid($id)) {
$id = $this->Toolbox->findIdByUuid($this->Inbox, $id);
if (!is_array($id)) {
$id = array($id);
}
foreach ($id as $k => $v) {
if (Validation::uuid($v)) {
$id[$k] = $this->Toolbox->findIdByUuid($this->Inbox, $v);
}
}
$registrations = $this->Inbox->find('all', array(
'recursive' => -1,
@ -2400,7 +2504,7 @@ class UsersController extends AppController
$this->Log->save(array(
'org' => $this->Auth->user('Organisation')['name'],
'model' => 'User',
'model_id' => $id,
'model_id' => 0,
'email' => $this->Auth->user('email'),
'action' => 'discardRegistrations',
'title' => $message,
@ -2466,6 +2570,9 @@ class UsersController extends AppController
'perm_admin' => $role['Role']['perm_admin']
);
}
if (empty($this->request->data['User'])) {
$this->request->data = array('User' => $this->request->data);
}
if (!empty($default_role)) {
$this->request->data['User']['role_id'] = $default_role['Role']['id'];
}
@ -2492,11 +2599,11 @@ class UsersController extends AppController
if (!empty($default_role)) {
$this->request->data['User']['role_id'] = $default_role['Role']['id'];
} else {
throw new InvalidArgumentException(__('Role ID not provided and no default role exist on the instance'));
throw new BadRequestException(__('Role ID not provided and no default role exist on the instance'));
}
}
if (!isset($this->request->data['User']['org_id'])) {
throw new InvalidArgumentException(__('No organisation selected. Supply an Organisation ID'));
throw new BadRequestException(__('No organisation selected. Supply an Organisation ID'));
} else {
if (Validation::uuid($this->request->data['User']['org_id'])) {
$id = $this->Toolbox->findIdByUuid($this->User->Organisation, $this->request->data['User']['org_id']);

2
app/Lib/Export/JsonExport.php
View File

@ -12,7 +12,7 @@ class JsonExport
} else if($options['scope'] === 'Event') {
return $this->__eventHandler($data, $options);
} else if($options['scope'] === 'Object') {
return $this->__eventHandler($data, $options);
return $this->__objectHandler($data, $options);
} else if($options['scope'] === 'Sighting') {
return $this->__sightingsHandler($data, $options);
}

130
app/Lib/Export/OpendataExport.php Normal file
View File

@ -0,0 +1,130 @@
<?php
class OpendataExport
{
public $non_restrictive_export = true;
public $use_default_filters = true;
public $mock_query_only = true;
private $__default_filters = null;
private $__auth = null;
private $__delete = false;
private $__scope = null;
private $__setup = array();
private $__url = null;
private $__scripts_dir = APP . 'files/scripts/';
private $__script_name = 'misp-opendata/opendata.py';
public function setDefaultFilters($filters)
{
$this->__default_filters = $filters;
}
public function header($options = array())
{
$this->__scope = $options['scope'];
if (isset($this->__default_filters['auth'])) {
$this->__auth = $this->__default_filters['auth'];
unset($this->__default_filters['auth']);
}
if (isset($this->__default_filters['setup'])) {
$this->__setup = $this->__default_filters['setup'];
$this->__check_setup_filter();
unset($this->__default_filters['setup']);
} else {
throw new Exception(__('Missing "setup" filter containing the dataset and resource(s) information.'));
}
if (isset($this->__default_filters['url'])) {
$this->__url = $this->__default_filters['url'];
unset($this->__default_filters['url']);
} else {
$external_baseurl = Configure::read('MISP.external_baseurl');
$baseurl = !empty($external_baseurl) ? $external_baseurl : Configure::read('MISP.baseurl');
if (empty($baseurl)) {
throw new Exception(__('Missing url of the MISP instance, and baseurl is not set.'));
}
$this->__url = $baseurl;
}
if (!empty($this->__default_filters['delete'])) {
$this->__delete = true;
unset($this->__default_filters['delete']);
}
return '';
}
public function footer()
{
$authParam = ' --auth ' . $this->__auth;
$my_server = ClassRegistry::init('Server');
$cmd = $my_server->getPythonVersion() . ' ' . $this->__scripts_dir . $this->__script_name . $authParam;
return $this->__delete ? $this->__delete_query($cmd) : $this->__add_query($cmd);
}
public function separator()
{
return '';
}
private function __add_query($cmd)
{
unset($this->__default_filters['returnFormat']);
$body = json_encode($this->__default_filters);
$bodyFilename = $this->__generateSetupFile($body);
$bodyParam = ' --body ' . $bodyFilename;
$levelParam = ' --level ' . strtolower($this->__scope) . 's';
$setup = json_encode($this->__setup);
$setupFilename = $this->__generateSetupFile($setup);
$setupParam = ' --setup ' . $setupFilename;
$urlParam = ' --url ' . $this->__url;
$cmd .= $bodyParam . $setupParam . $levelParam . $urlParam;
$results = shell_exec($cmd);
unlink($bodyFilename);
unlink($setupFilename);
return $results;
}
private function __check_setup_filter()
{
if (empty($this->__setup['dataset'])) {
throw new Exception(__('Missing dataset filter in the setup filter. Please provide the dataset setup.'));
}
if (!empty($this->__setup['resources']) && !empty($this->__setup['resource'])) {
throw new Exception(__('Please provide the resource setup in a single field called "resources".'));
}
if (!empty($this->__setup['resource']) && empty($this->__setup['resources'])) {
$this->__setup['resources'] = $this->__setup['resource'];
unset($this->__setup['resource']);
}
}
private function __delete_query($cmd)
{
$cmd .= " -d '" . $this->__setup['dataset'] . "'";
if (!empty($this->__setup['resources'])) {
if (is_array($this->__setup['resources'])) {
foreach ($this->__setup['resources'] as $resource) {
$cmd .= ' ' . $resource;
}
} else {
$cmd .= " '" . $this->__setup['resources'] . "'";
}
}
return shell_exec($cmd);
}
private function __generateRandomFileName()
{
return (new RandomTool())->random_str(false, 12);
}
private function __generateSetupFile($to_write)
{
$filename = $this->__scripts_dir . 'tmp/' . $this->__generateRandomFileName();
$tmpFile = new File($filename, true, 0644);
$tmpFile->write($to_write);
$tmpFile->close();
return $filename;
}
}

141
app/Lib/Tools/EventTimelineTool.php
View File

@ -135,4 +135,145 @@
return $this->__json;
}
/*
* Extrapolation strategy:
* - If only positive sightings: Will be from first to last sighting
* - If both positive and false positive: False positive get priority. It will be marked as false positive until next positive sighting
*/
public function get_sighting_timeline($id)
{
$event = $this->__eventModel->fetchEvent($this->__user, array(
'eventid' => $id,
'flatten' => 1,
'includeTagRelations' => 1,
'extended' => $this->__extended_view
));
$this->__json['items'] = array();
if (empty($event)) {
return $this->__json;
} else {
$event = $event[0];
}
$lookupAttribute = array();
foreach ($event['Attribute'] as $k => $attribute) {
$lookupAttribute[$attribute['id']] = &$event['Attribute'][$k];
}
// regroup sightings per attribute
$regroupedSightings = array();
foreach ($event['Sighting'] as $k => $sighting) {
$event['Sighting'][$k]['date_sighting'] *= 1000; // adapt to use micro
$regroupedSightings[$sighting['attribute_id']][] = &$event['Sighting'][$k];
}
// make sure sightings are ordered
uksort($regroupedSightings, function ($a, $b) {
return $a['date_sighting'] > $b['date_sighting'];
});
// generate extrapolation
$now = time()*1000;
foreach ($regroupedSightings as $attributeId => $sightings) {
$i = 0;
while ($i < count($sightings)) {
$sighting = $sightings[$i];
$attribute = $lookupAttribute[$attributeId];
$fpSightingIndex = $this->getNextFalsePositiveSightingIndex($sightings, $i+1);
if ($fpSightingIndex === false) { // No next FP, extrapolate to now
$this->__json['items'][] = array(
'attribute_id' => $attributeId,
'id' => sprintf('%s-%s', $attributeId, $sighting['id']),
'uuid' => $sighting['uuid'],
'content' => $attribute['value'],
'event_id' => $attribute['event_id'],
'group' => 'sighting_positive',
'timestamp' => $attribute['timestamp'],
'first_seen' => $sighting['date_sighting'],
'last_seen' => $now,
);
break;
} else {
// set up until last positive
$pSightingIndex = $fpSightingIndex - 1;
$halfTime = 0;
if ($pSightingIndex == $i) {
// we have only one positive sighting, thus the UP time should be take from a pooling frequence
// for now, consider it UP only for half the time until the next FP
$halfTime = ($sightings[$i+1]['date_sighting'] - $sighting['date_sighting'])/2;
}
$pSighting = $sightings[$pSightingIndex];
$this->__json['items'][] = array(
'attribute_id' => $attributeId,
'id' => sprintf('%s-%s', $attributeId, $sighting['id']),
'uuid' => $sighting['uuid'],
'content' => $attribute['value'],
'event_id' => $attribute['event_id'],
'group' => 'sighting_positive',
'timestamp' => $attribute['timestamp'],
'first_seen' => $sighting['date_sighting'],
'last_seen' => $pSighting['date_sighting'] + $halfTime,
);
// No next FP, extrapolate to now
$fpSighting = $sightings[$fpSightingIndex];
$secondNextPSightingIndex = $this->getNextPositiveSightingIndex($sightings, $fpSightingIndex+1);
if ($secondNextPSightingIndex === false) { // No next P, extrapolate to now
$this->__json['items'][] = array(
'attribute_id' => $attributeId,
'id' => sprintf('%s-%s', $attributeId, $sighting['id']),
'uuid' => $sighting['uuid'],
'content' => $attribute['value'],
'event_id' => $attribute['event_id'],
'group' => 'sighting_negative',
'timestamp' => $attribute['timestamp'],
'first_seen' => $pSighting['date_sighting'] - $halfTime,
'last_seen' => $now,
);
break;
} else {
if ($halfTime > 0) { // We need to fake a previous P
$pSightingIndex = $pSightingIndex+1;
$pSighting = $sightings[$pSightingIndex];
}
// set down until next postive
$secondNextPSighting = $sightings[$secondNextPSightingIndex];
$this->__json['items'][] = array(
'attribute_id' => $attributeId,
'id' => sprintf('%s-%s', $attributeId, $sighting['id']),
'uuid' => $pSighting['uuid'],
'content' => $attribute['value'],
'event_id' => $attribute['event_id'],
'group' => 'sighting_negative',
'timestamp' => $attribute['timestamp'],
'first_seen' => $pSighting['date_sighting'] - $halfTime,
'last_seen' => $secondNextPSighting['date_sighting'],
);
$i = $secondNextPSightingIndex;
}
}
}
}
return $this->__json;
}
private function getNextFalsePositiveSightingIndex($sightings, $startIndex)
{
for ($i=$startIndex; $i < count($sightings) ; $i++) {
$sighting = $sightings[$i];
if ($sighting['type'] == 1) { // is false positive
return $i;
}
}
return false;
}
private function getNextPositiveSightingIndex($sightings, $startIndex)
{
for ($i=$startIndex; $i < count($sightings) ; $i++) {
$sighting = $sightings[$i];
if ($sighting['type'] == 0) { // is false positive
return $i;
}
}
return false;
}
}

10
app/Lib/Tools/JSONConverterTool.php
View File

@ -45,10 +45,12 @@ class JSONConverterTool
unset($event['Event']['SharingGroup']);
}
if ($object == 'Galaxy') {
foreach ($event['Event']['Galaxy'] as $k => $galaxy) {
foreach ($galaxy['GalaxyCluster'] as $k2 => $cluster) {
if (empty($cluster['meta'])) {
$event['Event']['Galaxy'][$k]['GalaxyCluster'][$k2]['meta'] = new stdclass();
if (!empty($event['Event']['Galaxy'])) {
foreach ($event['Event']['Galaxy'] as $k => $galaxy) {
foreach ($galaxy['GalaxyCluster'] as $k2 => $cluster) {
if (empty($cluster['meta'])) {
$event['Event']['Galaxy'][$k]['GalaxyCluster'][$k2]['meta'] = new stdclass();
}
}
}
}

227
app/Lib/Tools/MalwareTool.php Normal file
View File

@ -0,0 +1,227 @@
<?php
class MalwareTool
{
const ZIP_PASSWORD = 'infected';
const ADVANCED_EXTRACTION_SCRIPT_PATH = APP . 'files/scripts/generate_file_objects.py';
/**
* @param string $originalFilename
* @param string $content
* @param string $md5
* @return string Content of zipped file
* @throws Exception
*/
public function encrypt($originalFilename, $content, $md5)
{
if (method_exists("ZipArchive", "setEncryptionName")) {
// When PHP zip extension is installed and supports creating encrypted archives.
return $this->encryptByExtension($originalFilename, $content, $md5);
} else {
return $this->encryptByCommand($originalFilename, $content, $md5);
}
}
/**
* @param string $originalFilename
* @param string $content
* @param string $md5
* @return string Content of zipped file
* @throws Exception
*/
private function encryptByCommand($originalFilename, $content, $md5)
{
$tempDir = $this->tempDir();
$contentsFile = new File($tempDir . DS . $md5, true);
if (!$contentsFile->write($content)) {
throw new Exception("Could not write content to file '{$contentsFile->path}'.");
}
$contentsFile->close();
$fileNameFile = new File($tempDir . DS . $md5 . '.filename.txt', true);
if (!$fileNameFile->write($originalFilename)) {
throw new Exception("Could not write original file name to file '{$fileNameFile->path}'.");
}
$fileNameFile->close();
$zipFile = new File($tempDir . DS . $md5 . '.zip');
$exec = [
'zip',
'-j', // junk (don't record) directory names
'-P', // use standard encryption
self::ZIP_PASSWORD,
escapeshellarg($zipFile->path),
escapeshellarg($contentsFile->path),
escapeshellarg($fileNameFile->path),
];
try {
$this->execute($exec);
$zipContent = $zipFile->read();
if ($zipContent === false) {
throw new Exception("Could not read content of newly created ZIP file.");
}
return $zipContent;
} catch (Exception $e) {
throw new Exception("Could not create encrypted ZIP file '{$zipFile->path}'.", 0, $e);
} finally {
$fileNameFile->delete();
$contentsFile->delete();
$zipFile->delete();
}
}
/**
* @param string $originalFilename
* @param string $content
* @param string $md5
* @return string Content of zipped file
* @throws Exception
*/
private function encryptByExtension($originalFilename, $content, $md5)
{
$zipFilePath = $this->tempFileName();
$zip = new ZipArchive();
$result = $zip->open($zipFilePath, ZipArchive::CREATE);
if ($result === true) {
$zip->setPassword(self::ZIP_PASSWORD);
$zip->addFromString($md5, $content);
$zip->setEncryptionName($md5, ZipArchive::EM_AES_128);
$zip->addFromString("$md5.filename.txt", $originalFilename);
$zip->setEncryptionName("$md5.filename.txt", ZipArchive::EM_AES_128);
$zip->close();
} else {
throw new Exception("Could not create encrypted ZIP file '$zipFilePath'. Error code: $result");
}
$zipFile = new File($zipFilePath);
$zipContent = $zipFile->read();
if ($zipContent === false) {
throw new Exception("Could not read content of newly created ZIP file.");
}
$zipFile->delete();
return $zipContent;
}
/**
* @param string $content
* @param array $hashTypes
* @return array
* @throws InvalidArgumentException
*/
public function computeHashes($content, array $hashTypes = array())
{
$validHashes = array('md5', 'sha1', 'sha256');
$hashes = [];
foreach ($hashTypes as $hashType) {
if (!in_array($hashType, $validHashes)) {
throw new InvalidArgumentException("Hash type '$hashType' is not valid hash type.");
}
$hashes[$hashType] = hash($hashType, $content);
}
return $hashes;
}
/**
* @param string $pythonBin
* @param string $filePath
* @return array
* @throws Exception
*/
public function advancedExtraction($pythonBin, $filePath)
{
return $this->executeAndParseJsonOutput([
$pythonBin,
self::ADVANCED_EXTRACTION_SCRIPT_PATH,
'-p',
escapeshellarg($filePath),
]);
}
/**
* @param string $pythonBin
* @return array
* @throws Exception
*/
public function checkAdvancedExtractionStatus($pythonBin)
{
return $this->executeAndParseJsonOutput([$pythonBin, self::ADVANCED_EXTRACTION_SCRIPT_PATH, '-c']);
}
private function tempFileName()
{
$randomName = (new RandomTool())->random_str(false, 12);
return $this->tempDir() . DS . $randomName;
}
/**
* @return string
*/
private function tempDir()
{
return Configure::read('MISP.tmpdir') ?: sys_get_temp_dir();
}
/**
* @param array $command
* @return array
* @throws Exception
*/
private function executeAndParseJsonOutput(array $command)
{
$output = $this->execute($command);
$json = json_decode($output, true);
if ($json === null) {
throw new Exception("Command output is not valid JSON: " . json_last_error_msg());
}
return $json;
}
/**
* This method is much more complicated than just `exec`, but it also provide stderr output, so Exceptions
* can be much more specific.
*
* @param array $command
* @return string
* @throws Exception
*/
private function execute(array $command)
{
$descriptorspec = [
1 => ["pipe", "w"], // stdout
2 => ["pipe", "w"], // stderr
];
$command = implode(' ', $command);
$process = proc_open($command, $descriptorspec, $pipes);
if (!$process) {
throw new Exception("Command '$command' could be started.");
}
$stdout = stream_get_contents($pipes[1]);
if ($stdout === false) {
throw new Exception("Could not get STDOUT of command.");
}
fclose($pipes[1]);
$stderr = stream_get_contents($pipes[2]);
fclose($pipes[2]);
$returnCode = proc_close($process);
if ($returnCode !== 0) {
throw new Exception("Command '$command' return error code $returnCode. STDERR: '$stderr', STDOUT: '$stdout'");
}
return $stdout;
}
}

263
app/Lib/Tools/PubSubTool.php
View File

@ -1,66 +1,53 @@
<?php
class PubSubTool
{
private $__redis = false;
private $__settings = false;
const SCRIPTS_TMP = APP . 'files' . DS . 'scripts' . DS . 'tmp' . DS;
const OLD_PID_LOCATION = APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmq.pid';
private function __getSetSettings()
{
$settings = array(
'redis_host' => 'localhost',
'redis_port' => '6379',
'redis_password' => '',
'redis_database' => '1',
'redis_namespace' => 'mispq',
'port' => '50000',
);
/**
* @var Redis
*/
private $redis;
foreach ($settings as $key => $setting) {
$temp = Configure::read('Plugin.ZeroMQ_' . $key);
if ($temp) {
$settings[$key] = $temp;
}
}
$settingsFile = new File(APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'settings.json', true, 0644);
$settingsFile->write(json_encode($settings, true));
$settingsFile->close();
return $settings;
}
/**
* @var array
*/
private $settings;
public function initTool()
{
if (!$this->__redis) {
$settings = $this->__setupPubServer();
$redis = new Redis();
$redis->connect($settings['redis_host'], $settings['redis_port']);
$redis_pwd = $settings['redis_password'];
if (!empty($redis_pwd)) {
$redis->auth($redis_pwd);
}
$redis->select($settings['redis_database']);
$this->__redis = $redis;
$this->__settings = $settings;
} else {
$settings = $this->__settings;
if (!$this->redis) {
$settings = $this->getSetSettings();
$this->setupPubServer($settings);
$this->redis = $this->createRedisConnection($settings);
$this->settings = $settings;
}
return $settings;
}
// read the pid file, if it exists, check if the process is actually running
// if either the pid file doesn't exists or the process is not running return false
// otherwise return the pid
public function checkIfRunning()
/**
* Read the pid file, if it exists, check if the process is actually running
* if either the pid file doesn't exists or the process is not running return false
* otherwise return the pid.
*
* @param string|null $pidFilePath
* @return bool|int False when process is not running, PID otherwise.
* @throws Exception
*/
public function checkIfRunning($pidFilePath = null)
{
$pidFile = new File(APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmq.pid');
$pid = $pidFile->read(true, 'r');
$pidFile = new File($pidFilePath ?: self::SCRIPTS_TMP . 'mispzmq.pid');
if (!$pidFile->exists()) {
return false;
}
$pid = $pidFile->read();
if ($pid === false || $pid === '') {
return false;
}
if (!is_numeric($pid)) {
throw new Exception('Internal error (invalid PID file for the MISP zmq script)');
}
$result = trim(shell_exec('ps aux | awk \'{print $2}\' | grep "^' . $pid . '$"'));
if (empty($result)) {
$result = file_exists("/proc/$pid");
if ($result === false) {
return false;
}
return $pid;
@ -68,18 +55,14 @@ class PubSubTool
public function statusCheck()
{
$redis = new Redis();
$settings = $this->__getSetSettings();
$redis->connect($settings['redis_host'], $settings['redis_port']);
$redis_pwd = $settings['redis_password'];
if (!empty($redis_pwd)) {
$redis->auth($redis_pwd);
}
$redis->select($settings['redis_database']);
$settings = $this->getSetSettings();
$redis = $this->createRedisConnection($settings);
$redis->rPush($settings['redis_namespace'] . ':command', 'status');
sleep(1);
$response = trim($redis->lPop($settings['redis_namespace'] . ':status'));
return json_decode($response, true);
$response = $redis->blPop($settings['redis_namespace'] . ':status', 5);
if ($response === null) {
throw new Exception("No response from status command returned after 5 seconds.");
}
return json_decode(trim($response[1]), true);
}
public function checkIfPythonLibInstalled()
@ -92,23 +75,12 @@ class PubSubTool
return false;
}
private function __setupPubServer()
{
App::uses('File', 'Utility');
$my_server = ClassRegistry::init('Server');
$settings = $this->__getSetSettings();
if ($this->checkIfRunning() === false) {
shell_exec($my_server->getPythonVersion() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmq.py > ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.log 2> ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.error.log &');
}
return $settings;
}
public function publishEvent($event)
{
App::uses('JSONConverterTool', 'Tools');
$jsonTool = new JSONConverterTool();
$json = $jsonTool->convert($event);
return $this->__pushToRedis(':data:misp_json', $json);
return $this->pushToRedis(':data:misp_json', $json);
}
public function event_save($event, $action)
@ -116,7 +88,7 @@ class PubSubTool
if (!empty($action)) {
$event['action'] = $action;
}
return $this->__pushToRedis(':data:misp_json_event', json_encode($event, JSON_PRETTY_PRINT));
return $this->pushToRedis(':data:misp_json_event', json_encode($event, JSON_PRETTY_PRINT));
}
public function object_save($object, $action)
@ -124,7 +96,7 @@ class PubSubTool
if (!empty($action)) {
$object['action'] = $action;
}
return $this->__pushToRedis(':data:misp_json_object', json_encode($object, JSON_PRETTY_PRINT));
return $this->pushToRedis(':data:misp_json_object', json_encode($object, JSON_PRETTY_PRINT));
}
public function object_reference_save($object_reference, $action)
@ -132,18 +104,12 @@ class PubSubTool
if (!empty($action)) {
$object_reference['action'] = $action;
}
return $this->__pushToRedis(':data:misp_json_object_reference', json_encode($object_reference, JSON_PRETTY_PRINT));
return $this->pushToRedis(':data:misp_json_object_reference', json_encode($object_reference, JSON_PRETTY_PRINT));
}
public function publishConversation($message)
{
return $this->__pushToRedis(':data:misp_json_conversation', json_encode($message, JSON_PRETTY_PRINT));
}
private function __pushToRedis($ns, $data)
{
$this->__redis->rPush($this->__settings['redis_namespace'] . $ns, $data);
return true;
return $this->pushToRedis(':data:misp_json_conversation', json_encode($message, JSON_PRETTY_PRINT));
}
public function attribute_save($attribute, $action = false)
@ -151,7 +117,7 @@ class PubSubTool
if (!empty($action)) {
$attribute['action'] = $action;
}
return $this->__pushToRedis(':data:misp_json_attribute', json_encode($attribute, JSON_PRETTY_PRINT));
return $this->pushToRedis(':data:misp_json_attribute', json_encode($attribute, JSON_PRETTY_PRINT));
}
public function tag_save($tag, $action = false)
@ -159,7 +125,7 @@ class PubSubTool
if (!empty($action)) {
$tag['action'] = $action;
}
return $this->__pushToRedis(':data:misp_json_tag', json_encode($tag, JSON_PRETTY_PRINT));
return $this->pushToRedis(':data:misp_json_tag', json_encode($tag, JSON_PRETTY_PRINT));
}
public function sighting_save($sighting, $action = false)
@ -167,7 +133,7 @@ class PubSubTool
if (!empty($action)) {
$sighting['action'] = $action;
}
return $this->__pushToRedis(':data:misp_json_sighting', json_encode($sighting, JSON_PRETTY_PRINT));
return $this->pushToRedis(':data:misp_json_sighting', json_encode($sighting, JSON_PRETTY_PRINT));
}
public function modified($data, $type, $action = false)
@ -175,7 +141,7 @@ class PubSubTool
if (!empty($action)) {
$data['action'] = $action;
}
return $this->__pushToRedis(':data:misp_json_' . $type, json_encode($data, JSON_PRETTY_PRINT));
return $this->pushToRedis(':data:misp_json_' . $type, json_encode($data, JSON_PRETTY_PRINT));
}
public function publish($data, $type, $action = false)
@ -183,48 +149,39 @@ class PubSubTool
if (!empty($action)) {
$data['action'] = $action;
}
return $this->__pushToRedis(':data:misp_json_' . $type, json_encode($data, JSON_PRETTY_PRINT));
return $this->pushToRedis(':data:misp_json_' . $type, json_encode($data, JSON_PRETTY_PRINT));
}
public function killService($settings = false)
public function killService()
{
$redis = new Redis();
if ($this->checkIfRunning()) {
if ($settings == false) {
$settings = $this->__getSetSettings();
}
$redis->connect($settings['redis_host'], $settings['redis_port']);
$redis_pwd = $settings['redis_password'];
if (!empty($redis_pwd)) {
$redis->auth($redis_pwd);
}
$redis->select($settings['redis_database']);
$settings = $this->getSetSettings();
$redis = $this->createRedisConnection($settings);
$redis->rPush($settings['redis_namespace'] . ':command', 'kill');
sleep(1);
if ($this->checkIfRunning()) {
// Still running.
return false;
}
}
return true;
}
// reload the server if it is running, if not, start it
/**
* Reload the server if it is running, if not, start it.
*
* @return bool|string
* @throws Exception
*/
public function reloadServer()
{
if (!$this->checkIfRunning()) {
$settings = $this->__setupPubServer();
} else {
$settings = $this->__getSetSettings();
$redis = new Redis();
$redis->connect($settings['redis_host'], $settings['redis_port']);
$redis_pwd = $settings['redis_password'];
if (!empty($redis_pwd)) {
$redis->auth($redis_pwd);
}
$redis->select($settings['redis_database']);
$settings = $this->getSetSettings();
$this->saveSettingToFile($settings);
if ($this->checkIfRunning()) {
$redis = $this->createRedisConnection($settings);
$redis->rPush($settings['redis_namespace'] . ':command', 'reload');
}
if (!$this->checkIfRunning()) {
} else {
return 'Setting saved, but something is wrong with the ZeroMQ server. Please check the diagnostics page for more information.';
}
return true;
@ -237,10 +194,96 @@ class PubSubTool
return 'Could not kill the previous instance of the ZeroMQ script.';
}
}
$this->__setupPubServer();
if (!is_numeric($this->checkIfRunning())) {
$settings = $this->getSetSettings();
$this->setupPubServer($settings);
if ($this->checkIfRunning() === false) {
return 'Failed starting the ZeroMQ script.';
}
return true;
}
/**
* @param array $settings
* @throws Exception
*/
private function setupPubServer(array $settings)
{
if ($this->checkIfRunning() === false) {
if ($this->checkIfRunning(self::OLD_PID_LOCATION)) {
// Old version is running, kill it and start again new one.
$redis = $this->createRedisConnection($settings);
$redis->rPush($settings['redis_namespace'] . ':command', 'kill');
sleep(1);
}
$this->saveSettingToFile($settings);
$server = ClassRegistry::init('Server');
shell_exec($server->getPythonVersion() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmq.py >> ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.log 2>> ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.error.log &');
}
}
private function pushToRedis($ns, $data)
{
$this->redis->rPush($this->settings['redis_namespace'] . $ns, $data);
return true;
}
/**
* @param array $settings
* @return Redis
*/
private function createRedisConnection(array $settings)
{
$redis = new Redis();
$redis->connect($settings['redis_host'], $settings['redis_port']);
$redisPassword = $settings['redis_password'];
if (!empty($redisPassword)) {
$redis->auth($redisPassword);
}
$redis->select($settings['redis_database']);
return $redis;
}
/**
* @param array $settings
* @throws Exception
*/
private function saveSettingToFile(array $settings)
{
$settingFilePath = self::SCRIPTS_TMP . 'mispzmq_settings.json';
$settingsFile = new File($settingFilePath, true, 0644);
if (!$settingsFile->exists()) {
throw new Exception("Could not create zmq config file '$settingFilePath'.");
}
// Because setting file contains secrets, it should be readable just by owner. But because in Travis test,
// config file is created under one user and then changed under other user, file must be readable and writable
// also by group.
chmod($settingsFile->pwd(), 0660);
if (!$settingsFile->write(json_encode($settings))) {
throw new Exception("Could not write zmq config file '$settingFilePath'.");
}
$settingsFile->close();
}
private function getSetSettings()
{
$settings = array(
'redis_host' => 'localhost',
'redis_port' => '6379',
'redis_password' => '',
'redis_database' => '1',
'redis_namespace' => 'mispq',
'port' => '50000',
'username' => null,
'password' => null,
);
foreach ($settings as $key => $setting) {
$temp = Configure::read('Plugin.ZeroMQ_' . $key);
if ($temp) {
$settings[$key] = $temp;
}
}
return $settings;
}
}

4681
app/Locale/zh-s/LC_MESSAGES/default.po
View File

File diff suppressed because it is too large Load Diff

72
app/Model/AppModel.php
View File

@ -28,7 +28,10 @@ class AppModel extends Model
{
public $name;
public $loadedPubSubTool = false;
/**
* @var PubSubTool
*/
private $loadedPubSubTool;
public $loadedKafkaPubTool = false;
@ -78,7 +81,7 @@ class AppModel extends Model
33 => false, 34 => false, 35 => false, 36 => false, 37 => false, 38 => false,
39 => false, 40 => false, 41 => false, 42 => false, 43 => false, 44 => false,
45 => false, 46 => false, 47 => false, 48 => false, 49 => false, 50 => false,
51 => false
51 => false, 52 => false, 53 => false, 54 => false
);
public $advanced_updates_description = array(
@ -1268,7 +1271,7 @@ class AppModel extends Model
case 39:
$sqlArray[] = "CREATE TABLE IF NOT EXISTS user_settings (
`id` int(11) NOT NULL AUTO_INCREMENT,
`key` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL,
`setting` varchar(255) COLLATE utf8_bin NOT NULL,
`value` text,
`user_id` int(11) NOT NULL,
`timestamp` int(11) NOT NULL,
@ -1377,6 +1380,21 @@ class AppModel extends Model
$sqlArray[] = "ALTER TABLE `feeds` ADD `orgc_id` int(11) NOT NULL DEFAULT 0";
$this->__addIndex('feeds', 'orgc_id');
break;
case 52:
if (!empty($this->query("SHOW COLUMNS FROM `admin_settings` LIKE 'key';"))) {
$sqlArray[] = "ALTER TABLE admin_settings CHANGE `key` `setting` varchar(255) COLLATE utf8_bin NOT NULL;";
$this->__addIndex('admin_settings', 'setting');
}
break;
case 53:
if (!empty($this->query("SHOW COLUMNS FROM `user_settings` LIKE 'key';"))) {
$sqlArray[] = "ALTER TABLE user_settings CHANGE `key` `setting` varchar(255) COLLATE utf8_bin NOT NULL;";
$this->__addIndex('user_settings', 'setting');
}
break;
case 54:
$sqlArray[] = "ALTER TABLE `sightingdbs` MODIFY `timestamp` int(11) NOT NULL DEFAULT 0;";
break;
case 'fixNonEmptySharingGroupID':
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@ -2352,20 +2370,14 @@ class AppModel extends Model
public function getPubSubTool()
{
if (!$this->loadedPubSubTool) {
$this->loadPubSubTool();
App::uses('PubSubTool', 'Tools');
$pubSubTool = new PubSubTool();
$pubSubTool->initTool();
$this->loadedPubSubTool = $pubSubTool;
}
return $this->loadedPubSubTool;
}
public function loadPubSubTool()
{
App::uses('PubSubTool', 'Tools');
$pubSubTool = new PubSubTool();
$pubSubTool->initTool();
$this->loadedPubSubTool = $pubSubTool;
return true;
}
public function getElasticSearchTool()
{
if (!$this->elasticSearchClient) {
@ -2839,12 +2851,34 @@ class AppModel extends Model
*/
protected function logException($message, Exception $exception, $type = LOG_ERR)
{
$message = sprintf("%s\n[%s] %s",
$message,
get_class($exception),
$exception->getMessage()
);
$message .= "\nStack Trace:\n" . $exception->getTraceAsString();
$message .= "\n";
do {
$message .= sprintf("[%s] %s",
get_class($exception),
$exception->getMessage()
);
$message .= "\nStack Trace:\n" . $exception->getTraceAsString();
$exception = $exception->getPrevious();
} while ($exception !== null);
return $this->log($message, $type);
}
/**
* Generates random file name in tmp dir.
* @return string
*/
protected function tempFileName()
{
return $this->tempDir() . DS . $this->generateRandomFileName();
}
/**
* @return string
*/
protected function tempDir()
{
return Configure::read('MISP.tmpdir') ?: sys_get_temp_dir();
}
}

312
app/Model/Attribute.php
View File

@ -5,6 +5,7 @@ App::uses('Folder', 'Utility');
App::uses('File', 'Utility');
App::uses('FinancialTool', 'Tools');
App::uses('RandomTool', 'Tools');
App::uses('MalwareTool', 'Tools');
class Attribute extends AppModel
{
@ -400,20 +401,21 @@ class Attribute extends AppModel
);
public $validFormats = array(
'json' => array('json', 'JsonExport', 'json'),
'openioc' => array('xml', 'OpeniocExport', 'ioc'),
'xml' => array('xml', 'XmlExport', 'xml'),
'suricata' => array('txt', 'NidsSuricataExport', 'rules'),
'snort' => array('txt', 'NidsSnortExport', 'rules'),
'text' => array('txt', 'TextExport', 'txt'),
'hashes' => array('txt', 'HashesExport', 'txt'),
'yara' => array('txt', 'YaraExport', 'yara'),
'yara-json' => array('json', 'YaraExport', 'json'),
'rpz' => array('txt', 'RPZExport', 'rpz'),
'csv' => array('csv', 'CsvExport', 'csv'),
'cache' => array('txt', 'CacheExport', 'cache'),
'attack-sightings' => array('json', 'AttackSightingsExport', 'json'),
'netfilter' => array('txt', 'NetfilterExport', 'sh')
'cache' => array('txt', 'CacheExport', 'cache'),
'csv' => array('csv', 'CsvExport', 'csv'),
'hashes' => array('txt', 'HashesExport', 'txt'),
'json' => array('json', 'JsonExport', 'json'),
'netfilter' => array('txt', 'NetfilterExport', 'sh'),
'opendata' => array('txt', 'OpendataExport', 'txt'),
'openioc' => array('xml', 'OpeniocExport', 'ioc'),
'rpz' => array('txt', 'RPZExport', 'rpz'),
'snort' => array('txt', 'NidsSnortExport', 'rules'),
'suricata' => array('txt', 'NidsSuricataExport', 'rules'),
'text' => array('txt', 'TextExport', 'txt'),
'xml' => array('xml', 'XmlExport', 'xml'),
'yara' => array('txt', 'YaraExport', 'yara'),
'yara-json' => array('json', 'YaraExport', 'json')
);
// FIXME we need a better way to list the defaultCategories knowing that new attribute types will continue to appear in the future. We should generate this dynamically or use a function using the default_category of the $typeDefinitions
@ -658,20 +660,10 @@ class Attribute extends AppModel
private function __alterAttributeCount($event_id, $increment = true)
{
$event = $this->Event->find('first', array(
'recursive' => -1,
'conditions' => array('Event.id' => $event_id)
));
if (!empty($event)) {
if ($increment) {
$event['Event']['attribute_count'] = $event['Event']['attribute_count'] + 1;
} else {
$event['Event']['attribute_count'] = $event['Event']['attribute_count'] - 1;
}
if ($event['Event']['attribute_count'] >= 0) {
$this->Event->save($event, array('callbacks' => false));
}
}
return $this->Event->updateAll(
array('Event.attribute_count' => $increment ? 'Event.attribute_count+1' : 'GREATEST(Event.attribute_count-1, 0)'),
array('Event.id' => $event_id)
);
}
public function afterSave($created, $options = array())
@ -691,14 +683,14 @@ class Attribute extends AppModel
if (isset($this->data['Attribute']['deleted']) && $this->data['Attribute']['deleted']) {
$this->__beforeSaveCorrelation($this->data['Attribute']);
if (isset($this->data['Attribute']['event_id'])) {
$this->__alterAttributeCount($this->data['Attribute']['event_id'], false, $passedEvent);
$this->__alterAttributeCount($this->data['Attribute']['event_id'], false);
}
} else {
/*
* Only recorrelate if:
* - We are dealing with a new attribute OR
* - The existing attribute's previous state is known AND
* value, type or disable correlation have changed
* value, type, disable correlation or distribution have changed
* This will avoid recorrelations when it's not really needed, such as adding a tag
*/
if (!$created) {
@ -706,7 +698,9 @@ class Attribute extends AppModel
empty($this->old) ||
$this->data['Attribute']['value'] != $this->old['Attribute']['value'] ||
$this->data['Attribute']['disable_correlation'] != $this->old['Attribute']['disable_correlation'] ||
$this->data['Attribute']['type'] != $this->old['Attribute']['type']
$this->data['Attribute']['type'] != $this->old['Attribute']['type'] ||
$this->data['Attribute']['distribution'] != $this->old['Attribute']['distribution'] ||
$this->data['Attribute']['sharing_group_id'] != $this->old['Attribute']['sharing_group_id']
) {
$this->__beforeSaveCorrelation($this->data['Attribute']);
$this->__afterSaveCorrelation($this->data['Attribute'], false, $passedEvent);
@ -757,7 +751,7 @@ class Attribute extends AppModel
}
}
}
if (Configure::read('MISP.enable_advanced_correlations') && in_array($this->data['Attribute']['type'], array('ip-src', 'ip-dst', 'domain-ip')) && strpos($this->data['Attribute']['value'], '/')) {
if (Configure::read('MISP.enable_advanced_correlations') && in_array($this->data['Attribute']['type'], array('ip-src', 'ip-dst')) && strpos($this->data['Attribute']['value'], '/')) {
$this->setCIDRList();
}
if ($created && isset($this->data['Attribute']['event_id']) && empty($this->data['Attribute']['skip_auto_increment'])) {
@ -810,7 +804,7 @@ class Attribute extends AppModel
public function afterDelete()
{
if (Configure::read('MISP.enable_advanced_correlations') && in_array($this->data['Attribute']['type'], array('ip-src', 'ip-dst', 'domain-ip')) && strpos($this->data['Attribute']['value'], '/')) {
if (Configure::read('MISP.enable_advanced_correlations') && in_array($this->data['Attribute']['type'], array('ip-src', 'ip-dst')) && strpos($this->data['Attribute']['value'], '/')) {
$this->setCIDRList();
}
if (isset($this->data['Attribute']['event_id'])) {
@ -1497,8 +1491,8 @@ class Attribute extends AppModel
if (filter_var($parts[1], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) {
// convert IPv6 address to compressed format
$parts[1] = inet_ntop(inet_pton($value));
$value = implode('|', $parts);
}
$value = implode('|', $parts);
break;
case 'filename|md5':
case 'filename|sha1':
@ -1932,67 +1926,46 @@ class Attribute extends AppModel
return ($ip & $mask) == $subnet;
}
// using Snifff's solution from http://stackoverflow.com/questions/7951061/matching-ipv6-address-to-a-cidr-subnet
// Using solution from https://github.com/symfony/symfony/blob/master/src/Symfony/Component/HttpFoundation/IpUtils.php
private function __ipv6InCidr($ip, $cidr)
{
$ip = $this->__expandIPv6Notation($ip);
$binaryip = $this->__inet_to_bits($ip);
list($net, $maskbits) = explode('/', $cidr);
$net = $this->__expandIPv6Notation($net);
if (substr($net, -1) == ':') {
$net .= '0';
}
$binarynet = $this->__inet_to_bits($net);
$ip_net_bits = substr($binaryip, 0, $maskbits);
$net_bits = substr($binarynet, 0, $maskbits);
return ($ip_net_bits === $net_bits);
}
list($address, $netmask) = explode('/', $cidr);
private function __expandIPv6Notation($ip)
{
if (strpos($ip, '::') !== false) {
$ip = str_replace('::', str_repeat(':0', 8 - substr_count($ip, ':')).':', $ip);
}
if (strpos($ip, ':') === 0) {
$ip = '0'.$ip;
}
return $ip;
}
$bytesAddr = unpack('n*', inet_pton($address));
$bytesTest = unpack('n*', inet_pton($ip));
private function __inet_to_bits($inet)
{
$unpacked = unpack('A16', $inet);
$unpacked = str_split($unpacked[1]);
$binaryip = '';
foreach ($unpacked as $char) {
$binaryip .= str_pad(decbin(ord($char)), 8, '0', STR_PAD_LEFT);
for ($i = 1, $ceil = ceil($netmask / 16); $i <= $ceil; ++$i) {
$left = $netmask - 16 * ($i - 1);
$left = ($left <= 16) ? $left : 16;
$mask = ~(0xffff >> $left) & 0xffff;
if (($bytesAddr[$i] & $mask) != ($bytesTest[$i] & $mask)) {
return false;
}
}
return $binaryip;
return true;
}
private function __cidrCorrelation($a)
{
$ipValues = array();
$ip = $a['type'] == 'domain-ip' ? $a['value2'] : $a['value1'];
if (strpos($ip, '/') !== false) {
$ip = $a['value1'];
if (strpos($ip, '/') !== false) { // IP is CIDR
$ip_array = explode('/', $ip);
$ip_version = filter_var($ip_array[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) ? 4 : 6;
$ipList = $this->find('list', array(
'conditions' => array(
'type' => array('ip-src', 'ip-dst', 'domain_ip'),
'type' => array('ip-src', 'ip-dst'),
'value1 NOT LIKE' => '%/%', // do not return CIDR, just plain IPs
),
'fields' => array('value1', 'value2'),
'group' => 'value1', // return just unique values
'fields' => array('value1'),
'order' => false
));
$ipList = array_merge(array_keys($ipList), array_values($ipList));
foreach ($ipList as $key => $value) {
if ($value == '') {
unset($ipList[$key]);
}
}
foreach ($ipList as $ipToCheck) {
if (filter_var($ipToCheck, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) && $ip_version == 4) {
if ($ip_version == 4) {
$ipToCheckVersion = filter_var($ipToCheck, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) ? 4 : 6;
if ($ipToCheckVersion === $ip_version) {
if ($ip_version === 4) {
if ($this->__ipv4InCidr($ipToCheck, $ip)) {
$ipValues[] = $ipToCheck;
}
@ -2004,19 +1977,18 @@ class Attribute extends AppModel
}
}
} else {
$ip = $a['value1'];
$ip_version = filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) ? 4 : 6;
$cidrList = $this->getSetCIDRList();
foreach ($cidrList as $cidr) {
$cidr_ip = explode('/', $cidr)[0];
if (filter_var($cidr_ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) {
if ($ip_version == 4) {
if ($ip_version === 4) {
if ($this->__ipv4InCidr($ip, $cidr)) {
$ipValues[] = $cidr;
}
}
} else {
if ($ip_version == 6) {
if ($ip_version === 6) {
if ($this->__ipv6InCidr($ip, $cidr)) {
$ipValues[] = $cidr;
}
@ -2052,7 +2024,7 @@ class Attribute extends AppModel
if (!empty($event['Event']['disable_correlation']) && $event['Event']['disable_correlation']) {
return true;
}
if (Configure::read('MISP.enable_advanced_correlations') && in_array($a['type'], array('ip-src', 'ip-dst', 'domain-ip'))) {
if (Configure::read('MISP.enable_advanced_correlations') && in_array($a['type'], array('ip-src', 'ip-dst'))) {
$extraConditions = $this->__cidrCorrelation($a);
}
if ($a['type'] == 'ssdeep') {
@ -3567,63 +3539,40 @@ class Attribute extends AppModel
if (!is_numeric($event_id)) {
throw new Exception(__('Something went wrong. Received a non-numeric event ID while trying to create a zip archive of an uploaded malware sample.'));
}
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$attachments_dir = $this->getDefaultAttachments_dir();
$content = base64_decode($base64);
$malwareTool = new MalwareTool();
$hashes = $malwareTool->computeHashes($content, $hash_types);
try {
$encrypted = $malwareTool->encrypt($original_filename, $content, $hashes['md5']);
} catch (Exception $e) {
$this->logException("Could not create encrypted malware sample.", $e);
return array('success' => false);
}
// If we've set attachments to S3, we can't write there
if ($this->attachmentDirIsS3()) {
$attachments_dir = Configure::read('MISP.tmpdir');
// Sometimes it's not set?
if (empty($attachments_dir)) {
// Get a default tmpdir
$attachments_dir = $this->getDefaultTmp_dir();
}
}
if ($proposal) {
$dir = new Folder($attachments_dir . DS . $event_id . DS . 'shadow', true);
} else {
$dir = new Folder($attachments_dir . DS . $event_id, true);
}
$tmpFile = new File($dir->path . DS . $this->generateRandomFileName(), true, 0600);
$tmpFile->write(base64_decode($base64));
$hashes = array();
foreach ($hash_types as $hash) {
$hashes[$hash] = $this->__hashRouter($hash, $tmpFile->path);
}
$contentsFile = new File($dir->path . DS . $hashes['md5']);
rename($tmpFile->path, $contentsFile->path);
$fileNameFile = new File($dir->path . DS . $hashes['md5'] . '.filename.txt');
$fileNameFile->write($original_filename);
$fileNameFile->close();
$zipFile = new File($dir->path . DS . $hashes['md5'] . '.zip');
exec('zip -j -P infected ' . escapeshellarg($zipFile->path) . ' ' . escapeshellarg($contentsFile->path) . ' ' . escapeshellarg($fileNameFile->path), $execOutput, $execRetval);
if ($execRetval != 0) {
$result = array('success' => false);
} else {
$result = array_merge(array('data' => base64_encode($zipFile->read()), 'success' => true), $hashes);
}
$fileNameFile->delete();
$zipFile->delete();
$contentsFile->delete();
$result = array_merge(array('data' => base64_encode($encrypted), 'success' => true), $hashes);
return $result;
}
private function __hashRouter($hashType, $file)
/**
* @return bool Return true if at least one advanced extraction tool is available
*/
public function isAdvancedExtractionAvailable()
{
$validHashes = array('md5', 'sha1', 'sha256');
if (!in_array($hashType, $validHashes)) {
$malwareTool = new MalwareTool();
try {
$types = $malwareTool->checkAdvancedExtractionStatus($this->getPythonVersion());
} catch (Exception $e) {
return false;
}
switch ($hashType) {
case 'md5':
case 'sha1':
case 'sha256':
return hash_file($hashType, $file);
break;
foreach ($types as $type => $missing) {
if ($missing === false) {
return true;
}
}
return false;
}
@ -3865,6 +3814,7 @@ class Attribute extends AppModel
'value1 LIKE' => '%/%'
),
'fields' => array('value1'),
'group' => 'value1', // return just unique value
'order' => false
));
}
@ -3876,12 +3826,15 @@ class Attribute extends AppModel
if ($redis) {
$redis->del('misp:cidr_cache_list');
$cidrList = $this->__getCIDRList();
$pipeline = $redis->multi(Redis::PIPELINE);
foreach ($cidrList as $cidr) {
$pipeline->sadd('misp:cidr_cache_list', $cidr);
if (method_exists($redis, 'saddArray')) {
$redis->sAddArray('misp:cidr_cache_list', $cidrList);
} else {
$pipeline = $redis->multi(Redis::PIPELINE);
foreach ($cidrList as $cidr) {
$pipeline->sadd('misp:cidr_cache_list', $cidr);
}
$pipeline->exec();
}
$pipeline->exec();
$redis->smembers('misp:cidr_cache_list');
}
return $cidrList;
}
@ -3890,8 +3843,8 @@ class Attribute extends AppModel
{
$redis = $this->setupRedis();
if ($redis) {
if (!$redis->exists('misp:cidr_cache_list') || $redis->sCard('misp:cidr_cache_list') == 0) {
$cidrList = $this->setCIDRList($redis);
if ($redis->sCard('misp:cidr_cache_list') === 0) {
$cidrList = $this->setCIDRList();
} else {
$cidrList = $redis->smembers('misp:cidr_cache_list');
}
@ -3931,7 +3884,7 @@ class Attribute extends AppModel
'size-in-bytes' => array('type' => 'size-in-bytes', 'category' => 'Other', 'to_ids' => 0, 'disable_correlation' => 1, 'object_relation' => 'size-in-bytes')
);
$hashes = array('md5', 'sha1', 'sha256');
$this->Object = ClassRegistry::init('Object');
$this->Object = ClassRegistry::init('MispObject');
$this->ObjectTemplate = ClassRegistry::init('ObjectTemplate');
$current = $this->ObjectTemplate->find('first', array(
'fields' => array('MAX(version) AS version', 'uuid'),
@ -3970,7 +3923,7 @@ class Attribute extends AppModel
'event_id' => $event_id,
'comment' => !empty($attribute_settings['comment']) ? $attribute_settings['comment'] : ''
);
$result = $this->Event->Attribute->handleMaliciousBase64($event_id, $filename, base64_encode($tmpfile->read()), $hashes);
$result = $this->handleMaliciousBase64($event_id, $filename, base64_encode($tmpfile->read()), $hashes);
foreach ($attributes as $k => $v) {
$attribute = array(
'distribution' => 5,
@ -4001,33 +3954,34 @@ class Attribute extends AppModel
public function advancedAddMalwareSample($event_id, $attribute_settings, $filename, $tmpfile)
{
$execRetval = '';
$execOutput = array();
$result = shell_exec($this->getPythonVersion() . ' ' . APP . 'files/scripts/generate_file_objects.py -p ' . $tmpfile->path);
if (!empty($result)) {
$result = json_decode($result, true);
if (isset($result['objects'])) {
$result['Object'] = $result['objects'];
unset($result['objects']);
}
if (isset($result['references'])) {
$result['ObjectReference'] = $result['references'];
unset($result['references']);
}
foreach ($result['Object'] as $k => $object) {
$result['Object'][$k]['distribution'] = $attribute_settings['distribution'];
$result['Object'][$k]['sharing_group_id'] = isset($attribute_settings['distribution']) ? $attribute_settings['distribution'] : 0;
if (!empty($result['Object'][$k]['Attribute'])) {
foreach ($result['Object'][$k]['Attribute'] as $k2 => $attribute) {
if ($attribute['value'] == $tmpfile->name) {
$result['Object'][$k]['Attribute'][$k2]['value'] = $filename;
}
$malwareTool = new MalwareTool();
try {
$result = $malwareTool->advancedExtraction($this->getPythonVersion(), $tmpfile->path);
} catch (Exception $e) {
$this->logException("Could not finish advanced extraction", $e);
return $this->simpleAddMalwareSample($event_id, $attribute_settings, $filename, $tmpfile);
}
if (isset($result['objects'])) {
$result['Object'] = $result['objects'];
unset($result['objects']);
}
if (isset($result['references'])) {
$result['ObjectReference'] = $result['references'];
unset($result['references']);
}
foreach ($result['Object'] as $k => $object) {
$result['Object'][$k]['distribution'] = $attribute_settings['distribution'];
$result['Object'][$k]['sharing_group_id'] = isset($attribute_settings['distribution']) ? $attribute_settings['distribution'] : 0;
if (!empty($result['Object'][$k]['Attribute'])) {
foreach ($result['Object'][$k]['Attribute'] as $k2 => $attribute) {
if ($attribute['value'] == $tmpfile->name) {
$result['Object'][$k]['Attribute'][$k2]['value'] = $filename;
}
}
}
} else {
$result = $this->simpleAddMalwareSample($event_id, $attribute_settings, $filename, $tmpfile);
}
return $result;
}
@ -4142,7 +4096,7 @@ class Attribute extends AppModel
return $attribute;
}
public function editAttribute($attribute, $eventId, $user, $objectId, $log = false)
public function editAttribute($attribute, $eventId, $user, $objectId, $log = false, $force = false)
{
$attribute['event_id'] = $eventId;
$attribute['object_id'] = $objectId;
@ -4186,7 +4140,7 @@ class Attribute extends AppModel
// If yes, it means that it's newer, so insert it. If no, it means that it's the same attribute or older - don't insert it, insert the old attribute.
// Alternatively, we could unset this attribute from the request, but that could lead with issues if we decide that we want to start deleting attributes that don't exist in a pushed event.
if (isset($attribute['timestamp'])) {
if ($attribute['timestamp'] <= $existingAttribute['Attribute']['timestamp']) {
if (!$force && $attribute['timestamp'] <= $existingAttribute['Attribute']['timestamp']) {
return true;
}
} else {
@ -4435,7 +4389,8 @@ class Attribute extends AppModel
'event_timestamp' => array('function' => 'set_filter_timestamp', 'pop' => true),
'publish_timestamp' => array('function' => 'set_filter_timestamp'),
'org' => array('function' => 'set_filter_org'),
'published' => array('function' => 'set_filter_published')
'published' => array('function' => 'set_filter_published'),
'threat_level_id' => array('function' => 'set_filter_threat_level_id')
),
'Object' => array(
'object_name' => array('function' => 'set_filter_object_name'),
@ -4466,6 +4421,9 @@ class Attribute extends AppModel
}
App::uses($this->validFormats[$returnFormat][1], 'Export');
$exportTool = new $this->validFormats[$returnFormat][1]();
if (!empty($exportTool->use_default_filters)) {
$exportTool->setDefaultFilters($filters);
}
if (empty($exportTool->non_restrictive_export)) {
if (!isset($filters['to_ids'])) {
$filters['to_ids'] = 1;
@ -4576,7 +4534,9 @@ class Attribute extends AppModel
$loop = true;
$params['page'] = 1;
}
$this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams, $elementCounter);
if (empty($exportTool->mock_query_only)) {
$this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams, $elementCounter);
}
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
if (fstat($tmpfile)['size']) {
@ -4648,4 +4608,28 @@ class Attribute extends AppModel
}
return $conditions;
}
/**
* @param array $attribute
*/
public function removeGalaxyClusterTags(array &$attribute)
{
$galaxyTagIds = array();
foreach ($attribute['Galaxy'] as $galaxy) {
foreach ($galaxy['GalaxyCluster'] as $galaxyCluster) {
$galaxyTagIds[$galaxyCluster['tag_id']] = true;
}
}
if (empty($galaxyTagIds)) {
return;
}
foreach ($attribute['AttributeTag'] as $k => $attributeTag) {
$tagId = $attributeTag['Tag']['id'];
if (isset($galaxyTagIds[$tagId])) {
unset($attribute['AttributeTag'][$k]);
}
}
}
}

221
app/Model/Event.php
View File

@ -173,24 +173,25 @@ class Event extends AppModel
);
public $validFormats = array(
'json' => array('json', 'JsonExport', 'json'),
'openioc' => array('xml', 'OpeniocExport', 'ioc'),
'xml' => array('xml', 'XmlExport', 'xml'),
'suricata' => array('txt', 'NidsSuricataExport', 'rules'),
'snort' => array('txt', 'NidsSnortExport', 'rules'),
'rpz' => array('txt', 'RPZExport', 'rpz'),
'text' => array('text', 'TextExport', 'txt'),
'hashes' => array('txt', 'HashesExport', 'txt'),
'attack' => array('html', 'AttackExport', 'html'),
'attack-sightings' => array('json', 'AttackSightingsExport', 'json'),
'cache' => array('txt', 'CacheExport', 'cache'),
'csv' => array('csv', 'CsvExport', 'csv'),
'hashes' => array('txt', 'HashesExport', 'txt'),
'json' => array('json', 'JsonExport', 'json'),
'netfilter' => array('txt', 'NetfilterExport', 'sh'),
'opendata' => array('txt', 'OpendataExport', 'txt'),
'openioc' => array('xml', 'OpeniocExport', 'ioc'),
'rpz' => array('txt', 'RPZExport', 'rpz'),
'snort' => array('txt', 'NidsSnortExport', 'rules'),
'stix' => array('xml', 'Stix1Export', 'xml'),
'stix-json' => array('json', 'Stix1Export', 'json'),
'stix2' => array('json', 'Stix2Export', 'json'),
'suricata' => array('txt', 'NidsSuricataExport', 'rules'),
'text' => array('text', 'TextExport', 'txt'),
'xml' => array('xml', 'XmlExport', 'xml'),
'yara' => array('txt', 'YaraExport', 'yara'),
'yara-json' => array('json', 'YaraExport', 'json'),
'cache' => array('txt', 'CacheExport', 'cache'),
'attack' => array('html', 'AttackExport', 'html'),
'attack-sightings' => array('json', 'AttackSightingsExport', 'json'),
'netfilter' => array('txt', 'NetfilterExport', 'sh')
'yara-json' => array('json', 'YaraExport', 'json')
);
public $csv_event_context_fields_to_fetch = array(
@ -641,6 +642,12 @@ class Event extends AppModel
if (isset($this->data['Event']['info'])) {
$this->Correlation->updateAll(array('Correlation.info' => $db->value($this->data['Event']['info'])), array('Correlation.event_id' => intval($this->data['Event']['id'])));
}
if (isset($this->data['Event']['distribution'])) {
$this->Correlation->updateAll(array('Correlation.distribution' => $db->value($this->data['Event']['distribution'])), array('Correlation.event_id' => intval($this->data['Event']['id'])));
}
if (isset($this->data['Event']['sharing_group_id'])) {
$this->Correlation->updateAll(array('Correlation.sharing_group_id' => $db->value($this->data['Event']['sharing_group_id'])), array('Correlation.event_id' => intval($this->data['Event']['id'])));
}
}
if (empty($this->data['Event']['unpublishAction']) && empty($this->data['Event']['skip_zmq']) && Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
@ -1652,7 +1659,8 @@ class Event extends AppModel
'publish_timestamp' => array('function' => 'set_filter_timestamp', 'pop' => true),
'org' => array('function' => 'set_filter_org', 'pop' => true),
'uuid' => array('function' => 'set_filter_uuid', 'pop' => true),
'published' => array('function' => 'set_filter_published', 'pop' => true)
'published' => array('function' => 'set_filter_published', 'pop' => true),
'threat_level_id' => array('function' => 'set_filter_threat_level_id', 'pop' => true)
),
'Object' => array(
'object_name' => array('function' => 'set_filter_object_name'),
@ -2150,6 +2158,22 @@ class Event extends AppModel
'Object' => array('name', 'meta-category')
);
foreach ($results as $eventKey => &$event) {
if ($event['Event']['distribution'] == 4 && !in_array($event['Event']['sharing_group_id'], $sgids)) {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Event',
'model_id' => $event['Event']['id'],
'email' => $user['email'],
'action' => 'fetchEvent',
'user_id' => $user['id'],
'title' => 'User was able to fetch the event but not the sharing_group it belongs to',
'change' => ''
));
unset($results[$eventKey]); // Current user cannot access sharing_group associated to this event
continue;
}
$this->__attachReferences($user, $event, $sgids, $fields);
$event = $this->Orgc->attachOrgsToEvent($event, $fieldsOrg);
if (!$options['sgReferenceOnly'] && $event['Event']['sharing_group_id']) {
@ -2208,6 +2232,8 @@ class Event extends AppModel
}
$event = $this->__filterBlockedAttributesByTags($event, $options, $user);
$event['Attribute'] = $this->__attachSharingGroups(!$options['sgReferenceOnly'], $event['Attribute'], $sharingGroupData);
$proposalBlockAttributes = Configure::read('MISP.proposals_block_attributes');
// move all object attributes to a temporary container
$tempObjectAttributeContainer = array();
foreach ($event['Attribute'] as $key => $attribute) {
@ -2262,10 +2288,7 @@ class Event extends AppModel
}
}
}
if (
Configure::read('MISP.proposals_block_attributes') &&
!empty($options['allow_proposal_blocking'])
) {
if ($proposalBlockAttributes && !empty($options['allow_proposal_blocking'])) {
foreach ($results[$eventKey]['Attribute'][$key]['ShadowAttribute'] as $sa) {
if ($sa['proposal_to_delete'] || $sa['to_ids'] == 0) {
unset($results[$eventKey]['Attribute'][$key]);
@ -2299,15 +2322,15 @@ class Event extends AppModel
}
$event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit);
}
}
if (!empty($options['includeServerCorrelations']) && $user['org_id'] == Configure::read('MISP.host_org_id')) {
$this->Feed = ClassRegistry::init('Feed');
if (!empty($options['overrideLimit'])) {
$overrideLimit = true;
} else {
$overrideLimit = false;
if (!empty($options['includeServerCorrelations']) && $user['org_id'] == Configure::read('MISP.host_org_id')) {
$this->Feed = ClassRegistry::init('Feed');
if (!empty($options['overrideLimit'])) {
$overrideLimit = true;
} else {
$overrideLimit = false;
}
$event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit, 'Server');
}
$event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit, 'Server');
}
if (empty($options['metadata'])) {
$this->Sighting = ClassRegistry::init('Sighting');
@ -2448,7 +2471,11 @@ class Event extends AppModel
}
foreach ($data as $k => $v) {
if ($v['distribution'] == 4) {
$data[$k]['SharingGroup'] = $sharingGroupData[$v['sharing_group_id']]['SharingGroup'];
if (isset($sharingGroupData[$v['sharing_group_id']])) {
$data[$k]['SharingGroup'] = $sharingGroupData[$v['sharing_group_id']]['SharingGroup'];
} else {
unset($data[$k]); // current user could not fetch the sharing_group
}
}
}
return $data;
@ -2689,6 +2716,14 @@ class Event extends AppModel
return $conditions;
}
public function set_filter_threat_level_id(&$params, $conditions, $options)
{
if (isset($params['threat_level_id'])) {
$conditions['AND']['Event.threat_level_id'] = $params['threat_level_id'];
}
return $conditions;
}
public function set_filter_tags(&$params, $conditions, $options)
{
if (!empty($params['tags'])) {
@ -3237,10 +3272,10 @@ class Event extends AppModel
return array($bodyevent, $body);
}
private function __captureSGForElement($element, $user)
private function __captureSGForElement($element, $user, $syncLocal=false)
{
if (isset($element['SharingGroup'])) {
$sg = $this->SharingGroup->captureSG($element['SharingGroup'], $user);
$sg = $this->SharingGroup->captureSG($element['SharingGroup'], $user, $syncLocal);
unset($element['SharingGroup']);
} elseif (isset($element['sharing_group_id'])) {
$sg = $this->SharingGroup->checkIfAuthorised($user, $element['sharing_group_id']) ? $element['sharing_group_id'] : false;
@ -3257,17 +3292,17 @@ class Event extends AppModel
// When we receive an event via REST, we might end up with organisations, sharing groups, tags that we do not know
// or which we need to update. All of that is controlled in this method.
private function __captureObjects($data, $user)
private function __captureObjects($data, $user, $syncLocal=false)
{
// First we need to check whether the event or any attributes are tied to a sharing group and whether the user is even allowed to create the sharing group / is part of it
if (isset($data['Event']['distribution']) && $data['Event']['distribution'] == 4) {
$data['Event'] = $this->__captureSGForElement($data['Event'], $user);
$data['Event'] = $this->__captureSGForElement($data['Event'], $user, $syncLocal);
}
if (!empty($data['Event']['Attribute'])) {
foreach ($data['Event']['Attribute'] as $k => $a) {
unset($data['Event']['Attribute']['id']);
if (isset($a['distribution']) && $a['distribution'] == 4) {
$data['Event']['Attribute'][$k] = $this->__captureSGForElement($a, $user);
$data['Event']['Attribute'][$k] = $this->__captureSGForElement($a, $user, $syncLocal);
if ($data['Event']['Attribute'][$k] === false) {
unset($data['Event']['Attribute']);
}
@ -3277,7 +3312,7 @@ class Event extends AppModel
if (!empty($data['Event']['Object'])) {
foreach ($data['Event']['Object'] as $k => $o) {
if (isset($o['distribution']) && $o['distribution'] == 4) {
$data['Event']['Object'][$k] = $this->__captureSGForElement($o, $user);
$data['Event']['Object'][$k] = $this->__captureSGForElement($o, $user, $syncLocal);
if ($data['Event']['Object'][$k] === false) {
unset($data['Event']['Object'][$k]);
continue;
@ -3285,7 +3320,7 @@ class Event extends AppModel
}
foreach ($o['Attribute'] as $k2 => $a) {
if (isset($a['distribution']) && $a['distribution'] == 4) {
$data['Event']['Object'][$k]['Attribute'][$k2] = $this->__captureSGForElement($a, $user);
$data['Event']['Object'][$k]['Attribute'][$k2] = $this->__captureSGForElement($a, $user, $syncLocal);
if ($data['Event']['Object'][$k]['Attribute'][$k2] === false) {
unset($data['Event']['Object'][$k]['Attribute'][$k2]);
}
@ -3453,6 +3488,24 @@ class Event extends AppModel
return 'blocked';
}
}
if ($passAlong) {
$this->Server = ClassRegistry::init('Server');
$server = $this->Server->find('first', array(
'conditions' => array(
'Server.id' => $passAlong
),
'recursive' => -1,
'fields' => array(
'Server.name',
'Server.id',
'Server.unpublish_event',
'Server.publish_without_email',
'Server.internal'
)
));
} else {
$server['Server']['internal'] = false;
}
if ($fromXml) {
// Workaround for different structure in XML/array than what CakePHP expects
$data = $this->cleanupEventArrayFromXML($data);
@ -3479,7 +3532,7 @@ class Event extends AppModel
return $existingEvent['Event']['id'];
} else {
if ($fromXml) {
$data = $this->__captureObjects($data, $user);
$data = $this->__captureObjects($data, $user, $server['Server']['internal']);
}
if ($data === false) {
$failedCapture = true;
@ -3487,7 +3540,7 @@ class Event extends AppModel
}
} else {
if ($fromXml) {
$data = $this->__captureObjects($data, $user);
$data = $this->__captureObjects($data, $user, $server['Server']['internal']);
}
if ($data === false) {
$failedCapture = true;
@ -3548,19 +3601,6 @@ class Event extends AppModel
$this->Log = ClassRegistry::init('Log');
if ($saveResult) {
if ($passAlong) {
$this->Server = ClassRegistry::init('Server');
$server = $this->Server->find('first', array(
'conditions' => array(
'Server.id' => $passAlong
),
'recursive' => -1,
'fields' => array(
'Server.name',
'Server.id',
'Server.unpublish_event',
'Server.publish_without_email'
)
));
if ($server['Server']['publish_without_email'] == 0) {
$st = "enabled";
} else {
@ -3691,7 +3731,7 @@ class Event extends AppModel
}
}
public function _edit(&$data, $user, $id, $jobId = null, $passAlong = null)
public function _edit(&$data, $user, $id, $jobId = null, $passAlong = null, $force = false)
{
$data = $this->cleanupEventArrayFromXML($data);
unset($this->Attribute->validate['event_id']);
@ -3703,6 +3743,23 @@ class Event extends AppModel
} else {
$existingEvent = $this->findById($id);
}
if ($passAlong) {
$this->Server = ClassRegistry::init('Server');
$server = $this->Server->find('first', array(
'conditions' => array(
'Server.id' => $passAlong
),
'recursive' => -1,
'fields' => array(
'Server.name',
'Server.id',
'Server.unpublish_event',
'Server.publish_without_email'
)
));
} else {
$server['Server']['internal'] = false;
}
// If the event exists...
$dateObj = new DateTime();
$date = $dateObj->getTimestamp();
@ -3712,7 +3769,7 @@ class Event extends AppModel
// Conditions affecting all:
// user.org == event.org
// edit timestamp newer than existing event timestamp
if (!isset($data['Event']['timestamp']) || $data['Event']['timestamp'] > $existingEvent['Event']['timestamp']) {
if ($force || !isset($data['Event']['timestamp']) || $data['Event']['timestamp'] > $existingEvent['Event']['timestamp']) {
if (!isset($data['Event']['timestamp'])) {
$data['Event']['timestamp'] = $date;
}
@ -3725,7 +3782,7 @@ class Event extends AppModel
return(array('error' => 'Event could not be saved: Invalid sharing group or you don\'t have access to that sharing group.'));
}
} else {
$data['Event']['sharing_group_id'] = $this->SharingGroup->captureSG($data['Event']['SharingGroup'], $user);
$data['Event']['sharing_group_id'] = $this->SharingGroup->captureSG($data['Event']['SharingGroup'], $user, $server['Server']['internal']);
unset($data['Event']['SharingGroup']);
if ($data['Event']['sharing_group_id'] === false) {
return (array('error' => 'Event could not be saved: User not authorised to create the associated sharing group.'));
@ -3784,7 +3841,7 @@ class Event extends AppModel
if (isset($data['Event']['Attribute'])) {
$data['Event']['Attribute'] = array_values($data['Event']['Attribute']);
foreach ($data['Event']['Attribute'] as $k => $attribute) {
$result = $this->Attribute->editAttribute($attribute, $this->id, $user, 0, $this->Log);
$result = $this->Attribute->editAttribute($attribute, $this->id, $user, 0, $this->Log, $force);
if ($result !== true) {
$validationErrors['Attribute'][] = $result;
}
@ -3793,7 +3850,7 @@ class Event extends AppModel
if (isset($data['Event']['Object'])) {
$data['Event']['Object'] = array_values($data['Event']['Object']);
foreach ($data['Event']['Object'] as $k => $object) {
$result = $this->Object->editObject($object, $this->id, $user, $this->Log);
$result = $this->Object->editObject($object, $this->id, $user, $this->Log, $force);
if ($result !== true) {
$validationErrors['Object'][] = $result;
}
@ -3801,7 +3858,7 @@ class Event extends AppModel
foreach ($data['Event']['Object'] as $object) {
if (isset($object['ObjectReference'])) {
foreach ($object['ObjectReference'] as $objectRef) {
$result = $this->Object->ObjectReference->captureReference($objectRef, $this->id, $user, $this->Log);
$result = $this->Object->ObjectReference->captureReference($objectRef, $this->id, $user, $this->Log, $force);
}
}
}
@ -3846,19 +3903,6 @@ class Event extends AppModel
if ((!empty($data['Event']['published']) && 1 == $data['Event']['published'])) {
// The edited event is from a remote server ?
if ($passAlong) {
$this->Server = ClassRegistry::init('Server');
$server = $this->Server->find('first', array(
'conditions' => array(
'Server.id' => $passAlong
),
'recursive' => -1,
'fields' => array(
'Server.name',
'Server.id',
'Server.unpublish_event',
'Server.publish_without_email'
)
));
if ($server['Server']['publish_without_email'] == 0) {
$st = "enabled";
} else {
@ -6636,6 +6680,10 @@ class Event extends AppModel
$this->Job->id = $jobId;
}
if (!empty($exportTool->use_default_filters)) {
$exportTool->setDefaultFilters($filters);
}
if (empty($exportTool->non_restrictive_export)) {
if (!isset($filters['to_ids'])) {
$filters['to_ids'] = 1;
@ -6675,11 +6723,15 @@ class Event extends AppModel
$subqueryElements = $this->harvestSubqueryElements($filters);
$filters = $this->addFiltersFromSubqueryElements($filters, $subqueryElements);
$filters['include_attribute_count'] = 1;
$eventid = $this->filterEventIds($user, $filters, $elementCounter);
$eventCount = count($eventid);
$eventids_chunked = $this->__clusterEventIds($exportTool, $eventid);
unset($eventid);
if (empty($exportTool->mock_query_only)) {
$filters['include_attribute_count'] = 1;
$eventid = $this->filterEventIds($user, $filters, $elementCounter);
$eventCount = count($eventid);
$eventids_chunked = $this->__clusterEventIds($exportTool, $eventid);
unset($eventid);
} else {
$eventids_chunked = array();
}
if (!empty($exportTool->additional_params)) {
$filters = array_merge($filters, $exportTool->additional_params);
}
@ -6948,4 +7000,27 @@ class Event extends AppModel
}
return $filters;
}
/**
* @param array $event
*/
public function removeGalaxyClusterTags(array &$event)
{
$galaxyTagIds = array();
foreach ($event['Galaxy'] as $galaxy) {
foreach ($galaxy['GalaxyCluster'] as $galaxyCluster) {
$galaxyTagIds[$galaxyCluster['tag_id']] = true;
}
}
if (empty($galaxyTagIds)) {
return;
}
foreach ($event['EventTag'] as $k => $eventTag) {
if (isset($galaxyTagIds[$eventTag['tag_id']])) {
unset($event['EventTag'][$k]);
}
}
}
}

362
app/Model/Feed.php
View File

@ -247,10 +247,7 @@ class Feed extends AppModel
$data = $this->feedGetUri($feed, $feedUrl, $HttpSocket, true);
if (!$isLocal) {
$redis = $this->setupRedis();
if ($redis === false) {
throw new Exception('Could not reach Redis.');
}
$redis = $this->setupRedisWithException();
$redis->del('misp:feed_cache:' . $feed['Feed']['id']);
file_put_contents($feedCache, $data);
}
@ -331,109 +328,128 @@ class Feed extends AppModel
return $data;
}
/**
* Attach correlations from cached servers or feeds.
*
* @param array $objects
* @param array $user
* @param array $event
* @param bool $overrideLimit Override hardcoded limit for 10 000 attribute correlations.
* @param string $scope `Feed` or `Server`
* @return array
*/
public function attachFeedCorrelations($objects, $user, &$event, $overrideLimit = false, $scope = 'Feed')
{
$redis = $this->setupRedis();
if ($redis !== false) {
$pipe = $redis->multi(Redis::PIPELINE);
$hashTable = array();
$cachePrefix = 'misp:' . strtolower($scope) . '_cache:';
try {
$redis = $this->setupRedisWithException();
} catch (Exception $e) {
return $objects;
}
$this->Event = ClassRegistry::init('Event');
$compositeTypes = $this->Event->Attribute->getCompositeTypes();
$cachePrefix = 'misp:' . strtolower($scope) . '_cache:';
foreach ($objects as $k => $object) {
if (in_array($object['type'], $compositeTypes)) {
$value = explode('|', $object['value']);
$hashTable[$k] = md5($value[0]);
} else {
$hashTable[$k] = md5($object['value']);
}
$redis->sismember($cachePrefix . 'combined', $hashTable[$k]);
}
$results = $pipe->exec();
if (!$overrideLimit && count($objects) > 10000) {
foreach ($results as $k => $result) {
if ($result && empty($objects[$k]['disable_correlation'])) {
if (isset($event['FeedCount'])) {
$event['FeedCount']++;
} else {
$event['FeedCount'] = 1;
}
$objects[$k]['FeedHit'] = true;
}
}
// Redis cache for $scope is empty.
if ($redis->sCard($cachePrefix . 'combined') === 0) {
return $objects;
}
$pipe = $redis->multi(Redis::PIPELINE);
$hashTable = array();
$this->Event = ClassRegistry::init('Event');
$compositeTypes = $this->Event->Attribute->getCompositeTypes();
foreach ($objects as $k => $object) {
if (in_array($object['type'], $compositeTypes)) {
$value = explode('|', $object['value']);
$hashTable[$k] = md5($value[0]);
} else {
if ($scope === 'Feed') {
$params = array(
'recursive' => -1,
'fields' => array('id', 'name', 'url', 'provider', 'source_format')
);
if (!$user['Role']['perm_site_admin']) {
$params['conditions'] = array('Feed.lookup_visible' => 1);
$hashTable[$k] = md5($object['value']);
}
$redis->sismember($cachePrefix . 'combined', $hashTable[$k]);
}
$results = $pipe->exec();
if (!$overrideLimit && count($objects) > 10000) {
foreach ($results as $k => $result) {
if ($result && empty($objects[$k]['disable_correlation'])) {
if (isset($event['FeedCount'])) {
$event['FeedCount']++;
} else {
$event['FeedCount'] = 1;
}
$sources = $this->find('all', $params);
} else {
$params = array(
'recursive' => -1,
'fields' => array('id', 'name', 'url', 'caching_enabled')
);
if (!$user['Role']['perm_site_admin']) {
$params['conditions'] = array('Server.caching_enabled' => 1);
}
$this->Server = ClassRegistry::init('Server');
$sources = $this->Server->find('all', $params);
$objects[$k]['FeedHit'] = true;
}
}
} else {
if ($scope === 'Feed') {
$params = array(
'recursive' => -1,
'fields' => array('id', 'name', 'url', 'provider', 'source_format')
);
if (!$user['Role']['perm_site_admin']) {
$params['conditions'] = array('Feed.lookup_visible' => 1);
}
$sources = $this->find('all', $params);
} else {
$params = array(
'recursive' => -1,
'fields' => array('id', 'name', 'url', 'caching_enabled')
);
if (!$user['Role']['perm_site_admin']) {
$params['conditions'] = array('Server.caching_enabled' => 1);
}
$this->Server = ClassRegistry::init('Server');
$sources = $this->Server->find('all', $params);
}
$hitIds = array();
foreach ($results as $k => $result) {
if ($result && empty($objects[$k]['disable_correlation'])) {
$hitIds[] = $k;
$hitIds = array();
foreach ($results as $k => $result) {
if ($result && empty($objects[$k]['disable_correlation'])) {
$hitIds[] = $k;
}
}
foreach ($sources as $source) {
$sourceScopeId = $source[$scope]['id'];
$pipe = $redis->multi(Redis::PIPELINE);
foreach ($hitIds as $k) {
$redis->sismember($cachePrefix . $sourceScopeId, $hashTable[$k]);
}
$sourceHits = $pipe->exec();
foreach ($sourceHits as $k4 => $hit) {
if ($hit) {
if (!isset($event[$scope][$sourceScopeId]['id'])) {
if (!isset($event[$scope][$sourceScopeId])) {
$event[$scope][$sourceScopeId] = array();
}
$event[$scope][$sourceScopeId] = array_merge($event[$scope][$sourceScopeId], $source[$scope]);
}
$objects[$hitIds[$k4]][$scope][] = $source[$scope];
}
}
foreach ($sources as $source) {
$sourceScopeId = $source[$scope]['id'];
if ($scope === 'Server' || $source[$scope]['source_format'] == 'misp') {
$pipe = $redis->multi(Redis::PIPELINE);
foreach ($hitIds as $k) {
$redis->sismember($cachePrefix . $sourceScopeId, $hashTable[$k]);
}
$sourceHits = $pipe->exec();
foreach ($sourceHits as $k4 => $hit) {
if ($hit) {
if (!isset($event[$scope][$sourceScopeId]['id'])) {
if (!isset($event[$scope][$sourceScopeId])) {
$event[$scope][$sourceScopeId] = array();
}
$event[$scope][$sourceScopeId] = array_merge($event[$scope][$sourceScopeId], $source[$scope]);
}
$objects[$hitIds[$k4]][$scope][] = $source[$scope];
}
}
if ($scope === 'Server' || $source[$scope]['source_format'] == 'misp') {
$pipe = $redis->multi(Redis::PIPELINE);
$eventUuidHitPosition = array();
foreach ($objects as $k => $object) {
if (isset($object[$scope])) {
foreach ($object[$scope] as $currentFeed) {
if ($source[$scope]['id'] == $currentFeed['id']) {
$eventUuidHitPosition[] = $k;
$redis->smembers($cachePrefix . 'event_uuid_lookup:' . $hashTable[$k]);
}
$eventUuidHitPosition = array();
foreach ($objects as $k => $object) {
if (isset($object[$scope])) {
foreach ($object[$scope] as $currentFeed) {
if ($source[$scope]['id'] == $currentFeed['id']) {
$eventUuidHitPosition[] = $k;
$redis->smembers($cachePrefix . 'event_uuid_lookup:' . $hashTable[$k]);
}
}
}
$mispFeedHits = $pipe->exec();
foreach ($mispFeedHits as $sourcehitPos => $f) {
foreach ($f as $url) {
list($feedId, $eventUuid) = explode('/', $url);
if (empty($event[$scope][$feedId]['event_uuids']) || !in_array($eventUuid, $event[$scope][$feedId]['event_uuids'])) {
$event[$scope][$feedId]['event_uuids'][] = $eventUuid;
}
foreach ($objects[$eventUuidHitPosition[$sourcehitPos]][$scope] as $tempKey => $tempFeed) {
if ($tempFeed['id'] == $feedId) {
$objects[$eventUuidHitPosition[$sourcehitPos]][$scope][$tempKey]['event_uuids'][] = $eventUuid;
}
}
$mispFeedHits = $pipe->exec();
foreach ($mispFeedHits as $sourcehitPos => $f) {
foreach ($f as $url) {
list($feedId, $eventUuid) = explode('/', $url);
if (empty($event[$scope][$feedId]['event_uuids']) || !in_array($eventUuid, $event[$scope][$feedId]['event_uuids'])) {
$event[$scope][$feedId]['event_uuids'][] = $eventUuid;
}
foreach ($objects[$eventUuidHitPosition[$sourcehitPos]][$scope] as $tempKey => $tempFeed) {
if ($tempFeed['id'] == $feedId) {
$objects[$eventUuidHitPosition[$sourcehitPos]][$scope][$tempKey]['event_uuids'][] = $eventUuid;
}
}
}
@ -441,9 +457,11 @@ class Feed extends AppModel
}
}
}
if (!empty($event[$scope])) {
$event[$scope] = array_values($event[$scope]);
}
return $objects;
}
@ -502,12 +520,17 @@ class Feed extends AppModel
$result = array(
'header' => array(
'Accept' => array('application/json', 'text/plain'),
'Content-Type' => 'application/json',
'MISP-version' => $version,
'MISP-uuid' => Configure::read('MISP.uuid')
'Accept' => array('application/json', 'text/plain'),
'MISP-version' => $version,
'MISP-uuid' => Configure::read('MISP.uuid'),
)
);
// Enable gzipped responses if PHP has 'gzdecode' method
if (function_exists('gzdecode')) {
$result['header']['Accept-Encoding'] = 'gzip';
}
if ($commit) {
$result['header']['commit'] = $commit;
}
@ -844,6 +867,10 @@ class Feed extends AppModel
return false;
}
if (empty($temp)) {
return true;
}
$data = array();
foreach ($temp as $value) {
$data[] = array(
@ -853,14 +880,11 @@ class Feed extends AppModel
'to_ids' => $value['to_ids']
);
}
if (empty($data)) {
return true;
}
$this->jobProgress($jobId, 'Saving data.', 50);
try {
$result = $this->saveFreetextFeedData($this->data, $data, $user);
$result = $this->saveFreetextFeedData($this->data, $data, $user, $jobId);
} catch (Exception $e) {
$this->logException("Could not save freetext feed data for feed $feedId.", $e);
return false;
@ -934,7 +958,7 @@ class Feed extends AppModel
}
}
if ($feed['Feed']['fixed_event']) {
$temp = $this->Event->Attribute->find('all', array(
$existsAttributes = $this->Event->Attribute->find('all', array(
'conditions' => array(
'Attribute.deleted' => 0,
'Attribute.event_id' => $event['Event']['id']
@ -942,28 +966,38 @@ class Feed extends AppModel
'recursive' => -1,
'fields' => array('id', 'value1', 'value2')
));
$event['Attribute'] = array();
foreach ($temp as $t) {
$existsAttributesValueToId = array();
foreach ($existsAttributes as $t) {
if (!empty($t['Attribute']['value2'])) {
$value = $t['Attribute']['value1'] . '|' . $t['Attribute']['value2'];
} else {
$value = $t['Attribute']['value1'];
}
$event['Attribute'][$t['Attribute']['id']] = $value;
// Since event values are unique, it is OK to put value into key
$existsAttributesValueToId[$value] = $t['Attribute']['id'];
}
unset($temp);
unset($existsAttributes);
// Create event diff. After this cycle, `$data` will contains just attributes that do not exists in current
// event and in `$existsAttributesValueToId` will contains just attributes that do not exists in current feed.
foreach ($data as $k => $dataPoint) {
$finder = array_search($dataPoint['value'], $event['Attribute']);
if ($finder !== false) {
if (isset($existsAttributesValueToId[$dataPoint['value']])) {
unset($data[$k]);
unset($event['Attribute'][$finder]);
unset($existsAttributesValueToId[$dataPoint['value']]);
continue;
}
// Because some types can be saved in modified version (for example, IPv6 address is convert to compressed
// format, we should also check if current event contains modified value.
$modifiedValue = $this->Event->Attribute->modifyBeforeValidation($dataPoint['type'], $dataPoint['value']);
if (isset($existsAttributesValueToId[$modifiedValue])) {
unset($data[$k]);
unset($existsAttributesValueToId[$modifiedValue]);
}
}
if ($feed['Feed']['delta_merge']) {
$to_delete = array_keys($event['Attribute']);
if (!empty($to_delete)) {
$this->Event->Attribute->deleteAll(array('Attribute.id' => $to_delete, 'Attribute.deleted' => 0));
}
if ($feed['Feed']['delta_merge'] && !empty($existsAttributesValueToId)) {
$to_delete = array_values($existsAttributesValueToId);
$this->Event->Attribute->deleteAll(array('Attribute.id' => $to_delete, 'Attribute.deleted' => 0));
}
}
if (empty($data)) {
@ -987,8 +1021,8 @@ class Feed extends AppModel
foreach ($data as $k => $chunk) {
$this->Event->Attribute->create();
$this->Event->Attribute->save($chunk);
if ($k % 100 == 0) {
$this->jobProgress($jobId, null, 50 + round((50 * ((($k + 1) * 100) / count($data)))));
if ($k % 100 === 0) {
$this->jobProgress($jobId, null, 50 + round(($k + 1) / count($data) * 50));
}
}
if (!empty($data)) {
@ -1029,14 +1063,14 @@ class Feed extends AppModel
} elseif ($scope == 'freetext' || $scope == 'csv') {
$params['conditions']['source_format'] = array('csv', 'freetext');
} elseif ($scope == 'misp') {
$redis->del('misp:feed_cache:event_uuid_lookup:');
$redis->del($redis->keys('misp:feed_cache:event_uuid_lookup:*'));
$params['conditions']['source_format'] = 'misp';
} else {
throw new InvalidArgumentException("Invalid value for scope, it must be integer or 'freetext', 'csv', 'misp' or 'all' string.");
}
} else {
$redis->del('misp:feed_cache:combined');
$redis->del('misp:feed_cache:event_uuid_lookup:');
$redis->del($redis->keys('misp:feed_cache:event_uuid_lookup:*'));
}
$feeds = $this->find('all', $params);
$atLeastOneSuccess = false;
@ -1590,24 +1624,51 @@ class Feed extends AppModel
if ($data === false) {
throw new Exception("Could not read local file '$uri'.");
}
return $data;
} else {
throw new Exception("Local file '$uri' doesn't exists.");
}
}
$request = $this->__createFeedRequest($feed['Feed']['headers']);
if ($followRedirect) {
$response = $this->getFollowRedirect($HttpSocket, $uri, $request);
} else {
$request = $this->__createFeedRequest($feed['Feed']['headers']);
$response = $HttpSocket->get($uri, array(), $request);
}
if ($followRedirect) {
$response = $this->getFollowRedirect($HttpSocket, $uri, $request);
} else {
$response = $HttpSocket->get($uri, array(), $request);
}
if ($response === false) {
throw new Exception("Could not reach '$uri'.");
} else if ($response->code != 200) { // intentionally !=
throw new Exception("Fetching the '$uri' failed with HTTP error {$response->code}: {$response->reasonPhrase}");
}
if ($response === false) {
throw new Exception("Could not reach '$uri'.");
} else if ($response->code != 200) { // intentionally !=
throw new Exception("Fetching the '$uri' failed with HTTP error {$response->code}: {$response->reasonPhrase}");
$data = $response->body;
$contentEncoding = $response->getHeader('Content-Encoding');
if ($contentEncoding === 'gzip') {
$data = gzdecode($data);
if ($data === false) {
throw new Exception("Fetching the '$uri' failed, response should be gzip encoded, but gzip decoding failed.");
}
} else if ($contentEncoding) {
throw new Exception("Fetching the '$uri' failed, because remote server returns unsupported content encoding '$contentEncoding'");
}
$contentType = $response->getHeader('Content-Type');
if ($contentType === 'application/zip') {
$zipFile = new File($this->tempFileName());
$zipFile->write($data);
$zipFile->close();
try {
$data = $this->unzipFirstFile($zipFile);
} catch (Exception $e) {
throw new Exception("Fetching the '$uri' failed: {$e->getMessage()}");
} finally {
$zipFile->delete();
}
$data = $response->body;
}
return $data;
@ -1720,4 +1781,47 @@ class Feed extends AppModel
$this->save($feed);
return $count;
}
/**
* @param File $zipFile
* @return string Uncompressed data
* @throws Exception
*/
private function unzipFirstFile(File $zipFile)
{
if (!class_exists('ZipArchive')) {
throw new Exception("ZIP archive decompressing is not supported.");
}
$zip = new ZipArchive();
$result = $zip->open($zipFile->pwd());
if ($result !== true) {
throw new Exception("Remote server returns ZIP file, that cannot be open (error $result)");
}
if ($zip->numFiles !== 1) {
throw new Exception("Remote server returns ZIP file, that contains multiple files.");
}
$filename = $zip->getNameIndex(0);
if ($filename === false) {
throw new Exception("Remote server returns ZIP file, but there is a problem with reading filename.");
}
$zip->close();
$destinationFile = $this->tempFileName();
$result = copy("zip://{$zipFile->pwd()}#$filename", $destinationFile);
if ($result === false) {
throw new Exception("Remote server returns ZIP file, that contains '$filename' file, that cannot be extracted.");
}
$unzipped = new File($destinationFile);
$data = $unzipped->read();
if ($data === false) {
throw new Exception("Couldn't read extracted file content.");
}
$unzipped->delete();
return $data;
}
}

19
app/Model/Log.php
View File

@ -37,7 +37,6 @@ class Log extends AppModel
'enable',
'error',
'export',
'failed_registration',
'file_upload',
'galaxy',
'include_formula',
@ -52,13 +51,15 @@ class Log extends AppModel
'pull',
'purge_events',
'push',
'registration',
'registration_error',
'remove_dead_workers',
'request',
'request_delegation',
'reset_auth_key',
'send_mail',
'security',
'serverSettingsEdit',
'succeeded_registration',
'tag',
'undelete',
'update',
@ -96,18 +97,6 @@ class Log extends AppModel
'email' => array('values' => array('admin_email'))
);
public function beforeValidete()
{
parent::beforeValidate();
if (!isset($this->data['Log']['org']) || empty($this->data['Log']['org'])) {
$this->data['Log']['org'] = 'SYSTEM';
}
// truncate the description if it would exceed the allowed size in mysql
if (!empty($this->data['Log']['description'] && strlen($this->data['Log']['description']) > 65536)) {
$this->data['Log']['description'] = substr($this->data['Log']['description'], 0, 65535);
}
}
public function beforeSave($options = array())
{
if (!empty(Configure::read('MISP.log_skip_db_logs_completely'))) {
@ -125,7 +114,7 @@ class Log extends AppModel
if (!isset($this->data['Log']['created'])) {
$this->data['Log']['created'] = date('Y-m-d H:i:s');
}
if (!isset($this->data['Log']['org'])) {
if (!isset($this->data['Log']['org']) || empty($this->data['Log']['org'])) {
$this->data['Log']['org'] = 'SYSTEM';
}
$truncate_fields = array('title', 'change', 'description');

70
app/Model/MispObject.php
View File

@ -506,7 +506,10 @@ class MispObject extends AppModel
)
),
);
if (empty($options['includeAllTags'])) {
if (!empty($options['metadata'])) {
unset($params['contain']['Attribute']);
}
if (empty($options['metadata']) && empty($options['includeAllTags'])) {
$params['contain']['Attribute']['AttributeTag']['Tag']['conditions']['exportable'] = 1;
}
if (isset($options['contain'])) {
@ -514,7 +517,12 @@ class MispObject extends AppModel
} else {
$option['contain']['Event']['fields'] = array('id', 'info', 'org_id', 'orgc_id');
}
if (Configure::read('MISP.proposals_block_attributes') && isset($options['conditions']['AND']['Attribute.to_ids']) && $options['conditions']['AND']['Attribute.to_ids'] == 1) {
if (
empty($options['metadata']) &&
Configure::read('MISP.proposals_block_attributes') &&
isset($options['conditions']['AND']['Attribute.to_ids']) &&
$options['conditions']['AND']['Attribute.to_ids'] == 1
) {
$this->Attribute->bindModel(array('hasMany' => array('ShadowAttribute' => array('foreignKey' => 'old_id'))));
$proposalRestriction = array(
'ShadowAttribute' => array(
@ -544,7 +552,7 @@ class MispObject extends AppModel
if (!isset($options['enforceWarninglist'])) {
$options['enforceWarninglist'] = false;
}
if (!$user['Role']['perm_sync'] || !isset($options['deleted']) || !$options['deleted']) {
if (empty($options['metadata']) && (!$user['Role']['perm_sync'] || !isset($options['deleted']) || !$options['deleted'])) {
$params['contain']['Attribute']['conditions']['AND']['Attribute.deleted'] = 0;
}
if (isset($options['group'])) {
@ -566,23 +574,25 @@ class MispObject extends AppModel
}
$results = array_values($results);
$proposals_block_attributes = Configure::read('MISP.proposals_block_attributes');
foreach ($results as $key => $objects) {
foreach ($objects as $key2 => $attribute) {
if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttributes($warninglists, $attribute['Attribute'], $this->Warninglist)) {
unset($results[$key][$key2]);
continue;
}
if ($proposals_block_attributes) {
if (!empty($attribute['ShadowAttribute'])) {
if (empty($options['metadata'])) {
foreach ($results as $key => $objects) {
foreach ($objects as $key2 => $attribute) {
if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttributes($warninglists, $attribute['Attribute'], $this->Warninglist)) {
unset($results[$key][$key2]);
} else {
unset($results[$key][$key2]['ShadowAttribute']);
continue;
}
}
if ($options['withAttachments']) {
if ($this->typeIsAttachment($attribute['Attribute']['type'])) {
$encodedFile = $this->base64EncodeAttachment($attribute['Attribute']);
$results[$key][$key2]['Attribute']['data'] = $encodedFile;
if ($proposals_block_attributes) {
if (!empty($attribute['ShadowAttribute'])) {
unset($results[$key][$key2]);
} else {
unset($results[$key][$key2]['ShadowAttribute']);
}
}
if ($options['withAttachments']) {
if ($this->typeIsAttachment($attribute['Attribute']['type'])) {
$encodedFile = $this->base64EncodeAttachment($attribute['Attribute']);
$results[$key][$key2]['Attribute']['data'] = $encodedFile;
}
}
}
}
@ -947,7 +957,7 @@ class MispObject extends AppModel
return 'fail';
}
public function editObject($object, $eventId, $user, $log)
public function editObject($object, $eventId, $user, $log, $force = false)
{
$object['event_id'] = $eventId;
if (isset($object['uuid'])) {
@ -973,7 +983,7 @@ class MispObject extends AppModel
return true;
}
if (isset($object['timestamp'])) {
if ($existingObject['Object']['timestamp'] >= $object['timestamp']) {
if ($force || $existingObject['Object']['timestamp'] >= $object['timestamp']) {
return true;
}
} else {
@ -1023,7 +1033,7 @@ class MispObject extends AppModel
}
if (!empty($object['Attribute'])) {
foreach ($object['Attribute'] as $attribute) {
$result = $this->Attribute->editAttribute($attribute, $eventId, $user, $object['id'], $log);
$result = $this->Attribute->editAttribute($attribute, $eventId, $user, $object['id'], $log, $force);
}
}
return true;
@ -1344,7 +1354,8 @@ class MispObject extends AppModel
'includeCorrelations' => !empty($filters['includeCorrelations']) ? $filters['includeCorrelations'] : 0,
'includeDecayScore' => !empty($filters['includeDecayScore']) ? $filters['includeDecayScore'] : 0,
'includeFullModel' => !empty($filters['includeFullModel']) ? $filters['includeFullModel'] : 0,
'allow_proposal_blocking' => !empty($filters['allow_proposal_blocking']) ? $filters['allow_proposal_blocking'] : 0
'allow_proposal_blocking' => !empty($filters['allow_proposal_blocking']) ? $filters['allow_proposal_blocking'] : 0,
'metadata' => !empty($filters['metadata']) ? $filters['metadata'] : 0,
);
if (!empty($filters['attackGalaxy'])) {
$params['attackGalaxy'] = $filters['attackGalaxy'];
@ -1377,6 +1388,9 @@ class MispObject extends AppModel
if (!empty($filters['score'])) {
$params['score'] = $filters['score'];
}
if (!empty($filters['metadata'])) {
$params['metadata'] = $filters['metadata'];
}
if ($paramsOnly) {
return $params;
}
@ -1423,8 +1437,16 @@ class MispObject extends AppModel
{
$continue = true;
while ($continue) {
$temp = '';
$this->Whitelist = ClassRegistry::init('Whitelist');
$results = $this->fetchObjects($user, $params, $continue);
if (empty($results)) {
$loop = false;
return true;
}
if ($elementCounter !== 0 && !empty($results)) {
$temp .= $exportTool->separator($exportToolParams);
}
if ($params['includeSightingdb']) {
$this->Sightingdb = ClassRegistry::init('Sightingdb');
$results = $this->Sightingdb->attachToObjects($results, $user);
@ -1433,7 +1455,6 @@ class MispObject extends AppModel
$results = $this->Whitelist->removeWhitelistedFromArray($results, true);
$results = array_values($results);
$i = 0;
$temp = '';
foreach ($results as $object) {
$elementCounter++;
$handlerResult = $exportTool->handler($object, $exportToolParams);
@ -1448,9 +1469,6 @@ class MispObject extends AppModel
if (!$loop) {
$continue = false;
}
if ($continue) {
$temp .= $exportTool->separator($exportToolParams);
}
fwrite($tmpfile, $temp);
}
return true;

3
app/Model/Role.php
View File

@ -84,7 +84,8 @@ class Role extends AppModel
'id' => 'RolePermAuth',
'text' => 'Auth key access',
'readonlyenabled' => true,
'title' => 'Users with this permission have access to authenticating via their Auth keys, granting them access to the API.'
'title' => 'Users with this permission have access to authenticating via their Auth keys, granting them access to the API.',
'site_admin_optional' => true
),
'perm_regexp_access' => array(
'id' => 'RolePermRegexpAccess',

134
app/Model/Server.php
View File

@ -1270,6 +1270,54 @@ class Server extends AppModel
'type' => 'boolean',
'null' => true
),
'email_otp_enabled' => array(
'level'=> 2,
'description' => __('Enable two step authentication with a OTP sent by email. Requires e-mailing to be enabled. Warning: You cannot use it in combination with external authentication plugins.'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'beforeHook' => 'otpBeforeHook',
'type' => 'boolean',
'null' => true
),
'email_otp_length' => array (
'level' => 2,
'description' => __('Define the length of the OTP code sent by email'),
'value' => '6',
'errorMessage' => '',
'type' => 'numeric',
'test' => 'testForNumeric',
'null' => true,
),
'email_otp_validity' => array (
'level' => 2,
'description' => __('Define the validity (in minutes) of the OTP code sent by email'),
'value' => '5',
'errorMessage' => '',
'type' => 'numeric',
'test' => 'testForNumeric',
'null' => true,
),
'email_otp_text' => array(
'level' => 2,
'bigField' => true,
'description' => __('The message sent to the user when a new OTP is requested. Use \\n for line-breaks. The following variables will be automatically replaced in the text: $otp = the new OTP generated by MISP, $username = the user\'s e-mail address, $org the Organisation managing the instance, $misp = the url of this instance, $contact = the e-mail address used to contact the support team (as set in MISP.contact), $ip the IP used to complete the first step of the login and $validity the validity time in minutes.'),
'value' => 'Dear MISP user,\n\nYou have attempted to login to MISP ($misp) from $ip with username $username.\n\n Use the following OTP to log into MISP: $otp\n This code is valid for the next $validity minutes.\n\nIf you have any questions, don\'t hesitate to contact us at: $contact.\n\nBest regards,\nYour $org MISP support team',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string',
'null' => true,
),
'email_otp_exceptions' => array(
'level' => 2,
'bigField' => true,
'description' => __('A comma separated list of emails for which the OTP is disabled. Note that if you remove someone from this list, the OTP will only be asked at next login.'),
'value' => '',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string',
'null' => true,
),
'allow_self_registration' => array(
'level' => 1,
'description' => __('Enabling this setting will allow users to have access to the pre-auth registration form. This will create an inbox entry for administrators to review.'),
@ -1767,6 +1815,24 @@ class Server extends AppModel
'type' => 'numeric',
'afterHook' => 'zmqAfterHook',
),
'ZeroMQ_username' => array(
'level' => 2,
'description' => __('The username that client need to use to connect to ZeroMQ.'),
'value' => '',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string',
'afterHook' => 'zmqAfterHook',
),
'ZeroMQ_password' => array(
'level' => 2,
'description' => __('The password that client need to use to connect to ZeroMQ.'),
'value' => '',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string',
'afterHook' => 'zmqAfterHook',
),
'ZeroMQ_redis_host' => array(
'level' => 2,
'description' => __('Location of the Redis db used by MISP and the Python PUB script to queue data to be published.'),
@ -2325,11 +2391,11 @@ class Server extends AppModel
return true;
}
private function __getEventIdListBasedOnPullTechnique($technique, $server)
private function __getEventIdListBasedOnPullTechnique($technique, $server, $force = false)
{
if ("full" === $technique) {
// get a list of the event_ids on the server
$eventIds = $this->getEventIdsFromServer($server);
$eventIds = $this->getEventIdsFromServer($server, false, null, false, false, 'events', $force);
if ($eventIds === 403) {
return array('error' => array(1, null));
} elseif (is_string($eventIds)) {
@ -2341,7 +2407,7 @@ class Server extends AppModel
$eventIds = array_reverse($eventIds);
}
} elseif ("update" === $technique) {
$eventIds = $this->getEventIdsFromServer($server, false, null, true, true);
$eventIds = $this->getEventIdsFromServer($server, false, null, true, true, 'events', $force);
if ($eventIds === 403) {
return array('error' => array(1, null));
} elseif (is_string($eventIds)) {
@ -2436,7 +2502,7 @@ class Server extends AppModel
return false;
}
private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, $eventModel, $server, $user, $jobId)
private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, $eventModel, $server, $user, $jobId, $force = false)
{
// check if the event already exist (using the uuid)
$existingEvent = $eventModel->find('first', array('conditions' => array('Event.uuid' => $event['Event']['uuid'])));
@ -2453,7 +2519,7 @@ class Server extends AppModel
if (!$existingEvent['Event']['locked'] && !$server['Server']['internal']) {
$fails[$eventId] = __('Blocked an edit to an event that was created locally. This can happen if a synchronised event that was created on this instance was modified by an administrator on the remote side.');
} else {
$result = $eventModel->_edit($event, $user, $existingEvent['Event']['id'], $jobId, $passAlong);
$result = $eventModel->_edit($event, $user, $existingEvent['Event']['id'], $jobId, $passAlong, $force);
if ($result === true) {
$successes[] = $eventId;
} elseif (isset($result['error'])) {
@ -2465,7 +2531,7 @@ class Server extends AppModel
}
}
private function __pullEvent($eventId, &$successes, &$fails, $eventModel, $server, $user, $jobId)
private function __pullEvent($eventId, &$successes, &$fails, $eventModel, $server, $user, $jobId, $force = false)
{
$event = $eventModel->downloadEventFromServer(
$eventId,
@ -2480,7 +2546,7 @@ class Server extends AppModel
if (!$this->__checkIfEventSaveAble($event)) {
$fails[$eventId] = __('Empty event detected.');
} else {
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $server, $user, $jobId);
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $server, $user, $jobId, $force);
}
} else {
// error
@ -2545,7 +2611,7 @@ class Server extends AppModel
return $pulledProposals;
}
public function pull($user, $id = null, $technique=false, $server, $jobId = false)
public function pull($user, $id = null, $technique=false, $server, $jobId = false, $force = false)
{
if ($jobId) {
$job = ClassRegistry::init('Job');
@ -2559,7 +2625,7 @@ class Server extends AppModel
$eventIds = array();
// if we are downloading a single event, don't fetch all proposals
$conditions = is_numeric($technique) ? array('Event.id' => $technique) : array();
$eventIds = $this->__getEventIdListBasedOnPullTechnique($technique, $server);
$eventIds = $this->__getEventIdListBasedOnPullTechnique($technique, $server, $force);
$server['Server']['version'] = $this->getRemoteVersion($id);
if (!empty($eventIds['error'])) {
$errors = array(
@ -2588,7 +2654,7 @@ class Server extends AppModel
if (!empty($eventIds)) {
// download each event
foreach ($eventIds as $k => $eventId) {
$this->__pullEvent($eventId, $successes, $fails, $eventModel, $server, $user, $jobId);
$this->__pullEvent($eventId, $successes, $fails, $eventModel, $server, $user, $jobId, $force);
if ($jobId) {
if ($k % 10 == 0) {
$job->saveField('progress', 50 * (($k + 1) / count($eventIds)));
@ -2701,7 +2767,7 @@ class Server extends AppModel
}
// Get an array of event_ids that are present on the remote server
public function getEventIdsFromServer($server, $all = false, $HttpSocket=null, $force_uuid=false, $ignoreFilterRules = false, $scope = 'events')
public function getEventIdsFromServer($server, $all = false, $HttpSocket=null, $force_uuid=false, $ignoreFilterRules = false, $scope = 'events', $force = false)
{
$url = $server['Server']['url'];
if ($ignoreFilterRules) {
@ -2785,7 +2851,9 @@ class Server extends AppModel
}
}
}
$this->Event->removeOlder($eventArray, $scope);
if (!$force) {
$this->Event->removeOlder($eventArray, $scope);
}
if (!empty($eventArray)) {
foreach ($eventArray as $event) {
if ($force_uuid) {
@ -3122,15 +3190,6 @@ class Server extends AppModel
private function readModuleSettings($serverSettings, $moduleTypes)
{
$this->Module = ClassRegistry::init('Module');
$orgs = $this->Organisation->find('list', array(
'conditions' => array(
'Organisation.local' => 1
),
'fields' => array(
'Organisation.id', 'Organisation.name'
)
));
$orgs = array_merge(array('Unrestricted'), $orgs);
foreach ($moduleTypes as $moduleType) {
if (Configure::read('Plugin.' . $moduleType . '_services_enable')) {
$results = $this->Module->getModuleSettings($moduleType);
@ -3559,7 +3618,7 @@ class Server extends AppModel
if ($errorMessage) {
return $errorMessage;
}
return 'Value is not a boolean, make sure that you convert \'true\' to true for example.';
return __('Value is not a boolean, make sure that you convert \'true\' to true for example.');
}
return true;
}
@ -3749,6 +3808,14 @@ class Server extends AppModel
return true;
}
public function otpBeforeHook($setting, $value)
{
if ($value && !empty(Configure::read('MISP.disable_emailing'))) {
return __('Emailing is currently disabled. Enabling OTP without e-mailing being configured would lock all users out.');
}
return true;
}
public function testForRPZSerial($value)
{
if ($this->testForEmpty($value) !== true) {
@ -3910,7 +3977,6 @@ class Server extends AppModel
} else {
$serverSettings = $this->serverSettings;
}
$relevantSettings = (array_intersect_key(Configure::read(), $serverSettings));
$setting = false;
foreach ($serverSettings as $k => $s) {
if (isset($s['branch'])) {
@ -4564,26 +4630,28 @@ class Server extends AppModel
switch($field['error_type']) {
case 'missing_column':
$field['sql'] = sprintf(
'ALTER TABLE `%s` ADD COLUMN `%s` %s%s %s %s %s;',
'ALTER TABLE `%s` ADD COLUMN `%s` %s%s %s %s %s %s;',
$table,
$field['column_name'],
$field['expected']['data_type'],
$length !== null ? sprintf('(%d)', $length) : '',
isset($field['expected']['column_default']) ? 'DEFAULT "' . $field['expected']['column_default'] . '"' : '',
$field['expected']['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL',
empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name']
empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name'],
empty($field['expected']['extra']) ? '' : $field['expected']['extra']
);
break;
case 'column_different':
$field['sql'] = sprintf(
'ALTER TABLE `%s` MODIFY COLUMN `%s` %s%s %s %s %s;',
'ALTER TABLE `%s` MODIFY COLUMN `%s` %s%s %s %s %s %s;',
$table,
$field['column_name'],
$field['expected']['data_type'],
$length !== null ? sprintf('(%d)', $length) : '',
isset($field['expected']['column_default']) ? 'DEFAULT "' . $field['expected']['column_default'] . '"' : '',
$field['expected']['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL',
empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name']
empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name'],
empty($field['expected']['extra']) ? '' : $field['expected']['extra']
);
break;
}
@ -4600,13 +4668,14 @@ class Server extends AppModel
} elseif ($expectedField['data_type'] === 'text') {
$length = null;
}
$fieldSql = sprintf('`%s` %s%s %s %s %s',
$fieldSql = sprintf('`%s` %s%s %s %s %s %s',
$expectedField['column_name'],
$expectedField['data_type'],
$length !== null ? sprintf('(%d)', $length) : '',
isset($expectedField['column_default']) ? 'DEFAULT "' . $expectedField['column_default'] . '"' : '',
$expectedField['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL',
empty($expectedField['collation_name']) ? '' : 'COLLATE ' . $expectedField['collation_name']
empty($expectedField['collation_name']) ? '' : 'COLLATE ' . $expectedField['collation_name'],
empty($field['expected']['extra']) ? '' : $field['expected']['extra']
);
$allFields[] = $fieldSql;
}
@ -4657,7 +4726,8 @@ class Server extends AppModel
// 'datetime_precision', -- Only available on MySQL 5.6+
'collation_name',
'column_type',
'column_default'
'column_default',
'extra',
)
){
$dbActualSchema = array();
@ -4939,7 +5009,7 @@ class Server extends AppModel
public function stixDiagnostics(&$diagnostic_errors, &$stixVersion, &$cyboxVersion, &$mixboxVersion, &$maecVersion, &$stix2Version, &$pymispVersion)
{
$result = array();
$expected = array('stix' => '>1.2.0.6', 'cybox' => '>2.1.0.18.dev0', 'mixbox' => '1.0.3', 'maec' => '>4.1.0.14', 'stix2' => '>1.2.0', 'pymisp' => '>2.4.120');
$expected = array('stix' => '>1.2.0.9', 'cybox' => '>2.1.0.21', 'mixbox' => '1.0.3', 'maec' => '>4.1.0.14', 'stix2' => '>1.2.0', 'pymisp' => '>2.4.120');
// check if the STIX and Cybox libraries are working using the test script stixtest.py
$scriptResult = shell_exec($this->getPythonVersion() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'stixtest.py');
$scriptResult = json_decode($scriptResult, true);
@ -5761,7 +5831,7 @@ class Server extends AppModel
$params['conditions']['Server.id'] = $id;
} else {
$redis->del('misp:server_cache:combined');
$redis->del('misp:server_cache:event_uuid_lookup:');
$redis->del($redis->keys('misp:server_cache:event_uuid_lookup:*'));
}
$servers = $this->find('all', $params);
if ($jobId) {

39
app/Model/SharingGroup.php
View File

@ -54,6 +54,10 @@ class SharingGroup extends AppModel
);
private $__sgoCache = array();
private $__sgAuthorisationCache = array(
'save' => array(),
'access' => array()
);
public function beforeValidate($options = array())
@ -353,6 +357,9 @@ class SharingGroup extends AppModel
// returns true if the SG exists and the user is allowed to see it
public function checkIfAuthorised($user, $id, $adminCheck = true)
{
if (isset($this->__sgAuthorisationCache['access'][boolval($adminCheck)][$id])) {
return $this->__sgAuthorisationCache['access'][boolval($adminCheck)][$id];
}
if (Validation::uuid($id)) {
$sgid = $this->SharingGroup->find('first', array(
'conditions' => array('SharingGroup.uuid' => $id),
@ -372,8 +379,10 @@ class SharingGroup extends AppModel
return false;
}
if (($adminCheck && $user['Role']['perm_site_admin']) || $this->SharingGroupServer->checkIfAuthorised($id) || $this->SharingGroupOrg->checkIfAuthorised($id, $user['org_id'])) {
$this->__sgAuthorisationCache['access'][boolval($adminCheck)][$id] = true;
return true;
}
$this->__sgAuthorisationCache['access'][boolval($adminCheck)][$id] = false;
return false;
}
@ -485,7 +494,7 @@ class SharingGroup extends AppModel
return $results;
}
public function captureSG($sg, $user)
public function captureSG($sg, $user, $syncLocal=false)
{
$existingSG = !isset($sg['uuid']) ? null : $this->find('first', array(
'recursive' => -1,
@ -501,6 +510,34 @@ class SharingGroup extends AppModel
if (!$user['Role']['perm_sharing_group']) {
return false;
}
// check if current user is contained in the SG and we are in a local sync setup
if (!empty($sg['uuid'])) {
if (isset($this->__sgAuthorisationCache['save'][boolval($syncLocal)][$sg['uuid']])) {
$authorisedToSave = $this->__sgAuthorisationCache['save'][boolval($syncLocal)][$sg['uuid']];
} else {
$authorisedToSave = $this->checkIfAuthorisedToSave($user, $sg);
$this->__sgAuthorisationCache['save'][boolval($syncLocal)][$sg['uuid']] = $authorisedToSave;
}
} else {
$authorisedToSave = $this->checkIfAuthorisedToSave($user, $sg);
}
if (!$user['Role']['perm_site_admin'] &&
!($user['Role']['perm_sync'] && $syncLocal ) &&
!$authorisedToSave
) {
$this->Log->create();
$entry = array(
'org' => $user['Organisation']['name'],
'model' => 'SharingGroup',
'model_id' => $sg['SharingGroup']['uuid'],
'email' => $user['email'],
'action' => 'error',
'user_id' => $user['id'],
'title' => 'Tried to save a sharing group but the user does not belong to it.'
);
$this->Log->save($entry);
return false;
}
$this->create();
$newSG = array();
$attributes = array(

20
app/Model/User.php
View File

@ -910,7 +910,21 @@ class User extends AppModel
}
}
$Email->attachments($attachments);
$result = $Email->send($body);
try {
$result = $Email->send($body);
} catch (Exception $e) {
$this->Log = ClassRegistry::init('Log');
$this->Log->save(array(
'org' => 'SYSTEM',
'model' => 'User',
'model_id' => $user['User']['id'],
'email' => $user['User']['email'],
'action' => 'send_mail',
'title' => sprintf(__('Could not send mail. Reasons: %s'), $e->getMessage()),
'change' => null,
));
$result = false;
}
$Email->reset();
return $result;
}
@ -1540,7 +1554,7 @@ class User extends AppModel
'model' => 'User',
'model_id' => $added_by['id'],
'email' => $added_by['email'],
'action' => 'failed_registration',
'action' => 'registration_error',
'title' => 'User registration failed for ' . $user['email'] . '. Reason(s): ' . $error,
'change' => null,
));
@ -1555,7 +1569,7 @@ class User extends AppModel
'model' => 'User',
'model_id' => $added_by['id'],
'email' => $added_by['email'],
'action' => 'succeeded_registration',
'action' => 'registration',
'title' => sprintf('User registration success for %s (id=%s)', $user['User']['email'], $user['User']['id']),
'change' => null,
));

2
app/Plugin/SysLog/Lib/SysLog.php
View File

@ -68,10 +68,10 @@ class SysLog {
} else if (in_array($type, $debugTypes)) {
$priority = LOG_DEBUG;
}
$output = date('Y-m-d H:i:s') . ' ' . ucfirst($type) . ': ' . $message . "\n";
if (!openlog($this->_ident, LOG_PID | LOG_PERROR, $this->_facility)) {
return false;
}
$output = date('Y-m-d H:i:s') . ' ' . ucfirst($type) . ': ' . $message;
$result = syslog($priority, $output);
closelog();
return $result;

8
app/View/Attributes/add_attachment.ctp
View File

@ -67,9 +67,11 @@
<?php
echo $this->Form->input('advanced', array(
'type' => 'checkbox',
'checked' => false,
'checked' => true,
'disabled' => !$advancedExtractionAvailable,
'data-disabled-reason' => !$advancedExtractionAvailable ? __('Advanced extraction is not installed') : '',
'div' => array('id' => 'advanced_input', 'style' => 'display:none'),
'label' => __('Advanced extraction (if installed)'),
'label' => __('Advanced extraction'),
));
?>
</fieldset>
@ -131,7 +133,7 @@ $(document).ready(function() {
$("#AttributeCategory, #AttributeDistribution").change(function() {
initPopoverContent('Attribute');
});
$("#AttributeMalware").change(function () {
if (this.checked) {
$('#advanced_input').show();

34
app/View/Elements/eventdiscussion.ctp
View File

@ -1,3 +1,4 @@
<h3><?= __('Discussion') ?></h3>
<div id="top">
<div class="pagination">
<?php
@ -18,12 +19,11 @@
?>
</ul>
</div>
<div id = "posts">
<div id="posts">
<?php
foreach ($posts as $post) {
?>
<a name="message_<?php echo h($post['id']);?>"></a>
<table class="discussionBox" id=<?php echo '"' . h($post['id']) . '"';?> >
<table class="discussionBox" id="message_<?= h($post['id']) ?>">
<tr>
<td class="discussionBoxTD discussionBoxTDtop" colspan="2">
<div>
@ -36,7 +36,7 @@
</td>
<td style="text-align:right">
<a href="#top" class="whitelink">Top</a> |
<a href="<?php echo "#".$post['id']; ?>" class="whitelink">#<?php echo h($post['id'])?></a>
<a href="#message_<?php echo h($post['id']); ?>" class="whitelink">#<?php echo h($post['id'])?></a>
</td>
</tr>
</table>
@ -58,14 +58,14 @@
echo $this->Command->convertQuotes(nl2br(h($post['contents'])));
if ($post['post_id'] !=0 || ($post['date_created'] != $post['date_modified'])) {
?>
<br /><br />
<br><br>
<?php
}
if ($post['post_id'] != 0) {
?>
<span style="font-style:italic">
In reply to post
<a href="<?php echo "#".h($post['post_id']); ?>">#<?php echo h($post['post_id'])?></a>
<a href="#message_<?php echo h($post['post_id']); ?>">#<?php echo h($post['post_id'])?></a>
</span>
<?php
}
@ -76,7 +76,7 @@
</td>
</tr>
<tr>
<td class="discussionBoxTD discussionBoxTDbottom" colspan = "2">
<td class="discussionBoxTD discussionBoxTDbottom" colspan="2">
<table style="width:100%">
<tr>
<td>
@ -90,14 +90,14 @@
echo $this->Form->postLink('', array('controller' => 'posts', 'action' => 'delete', h($post['id']), h($context)), array('class' => 'fa fa-trash', 'title' => __('Delete'), 'aria-label' => __('Delete')), __('Are you sure you want to delete this post?'));
} else {
?>
<a href="<?php echo $baseurl.'/posts/add/post/'.h($post['id']); ?>" class="icon-comment" title = "<?php echo __('Reply');?>" aria-label = "<?php echo __('Reply');?>"></a>
<a href="<?php echo $baseurl.'/posts/add/post/'.h($post['id']); ?>" class="icon-comment" title="<?php echo __('Reply');?>" aria-label="<?php echo __('Reply');?>"></a>
<?php
}
} else {
echo $this->Html->link('', array('controller' => 'posts', 'action' => 'edit', h($post['id']), h($context)), array('class' => 'fa fa-edit', 'title' => __('Edit'), 'aria-label' => __('Edit')));
echo $this->Form->postLink('', array('controller' => 'posts', 'action' => 'delete', h($post['id']), h($context)), array('class' => 'fa fa-trash', 'title' => __('Delete'), 'aria-label' => __('Delete')), __('Are you sure you want to delete this post?'));
?>
<a href = "<?php echo $baseurl.'/posts/add/post/'.h($post['id']); ?>" class="icon-comment" title = "<?php echo __('Reply');?>" aria-label = "<?php echo __('Reply');?>"></a>
<a href="<?php echo $baseurl.'/posts/add/post/'.h($post['id']); ?>" class="icon-comment" title="<?php echo __('Reply');?>" aria-label="<?php echo __('Reply');?>"></a>
<?php
}
@ -108,7 +108,7 @@
</td>
</tr>
</table>
<br />
<br>
<?php
}
?>
@ -139,11 +139,11 @@
?>
<fieldset>
<div class="input clear">
<button type="button" title="<?php echo __('Insert a quote - just paste your quote between the [quote][/quote] tags.');?>" class="toggle-left btn btn-inverse qet" id = "quote" onclick="insertQuote()"><?php echo __('Quote');?></button>
<button type="button" title="<?php echo __('Insert a link to an event - just enter the event ID between the [event][/event] tags.');?>" class="toggle btn btn-inverse qet" id = "event" onclick="insertEvent()"><?php echo __('Event');?></button>
<button type="button" title="<?php echo __('Insert a link to a discussion thread - enter the thread\'s ID between the [thread][/thread] tags.');?>" class="toggle btn btn-inverse qet" id = "thread" onclick="insertThread()"><?php echo __('Thread');?></button>
<button type="button" title="<?php echo __('Insert a link [link][/link] tags.');?>" class="toggle btn btn-inverse qet" id = "link" onclick="insertLink()"><?php echo __('Link');?></button>
<button type="button" title="<?php echo __('Insert a code [code][/code] tags.');?>" class="toggle-right btn btn-inverse qet" id = "code" onclick="insertCode()"><?php echo __('Code');?></button>
<button type="button" title="<?php echo __('Insert a quote - just paste your quote between the [quote][/quote] tags.');?>" class="toggle-left btn btn-inverse qet" id="quote" onclick="insertQuote()"><?php echo __('Quote');?></button>
<button type="button" title="<?php echo __('Insert a link to an event - just enter the event ID between the [event][/event] tags.');?>" class="toggle btn btn-inverse qet" id="event" onclick="insertEvent()"><?php echo __('Event');?></button>
<button type="button" title="<?php echo __('Insert a link to a discussion thread - enter the thread\'s ID between the [thread][/thread] tags.');?>" class="toggle btn btn-inverse qet" id="thread" onclick="insertThread()"><?php echo __('Thread');?></button>
<button type="button" title="<?php echo __('Insert a link [link][/link] tags.');?>" class="toggle btn btn-inverse qet" id="link" onclick="insertLink()"><?php echo __('Link');?></button>
<button type="button" title="<?php echo __('Insert a code [code][/code] tags.');?>" class="toggle-right btn btn-inverse qet" id="code" onclick="insertCode()"><?php echo __('Code');?></button>
</div>
<?php
echo $this->Form->input('message', array(
@ -154,7 +154,7 @@
));
?>
</fieldset>
<button class="btn btn-primary" onClick="submitMessageForm('<?php echo $url;?>', 'PostViewForm', 'top'); return false;"><?php echo __('Send');?></button>
<button class="btn btn-primary" onclick="submitMessageForm('<?php echo $url;?>', 'PostViewForm', 'top'); return false;"><?php echo __('Send comment');?></button>
<?php
echo $this->Form->end();
?>
@ -177,7 +177,7 @@
document.getElementById("PostMessage").value+="[Code][/Code]";
}
<?php if (isset($post_id) && $post_id): ?>
$(document).ready(function() {
$(function() {
location.hash = "#message_<?php echo h($post_id); ?>";
});
<?php endif; ?>

3
app/View/Elements/genericElements/IndexTable/index_table.ctp
View File

@ -32,6 +32,7 @@
if (!$skipPagination) {
$paginationData = !empty($data['paginatorOptions']) ? $data['paginatorOptions'] : array();
echo $this->element('/genericElements/IndexTable/pagination', array('paginationOptions' => $paginationData));
echo $this->element('/genericElements/IndexTable/pagination_links');
}
if (!empty($data['top_bar'])) {
echo $this->element('/genericElements/ListTopBar/scaffold', array('data' => $data['top_bar']));
@ -81,7 +82,7 @@
echo '</div>';
if (!$skipPagination) {
echo $this->element('/genericElements/IndexTable/pagination_counter', $paginationData);
echo $this->element('/genericElements/IndexTable/pagination', $paginationData);
echo $this->element('/genericElements/IndexTable/pagination_links');
}
?>
<script type="text/javascript">

6
app/View/Elements/genericElements/IndexTable/pagination.ctp
View File

@ -9,10 +9,4 @@
$options = array_merge($options, $paginationOptions);
}
echo $this->Paginator->options($options);
echo sprintf(
'<div class="pagination"><ul>%s%s%s</ul></div>',
$this->Paginator->prev('&laquo; ' . __('previous'), array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'prev disabled', 'escape' => false, 'disabledTag' => 'span')),
$this->Paginator->numbers(array('modulus' => 20, 'separator' => '', 'tag' => 'li', 'currentClass' => 'active', 'currentTag' => 'span')),
$this->Paginator->next(__('next') . ' &raquo;', array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'next disabled', 'escape' => false, 'disabledTag' => 'span'))
);
?>

7
app/View/Elements/genericElements/IndexTable/pagination_links.ctp Normal file
View File

@ -0,0 +1,7 @@
<?php
echo sprintf(
'<div class="pagination"><ul>%s%s%s</ul></div>',
$this->Paginator->prev('&laquo; ' . __('previous'), array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'prev disabled', 'escape' => false, 'disabledTag' => 'span')),
$this->Paginator->numbers(array('modulus' => 20, 'separator' => '', 'tag' => 'li', 'currentClass' => 'active', 'currentTag' => 'span')),
$this->Paginator->next(__('next') . ' &raquo;', array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'next disabled', 'escape' => false, 'disabledTag' => 'span'))
);

36
app/View/Events/view.ctp
View File

@ -301,27 +301,27 @@
)
)
);
if (!Configure::read('MISP.completely_disable_correlation') && Configure::read('MISP.allow_disabling_correlation')) {
$table_data[] = array(
'key' => __('Correlation'),
'class' => $event['Event']['disable_correlation'] ? 'background-red bold' : '',
'html' => sprintf(
'%s%s',
$event['Event']['disable_correlation'] ? __('Disabled') : __('Enabled'),
(!$mayModify && !$isSiteAdmin) ? '' : sprintf(
}
if (!Configure::read('MISP.completely_disable_correlation') && Configure::read('MISP.allow_disabling_correlation')) {
$table_data[] = array(
'key' => __('Correlation'),
'class' => $event['Event']['disable_correlation'] ? 'background-red bold' : '',
'html' => sprintf(
'%s%s',
$event['Event']['disable_correlation'] ? __('Disabled') : __('Enabled'),
(!$mayModify && !$isSiteAdmin) ? '' : sprintf(
sprintf(
' (<a onClick="getPopup(%s);" style="%scursor:pointer;font-weight:normal;">%s</a>)',
sprintf(
' (<a onClick="getPopup(%s);" style="%scursor:pointer;font-weight:normal;">%s</a>)',
sprintf(
"'%s', 'events', 'toggleCorrelation', '', '#confirmation_box'",
h($event['Event']['id'])
),
$event['Event']['disable_correlation'] ? 'color:white;' : '',
$event['Event']['disable_correlation'] ? __('enable') : __('disable')
)
"'%s', 'events', 'toggleCorrelation', '', '#confirmation_box'",
h($event['Event']['id'])
),
$event['Event']['disable_correlation'] ? 'color:white;' : '',
$event['Event']['disable_correlation'] ? __('enable') : __('disable')
)
)
);
}
)
);
}
?>

6
app/View/Roles/admin_add.ctp
View File

@ -41,7 +41,11 @@
<?php
echo $this->Form->input($k, array(
'type' => 'checkbox',
'class' => 'checkbox ' . ($flag['readonlyenabled'] ? 'readonlyenabled' : 'readonlydisabled'),
'class' => sprintf(
'checkbox %s %s',
($flag['readonlyenabled'] ? 'readonlyenabled' : 'readonlydisabled'),
empty($flag['site_admin_optional']) ? 'site_admin_enforced' : 'site_admin_optional'
),
'checked' => false,
'label' => Inflector::humanize(substr($k, 5))
));

6
app/View/Roles/admin_edit.ctp
View File

@ -41,7 +41,11 @@
<?php
echo $this->Form->input($k, array(
'type' => 'checkbox',
'class' => 'checkbox ' . ($flag['readonlyenabled'] ? 'readonlyenabled' : 'readonlydisabled'),
'class' => sprintf(
'checkbox %s %s',
($flag['readonlyenabled'] ? 'readonlyenabled' : 'readonlydisabled'),
empty($flag['site_admin_optional']) ? 'site_admin_enforced' : 'site_admin_optional'
),
'label' => Inflector::humanize(substr($k, 5))
));
if ($counter%3 == 0) echo "<div class = 'input clear'></div>";

7
app/View/Servers/ajax/zeromqstatus.ctp
View File

@ -2,9 +2,10 @@
<legend><?php echo __('ZeroMQ Server Status');?></legend>
<div style="padding-left:5px;padding-right:5px;padding-bottom:5px;">
<?php if (isset($time)): ?>
<p><b><?php echo __('Start time');?></b>: <?php echo h($time); ?><br />
<b><?php echo __('Settings read at');?></b>: <?php echo h($time2); ?><br />
<b><?php echo __('Events processed');?></b>: <?php echo h($events); ?></p>
<p><b><?php echo __('Reply time');?></b>: <?php echo h($time); ?><br>
<b><?php echo __('Start time');?></b>: <?php echo h($time2); ?><br>
<b><?php echo __('Events processed');?></b>: <?php echo h($events); ?><br>
<b><?php echo __('Messages processed');?></b>: <?php echo h($messages); ?></p>
<?php else: ?>
<p><?php echo __('The ZeroMQ server is unreachable.');?></p>
<?php endif; ?>

30
app/View/Users/email_otp.ctp Normal file
View File

@ -0,0 +1,30 @@
<?php echo $this->Flash->render(); ?>
<div class="actions sideMenu">
<div style="padding: 10px;">
<p> <?php echo __("Your administrator has turned on an additional authentication step which
requires you to enter a OTP (one time password) you have received via email.");?>
</p>
<p> <?php echo __("Make sure to check your SPAM folder.");?> </p>
<a href='<?php echo $baseurl; ?>/users/email_otp'> <button class='btn'> <?php echo __("Resend"); ?> </button></a>
</div>
</div>
<?php
echo $this->element('/genericElements/Form/genericForm', array(
"form" => $this->Form,
"data" => array(
"title" => __("Validate your OTP"),
"fields" => array(
array(
"field" => "otp",
"label" => __("One Time Password"),
"type" => "text",
"placeholder" => __("Enter your OTP here"),
),
),
"submit" => array (
"action" => "EmailOtp",
),
)));
?>

2
app/View/Users/statistics_data.ctp
View File

@ -35,6 +35,8 @@
<dd><?php echo h($stats['org_count']); ?>&nbsp;</dd>
<dt><?php echo __('Local Organisations'); ?></dt>
<dd><?php echo h($stats['local_org_count']); ?>&nbsp;</dd>
<dt><?php echo __('Event creator orgs'); ?></dt>
<dd><?php echo h($stats['contributing_org_count']); ?>&nbsp;</dd>
<dt><?php echo __('Average Users / Org'); ?></dt>
<dd><?php echo h($stats['average_user_per_org']); ?>&nbsp;</dd>
<dt><?php echo __('Discussion threads'); ?></dt>

4
app/files/feed-metadata/defaults.json
View File

@ -232,7 +232,7 @@
"event_id": "5655",
"publish": false,
"override_ids": false,
"settings": "{\"csv\":{\"value\":\"\",\"delimiter\":\"\"},\"common\":{\"excluderegex\":\"\\/^http:\\\\\\/\\\\\\/www.phishtank.com\\/i\"}}",
"settings": "{\"csv\":{\"value\":\"2\",\"delimiter\":\",\"},\"common\":{\"excluderegex\":\"\\/^http:\\\\\\/\\\\\\/www.phishtank.com\\/i\"}}",
"input_source": "network",
"delete_local_file": false,
"lookup_visible": false,
@ -907,7 +907,7 @@
"id": "46",
"name": "All current domains belonging to known malicious DGAs",
"provider": "osint.bambenekconsulting.com",
"url": "http://osint.bambenekconsulting.com/feeds/dga-feed-high.csv",
"url": "https://osint.bambenekconsulting.com/feeds/dga-feed-high.csv",
"rules": "",
"enabled": true,
"distribution": "3",

2
app/files/misp-galaxy

@ -1 +1 @@
Subproject commit c7104e8819d6b789b24a45655aa28625a8c4c346
Subproject commit 881a5664acfbb0e3ef68c70b641a966859ed2f4f

2
app/files/misp-objects

@ -1 +1 @@
Subproject commit 3b5451c32518da3e29c575e868d245f27c18dcf4
Subproject commit 4f05fc5bcd322e5d983c7d8d7de81848f09de8ba

1
app/files/scripts/misp-opendata Submodule

@ -0,0 +1 @@
Subproject commit 597a6878f32b7416b0b910e8f2c4ae2cef086113

198
app/files/scripts/mispzmq/mispzmq.py
View File

@ -1,15 +1,20 @@
#!/usr/bin/env python3
import zmq
from zmq.auth.thread import ThreadAuthenticator
from zmq.utils.monitor import recv_monitor_message
import sys
import redis
import json
import os
import time
import threading
import logging
from pathlib import Path
logging.basicConfig(level=logging.INFO, format="%(asctime)s:%(levelname)s:%(name)s:%(message)s")
def check_pid(pid):
""" Check For the existence of a unix pid. """
if not pid:
@ -24,102 +29,185 @@ def check_pid(pid):
return True
class MISPZMQ:
EVENT_MAP = {}
for name in dir(zmq):
if name.startswith("EVENT_"):
value = getattr(zmq, name)
EVENT_MAP[value] = name
def event_monitor(monitor, logger):
while monitor.poll():
evt = recv_monitor_message(monitor)
if evt["event"] == zmq.EVENT_MONITOR_STOPPED:
break
evt.update({"description": EVENT_MAP[evt["event"]]})
logger.debug("ZMQ event: {}".format(evt))
monitor.close()
class MispZmq:
message_count = 0
publish_count = 0
monitor_thread = None
auth = None
socket = None
pidfile = None
def __init__(self):
self.current_location = Path(__file__).parent
self.pidfile = self.current_location / "mispzmq.pid"
self.publishCount = 0
self._logger = logging.getLogger()
self.tmp_location = Path(__file__).parent.parent / "tmp"
self.pidfile = self.tmp_location / "mispzmq.pid"
if self.pidfile.exists():
with open(self.pidfile.as_posix()) as f:
pid = f.read()
if check_pid(pid):
raise Exception('mispzmq already running on PID {}'.format(pid))
raise Exception("mispzmq already running on PID {}".format(pid))
else:
# Cleanup
self.pidfile.unlink()
if (self.current_location / 'settings.json').exists():
self.setup()
if (self.tmp_location / "mispzmq_settings.json").exists():
self._setup()
else:
raise Exception("The settings file is missing.")
def setup(self):
with open((self.current_location / 'settings.json').as_posix()) as settings_file:
def _setup(self):
with open((self.tmp_location / "mispzmq_settings.json").as_posix()) as settings_file:
self.settings = json.load(settings_file)
self.namespace = self.settings["redis_namespace"]
self.r = redis.StrictRedis(host=self.settings["redis_host"], db=self.settings["redis_database"],
password=self.settings["redis_password"], port=self.settings["redis_port"],
decode_responses=True)
self.timestampSettings = time.time()
self.timestamp_settings = time.time()
self._logger.debug("Connected to Redis {}:{}/{}".format(self.settings["redis_host"], self.settings["redis_port"],
self.settings["redis_database"]))
def handleCommand(self, command):
def _setup_zmq(self):
context = zmq.Context()
if "username" in self.settings and self.settings["username"]:
if "password" not in self.settings or not self.settings["password"]:
raise Exception("When username is set, password cannot be empty.")
self.auth = ThreadAuthenticator(context)
self.auth.start()
self.auth.configure_plain(domain="*", passwords={self.settings["username"]: self.settings["password"]})
else:
if self.auth:
self.auth.stop()
self.auth = None
self.socket = context.socket(zmq.PUB)
if self.settings["username"]:
self.socket.plain_server = True # must come before bind
self.socket.bind("tcp://*:{}".format(self.settings["port"]))
self._logger.debug("ZMQ listening on tcp://*:{}".format(self.settings["port"]))
if self._logger.isEnabledFor(logging.DEBUG):
monitor = self.socket.get_monitor_socket()
self.monitor_thread = threading.Thread(target=event_monitor, args=(monitor, self._logger))
self.monitor_thread.start()
else:
if self.monitor_thread:
self.socket.disable_monitor()
self.monitor_thread = None
def _handle_command(self, command):
if command == "kill":
print("Kill command received, shutting down.")
self.pidfile.unlink()
self._logger.info("Kill command received, shutting down.")
self.clean()
sys.exit()
elif command == "reload":
print("Reload command received, reloading settings from file.")
self.setup()
self._logger.info("Reload command received, reloading settings from file.")
self._setup()
self._setup_zmq()
elif command == "status":
print("Status command received, responding with latest stats.")
self._logger.info("Status command received, responding with latest stats.")
self.r.delete("{}:status".format(self.namespace))
self.r.lpush("{}:status".format(self.namespace),
json.dumps({"timestamp": time.time(),
"timestampSettings": self.timestampSettings,
"publishCount": self.publishCount}))
"timestampSettings": self.timestamp_settings,
"publishCount": self.publish_count,
"messageCount": self.message_count}))
else:
self._logger.warning("Received invalid command '{}'.".format(command))
def createPidFile(self):
with open(self.pidfile.as_posix(), 'w') as f:
def _create_pid_file(self):
with open(self.pidfile.as_posix(), "w") as f:
f.write(str(os.getpid()))
def pubMessage(self, topic, data, socket):
socket.send_string("{} {}".format(topic, data))
if topic is 'misp_json':
self.publishCount += 1
def _pub_message(self, topic, data):
self.socket.send_string("{} {}".format(topic, data))
def clean(self):
if self.monitor_thread:
self.socket.disable_monitor()
if self.auth:
self.auth.stop()
if self.socket:
self.socket.close()
if self.pidfile:
self.pidfile.unlink()
def main(self):
start_time = int(time.time())
self.createPidFile()
self._create_pid_file()
self._setup_zmq()
time.sleep(1)
status_array = [
"And when you're dead I will be still alive.",
"And believe me I am still alive.",
"I'm doing science and I'm still alive.",
"I feel FANTASTIC and I'm still alive.",
"While you're dying I'll be still alive."
"While you're dying I'll be still alive.",
]
topics = ["misp_json", "misp_json_event", "misp_json_attribute", "misp_json_sighting",
"misp_json_organisation", "misp_json_user", "misp_json_conversation",
"misp_json_object", "misp_json_object_reference", "misp_json_audit",
"misp_json_tag"
"misp_json_tag",
]
context = zmq.Context()
socket = context.socket(zmq.PUB)
socket.bind("tcp://*:{}".format(self.settings["port"]))
time.sleep(1)
lists = ["{}:command".format(self.namespace)]
for topic in topics:
lists.append("{}:data:{}".format(self.namespace, topic))
while True:
command = self.r.lpop("{}:command".format(self.namespace))
if command is not None:
self.handleCommand(command)
message_received = False
for topic in topics:
data = self.r.lpop("{}:data:{}".format(self.namespace, topic))
if data is not None:
self.pubMessage(topic, data, socket)
message_received = True
if not message_received:
time.sleep(0.1)
current_time = 10 * time.time()
temp_start_time = 10 * start_time
time_delta = int(current_time - temp_start_time)
if time_delta % 100 == 0:
status_entry = int(time_delta / 100 % 5)
data = self.r.blpop(lists, timeout=10)
if data is None:
# redis timeout expired
current_time = int(time.time())
time_delta = current_time - int(self.timestamp_settings)
status_entry = int(time_delta / 10 % 5)
status_message = {
'status': status_array[status_entry],
'uptime': int(time.time()) - start_time
"status": status_array[status_entry],
"uptime": current_time - int(self.timestamp_settings)
}
self.pubMessage('misp_json_self', json.dumps(status_message), socket)
self._pub_message("misp_json_self", json.dumps(status_message))
self._logger.debug("No message received for 10 seconds, sending ZMQ status message.")
else:
key, value = data
key = key.replace("{}:".format(self.namespace), "")
if key == "command":
self._handle_command(value)
elif key.startswith("data:"):
topic = key.split(":")[1]
self._logger.debug("Received data for topic '{}', sending to ZMQ.".format(topic))
self._pub_message(topic, value)
self.message_count += 1
if topic == "misp_json":
self.publish_count += 1
else:
self._logger.warning("Received invalid message '{}'.".format(key))
if __name__ == "__main__":
mzq = MISPZMQ()
mzq.main()
mzq = MispZmq()
try:
mzq.main()
except KeyboardInterrupt:
mzq.clean()

72
app/files/scripts/stix2/misp2stix2.py
View File

@ -20,8 +20,13 @@ import sys, json, os, datetime
import pymisp
import re
import uuid
from stix2 import *
from stix2.base import STIXJSONEncoder
from stix2.exceptions import InvalidValueError, TLPMarkingDefinitionError
from stix2.properties import DictionaryProperty, ListProperty, StringProperty, TimestampProperty
from stix2.v20.common import MarkingDefinition, TLP_WHITE, TLP_GREEN, TLP_AMBER, TLP_RED
from stix2.v20.observables import WindowsPESection
from stix2.v20.sdo import AttackPattern, CourseOfAction, CustomObject, Identity, Indicator, IntrusionSet, Malware, ObservedData, Report, ThreatActor, Tool, Vulnerability
from stix2.v20.sro import Relationship
from misp2stix2_mapping import *
from collections import defaultdict
from copy import deepcopy
@ -48,6 +53,7 @@ class StixBuilder():
self.orgs = []
self.galaxies = []
self.ids = {}
self.custom_objects = {}
def loadEvent(self, args):
pathname = os.path.dirname(args[0])
@ -465,19 +471,24 @@ class StixBuilder():
if markings:
markings = self.handle_tags(markings)
custom_object_args['object_marking_refs'] = markings
@CustomObject(custom_object_type, [('id', properties.StringProperty(required=True)),
('labels', properties.ListProperty(labels, required=True)),
('x_misp_value', properties.StringProperty(required=True)),
('created', properties.TimestampProperty(required=True, precision='millisecond')),
('modified', properties.TimestampProperty(required=True, precision='millisecond')),
('created_by_ref', properties.StringProperty(required=True)),
('object_marking_refs', properties.ListProperty(markings)),
('x_misp_comment', properties.StringProperty()),
('x_misp_category', properties.StringProperty())
])
class Custom(object):
def __init__(self, **kwargs):
return
if custom_object_type not in self.custom_objects:
@CustomObject(custom_object_type, [
('id', StringProperty(required=True)),
('labels', ListProperty(labels, required=True)),
('x_misp_value', StringProperty(required=True)),
('created', TimestampProperty(required=True, precision='millisecond')),
('modified', TimestampProperty(required=True, precision='millisecond')),
('created_by_ref', StringProperty(required=True)),
('object_marking_refs', ListProperty(markings)),
('x_misp_comment', StringProperty()),
('x_misp_category', StringProperty())
])
class Custom(object):
def __init__(self, **kwargs):
return
self.custom_objects[custom_object_type] = Custom
else:
Custom = self.custom_objects[custom_object_type]
custom_object = Custom(**custom_object_args)
self.append_object(custom_object)
@ -603,18 +614,23 @@ class StixBuilder():
'x_misp_category': category, 'created_by_ref': self.identity_id}
if hasattr(misp_object, 'comment') and misp_object.get('comment'):
custom_object_args['x_misp_comment'] = misp_object['comment']
@CustomObject(custom_object_type, [('id', properties.StringProperty(required=True)),
('labels', properties.ListProperty(labels, required=True)),
('x_misp_values', properties.DictionaryProperty(required=True)),
('created', properties.TimestampProperty(required=True, precision='millisecond')),
('modified', properties.TimestampProperty(required=True, precision='millisecond')),
('created_by_ref', properties.StringProperty(required=True)),
('x_misp_comment', properties.StringProperty()),
('x_misp_category', properties.StringProperty())
])
class Custom(object):
def __init__(self, **kwargs):
return
if custom_object_type not in self.custom_objects:
@CustomObject(custom_object_type, [
('id', StringProperty(required=True)),
('labels', ListProperty(labels, required=True)),
('x_misp_values', DictionaryProperty(required=True)),
('created', TimestampProperty(required=True, precision='millisecond')),
('modified', TimestampProperty(required=True, precision='millisecond')),
('created_by_ref', StringProperty(required=True)),
('x_misp_comment', StringProperty()),
('x_misp_category', StringProperty())
])
class Custom(object):
def __init__(self, **kwargs):
return
self.custom_objects[custom_object_type] = Custom
else:
Custom = self.custom_objects[custom_object_type]
custom_object = Custom(**custom_object_args)
self.append_object(custom_object)
@ -656,7 +672,7 @@ class StixBuilder():
observed_data_args.update(self.handle_time_fields(misp_object, timestamp, 'observed-data'))
try:
observed_data = ObservedData(**observed_data_args)
except exceptions.InvalidValueError:
except InvalidValueError:
observed_data = self.fix_enumeration_issues(name, observed_data_args)
self.append_object(observed_data)
@ -733,7 +749,7 @@ class StixBuilder():
'definition': {definition_type: definition}}
try:
self.markings[tag] = MarkingDefinition(**marking_definition)
except exceptions.TLPMarkingDefinitionError:
except (TLPMarkingDefinitionError, ValueError):
return
return marking_id

6
app/files/scripts/stix2misp.py
View File

@ -793,7 +793,7 @@ class StixParser():
@staticmethod
def fetch_uuid(object_id):
try:
return "-".join(object_id.split("-")[-5:])
return uuid.UUID('-'.join(object_id.split("-")[1:]))
except Exception:
return str(uuid.uuid4())
@ -841,7 +841,7 @@ class StixFromMISPParser(StixParser):
for reference in self.event.information_source.references:
self.misp_event.add_attribute(**{'type': 'link', 'value': reference})
if package.ttps:
for ttp in package.ttps.ttps:
for ttp in package.ttps.ttp:
ttp_id = '-'.join((part for part in ttp.id_.split('-')[-5:]))
ttp_type = 'galaxy' if ttp_id in self.galaxies_references else 'object'
self.parse_ttp(ttp, ttp_type, ttp_id)
@ -1205,7 +1205,7 @@ class ExternalStixParser(StixParser):
if self.event.observables:
self.parse_external_observable(self.event.observables.observables)
if self.event.ttps:
self.parse_ttps(self.event.ttps.ttps)
self.parse_ttps(self.event.ttps.ttp)
if self.event.courses_of_action:
self.parse_coa(self.event.courses_of_action)
if self.dns_objects:

2
app/files/warninglists

@ -1 +1 @@
Subproject commit 28687d90d575332776480cd5d683361e7485033c
Subproject commit 2c116cbd1ff50b2ed677ea6af08c8a6bf511a6fd

9
app/webroot/css/event-timeline.css
View File

@ -59,6 +59,15 @@
box-shadow: 0 0 20px rgba(82, 168, 236, 1);`
}
.vis-item.sighting_positive {
background-color: green;
border-color: white;
}
.vis-item.sighting_negative {
background-color: red;
border-color: white;
}
.vis-item.object {
background-color: #3465a4;
border-color: black;

59
app/webroot/js/event-timeline.js
View File

@ -29,10 +29,16 @@ var options = {
return build_object_template(item);
case "object_attribute":
console.log('Error');
console.log('Error: Group not valid');
break;
default:
if (item.className == "sighting_positive" || item.className == "sighting_negative") {
return build_sighting_template(item);
} else {
console.log(item)
console.log('Error: Unkown group');
}
break;
}
},
@ -199,6 +205,13 @@ function build_object_template(obj) {
return html;
}
function build_sighting_template(attr){
var span = $('<span data-itemID="'+attr.id+'">');
span.text(attr.content);
var html = span[0].outerHTML;
return html;
}
function contain_seen_attribute(obj) {
if (obj['Attribute'] === undefined) {
return false;
@ -372,6 +385,8 @@ function map_scope(val) {
return 'seen';
case 'Object relationship':
return 'relationship';
case 'Sightings':
return 'sightings';
default:
return 'seen';
}
@ -400,7 +415,8 @@ function update_badge() {
function reload_timeline() {
update_badge();
var payload = {scope: map_scope($('#select_timeline_scope').val())};
var selectedScope = map_scope($('#select_timeline_scope').val());
var payload = {scope: selectedScope};
$.ajax({
url: "/events/"+"getEventTimeline"+"/"+scope_id+"/"+extended_text+"event.json",
dataType: 'json',
@ -413,6 +429,8 @@ function reload_timeline() {
},
success: function( data, textStatus, jQxhr ){
items_timeline.clear();
mapping_text_to_id = new Map();
var itemIds = {};
for (var item of data.items) {
item.className = item.group;
item.orig_id = item.id;
@ -420,16 +438,45 @@ function reload_timeline() {
set_spanned_time(item);
if (item.group == 'object') {
for (var attr of item.Attribute) {
mapping_text_to_id.set(attr.contentType+': '+attr.content+' ('+item.orig_id+')', item.id);
if (selectedScope == 'sightings') {
var k = attr.contentType+': '+attr.content+' ('+item.orig_id.split('-')[0]+')'
if (!mapping_text_to_id.get(k)) {
mapping_text_to_id.set(k, item.id);
}
} else {
mapping_text_to_id.set(attr.contentType+': '+attr.content+' ('+item.orig_id+')', item.id);
}
adjust_text_length(attr);
}
} else {
mapping_text_to_id.set(item.content+' ('+item.orig_id+')', item.id);
if (selectedScope == 'sightings') {
var k = item.content+' ('+item.orig_id.split('-')[0]+')'
if (!mapping_text_to_id.get(k)) {
mapping_text_to_id.set(k, item.id);
}
} else {
mapping_text_to_id.set(item.content+' ('+item.orig_id+')', item.id);
}
adjust_text_length(item);
}
itemIds[item.attribute_id] = item.content;
if (selectedScope == 'sightings') {
item.group = item.attribute_id;
item.content = '';
}
}
items_timeline.add(data.items);
handle_not_seen_enabled($('#checkbox_timeline_display_hide_not_seen_enabled').prop('checked'), false)
if (selectedScope == 'sightings') {
var groups = Object.keys(itemIds).map(function(id) {
return {id: id, content: itemIds[id]}
})
eventTimeline.setGroups(groups);
eventTimeline.setOptions({selectable: false});
} else {
eventTimeline.setOptions({selectable: true});
eventTimeline.setGroups([]);
}
},
error: function( jqXhr, textStatus, errorThrown ){
console.log( errorThrown );
@ -610,11 +657,11 @@ function init_popover() {
label: "Scope",
tooltip: "The time scope represented by the timeline",
event: function(value) {
if (value == "First seen/Last seen") {
if (value == "First seen/Last seen" || value == "Sightings") {
reload_timeline();
}
},
options: ["First seen/Last seen"],
options: ["First seen/Last seen", "Sightings"],
default: "First seen/Last seen"
});

44
app/webroot/js/misp.js
View File

@ -2113,7 +2113,7 @@ function quickFilter(passedArgs, url) {
var passedArgs = [];
}
if( $('#quickFilterField').val().trim().length > 0){
passedArgs["searchall"] = $('#quickFilterField').val().trim();
passedArgs["searchall"] = encodeURIComponent($('#quickFilterField').val().trim());
for (var key in passedArgs) {
if (key !== 'page') {
url += "/" + key + ":" + passedArgs[key];
@ -2148,7 +2148,7 @@ function runIndexQuickFilter(preserveParams) {
searchKey = $('#quickFilterField').data('searchkey');
}
if ( $('#quickFilterField').val().trim().length > 0){
passedArgsArray[searchKey] = $('#quickFilterField').val().trim();
passedArgsArray[searchKey] = encodeURIComponent($('#quickFilterField').val().trim());
}
url = here;
if (typeof preserveParams !== "undefined") {
@ -2168,7 +2168,7 @@ function executeFilter(passedArgs, url) {
}
function quickFilterTaxonomy(taxonomy_id, passedArgs) {
var url = "/taxonomies/view/" + taxonomy_id + "/filter:" + $('#quickFilterField').val();
var url = "/taxonomies/view/" + taxonomy_id + "/filter:" + encodeURIComponent($('#quickFilterField').val());
window.location.href=url;
}
@ -2176,7 +2176,7 @@ function quickFilterRemoteEvents(passedArgs, id) {
passedArgs["searchall"] = $('#quickFilterField').val();
var url = "/servers/previewIndex/" + id;
for (var key in passedArgs) {
url += "/" + key + ":" + passedArgs[key];
url += "/" + key + ":" + encodeURIComponent(passedArgs[key]);
}
window.location.href=url;
}
@ -2297,28 +2297,28 @@ function indexAddRule(param) {
var found = false;
if (filterContext == 'event') {
if (param.data.param1 == "date") {
var val1 = escape($('#EventSearch' + param.data.param1 + 'from').val());
var val2 = escape($('#EventSearch' + param.data.param1 + 'until').val());
var val1 = encodeURIComponent($('#EventSearch' + param.data.param1 + 'from').val());
var val2 = encodeURIComponent($('#EventSearch' + param.data.param1 + 'until').val());
if (val1 != "") filtering.date.from = val1;
if (val2 != "") filtering.date.until = val2;
} else if (param.data.param1 == "published") {
var value = escape($('#EventSearchpublished').val());
var value = encodeURIComponent($('#EventSearchpublished').val());
if (value != "") filtering.published = value;
} else if (param.data.param1 == "hasproposal") {
var value = escape($('#EventSearchhasproposal').val());
var value = encodeURIComponent($('#EventSearchhasproposal').val());
if (value != "") filtering.hasproposal = value;
} else {
var value = escape($('#EventSearch' + param.data.param1).val());
var operator = operators[escape($('#EventSearchbool').val())];
var value = encodeURIComponent($('#EventSearch' + param.data.param1).val());
var operator = operators[encodeURIComponent($('#EventSearchbool').val())];
if (value != "" && filtering[param.data.param1][operator].indexOf(value) < 0) filtering[param.data.param1][operator].push(value);
}
} else if (filterContext == 'user') {
if (differentFilters.indexOf(param.data.param1) != -1) {
var value = escape($('#UserSearch' + param.data.param1).val());
var value = encodeURIComponent($('#UserSearch' + param.data.param1).val());
if (value != "") filtering[param.data.param1] = value;
} else {
var value = escape($('#UserSearch' + param.data.param1).val());
var operator = operators[escape($('#UserSearchbool').val())];
var value = encodeURIComponent($('#UserSearch' + param.data.param1).val());
var operator = operators[encodeURIComponent($('#UserSearchbool').val())];
if (value != "" && filtering[param.data.param1][operator].indexOf(value) < 0) filtering[param.data.param1][operator].push(value);
}
}
@ -2537,7 +2537,7 @@ function serverSettingSubmitForm(name, setting, id) {
}
function updateOrgCreateImageField(string) {
string = escape(string);
string = encodeURIComponent(string);
$.ajax({
url:'/img/orgs/' + string + '.png',
type:'HEAD',
@ -3605,11 +3605,11 @@ function syncUserSelected() {
function filterAttributes(filter, id) {
url = "/events/viewEventAttributes/" + id;
if(filter === 'value'){
filter = $('#quickFilterField').val().trim();
filter = encodeURIComponent($('#quickFilterField').val().trim());
url += filter.length > 0 ? "/searchFor:" + filter : "";
} else if(filter !== 'all') {
url += "/attributeFilter:" + filter
filter = $('#quickFilterField').val().trim();
filter = encodeURIComponent($('#quickFilterField').val().trim());
url += filter.length > 0 ? "/searchFor:" + filter : "";
}
if (deleted) url += '/deleted:true';
@ -4215,7 +4215,7 @@ function checkRolePerms() {
$('.permFlags').show();
}
if ($("#RolePermSiteAdmin").prop('checked')) {
$('.checkbox').prop('checked', true);
$('.site_admin_enforced').prop('checked', true);
}
}
@ -4473,6 +4473,16 @@ function quickSelect(target) {
}
$(document).ready(function() {
// Show popover for disabled input that contains `data-disabled-reason`.
$('input:disabled[data-disabled-reason]').popover("destroy").popover({
placement: 'right',
html: 'true',
trigger: 'hover',
content: function () {
return $(this).data('disabled-reason');
}
});
$('#quickFilterField').bind("enterKey",function(e){
$('#quickFilterButton').trigger("click");
});

8
build-deb.sh
View File

@ -1,4 +1,12 @@
#!/bin/bash
if [ ! -d ./app/Lib/cakephp/app ]
then
echo "CakePHP has not been pulled."
echo "Make sure all submodules are intialized and updated. Please run:"
echo "git submodule init"
echo "git submodule update"
exit 1
fi
dpkg-buildpackage -b -rfakeroot -us -uc

1811
db_schema.json
View File

File diff suppressed because it is too large Load Diff

6
debian/changelog vendored
View File

@ -1,3 +1,9 @@
misp (2.4.125-1) UNRELEASED; urgency=low
* Bump to release 2.4.125
-- Sebastien Tricaud <sebastien.tricaud@devo.com> Fri, 1 May 2020 17:32:05 -0800
misp (2.4.123-1) UNRELEASED; urgency=low
* Bump to release 2.4.123

1
debian/install vendored
View File

@ -6,3 +6,4 @@ cti-python-stix2 usr/share/misp
PyMISP usr/share/misp
INSTALL/MYSQL.sql usr/share/doc/misp
INSTALL/misp-workers.service usr/share/doc/misp
INSTALL/setup/config.php usr/share/misp/app/Plugin/CakeResque/Config

8
debian/postinst vendored
View File

@ -36,8 +36,6 @@ if [ "$1" = "configure" ] ; then
a2dissite 000-default || true
a2ensite misp.apache2 || true
a2enmod rewrite
a2enmod headers
db_get misp/mariadb_host
HOST=$RET
@ -53,8 +51,8 @@ if [ "$1" = "configure" ] ; then
BASEURL=$RET
db_stop
mysql -h$HOST -uroot -p$ROOTPWD -e "CREATE USER IF NOT EXISTS '$MISPDBUSER'@'localhost' IDENTIFIED BY '$MISPDBUSERPWD';"
mysql -h$HOST -uroot -p$ROOTPWD -e "GRANT ALL PRIVILEGES ON misp.* TO '$MISPDBUSER'@'localhost';"
mysql -h$HOST -uroot -p$ROOTPWD -e "CREATE USER IF NOT EXISTS '$MISPDBUSER'@'%' IDENTIFIED BY '$MISPDBUSERPWD';"
mysql -h$HOST -uroot -p$ROOTPWD -e "GRANT ALL PRIVILEGES ON misp.* TO '$MISPDBUSER'@'%';"
mysql -h$HOST -uroot -p$ROOTPWD -e "FLUSH PRIVILEGES;"
echo "Creating MISP Database..."
mysql -h$HOST -uroot -p$ROOTPWD -e "CREATE DATABASE $MISPDB;" && gunzip < /usr/share/doc/misp/MYSQL.sql.gz | mysql -h$HOST -u$MISPDBUSER -p$MISPDBUSERPWD $MISPDB || true
@ -80,7 +78,7 @@ if [ "$1" = "configure" ] ; then
chmod +x /usr/share/misp/app/Console/worker/start.sh
cat /usr/share/doc/misp/misp-workers.service > /etc/systemd/system/misp-workers.service
sed -i -E "s/\/var\/www\/MISP/\/usr\/share\/misp/" /etc/systemd/system/misp-workers.service
# systemctl daemon-reload
# systemctl enable --now misp-workers
fi

10
debian/preinst vendored Normal file
View File

@ -0,0 +1,10 @@
#!/bin/sh
set -e
#DEBHELPER#
. /usr/share/debconf/confmodule
a2enmod rewrite || true
a2enmod headers || true

4105
docs/Changelog.md
View File

File diff suppressed because it is too large Load Diff

22
docs/INSTALL.rhel7.md
View File

@ -243,9 +243,6 @@ installCoreRHEL () {
cd $PATH_TO_MISP/app/files/scripts/mixbox
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
# FIXME: Remove once stix-fixed
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
# install STIX2.0 library to support STIX 2.0 export:
cd $PATH_TO_MISP/cti-python-stix2
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
@ -297,6 +294,25 @@ installCoreRHEL () {
cd $PATH_TO_MISP/PyMISP
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U .
# FIXME: Remove libfaup etc once the egg has the library baked-in
# BROKEN: This needs to be tested on RHEL/CentOS
##sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
cd /tmp
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
cd gtcaca
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
# Enable dependencies detection in the diagnostics page
# This allows MISP to detect GnuPG, the Python modules' versions and to read the PHP settings.
# The LD_LIBRARY_PATH setting is needed for rh-git218 to work

30
docs/INSTALL.rhel8.md
View File

@ -222,9 +222,6 @@ installCoreRHEL () {
cd $PATH_TO_MISP/app/files/scripts/mixbox
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
# FIXME: Remove once stix-fixed
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
# install STIX2.0 library to support STIX 2.0 export:
cd $PATH_TO_MISP/cti-python-stix2
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
@ -277,6 +274,25 @@ installCoreRHEL () {
cd $PATH_TO_MISP/PyMISP
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U .
# FIXME: Remove libfaup etc once the egg has the library baked-in
# BROKEN: This needs to be tested on RHEL/CentOS
##sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
cd /tmp
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
cd gtcaca
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
# Enable python3 for php-fpm
sudo sed -i.org -e 's/^;\(clear_env = no\)/\1/' /etc/php-fpm.d/www.conf
sudo systemctl restart php-fpm.service
@ -306,10 +322,10 @@ installCake_RHEL ()
sudo chown $WWW_USER:$WWW_USER /usr/share/httpd/.composer
cd $PATH_TO_MISP/app
# Update composer.phar (optional)
$SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === 'baf1608c33254d00611ac1705c1d9958c817a1a33bce370c0595974b342601bd80b92a3f46067da89e3b06bff421f182') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
$SUDO_WWW php composer-setup.php
$SUDO_WWW php -r "unlink('composer-setup.php');"
#$SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
#$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === 'baf1608c33254d00611ac1705c1d9958c817a1a33bce370c0595974b342601bd80b92a3f46067da89e3b06bff421f182') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
#$SUDO_WWW php composer-setup.php
#$SUDO_WWW php -r "unlink('composer-setup.php');"
$SUDO_WWW php composer.phar install
## sudo yum install php-redis -y

21
docs/INSTALL.ubuntu1804.md
View File

@ -20,7 +20,7 @@ bash /tmp/INSTALL.sh -c
### 0/ MISP Ubuntu 18.04-server install - status
-------------------------
!!! notice
Installer tested working by [@SteveClement](https://twitter.com/SteveClement) on 20190513 (works with **Ubuntu 18.10/19.04** too)
Installer tested working by [@SteveClement](https://twitter.com/SteveClement) on 20200501 (works with **Ubuntu 18.10/19.04** too)
!!! notice
This document also serves as a source for the [INSTALL-misp.sh](https://github.com/MISP/MISP/blob/2.4/INSTALL/INSTALL.sh) script.
@ -167,8 +167,6 @@ installCore () {
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
cd $PATH_TO_MISP/app/files/scripts/python-maec
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
# FIXME: Remove once stix-fixed
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
# install STIX2.0 library to support STIX 2.0 export:
cd ${PATH_TO_MISP}/cti-python-stix2
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
@ -176,6 +174,23 @@ installCore () {
# install PyMISP
cd ${PATH_TO_MISP}/PyMISP
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
# FIXME: Remove libfaup etc once the egg has the library baked-in
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
cd /tmp
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
cd gtcaca
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
# install pydeep
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/pydeep.git

21
docs/INSTALL.ubuntu2004.md
View File

@ -20,7 +20,7 @@ bash /tmp/INSTALL.sh -c
### 0/ MISP Ubuntu 20.04-server install - status
-------------------------
!!! notice
Installer tested working by [@SteveClement](https://twitter.com/SteveClement) on 20200427
Installer tested working by [@SteveClement](https://twitter.com/SteveClement) on 20200501
!!! notice
This document also serves as a source for the [INSTALL-misp.sh](https://github.com/MISP/MISP/blob/2.4/INSTALL/INSTALL.sh) script.
@ -167,8 +167,6 @@ installCore () {
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
cd $PATH_TO_MISP/app/files/scripts/python-maec
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
# FIXME: Remove once stix-fixed
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
# install STIX2.0 library to support STIX 2.0 export:
cd ${PATH_TO_MISP}/cti-python-stix2
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
@ -176,6 +174,23 @@ installCore () {
# install PyMISP
cd ${PATH_TO_MISP}/PyMISP
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
# FIXME: Remove libfaup etc once the egg has the library baked-in
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
cd /tmp
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
cd gtcaca
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
# install pydeep
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/pydeep.git

4
docs/generic/mail_to_misp-debian.md
View File

@ -9,8 +9,8 @@ mail2misp () {
cd /usr/local/src/
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
$SUDO_CMD git clone https://github.com/MISP/mail_to_misp.git
$SUDO_CMD git clone git://github.com/stricaud/faup.git faup
$SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
sudo chown -R ${MISP_USER}:${MISP_USER} faup mail_to_misp gtcaca
cd gtcaca
$SUDO_CMD mkdir -p build

21
docs/generic/misp-modules-debian.md
View File

@ -5,15 +5,32 @@
# Main MISP Modules install function
mispmodules () {
cd /usr/local/src/
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
## TODO: checkUsrLocalSrc in main doc
debug "Cloning misp-modules"
$SUDO_CMD git clone https://github.com/MISP/misp-modules.git
cd misp-modules
$SUDO_CMD git clone git://github.com/stricaud/gtcaca.git
$SUDO_CMD git clone git://github.com/stricaud/faup.git
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
# Install gtcaca
cd gtcaca
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
# Install faup
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
cd ../../misp-modules
# some misp-modules dependencies
sudo apt install libpq5 libjpeg-dev tesseract-ocr libpoppler-cpp-dev imagemagick libopencv-dev zbar-tools libzbar0 libzbar-dev libfuzzy-dev -y
# If you build an egg, the user you build it as need write permissions in the CWD
sudo chgrp $WWW_USER .
sudo chmod g+w .
sudo chmod og+w .
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I -r REQUIREMENTS
sudo chgrp staff .
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I .

30
docs/generic/supportFunctions.md
View File

@ -217,18 +217,19 @@ EOF
checkInstaller () {
# Workaround: shasum is not available on RHEL, only checking sha512
if [[ $FLAVOUR == "rhel" ]] || [[ $FLAVOUR == "centos" ]]; then
INSTsum=$(sha512sum ${0} | cut -f1 -d\ )
/usr/bin/wget --no-cache -q -O /tmp/INSTALL.sh.sha512 https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/INSTALL.sh.sha512
INSTsum=$(sha512sum ${0} | cut -f1 -d\ )
/usr/bin/wget --no-cache -q -O /tmp/INSTALL.sh.sha512 https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/INSTALL.sh.sha512
chsum=$(cat /tmp/INSTALL.sh.sha512)
if [[ "${chsum}" == "${INSTsum}" ]]; then
echo "SHA512 matches"
else
echo "SHA512: ${chsum} does not match the installer sum of: ${INSTsum}"
# exit 1 # uncomment when/if PR is merged
fi
if [[ "${chsum}" == "${INSTsum}" ]]; then
echo "SHA512 matches"
else
echo "SHA512: ${chsum} does not match the installer sum of: ${INSTsum}"
# exit 1 # uncomment when/if PR is merged
fi
else
# TODO: Implement $FLAVOUR checks and install depending on the platform we are on
if [[ $(which shasum > /dev/null 2>&1 ; echo $?) != 0 ]]; then
sudo apt update
sudo apt install libdigest-sha-perl -qyy
fi
# SHAsums to be computed, not the -- notatiation is for ease of use with rhash
@ -512,8 +513,7 @@ setBaseURL () {
CONN=$(ip -br -o -4 a |grep UP |head -1 |tr -d "UP")
IFACE=`echo $CONN |awk {'print $1'}`
IP=`echo $CONN |awk {'print $2'}| cut -f1 -d/`
# TODO: Consider "QEMU"
if [[ "$(checkManufacturer)" != "innotek GmbH" ]] && [[ "$(checkManufacturer)" != "VMware, Inc." ]]; then
if [[ "$(checkManufacturer)" != "innotek GmbH" ]] && [[ "$(checkManufacturer)" != "VMware, Inc." ]] && [[ "$(checkManufacturer)" != "QEMU" ]]; then
debug "We guess that this is a physical machine and cannot possibly guess what the MISP_BASEURL might be."
if [[ "$UNATTENDED" != "1" ]]; then
echo "You can now enter your own MISP_BASEURL, if you wish to NOT do that, the MISP_BASEURL will be empty, which will work, but ideally you configure it afterwards."
@ -541,12 +541,20 @@ setBaseURL () {
MISP_BASEURL="https://misp.local"
# Webserver configuration
FQDN='misp.local'
else
elif [[ "$(checkManufacturer)" == "innotek GmbH" ]]; then
MISP_BASEURL='https://localhost:8443'
IP=$(ip addr show | awk '$1 == "inet" {gsub(/\/.*$/, "", $2); print $2}' |grep -v "127.0.0.1" |tail -1)
sudo iptables -t nat -A OUTPUT -p tcp --dport 8443 -j DNAT --to ${IP}:443
# Webserver configuration
FQDN='localhost.localdomain'
elif [[ "$(checkManufacturer)" == "VMware, Inc." ]]; then
MISP_BASEURL='""'
# Webserver configuration
FQDN='misp.local'
else
MISP_BASEURL='""'
# Webserver configuration
FQDN='misp.local'
fi
}

10
docs/xINSTALL.NetBSD.md
View File

@ -233,11 +233,11 @@ sudo -u www HOME=/tmp $PATH_TO_MISP/venv/bin/pip install plyara
# CakePHP is included as a submodule of MISP and has been fetched earlier.
# Install CakeResque along with its dependencies if you intend to use the built in background jobs:
cd $PATH_TO_MISP/app
sudo -u www php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
sudo -u www php -r "if (hash_file('SHA384', 'composer-setup.php') === 'baf1608c33254d00611ac1705c1d9958c817a1a33bce370c0595974b342601bd80b92a3f46067da89e3b06bff421f182') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink
('composer-setup.php'); } echo PHP_EOL;"
sudo -u www env HOME=/tmp php composer-setup.php
sudo -u www php -r "unlink('composer-setup.php');"
#sudo -u www php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
#sudo -u www php -r "if (hash_file('SHA384', 'composer-setup.php') === 'baf1608c33254d00611ac1705c1d9958c817a1a33bce370c0595974b342601bd80b92a3f46067da89e3b06bff421f182') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink
#('composer-setup.php'); } echo PHP_EOL;"
#sudo -u www env HOME=/tmp php composer-setup.php
#sudo -u www php -r "unlink('composer-setup.php');"
sudo -u www HOME=/tmp php composer.phar install
# To use the scheduler worker for scheduled tasks, do the following:

13
docs/xINSTALL.OpenBSD.md
View File

@ -383,9 +383,6 @@ doas /usr/local/virtualenvs/MISP/bin/python setup.py install
cd /var/www/htdocs/MISP/PyMISP
doas /usr/local/virtualenvs/MISP/bin/python setup.py install
# FIXME: Remove once stix-fixed
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
# install support for STIX 2.0
cd /var/www/htdocs/MISP/cti-python-stix2
doas /usr/local/virtualenvs/MISP/bin/python setup.py install
@ -402,11 +399,11 @@ doas /usr/local/virtualenvs/MISP/bin/pip install git+https://github.com/kbandla/
# Install CakeResque along with its dependencies if you intend to use the built in background jobs:
cd /var/www/htdocs/MISP/app
doas mkdir /var/www/.composer ; doas chown www:www /var/www/.composer
EXPECTED_SIGNATURE="$(wget -q -O - https://composer.github.io/installer.sig)"
doas -u www php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
doas -u www php -r "if (hash_file('SHA384', 'composer-setup.php') === '$EXPECTED_SIGNATURE') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
doas -u www env HOME=/var/www php composer-setup.php
doas -u www php -r "unlink('composer-setup.php');"
#EXPECTED_SIGNATURE="$(wget -q -O - https://composer.github.io/installer.sig)"
#doas -u www php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
#doas -u www php -r "if (hash_file('SHA384', 'composer-setup.php') === '$EXPECTED_SIGNATURE') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
#doas -u www env HOME=/var/www php composer-setup.php
#doas -u www php -r "unlink('composer-setup.php');"
doas -u www env HOME=/var/www php composer.phar install
# To use the scheduler worker for scheduled tasks, do the following:

22
docs/xINSTALL.centos6.md
View File

@ -209,9 +209,6 @@ $SUDO_WWW git clone https://github.com/CybOXProject/mixbox.git
cd $PATH_TO_MISP/app/files/scripts/mixbox
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
# FIXME: Remove once stix-fixed
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
# install STIX2.0 library to support STIX 2.0 export:
cd $PATH_TO_MISP/cti-python-stix2
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
@ -220,6 +217,25 @@ $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
cd $PATH_TO_MISP/PyMISP
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
# FIXME: Remove libfaup etc once the egg has the library baked-in
# BROKEN: This needs to be tested on RHEL/CentOS
##sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
cd /tmp
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
cd gtcaca
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
# Enable python3 for php-fpm
echo 'source scl_source enable rh-python36' | sudo tee -a /etc/opt/rh/rh-php70/sysconfig/php-fpm
sudo sed -i.org -e 's/^;\(clear_env = no\)/\1/' /etc/opt/rh/rh-php70/php-fpm.d/www.conf

22
docs/xINSTALL.centos7.md
View File

@ -196,9 +196,6 @@ $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U python-magic git+https://github.
cd $PATH_TO_MISP/app/files/scripts/mixbox
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
# FIXME: Remove once stix-fixed
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
# install STIX2.0 library to support STIX 2.0 export:
cd $PATH_TO_MISP/cti-python-stix2
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
@ -207,6 +204,25 @@ $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
cd $PATH_TO_MISP/PyMISP
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
# FIXME: Remove libfaup etc once the egg has the library baked-in
# BROKEN: This needs to be tested on RHEL/CentOS
##sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
cd /tmp
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
cd gtcaca
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
# Enable dependencies detection in the diagnostics page
# This allows MISP to detect GnuPG, the Python modules' versions and to read the PHP settings.
echo "env[PATH] =/opt/rh/rh-php72/root/usr/bin:/usr/local/bin:/usr/bin:/bin" |sudo tee -a /etc/opt/rh/rh-php72/php-fpm.d/www.conf

37
docs/xINSTALL.debian10.md
View File

@ -1,16 +1,12 @@
# INSTALLATION INSTRUCTIONS
## for Debian 10.2 "buster"
## for Debian 10.3 "buster"
### 0/ MISP debian stable install - Status
------------------------------------
!!! notice
This is mostly the install [@SteveClement](https://twitter.com/SteveClement) uses for testing, qc and random development.
Maintained and tested by @SteveClement on 20191122
!!! warning
PHP 7.3.4-2 is not working at the moment with the packaged composer.phar<br />
You need to manually update composer.phar as outlined below.
Maintained and tested by @SteveClement on 20200405
{!generic/known-issues-debian.md!}
@ -64,7 +60,7 @@ sudo postfix reload
#### Install all the dependencies (some might already be installed)
```bash
sudo apt install -y \
sudo apt install \
curl gcc git gnupg-agent make openssl redis-server neovim zip libyara-dev \
python3-setuptools python3-dev python3-pip python3-redis python3-zmq virtualenv \
mariadb-client \
@ -158,8 +154,6 @@ cd $PATH_TO_MISP/app/files/scripts/python-stix
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
cd $PATH_TO_MISP/app/files/scripts/python-maec
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
# FIXME: Remove once stix-fixed
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
# install STIX2.0 library to support STIX 2.0 export:
cd ${PATH_TO_MISP}/cti-python-stix2
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
@ -167,6 +161,23 @@ $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
# install PyMISP
cd $PATH_TO_MISP/PyMISP
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
# FIXME: Remove libfaup etc once the egg has the library baked-in
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
cd /tmp
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
cd gtcaca
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
# install pydeep
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/pydeep.git
@ -199,10 +210,10 @@ cd $PATH_TO_MISP/app
# Make composer cache happy
sudo mkdir /var/www/.composer ; sudo chown $WWW_USER:$WWW_USER /var/www/.composer
# Update composer.phar
$SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === 'baf1608c33254d00611ac1705c1d9958c817a1a33bce370c0595974b342601bd80b92a3f46067da89e3b06bff421f182') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
$SUDO_WWW php composer-setup.php
$SUDO_WWW php -r "unlink('composer-setup.php');"
#$SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
#$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === 'baf1608c33254d00611ac1705c1d9958c817a1a33bce370c0595974b342601bd80b92a3f46067da89e3b06bff421f182') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
#$SUDO_WWW php composer-setup.php
#$SUDO_WWW php -r "unlink('composer-setup.php');"
$SUDO_WWW php composer.phar install
# The following is potentially not needed, but just here in case of Keyboard/Chair failures
$SUDO_WWW php composer.phar update

19
docs/xINSTALL.debian9.md
View File

@ -183,8 +183,6 @@ cd $PATH_TO_MISP/app/files/scripts/python-stix
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
cd $PATH_TO_MISP/app/files/scripts/python-maec
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
# FIXME: Remove once stix-fixed
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
# install STIX2.0 library to support STIX 2.0 export:
cd ${PATH_TO_MISP}/cti-python-stix2
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
@ -192,6 +190,23 @@ $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
# install PyMISP
cd $PATH_TO_MISP/PyMISP
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
# FIXME: Remove libfaup etc once the egg has the library baked-in
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
cd /tmp
[[ ! -d "faup" ]] && $SUDO_CMD git clone git://github.com/stricaud/faup.git faup
[[ ! -d "gtcaca" ]] && $SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
cd gtcaca
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
$SUDO_CMD mkdir -p build
cd build
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
# install pydeep
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/pydeep.git

10
preinst Normal file
View File

@ -0,0 +1,10 @@
#!/bin/sh
set -e
#DEBHELPER#
. /usr/share/debconf/confmodule
a2enmod rewrite || true
a2enmod headers || true

226
tools/misp-config Executable file
View File

@ -0,0 +1,226 @@
#!/usr/bin/env perl
#
# Author: Sebastien Tricaud <sebastien@honeynet.org>
# This script is written in Perl because it is default to most distributions
# without any dependency.
#
# MISP Configuration is handled mostly via its UI. This script helps to configure
# MISP for anything that can be configured. For now this is mostly CakePHP stuff.
#
# How to use?
# 1) dump the configuration options that are already set:
# misp-config --create
# -> Will dump in /etc/misp/misp.conf.d/ unless to add a --prefix option, such as:
# misp-config --misp-path /var/www/MISP --prefix /var/www/MISP/etc/ --create
#
# 2) set the configuration manually from /etc/misp/misp.conf.d/ with your favorite text editor (emacs)
# then load to MISP using:
# misp-config --apply
# Of course, if your MISP instance is installed in /var/www/MISP you should do:
# misp-config --misp-path /var/www/MISP --apply
#
# Note there is a configuration, Internal.conf, which where CakePHP has no control, for example
# it will change the source code where there was a lack of configuration capability.
#
use strict;
use warnings;
use File::Path qw(make_path);
my $misp_user="www-data";
my $misp_path="/usr/share/misp";
my $misp_config_path="/etc/misp/";
my $prefix="";
my $create=0;
my $apply=0;
my %CATEGORIES_CREATED = ();
sub print_help_exit {
print "\nUsage: misp-config --user www-data
\t --misp-path /usr/share/misp
\t --prefix /usr/local
\t --apply: Applies configuration from $misp_config_path
\t --create: Create configuration from all available options\n\n";
exit;
}
if ($#ARGV < 0) {
print_help_exit();
}
sub replace_inplace_content {
my ($filename, $buftoreplace, $replacement) = @_;
open my $fp, "<:encoding(UTF-8)", $filename or die "unable to open $filename!";
local $/ = undef;
my $data = <$fp>;
close $fp;
$data =~ s/$buftoreplace/$replacement/g;
open $fp, ">:encoding(UTF-8)", $filename or die "Could not write to '$filename'";
print $fp $data;
close $fp;
}
while (my ($i, $arg) = each @ARGV) {
if ($arg eq "--user") {
$misp_user = $ARGV[$i+1];
}
if ($arg eq "--misp-path") {
$misp_path = $ARGV[$i+1];
}
if ($arg eq "--prefix") {
$prefix = $ARGV[$i+1];
$misp_config_path = "$prefix/$misp_config_path"
}
if ($arg eq "--create") {
$create = 1;
}
if ($arg eq "--apply") {
$apply = 1;
}
if ($arg eq "--help") {
print_help_exit();
}
}
if ( ! $apply && ! $create ) {
print "Argument missing: Please use either --apply or --create.\n";
exit;
}
if ( $apply && $create ) {
print "Argument error: Cannot use both --apply and --create.\n";
exit;
}
if ($create) {
my $misp_config_path = "$misp_config_path/misp.conf.d";
make_path($misp_config_path);
unless(-e "$misp_config_path/Internal.conf") {
open(my $internalfh, ">", "$misp_config_path/Internal.conf");
print $internalfh "redis_host localhost\n";
close $internalfh;
}
die "foo";
my $settings = `sudo -u $misp_user $misp_path/app/Console/cake admin getSetting all`;
my @lines = split("\n",$settings);
my $value = 0;
my $setting = 0;
foreach(@lines) {
if ($_ =~ m/\"value\": (.*)/) {
$value = substr $1, 0, -1;
}
if ($_ =~ m/\"setting\": \"(.*)\"/) {
$setting = $1;
}
if ($value && $setting) {
#print "$setting -> $value\n";
my $category = "";
my $key = "";
my $section = "";
# Plugin.S3_aws_secret_key -> false
# ^ ^
# | |- Section (S3)
# | |- Key (S3_aws_secret_key)
# |-- Category
#
# We also have:
# Proxy.host -> ""
# Which won't create a section because of the missing _
if ($setting =~ m/(\w+)\.(\w+)/) {
$category = $1;
$key = $2;
# print "Setting category: $1 and key: $2\n";
if ($key =~ m/(.*?)_/) {
$section = $1
# print "Section: $1\n";
} else { # There is no _, so the section is the key
$section = $key;
}
} else {
# Orphan settings are debug settings:
$category = "Debug";
$key = $setting;
# print "Orphan setting:$setting\n";
}
#
# We have all we needed, we can write the files
#
my $conf_to_write = "$misp_config_path/$category.conf";
unless($CATEGORIES_CREATED{"$category"}) {
if (-e $conf_to_write) {
die "We cannot overwrite $conf_to_write, it already exists. Remove it manually.\n";
}
}
$CATEGORIES_CREATED{"$category"} = 1;
open(my $fh, ">>", "$conf_to_write");
print $fh "#$key $value\n";
close $fh;
# Unset those two variables to be filled again!
$value = 0;
$setting = 0;
}
}
}
if ($apply) {
my $misp_config_path = "$misp_config_path/misp.conf.d";
unless(-d $misp_config_path) {
die "MISP Configuration Path does not exists: $misp_config_path\n";
}
foreach my $fp (glob("$misp_config_path/*.conf")) {
my $category = substr($fp, length($misp_config_path)+1, -5);
my $cake_config = $category eq "Internal" ? 0 : 1;
# print "$category:$cake_config\n";
# print "$fp\n";
open(my $fh, "<", $fp) or die "$!";
while (my $line = <$fh>) {
chomp $line;
$line =~ /(\S+) (.*)/;
my $key = "";
my $config_el = $1;
my $first_config_char = substr($config_el, 0, 1);
my $is_comment = 0;
if ($first_config_char eq "#") {
$is_comment = 1;
}
# print("config element: $config_el\n");
if ($category eq "Debug") {
# The Debug category does not... have a category
$key = $config_el;
} elsif ($category eq "Internal") {
# Internal means we are patching portions of the code to configure
if ($config_el eq "redis_host") {
replace_inplace_content("$misp_path/app/Lib/Tools/PubSubTool.php", "localhost", $2);
replace_inplace_content("$misp_path/app/Plugin/CakeResque/Config/config.php", "localhost", $2);
replace_inplace_content("$misp_path/app/Vendor/kamisama/php-resque-ex/lib/Resque.php", "localhost", $2);
replace_inplace_content("$misp_path/app/Vendor/monolog/monolog/src/Monolog/Handler/RedisHandler.php", "localhost", $2);
}
} else {
$key = "$category.$config_el";
}
if ($cake_config && !$is_comment) {
# print("sudo -u $misp_user $misp_path/app/Console/cake admin setSetting $key $2\n");
system("sudo -u $misp_user $misp_path/app/Console/cake admin setSetting $key $2");
}
}
close($fh);
}
}

6
travis/email.php Normal file
View File

@ -0,0 +1,6 @@
<?php
class EmailConfig {
public $default = array(
'transport' => 'Debug',
);
}