MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge remote-tracking branch 'origin/2.4' into pr-selective_import_v2

pull/5540/head
mokaddem 2020-01-23 14:58:19 +01:00
parent 1c6be34bb1 1e49cc6aab
commit e3229c423e
No known key found for this signature in database
GPG Key ID: 164C473F627A06FA
112 changed files with 9127 additions and 744 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
.travis.yml
View File

@ -1,9 +1,9 @@
language: php
php:
- 7.1
- 7.2
- 7.3
- 7.4
- nightly
services:
@ -11,7 +11,6 @@ services:
sudo: required
dist: bionic
group: edge
addons:
mariadb: '10.2'
@ -96,7 +95,6 @@ install:
- sudo usermod -a -G www-data $USER
- sudo -E su $USER -c 'app/Console/cake userInit -q | sudo tee ./key.txt'
- sudo -E su $USER -c 'app/Console/cake Admin runUpdates'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "/usr/bin/python3"'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.autoRegenerate" 0'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.timeout" 600'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.cookieTimeout" 3600'
@ -123,27 +121,18 @@ install:
- app/Console/worker/start.sh &
- sleep 10
# Dirty install python stuff
- virtualenv -p python3.6 ./venv
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$TRAVIS_BUILD_DIR/venv/bin/python"'
- . ./venv/bin/activate
- pushd cti-python-stix2
- sudo pip3 install .
- popd
- pushd app/files/scripts
- sudo -H -u $USER git clone https://github.com/CybOXProject/python-cybox.git
- sudo -H -u $USER git clone https://github.com/STIXProject/python-stix.git
- sudo -H -u $USER git clone https://github.com/MAECProject/python-maec.git
- sudo -H -u $USER git clone https://github.com/CybOXProject/mixbox.git
- pushd mixbox
- sudo pip3 install .
- popd
- pushd python-cybox
- sudo pip3 install .
- popd
- pushd python-stix
- sudo pip3 install .
- popd
- pushd python-maec
- sudo pip3 install .
- pip install .
- popd
- pushd PyMISP
- pip install .[fileobjects]
- popd
- pip install stix zmq redis plyara
- pip install -I antlr4-python3-runtime==4.7.2 # antlr4-python3-runtime==4.8 causes issues
- deactivate
before_script:
- curl http://misp.local
@ -164,7 +153,6 @@ script:
- git submodule update
- pipenv install -d
- pipenv run python tests/testlive_comprehensive.py
- pipenv run python tests/test.py
- pipenv run python tests/test_mispevent.py
- popd
- cp PyMISP/tests/keys.py PyMISP/examples/events/

87
INSTALL/INSTALL.sh
View File

@ -148,9 +148,9 @@ MISPvars () {
# sudo config to run $LUSER commands
if [[ "$(groups ${MISP_USER} |grep -o 'staff')" == "staff" ]]; then
SUDO_USER="sudo -H -u ${MISP_USER} -g staff"
SUDO_CMD="sudo -H -u ${MISP_USER} -g staff"
else
SUDO_USER="sudo -H -u ${MISP_USER}"
SUDO_CMD="sudo -H -u ${MISP_USER}"
fi
SUDO_WWW="sudo -H -u ${WWW_USER} "
@ -544,12 +544,12 @@ checkID () {
sudo adduser $MISP_USER $WWW_USER
fi
# FIXME: the below SUDO_USER check is a duplicate from global variables, try to have just one check
# FIXME: the below SUDO_CMD check is a duplicate from global variables, try to have just one check
# sudo config to run $LUSER commands
if [[ "$(groups ${MISP_USER} |grep -o 'staff')" == "staff" ]]; then
SUDO_USER="sudo -H -u ${MISP_USER} -g staff"
SUDO_CMD="sudo -H -u ${MISP_USER} -g staff"
else
SUDO_USER="sudo -H -u ${MISP_USER}"
SUDO_CMD="sudo -H -u ${MISP_USER}"
fi
}
@ -953,7 +953,8 @@ composer73 () {
# Update composer.phar
# If hash changes, check here: https://getcomposer.org/download/ and replace with the correct one
# Current Sum for: v1.8.3
SHA384_SUM='48e3236262b34d30969dca3c37281b3b4bbe3221bda826ac6a9a62d6444cdb0dcd0615698a5cbe587c3f0fe57a54d8f5'
SHA384_SUM="$(wget -q -O - https://composer.github.io/installer.sig)"
php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
$SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === '$SHA384_SUM') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); exit(137); } echo PHP_EOL;"
checkFail "composer.phar checksum failed, please investigate manually. " $?
@ -1008,11 +1009,12 @@ nuke () {
# Final function to let the user know what happened
theEnd () {
space
echo "Admin (root) DB Password: $DBPASSWORD_ADMIN" |$SUDO_USER tee /home/${MISP_USER}/mysql.txt
echo "User (misp) DB Password: $DBPASSWORD_MISP" |$SUDO_USER tee -a /home/${MISP_USER}/mysql.txt
echo "Authkey: $AUTH_KEY" |$SUDO_USER tee -a /home/${MISP_USER}/MISP-authkey.txt
echo "Admin (root) DB Password: $DBPASSWORD_ADMIN" |$SUDO_CMD tee /home/${MISP_USER}/mysql.txt
echo "User (misp) DB Password: $DBPASSWORD_MISP" |$SUDO_CMD tee -a /home/${MISP_USER}/mysql.txt
echo "Authkey: $AUTH_KEY" |$SUDO_CMD tee -a /home/${MISP_USER}/MISP-authkey.txt
clear
# Commenting out, see: https://github.com/MISP/MISP/issues/5368
# clear -x
space
echo -e "${LBLUE}MISP${NC} Installed, access here: ${MISP_BASEURL}"
echo
@ -1224,7 +1226,7 @@ EOF
}
apacheConfig () {
debug "Generating Apache config"
debug "Generating Apache config, if this hangs, make sure you have enough entropy (install: haveged or wait)"
sudo cp ${PATH_TO_MISP}/INSTALL/apache.24.misp.ssl /etc/apache2/sites-available/misp-ssl.conf
if [[ ! -z ${MISP_BASEURL} ]] && [[ "$(echo $MISP_BASEURL|cut -f 1 -d :)" == "http" || "$(echo $MISP_BASEURL|cut -f 1 -d :)" == "https" ]]; then
@ -1293,6 +1295,8 @@ installCore () {
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
cd $PATH_TO_MISP/app/files/scripts/python-maec
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
# FIXME: Remove once stix-fixed
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
# install STIX2.0 library to support STIX 2.0 export:
cd ${PATH_TO_MISP}/cti-python-stix2
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
@ -1607,7 +1611,7 @@ mispmodules () {
cd /usr/local/src/
## TODO: checkUsrLocalSrc in main doc
debug "Cloning misp-modules"
$SUDO_USER git clone https://github.com/MISP/misp-modules.git
$SUDO_CMD git clone https://github.com/MISP/misp-modules.git
cd misp-modules
# some misp-modules dependencies
sudo apt install libpq5 libjpeg-dev tesseract-ocr libpoppler-cpp-dev imagemagick libopencv-dev zbar-tools libzbar0 libzbar-dev libfuzzy-dev -y
@ -1762,39 +1766,39 @@ mail2misp () {
debug "Installing Mail2${LBLUE}MISP${NC}"
cd /usr/local/src/
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
$SUDO_USER git clone https://github.com/MISP/mail_to_misp.git
$SUDO_USER git clone git://github.com/stricaud/faup.git faup
$SUDO_USER git clone git://github.com/stricaud/gtcaca.git gtcaca
$SUDO_CMD git clone https://github.com/MISP/mail_to_misp.git
$SUDO_CMD git clone git://github.com/stricaud/faup.git faup
$SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
sudo chown -R ${MISP_USER}:${MISP_USER} faup mail_to_misp gtcaca
cd gtcaca
$SUDO_USER mkdir -p build
$SUDO_CMD mkdir -p build
cd build
$SUDO_USER cmake .. && $SUDO_USER make
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
$SUDO_USER mkdir -p build
$SUDO_CMD mkdir -p build
cd build
$SUDO_USER cmake .. && $SUDO_USER make
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
cd ../../mail_to_misp
$SUDO_USER virtualenv -p python3 venv
$SUDO_USER ./venv/bin/pip install https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
$SUDO_USER ./venv/bin/pip install -r requirements.txt
$SUDO_USER cp mail_to_misp_config.py-example mail_to_misp_config.py
$SUDO_CMD virtualenv -p python3 venv
$SUDO_CMD ./venv/bin/pip install https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
$SUDO_CMD ./venv/bin/pip install -r requirements.txt
$SUDO_CMD cp mail_to_misp_config.py-example mail_to_misp_config.py
##$SUDO cp mail_to_misp_config.py-example mail_to_misp_config.py
$SUDO_USER sed -i "s/^misp_url\ =\ 'YOUR_MISP_URL'/misp_url\ =\ 'https:\/\/localhost'/g" /usr/local/src/mail_to_misp/mail_to_misp_config.py
$SUDO_USER sed -i "s/^misp_key\ =\ 'YOUR_KEY_HERE'/misp_key\ =\ '${AUTH_KEY}'/g" /usr/local/src/mail_to_misp/mail_to_misp_config.py
$SUDO_CMD sed -i "s/^misp_url\ =\ 'YOUR_MISP_URL'/misp_url\ =\ 'https:\/\/localhost'/g" /usr/local/src/mail_to_misp/mail_to_misp_config.py
$SUDO_CMD sed -i "s/^misp_key\ =\ 'YOUR_KEY_HERE'/misp_key\ =\ '${AUTH_KEY}'/g" /usr/local/src/mail_to_misp/mail_to_misp_config.py
}
ssdeep () {
debug "Install ssdeep 2.14.1"
cd /usr/local/src
$SUDO_USER wget https://github.com/ssdeep-project/ssdeep/releases/download/release-2.14.1/ssdeep-2.14.1.tar.gz
$SUDO_USER tar zxvf ssdeep-2.14.1.tar.gz
$SUDO_CMD wget https://github.com/ssdeep-project/ssdeep/releases/download/release-2.14.1/ssdeep-2.14.1.tar.gz
$SUDO_CMD tar zxvf ssdeep-2.14.1.tar.gz
cd ssdeep-2.14.1
$SUDO_USER ./configure --datadir=/usr --prefix=/usr --localstatedir=/var --sysconfdir=/etc
$SUDO_USER make
$SUDO_CMD ./configure --datadir=/usr --prefix=/usr --localstatedir=/var --sysconfdir=/etc
$SUDO_CMD make
sudo make install
#installing ssdeep_php
@ -1822,25 +1826,25 @@ viper () {
fi
fi
echo "Cloning Viper"
$SUDO_USER git clone https://github.com/viper-framework/viper.git
$SUDO_USER git clone https://github.com/viper-framework/viper-web.git
$SUDO_CMD git clone https://github.com/viper-framework/viper.git
$SUDO_CMD git clone https://github.com/viper-framework/viper-web.git
sudo chown -R $MISP_USER:$MISP_USER viper
sudo chown -R $MISP_USER:$MISP_USER viper-web
cd viper
echo "Creating virtualenv"
$SUDO_USER virtualenv -p python3 venv
$SUDO_CMD virtualenv -p python3 venv
echo "Submodule update"
# TODO: Check for current user install permissions
$SUDO_USER git submodule update --init --recursive
$SUDO_CMD git submodule update --init --recursive
echo "pip install deps"
$SUDO_USER ./venv/bin/pip install pefile olefile jbxapi Crypto pypdns pypssl r2pipe pdftools virustotal-api SQLAlchemy PrettyTable python-magic scrapy https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
$SUDO_USER ./venv/bin/pip install .
$SUDO_CMD ./venv/bin/pip install pefile olefile jbxapi Crypto pypdns pypssl r2pipe pdftools virustotal-api SQLAlchemy PrettyTable python-magic scrapy https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
$SUDO_CMD ./venv/bin/pip install .
echo 'update-modules' |/usr/local/src/viper/venv/bin/viper
cd /usr/local/src/viper-web
$SUDO_USER sed -i '1 s/^.*$/\#!\/usr\/local\/src\/viper\/venv\/bin\/python/' viper-web
$SUDO_USER /usr/local/src/viper/venv/bin/pip install -r requirements.txt
$SUDO_CMD sed -i '1 s/^.*$/\#!\/usr\/local\/src\/viper\/venv\/bin\/python/' viper-web
$SUDO_CMD /usr/local/src/viper/venv/bin/pip install -r requirements.txt
echo "Launching viper-web"
$SUDO_USER /usr/local/src/viper-web/viper-web -p 8888 -H 0.0.0.0 &
$SUDO_CMD /usr/local/src/viper-web/viper-web -p 8888 -H 0.0.0.0 &
echo 'PATH="/home/misp/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/usr/local/src/viper:/var/www/MISP/app/Console"' |sudo tee /etc/environment
echo ". /etc/environment" >> /home/${MISP_USER}/.profile
@ -1852,8 +1856,8 @@ viper () {
fi
echo "Setting misp_url/misp_key"
$SUDO_USER sed -i "s/^misp_url\ =/misp_url\ =\ http:\/\/localhost/g" ${VIPER_HOME}/viper.conf
$SUDO_USER sed -i "s/^misp_key\ =/misp_key\ =\ $AUTH_KEY/g" ${VIPER_HOME}/viper.conf
$SUDO_CMD sed -i "s/^misp_url\ =/misp_url\ =\ http:\/\/localhost/g" ${VIPER_HOME}/viper.conf
$SUDO_CMD sed -i "s/^misp_key\ =/misp_key\ =\ $AUTH_KEY/g" ${VIPER_HOME}/viper.conf
# Reset admin password to: admin/Password1234
echo "Fixing admin.db with default password"
VIPER_COUNT=0
@ -2004,7 +2008,7 @@ installSupported () {
# TODO: Double check how the user is added and subsequently used during the install.
# TODO: Work on possibility to install as user X and install MISP for user Y
# TODO: Check if logout needed. (run SUDO_USER in installer)
# TODO: Check if logout needed. (run SUDO_CMD in installer)
# <snippet-begin add-user.sh>
# TODO: Double check how to properly handle postfix
# <snippet-begin postfix.sh>
@ -2228,6 +2232,7 @@ installMISPonKali () {
# install STIX2.0 library to support STIX 2.0 export
debug "Installing cti-python-stix2"
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I antlr4-python3-runtime==4.7.2 2> /dev/null > /dev/null
cd ${PATH_TO_MISP}/cti-python-stix2
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I . 2> /dev/null > /dev/null

6
INSTALL/INSTALL.sh.sfv
View File

@ -1,5 +1,5 @@
; Generated by RHash v1.3.8 on 2019-11-24 at 17:30.46
; Generated by RHash v1.3.9 on 2020-01-21 at 17:22.04
; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
;
; 99082 17:30.46 2019-11-24 INSTALL.sh
INSTALL.sh 5BB0CEB0AB45AF769C8A3B044F9A494E8733B1CB 9402BCF66DD2C8A82B8871C5C414A5710D5FAA0B1AD40BB0EDEC57A8883F52F7 616975D3EC3CA34C590570F272AC244535ECECCF535B66AA765B4E36C68E78649E65E5D719977D18B6D69FF59F709CC0 5BAA423F8306B0B2E16FC91380E1E551550F31ED013A38233B049040AAB81579BAD4E3C81203C0EC324BC17010BC1063E1D67BDDF45A922EC7CEC3A551AA49EE
; 99445 17:22.04 2020-01-21 INSTALL.sh
INSTALL.sh 6E1976E8429DE5E28AD7B4F286904F0D6AE278FE 0032F7224EF5CAC309A1E3EDC4D185D6014BAE6DCCC07FBD12528D7574C6167E 1EEEA5E9C5FC2DDF6198442E8B301CB30DEC443798BD597B9D6DA3C8FD5F1C77AB5EB0F44988876BBA7314B3BEC4C297 8C950F3C6A5A8D5CCCC298551555CDC7C631E2A93177DF7BEA2E5C0A80D651C2D459A4328832BE96CDEB874C3CC2214FAB9DA0FA1B97DD642E7488ADB558E748

2
INSTALL/INSTALL.sh.sha1
View File

@ -1 +1 @@
5bb0ceb0ab45af769c8a3b044f9a494e8733b1cb INSTALL.sh
6e1976e8429de5e28ad7b4f286904f0d6ae278fe INSTALL.sh

2
INSTALL/INSTALL.sh.sha256
View File

@ -1 +1 @@
9402bcf66dd2c8a82b8871c5c414a5710d5faa0b1ad40bb0edec57a8883f52f7 INSTALL.sh
0032f7224ef5cac309a1e3edc4d185d6014bae6dccc07fbd12528d7574c6167e INSTALL.sh

2
INSTALL/INSTALL.sh.sha384
View File

@ -1 +1 @@
616975d3ec3ca34c590570f272ac244535ececcf535b66aa765b4e36c68e78649e65e5d719977d18b6d69ff59f709cc0 INSTALL.sh
1eeea5e9c5fc2ddf6198442e8b301cb30dec443798bd597b9d6da3c8fd5f1c77ab5eb0f44988876bba7314b3bec4c297 INSTALL.sh

2
INSTALL/INSTALL.sh.sha512
View File

@ -1 +1 @@
5baa423f8306b0b2e16fc91380e1e551550f31ed013a38233b049040aab81579bad4e3c81203c0ec324bc17010bc1063e1d67bddf45a922ec7cec3a551aa49ee INSTALL.sh
8c950f3c6a5a8d5cccc298551555cdc7c631e2a93177df7bea2e5c0a80d651c2d459a4328832be96cdeb874c3cc2214fab9da0fa1b97dd642e7488adb558e748 INSTALL.sh

3
INSTALL/INSTALL.tpl.sh
View File

@ -220,7 +220,7 @@ installSupported () {
# TODO: Double check how the user is added and subsequently used during the install.
# TODO: Work on possibility to install as user X and install MISP for user Y
# TODO: Check if logout needed. (run SUDO_USER in installer)
# TODO: Check if logout needed. (run SUDO_CMD in installer)
# <snippet-begin add-user.sh>
# TODO: Double check how to properly handle postfix
# <snippet-begin postfix.sh>
@ -444,6 +444,7 @@ installMISPonKali () {
# install STIX2.0 library to support STIX 2.0 export
debug "Installing cti-python-stix2"
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I antlr4-python3-runtime==4.7.2 2> /dev/null > /dev/null
cd ${PATH_TO_MISP}/cti-python-stix2
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I . 2> /dev/null > /dev/null

BIN
INSTALL/screenshots/sh-misp-attack.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 218 KiB

BIN
INSTALL/screenshots/sh-misp-decaying-of-indicators.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 160 KiB

BIN
INSTALL/screenshots/sh-misp-eventgraph.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 248 KiB

BIN
INSTALL/screenshots/sh-misp-threat-actor-attack.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 215 KiB

2
PyMISP

@ -1 +1 @@
Subproject commit a26a8e450b14d48bb0c8ef46b32bff2f1eadc514
Subproject commit eabc6481d039940ad30b7342a477ccf56c878419

2
VERSION.json
View File

@ -1 +1 @@
{"major":2, "minor":4, "hotfix":119}
{"major":2, "minor":4, "hotfix":120}

9
app/Console/Command/AdminShell.php
View File

@ -516,18 +516,19 @@ class AdminShell extends AppShell
public function dumpCurrentDatabaseSchema()
{
$dbActualSchema = $this->Server->getActualDBSchema()['schema'];
$dbActualSchema = $this->Server->getActualDBSchema();
$dbVersion = $this->AdminSetting->find('first', array(
'conditions' => array('setting' => 'db_version')
));
if (!empty($dbVersion) && !empty($dbActualSchema)) {
if (!empty($dbVersion) && !empty($dbActualSchema['schema'])) {
$dbVersion = $dbVersion['AdminSetting']['value'];
$data = array(
'schema' => $dbActualSchema,
'schema' => $dbActualSchema['schema'],
'indexes' => $dbActualSchema['indexes'],
'db_version' => $dbVersion
);
$file = new File(ROOT . DS . 'db_schema.json', true);
$file->write(json_encode($data));
$file->write(json_encode($data, JSON_PRETTY_PRINT));
$file->close();
echo __("> Database schema dumped on disk") . PHP_EOL;
} else {

5
app/Console/Command/ServerShell.php
View File

@ -6,7 +6,7 @@ class ServerShell extends AppShell
{
public $uses = array('Server', 'Task', 'Job', 'User', 'Feed');
public function list() {
public function listServers() {
$res = ['servers'=>[]];
$servers = $this->Server->find('all', [
@ -38,6 +38,9 @@ class ServerShell extends AppShell
}
$userId = $this->args[0];
$user = $this->User->getAuthUser($userId);
if (empty($user)) {
die('User ID do not match an existing user.' . PHP_EOL);
}
if (empty($this->args[1])) die();
$serverId = $this->args[1];
if (!empty($this->args[2])) {

17
app/Controller/AppController.php
View File

@ -46,10 +46,12 @@ class AppController extends Controller
public $helpers = array('Utility', 'OrgImg', 'FontAwesome', 'UserName');
private $__queryVersion = '94';
public $pyMispVersion = '2.4.119';
public $phpmin = '7.0';
public $phprec = '7.2';
private $__queryVersion = '97';
public $pyMispVersion = '2.4.120';
public $phpmin = '7.2';
public $phprec = '7.4';
public $pythonmin = '3.6';
public $pythonrec = '3.7';
public $isApiAuthed = false;
public $baseurl = '';
@ -289,7 +291,9 @@ class AppController extends Controller
if ($this->Auth->user()) {
// update script
$this->{$this->modelClass}->runUpdates();
if ($this->Auth->user('Role')['perm_site_admin'] || (Configure::read('MISP.live') && !$this->_isRest())) {
$this->{$this->modelClass}->runUpdates();
}
$user = $this->Auth->user();
if (!isset($user['force_logout']) || $user['force_logout']) {
$this->loadModel('User');
@ -1188,6 +1192,9 @@ class AppController extends Controller
if ($returnFormat === 'download') {
$returnFormat = 'json';
}
if ($returnFormat === 'stix' && $this->_isJson()) {
$returnFormat = 'stix-json';
}
$elementCounter = 0;
$renderView = false;
$final = $this->$scope->restSearch($user, $returnFormat, $filters, false, false, $elementCounter, $renderView);

21
app/Controller/AttributesController.php
View File

@ -864,7 +864,7 @@ class AttributesController extends AppController
$skipTimeCheck = true;
}
if ($skipTimeCheck || $this->request->data['Attribute']['timestamp'] > $existingAttribute['Attribute']['timestamp']) {
$recoverFields = array('value', 'to_ids', 'distribution', 'category', 'type', 'comment');
$recoverFields = array('value', 'to_ids', 'distribution', 'category', 'type', 'comment', 'first_seen', 'last_seen');
foreach ($recoverFields as $rF) {
if (!isset($this->request->data['Attribute'][$rF])) {
$this->request->data['Attribute'][$rF] = $existingAttribute['Attribute'][$rF];
@ -1064,7 +1064,7 @@ class AttributesController extends AppController
if (!$this->_isRest()) {
$this->Attribute->Event->insertLock($this->Auth->user(), $this->Attribute->data['Attribute']['event_id']);
}
$validFields = array('value', 'category', 'type', 'comment', 'to_ids', 'distribution');
$validFields = array('value', 'category', 'type', 'comment', 'to_ids', 'distribution', 'first_seen', 'last_seen');
$changed = false;
if (empty($this->request->data['Attribute'])) {
$this->request->data = array('Attribute' => $this->request->data);
@ -1098,6 +1098,9 @@ class AttributesController extends AppController
$event['Event']['timestamp'] = $date->getTimestamp();
$event['Event']['published'] = 0;
$this->Attribute->Event->save($event, array('fieldList' => array('published', 'timestamp', 'info')));
if ($attribute['Attribute']['object_id'] != 0) {
$this->Attribute->Object->updateTimestamp($attribute['Attribute']['object_id'], $date->getTimestamp());
}
$this->autoRender = false;
return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => 'Field updated.', 'check_publish' => true)), 'status'=>200, 'type' => 'json'));
} else {
@ -1649,7 +1652,7 @@ class AttributesController extends AppController
if (isset($this->request->data['Attribute'])) {
$this->request->data = $this->request->data['Attribute'];
}
$checkForEmpty = array('value', 'tags', 'uuid', 'org', 'type', 'category');
$checkForEmpty = array('value', 'tags', 'uuid', 'org', 'type', 'category', 'first_seen', 'last_seen');
foreach ($checkForEmpty as $field) {
if (empty($this->request->data[$field]) || $this->request->data[$field] === 'ALL') {
unset($this->request->data[$field]);
@ -1659,7 +1662,7 @@ class AttributesController extends AppController
unset($this->request->data['to_ids']);
$this->request->data['ignore'] = 1;
}
$paramArray = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags');
$paramArray = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags', 'first_seen', 'last_seen');
$filterData = array(
'request' => $this->request,
'named_params' => $this->params['named'],
@ -2149,7 +2152,7 @@ class AttributesController extends AppController
public function fetchViewValue($id, $field = null)
{
$validFields = array('value', 'comment', 'type', 'category', 'to_ids', 'distribution', 'timestamp');
$validFields = array('value', 'comment', 'type', 'category', 'to_ids', 'distribution', 'timestamp', 'first_seen', 'last_seen');
if (!isset($field) || !in_array($field, $validFields)) {
throw new MethodNotAllowedException(__('Invalid field requested.'));
}
@ -2196,7 +2199,7 @@ class AttributesController extends AppController
public function fetchEditForm($id, $field = null)
{
$validFields = array('value', 'comment', 'type', 'category', 'to_ids', 'distribution');
$validFields = array('value', 'comment', 'type', 'category', 'to_ids', 'distribution', 'first_seen', 'last_seen');
if (!isset($field) || !in_array($field, $validFields)) {
throw new MethodNotAllowedException(__('Invalid field requested.'));
}
@ -2948,6 +2951,9 @@ class AttributesController extends AppController
$event['Event']['timestamp'] = $date->getTimestamp();
$result = $this->Attribute->Event->save($event);
$attribute['Attribute']['timestamp'] = $date->getTimestamp();
if ($attribute['Attribute']['object_id'] != 0) {
$this->Attribute->Object->updateTimestamp($attribute['Attribute']['object_id'], $date->getTimestamp());
}
$this->Attribute->save($attribute);
}
$log = ClassRegistry::init('Log');
@ -3094,6 +3100,9 @@ class AttributesController extends AppController
$date = new DateTime();
$event['Event']['timestamp'] = $date->getTimestamp();
$this->Attribute->Event->save($event);
if ($this->Attribute->data['Attribute']['object_id'] != 0) {
$this->Attribute->Object->updateTimestamp($this->Attribute->data['Attribute']['object_id'], $date->getTimestamp());
}
$this->Attribute->data['Attribute']['timestamp'] = $date->getTimestamp();
$this->Attribute->save($this->Attribute->data);
}

6
app/Controller/Component/ACLComponent.php
View File

@ -144,6 +144,7 @@ class ACLComponent extends Component
'getEventGraphReferences' => array('*'),
'getEventGraphTags' => array('*'),
'getEventGraphGeneric' => array('*'),
'getEventTimeline' => array('*'),
'genDistributionGraph' => array('*'),
'getDistributionGraph' => array('*'),
'getReferenceData' => array('*'),
@ -277,6 +278,11 @@ class ACLComponent extends Component
'edit' => array('perm_add'),
'get_row' => array('perm_add'),
'orphanedObjectDiagnostics' => array(),
'editField' => array('perm_add'),
'fetchEditForm' => array('perm_add'),
'fetchViewValue' => array('*'),
'quickAddAttributeForm' => array('perm_add'),
'quickFetchTemplateWithValidObjectAttributes' => array('perm_add'),
'proposeObjectsFromAttributes' => array('*'),
'groupAttributesIntoObject' => array('perm_add'),
'revise_object' => array('perm_add'),

62
app/Controller/Component/RestResponseComponent.php
View File

@ -22,13 +22,13 @@ class RestResponseComponent extends Component
'add' => array(
'description' => "POST a MISP Attribute JSON to this API to create an Attribute.",
'mandatory' => array('value', 'type'),
'optional' => array('category', 'to_ids', 'uuid', 'distribution', 'sharing_group_id', 'timestamp', 'comment', 'data', 'encrypt'),
'optional' => array('category', 'to_ids', 'uuid', 'distribution', 'sharing_group_id', 'timestamp', 'comment', 'data', 'encrypt', 'first_seen', 'last_seen'),
'params' => array('event_id')
),
'edit' => array(
'description' => "POST a MISP Attribute JSON to this API to update an Attribute. If the timestamp is set, it has to be newer than the existing Attribute.",
'mandatory' => array(),
'optional' => array('value', 'type', 'category', 'to_ids', 'uuid', 'distribution', 'sharing_group_id', 'timestamp', 'comment', 'date', 'encrypt'),
'optional' => array('value', 'type', 'category', 'to_ids', 'uuid', 'distribution', 'sharing_group_id', 'timestamp', 'comment', 'date', 'encrypt', 'first_seen', 'last_seen'),
'params' => array('attribute_id')
),
'deleteSelected' => array(
@ -40,7 +40,7 @@ class RestResponseComponent extends Component
'restSearch' => array(
'description' => "Search MISP using a list of filter parameters and return the data in the selected format. The search is available on an event and an attribute level, just select the scope via the URL (/events/restSearch vs /attributes/restSearch). Besides the parameters listed, other, format specific ones can be passed along (for example: requested_attributes and includeContext for the CSV export). This API allows pagination via the page and limit parameters.",
'mandatory' => array('returnFormat'),
'optional' => array('page', 'limit', 'value' , 'type', 'category', 'org', 'tags', 'date', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'attribute_timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'includeEventTags', 'event_timestamp', 'threat_level_id', 'eventinfo', 'includeProposals', 'includeDecayScore', 'includeFullModel', 'decayingModel', 'excludeDecayed', 'score'),
'optional' => array('page', 'limit', 'value' , 'type', 'category', 'org', 'tags', 'date', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'attribute_timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'includeEventTags', 'event_timestamp', 'threat_level_id', 'eventinfo', 'includeProposals', 'includeDecayScore', 'includeFullModel', 'decayingModel', 'excludeDecayed', 'score', 'first_seen', 'last_seen'),
'params' => array()
)
),
@ -392,6 +392,7 @@ class RestResponseComponent extends Component
if (isset($this->__convertActionToMessage[$controller][$action['action']])) {
$stringifiedAction = $this->__convertActionToMessage[$controller][$action['action']];
}
$response['saved'] = false;
$response['name'] = 'Could not ' . $stringifiedAction . ' ' . Inflector::singularize($controller);
$response['message'] = $response['name'];
$response['url'] = $this->__generateURL($action, $controller, $id);
@ -405,6 +406,8 @@ class RestResponseComponent extends Component
if (!$message) {
$message = Inflector::singularize($controller) . ' ' . $action['action'] . ((substr($action['action'], -1) == 'e') ? 'd' : 'ed');
}
$response['saved'] = true;
$response['success'] = true;
$response['name'] = $message;
$response['message'] = $response['name'];
$response['url'] = $this->__generateURL($action, $controller, $id);
@ -914,6 +917,12 @@ class RestResponseComponent extends Component
'operators' => array('equal'),
'help' => __('A valid external auth key')
),
'first_seen' => array(
'input' => 'text',
'type' => 'string',
'operators' => array('equal'),
'help' => 'A valid ISO 8601 datetime format, up to milli-seconds. i.e.: 2019-06-13T15:56:56.856074+02:00'
),
'fixed_event' => array(
'input' => 'select',
'type' => 'integer',
@ -1036,6 +1045,12 @@ class RestResponseComponent extends Component
'operators' => array('equal', 'not_equal'),
'help' => __('Events published within the last x amount of time, where x can be defined in days, hours, minutes (for example 5d or 12h or 30m)')
),
'last_seen' => array(
'input' => 'text',
'type' => 'string',
'operators' => array('equal'),
'help' => 'A valid ISO 8601 datetime format, up to milli-seconds. i.e.: 2019-06-13T15:56:56.856074+02:00'
),
'limit' => array(
'input' => 'number',
'type' => 'integer',
@ -1630,33 +1645,37 @@ class RestResponseComponent extends Component
// add dynamic data and overwrite name collisions
switch($field) {
case "returnFormat":
$this->__overwriteReturnFormat($scope, $fieldsConstraint[$field]);
$this->__overwriteReturnFormat($scope, $action, $fieldsConstraint[$field]);
break;
case "type":
$this->__overwriteType($scope, $fieldsConstraint[$field]);
$this->__overwriteType($scope, $action, $fieldsConstraint[$field]);
break;
case "category":
$this->__overwriteCategory($scope, $fieldsConstraint[$field]);
$this->__overwriteCategory($scope, $action, $fieldsConstraint[$field]);
break;
case "decayingModel":
$this->__overwriteDecayingModel($scope, $fieldsConstraint[$field]);
break;
case "distribution":
$this->__overwriteDistribution($scope, $fieldsConstraint[$field]);
$this->__overwriteDistribution($scope, $action, $fieldsConstraint[$field]);
break;
case "tag":
case "tags":
case "EventTag":
$this->__overwriteTags($scope, $fieldsConstraint[$field]);
$this->__overwriteTags($scope, $action, $fieldsConstraint[$field]);
break;
case "nationality":
$this->__overwriteNationality($scope, $fieldsConstraint[$field]);
$this->__overwriteNationality($scope, $action, $fieldsConstraint[$field]);
break;
case "action":
$this->__overwriteAction($scope, $fieldsConstraint[$field]);
$this->__overwriteAction($scope, $action, $fieldsConstraint[$field]);
break;
case "role_id":
$this->__overwriteRoleId($scope, $fieldsConstraint[$field]);
$this->__overwriteRoleId($scope, $action, $fieldsConstraint[$field]);
break;
case "first_seen":
case "last_seen":
$this->__overwriteSeen($scope, $action, $fieldsConstraint[$field]);
break;
default:
break;
@ -1672,7 +1691,7 @@ class RestResponseComponent extends Component
}
// Fetch the correct values based on the scope, then overwrite default value
private function __overwriteReturnFormat($scope, &$field) {
private function __overwriteReturnFormat($scope, $action, &$field) {
switch($scope) {
case "Attribute":
$field['values'] = array_keys(ClassRegistry::init($scope)->validFormats);
@ -1682,7 +1701,7 @@ class RestResponseComponent extends Component
break;
}
}
private function __overwriteType($scope, &$field) {
private function __overwriteType($scope, $action, &$field) {
$field['input'] = 'select';
switch($scope) {
case "Attribute":
@ -1700,10 +1719,10 @@ class RestResponseComponent extends Component
}
}
private function __overwriteCategory($scope, &$field) {
private function __overwriteCategory($scope, $action, &$field) {
$field['values'] = array_keys(ClassRegistry::init("Attribute")->categoryDefinitions);
}
private function __overwriteDistribution($scope, &$field) {
private function __overwriteDistribution($scope, $action, &$field) {
$field['values'] = array();
foreach(ClassRegistry::init("Attribute")->distributionLevels as $d => $text) {
$field['values'][] = array('label' => $text, 'value' => $d);
@ -1720,7 +1739,7 @@ class RestResponseComponent extends Component
$field['values'][] = array('label' => h($model_name), 'value' => $i);
}
}
private function __overwriteTags($scope, &$field) {
private function __overwriteTags($scope, $action, &$field) {
$this->{$scope} = ClassRegistry::init("Tag");
$tags = $this->{$scope}->find('list', array(
'recursive' => -1,
@ -1733,13 +1752,13 @@ class RestResponseComponent extends Component
}
$field['values'] = $tags;
}
private function __overwriteNationality($scope, &$field) {
private function __overwriteNationality($scope, $action, &$field) {
$field['values'] = ClassRegistry::init("Organisation")->countries;
}
private function __overwriteAction($scope, &$field) {
private function __overwriteAction($scope, $action, &$field) {
$field['values'] = array_keys(ClassRegistry::init("Log")->actionDefinitions);
}
private function __overwriteRoleId($scope, &$field) {
private function __overwriteRoleId($scope, $action, &$field) {
$this->{$scope} = ClassRegistry::init("Role");
$roles = $this->{$scope}->find('list', array(
'recursive' => -1,
@ -1747,5 +1766,10 @@ class RestResponseComponent extends Component
));
$field['values'] = $roles;
}
private function __overwriteSeen($scope, $action, &$field) {
if ($action == 'restSearch') {
$field['help'] = __('Seen within the last x amount of time, where x can be defined in days, hours, minutes (for example 5d or 12h or 30m)');
}
}
}

2
app/Controller/Component/RestSearchComponent.php
View File

@ -10,7 +10,7 @@ class RestSearchComponent extends Component
'published', 'timestamp','enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags',
'includeProposals', 'returnFormat', 'limit', 'page', 'requested_attributes', 'includeContext', 'headerless',
'includeWarninglistHits', 'attackGalaxy', 'object_relation', 'includeSightings', 'includeCorrelations', 'includeDecayScore',
'decayingModel', 'excludeDecayed', 'modelOverrides', 'includeFullModel', 'score', 'attribute_timestamp'
'decayingModel', 'excludeDecayed', 'modelOverrides', 'includeFullModel', 'score', 'attribute_timestamp', 'first_seen', 'last_seen'
),
'Event' => array(
'returnFormat', 'value', 'type', 'category', 'org', 'tags', 'searchall', 'from', 'to', 'last', 'eventid', 'withAttachments',

72
app/Controller/EventsController.php
View File

@ -1309,7 +1309,7 @@ class EventsController extends AppController
foreach ($event['Object'] as $k => $object) {
if (!empty($object['Attribute'])) {
foreach ($object['Attribute'] as $attribute) {
if ($oldest_timestamp == false || $oldest_timestamp < $attribute['timestamp']) {
if ($oldest_timestamp == false || $oldest_timestamp > $attribute['timestamp']) {
$oldest_timestamp = $attribute['timestamp'];
}
}
@ -1407,7 +1407,7 @@ class EventsController extends AppController
$startDate = null;
$modificationMap = array();
foreach ($event['Attribute'] as $k => $attribute) {
if ($oldest_timestamp == false || $oldest_timestamp < $attribute['timestamp']) {
if ($oldest_timestamp == false || $oldest_timestamp > $attribute['timestamp']) {
$oldest_timestamp = $attribute['timestamp'];
}
if ($startDate === null || $attribute['timestamp'] < $startDate) {
@ -2147,7 +2147,6 @@ class EventsController extends AppController
throw new UnauthorizedException(__('You do not have permission to do that.'));
}
if ($this->request->is('post')) {
$original_file = !empty($this->data['Event']['original_file']) ? $this->data['Event']['stix']['name'] : '';
if ($this->_isRest()) {
$randomFileName = $this->Event->generateRandomFileName();
$tmpDir = APP . "files" . DS . "scripts" . DS . "tmp";
@ -2158,8 +2157,8 @@ class EventsController extends AppController
$this->Auth->user(),
$randomFileName,
$stix_version,
$original_file,
$this->data['Event']['publish']
'uploaded_stix_file.' . ($stix_version == '1' ? 'xml' : 'json'),
false
);
if (is_array($result)) {
return $this->RestResponse->saveSuccessResponse('Events', 'upload_stix', false, $this->response->type(), 'STIX document imported, event\'s created: ' . implode(', ', $result) . '.');
@ -2174,6 +2173,7 @@ class EventsController extends AppController
return $this->RestResponse->saveFailResponse('Events', 'upload_stix', false, $result, $this->response->type());
}
} else {
$original_file = !empty($this->data['Event']['original_file']) ? $this->data['Event']['stix']['name'] : '';
if (isset($this->data['Event']['stix']) && $this->data['Event']['stix']['size'] > 0 && is_uploaded_file($this->data['Event']['stix']['tmp_name'])) {
$randomFileName = $this->Event->generateRandomFileName();
$tmpDir = APP . "files" . DS . "scripts" . DS . "tmp";
@ -4502,6 +4502,32 @@ class EventsController extends AppController
return $json;
}
public function getEventTimeline($id, $type = 'event')
{
$validTools = array('event');
if (!in_array($type, $validTools)) {
throw new MethodNotAllowedException('Invalid type.');
}
App::uses('EventTimelineTool', 'Tools');
$grapher = new EventTimelineTool();
$data = $this->request->is('post') ? $this->request->data : array();
$dataFiltering = array_key_exists('filtering', $data) ? $data['filtering'] : array();
$extended = isset($this->params['named']['extended']) ? 1 : 0;
$grapher->construct($this->Event, $this->Auth->user(), $dataFiltering, $extended);
$json = $grapher->get_timeline($id);
array_walk_recursive($json, function (&$item, $key) {
if (!mb_detect_encoding($item, 'utf-8', true)) {
$item = utf8_encode($item);
}
});
$this->response->type('json');
return new CakeResponse(array('body' => json_encode($json), 'status' => 200, 'type' => 'json'));
}
public function getDistributionGraph($id, $type = 'event')
{
$extended = isset($this->params['named']['extended']) ? 1 : 0;
@ -4665,22 +4691,24 @@ class EventsController extends AppController
throw new Exception("Invalid options.");
}
$event = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $eventId, 'metadata' => true));
if (empty($event)) {
throw new NotFoundException(__('Event not found or you are not authorised to view it.'));
if ($scope !== 'tag_collection') {
$event = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $eventId, 'metadata' => true));
if (empty($event)) {
throw new NotFoundException(__('Event not found or you are not authorised to view it.'));
}
$scoresDataAttr = $this->Event->Attribute->AttributeTag->getTagScores($this->Auth->user(), $eventId, $matrixTags);
$scoresDataEvent = $this->Event->EventTag->getTagScores($eventId, $matrixTags);
$maxScore = 0;
$scoresData = array();
foreach (array_keys($scoresDataAttr['scores'] + $scoresDataEvent['scores']) as $key) {
$sum = (isset($scoresDataAttr['scores'][$key]) ? $scoresDataAttr['scores'][$key] : 0) + (isset($scoresDataEvent['scores'][$key]) ? $scoresDataEvent['scores'][$key] : 0);
$scoresData[$key] = $sum;
$maxScore = max($maxScore, $sum);
}
$scores = $scoresData;
} else {
$scores = $scoresData = array();
}
$scoresDataAttr = $this->Event->Attribute->AttributeTag->getTagScores($this->Auth->user(), $eventId, $matrixTags);
$scoresDataEvent = $this->Event->EventTag->getTagScores($eventId, $matrixTags);
$maxScore = 0;
$scoresData = array();
foreach (array_keys($scoresDataAttr['scores'] + $scoresDataEvent['scores']) as $key) {
$sum = (isset($scoresDataAttr['scores'][$key]) ? $scoresDataAttr['scores'][$key] : 0) + (isset($scoresDataEvent['scores'][$key]) ? $scoresDataEvent['scores'][$key] : 0);
$scoresData[$key] = $sum;
$maxScore = max($maxScore, $sum);
}
$scores = $scoresData;
// FIXME: temporary fix: add the score of deprecated mitre galaxies to the new one (for the stats)
if ($matrixData['galaxy']['id'] == $mitreAttackGalaxyId) {
$mergedScore = array();
@ -4840,7 +4868,7 @@ class EventsController extends AppController
$options = array();
foreach ($enabledModules['modules'] as $temp) {
if ($temp['name'] == $module) {
$format = (isset($temp['mispattributes']['format']) ? $temp['mispattributes']['format'] : 'simplified');
$format = (!empty($temp['mispattributes']['format']) ? $temp['mispattributes']['format'] : 'simplified');
if (isset($temp['meta']['config'])) {
foreach ($temp['meta']['config'] as $conf) {
$options[$conf] = Configure::read('Plugin.' . $type . '_' . $module . '_' . $conf);
@ -5104,7 +5132,7 @@ class EventsController extends AppController
throw new Exception($result);
}
$importComment = !empty($result['comment']) ? $result['comment'] : 'Enriched via the ' . $module['name'] . ' module';
if (isset($module['mispattributes']['format']) && $module['mispattributes']['format'] === 'misp_standard') {
if (!empty($module['mispattributes']['format']) && $module['mispattributes']['format'] === 'misp_standard') {
$event = $this->Event->handleMispFormatFromModuleResult($result);
$event['Event'] = array('id' => $eventId);
if ($this->_isRest()) {

14
app/Controller/GalaxiesController.php
View File

@ -111,17 +111,19 @@ class GalaxiesController extends AppController
{
$mitreAttackGalaxyId = $this->Galaxy->getMitreAttackGalaxyId();
$local = !empty($this->params['named']['local']) ? $this->params['named']['local'] : '0';
$conditions = $namespace == '0' ? array() : array('namespace' => $namespace);
$conditions = $namespace === '0' ? array() : array('namespace' => $namespace);
$galaxies = $this->Galaxy->find('all', array(
'recursive' => -1,
'fields' => array('MAX(Galaxy.version) as latest_version', 'id', 'kill_chain_order', 'name', 'icon', 'description'),
'conditions' => $conditions,
'group' => array('name', 'id', 'kill_chain_order', 'icon', 'description'),
'order' => array('name asc')
));
$items = array();
$items[] = array(
'name' => __('All clusters'),
'value' => "/galaxies/selectCluster/" . h($target_id) . '/' . h($target_type) . '/0'. '/local:' . $local
$items = array(
array(
'name' => __('All clusters'),
'value' => "/galaxies/selectCluster/" . h($target_id) . '/' . h($target_type) . '/0'. '/local:' . $local
)
);
foreach ($galaxies as $galaxy) {
if (!isset($galaxy['Galaxy']['kill_chain_order'])) {

2
app/Controller/LogsController.php
View File

@ -416,7 +416,7 @@ class LogsController extends AppController
$this->set('actions', $actions);
// combobox for models
$models = array('Attribute', 'Event', 'EventBlacklist', 'EventTag', 'DecayingModel', 'MispObject', 'Organisation', 'Post', 'Regexp', 'Role', 'Server', 'ShadowAttribute', 'SharingGroup', 'Tag', 'Task', 'Taxonomy', 'Template', 'Thread', 'User', 'Whitelist');
$models = array('Attribute', 'Event', 'EventBlacklist', 'EventTag', 'Feed', 'DecayingModel', 'MispObject', 'Organisation', 'Post', 'Regexp', 'Role', 'Server', 'ShadowAttribute', 'SharingGroup', 'Tag', 'Task', 'Taxonomy', 'Template', 'Thread', 'User', 'Whitelist');
$models = array('' => 'ALL') + $this->_arrayToValuesIndexArray($models);
$this->set('models', $models);
$this->set('actionDefinitions', $this->{$this->defaultModel}->actionDefinitions);

530
app/Controller/ObjectsController.php
View File

@ -358,43 +358,20 @@ class ObjectsController extends AppController
$this->set('element', $element);
}
public function edit($id, $update_template_available=false)
public function edit($id, $update_template_available=false, $onlyAddNewAttribute=false)
{
if (Validation::uuid($id)) {
$conditions = array('Object.uuid' => $id);
} else {
$conditions = array('Object.id' => $id);
}
if (!$this->userRole['perm_modify']) {
throw new MethodNotAllowedException(__('You don\'t have permissions to edit objects.'));
}
$object = $this->MispObject->find('first', array(
'conditions' => $conditions,
'recursive' => -1,
'contain' => array(
'Attribute' => array(
'conditions' => array(
'Attribute.deleted' => 0
)
)
)
));
$id = $this->Toolbox->findIdByUuid($this->MispObject, $id);
$object = $this->MispObject->fetchObjects($this->Auth->user(), array('conditions' => array('Object.id' => $id)));
if (empty($object)) {
throw new NotFoundException(__('Invalid object.'));
}
$id = $object['Object']['id'];
$eventFindParams = array(
'recursive' => -1,
'fields' => array('Event.id', 'Event.uuid', 'Event.orgc_id'),
'conditions' => array('Event.id' => $object['Object']['event_id'])
);
$event = $this->MispObject->Event->find('first', $eventFindParams);
if (empty($event) || (!$this->_isSiteAdmin() && $event['Event']['orgc_id'] != $this->Auth->user('org_id'))) {
throw new NotFoundException(__('Invalid object.'));
$object = $object[0];
if ((!$this->_isSiteAdmin() && $object['Event']['orgc_id'] != $this->Auth->user('org_id'))) {
throw new MethodNotAllowedException(__('Insufficient permissions to edit this object.'));
}
$event = $this->MispObject->Event->fetchEvent($this->Auth->user(), array('eventid' => $object['Event']['id'], 'metadata' => 1))[0];
if (!$this->_isRest()) {
$this->MispObject->Event->insertLock($this->Auth->user(), $event['Event']['id']);
$this->MispObject->Event->insertLock($this->Auth->user(), $object['Event']['id']);
}
$template = $this->MispObject->ObjectTemplate->find('first', array(
'conditions' => array(
@ -406,130 +383,28 @@ class ObjectsController extends AppController
'ObjectTemplateElement'
)
));
if (empty($template)) {
if (empty($template) && !$this->_isRest()) {
$this->Flash->error('Object cannot be edited, no valid template found.');
$this->redirect(array('controller' => 'events', 'action' => 'view', $object['Object']['event_id']));
}
$newer_template = $this->MispObject->ObjectTemplate->find('first', array(
'conditions' => array(
'ObjectTemplate.uuid' => $object['Object']['template_uuid'],
'ObjectTemplate.version >' => $object['Object']['template_version'],
),
'recursive' => -1,
'contain' => array(
'ObjectTemplateElement'
),
'order' => array('ObjectTemplate.version DESC')
));
if (!empty($newer_template)) {
$newer_template_version = $newer_template['ObjectTemplate']['version'];
// ignore IDs for comparison
$cur_template_temp = Hash::remove(Hash::remove($template['ObjectTemplateElement'], '{n}.id'), '{n}.object_template_id');
$newer_template_temp = Hash::remove(Hash::remove($newer_template['ObjectTemplateElement'], '{n}.id'), '{n}.object_template_id');
$template_difference = array();
// check how current template is included in the newer
foreach ($cur_template_temp as $cur_obj_rel) {
$flag_sim = false;
foreach ($newer_template_temp as $newer_obj_rel) {
$tmp = Hash::diff($cur_obj_rel, $newer_obj_rel);
if (count($tmp) == 0) {
$flag_sim = true;
break;
}
}
if (!$flag_sim) {
$template_difference[] = $cur_obj_rel;
}
}
$updateable_attribute = $object['Attribute'];
$not_updateable_attribute = array();
if (!empty($template_difference)) { // older template not completely embeded in newer
foreach ($template_difference as $temp_diff_element) {
foreach ($object['Attribute'] as $i => $attribute) {
if (
$attribute['object_relation'] == $temp_diff_element['object_relation']
&& $attribute['type'] == $temp_diff_element['type']
) { // This attribute cannot be merged automatically
$attribute['merge-possible'] = false;
$not_updateable_attribute[] = $attribute;
unset($updateable_attribute[$i]);
}
}
}
}
$this->set('updateable_attribute', $updateable_attribute);
$this->set('not_updateable_attribute', $not_updateable_attribute);
if ($update_template_available) { // template version bump requested
$template = $newer_template; // bump the template version
}
} else {
$newer_template_version = false;
$templateData = $this->MispObject->resolveUpdatedTemplate($template, $object, $update_template_available);
$this->set('updateable_attribute', $templateData['updateable_attribute']);
$this->set('not_updateable_attribute', $templateData['not_updateable_attribute']);
if (isset($this->params['named']['revised_object'])) {
$revisedData = $this->MispObject->reviseObject($this->params['named']['revised_object'], $object);
$this->set('revised_object', $revisedData['revised_object_both']);
$object = $revisedData['object'];
}
if (isset($this->params['named']['revised_object'])) { // revised object data to be injected
$revised_object = json_decode(base64_decode($this->params['named']['revised_object']), true);
$revised_object_both = array('mergeable' => array(), 'notMergeable' => array());
// Loop through attributes to inject and perform the correct action
// (inject, duplicate, add warnings, ...) when applicable
foreach ($revised_object['Attribute'] as $attribute_to_inject) {
$flag_no_collision = true;
foreach ($object['Attribute'] as $attribute) {
if (
$attribute['object_relation'] == $attribute_to_inject['object_relation']
&& $attribute['type'] == $attribute_to_inject['type']
&& $attribute['value'] !== $attribute_to_inject['value']
) { // Collision on value
$multiple = !empty(Hash::extract($template['ObjectTemplateElement'], sprintf('{n}[object_relation=%s][type=%s][multiple=true]', $attribute['object_relation'], $attribute['type'])));
if ($multiple) { // if multiple is set, check if an entry exists already
$flag_entry_exists = false;
foreach ($object['Attribute'] as $attr) {
if (
$attr['object_relation'] == $attribute_to_inject['object_relation']
&& $attr['type'] == $attribute_to_inject['type']
&& $attr['value'] === $attribute_to_inject['value']
) {
$flag_entry_exists = true;
break;
}
}
if (!$flag_entry_exists) { // entry does no exists, can be duplicated
$attribute_to_inject['is_multiple'] = true;
$revised_object_both['mergeable'][] = $attribute_to_inject;
$object['Attribute'][] = $attribute_to_inject;
}
} else { // Collision on value, multiple not set => propose overwrite
$attribute_to_inject['current_value'] = $attribute['value'];
$attribute_to_inject['merge-possible'] = true; // the user can still swap value
$revised_object_both['notMergeable'][] = $attribute_to_inject;
}
$flag_no_collision = false;
} else if (
$attribute['object_relation'] == $attribute_to_inject['object_relation']
&& $attribute['type'] == $attribute_to_inject['type']
&& $attribute['value'] === $attribute_to_inject['value']
) { // all good, they are basically the same, do nothing
$revised_object_both['mergeable'][] = $attribute_to_inject;
$flag_no_collision = false;
}
}
if ($flag_no_collision) { // no collision, nor equalities => inject it straight away
$revised_object_both['mergeable'][] = $attribute_to_inject;
$object['Attribute'][] = $attribute_to_inject;
}
}
$this->set('revised_object', $revised_object_both);
if (!empty($templateData['template'])) {
$template = $this->MispObject->prepareTemplate($templateData['template'], $object);
}
$template = $this->MispObject->prepareTemplate($template, $object);
$enabledRows = false;
if ($this->request->is('post') || $this->request->is('put')) {
if (isset($this->request->data['request'])) {
$this->request->data = $this->request->data['request'];
}
if (empty($this->request->data['Object'])) {
$this->request->data['Object'] = $this->request->data;
}
if (isset($this->request->data['Object']['data'])) {
$this->request->data = json_decode($this->request->data['Object']['data'], true);
}
@ -538,34 +413,41 @@ class ObjectsController extends AppController
unset($this->request->data['Object']);
}
$objectToSave = $this->MispObject->attributeCleanup($this->request->data);
$objectToSave = $this->MispObject->deltaMerge($object, $objectToSave);
$objectToSave = $this->MispObject->deltaMerge($object, $objectToSave, $onlyAddNewAttribute);
// we pre-validate the attributes before we create an object at this point
// This allows us to stop the process and return an error (API) or return
// to the add form
if (empty($error)) {
if ($this->_isRest()) {
if ($this->_isRest()) {
if (is_numeric($objectToSave)) {
$objectToSave = $this->MispObject->fetchObjects($this->Auth->user(), array('conditions' => array('Object.id' => $id)));
if (!empty($objectToSave)) {
$objectToSave = $objectToSave[0];
$objectToSave['Object']['Attribute'] = $objectToSave['Attribute'];
unset($objectToSave['Attribute']);
}
$this->MispObject->Event->unpublishEvent($objectToSave['Object']['event_id']);
return $this->RestResponse->viewData($objectToSave, $this->response->type());
} else {
return $this->RestResponse->saveFailResponse('Objects', 'add', false, $id, $this->response->type());
}
} else {
$message = __('Object attributes saved.');
$error_message = __('Object attributes could not be saved.');
if ($this->request->is('ajax')) {
$this->autoRender = false;
if (is_numeric($objectToSave)) {
$objectToSave = $this->MispObject->find('first', array(
'recursive' => -1,
'conditions' => array('Object.id' => $id),
'contain' => array(
'Attribute' => array(
'fields' => $this->MispObject->Attribute->defaultFields
)
)
));
if (!empty($objectToSave)) {
$objectToSave['Object']['Attribute'] = $objectToSave['Attribute'];
unset($objectToSave['Attribute']);
}
$this->MispObject->Event->unpublishEvent($object['Object']['event_id']);
return $this->RestResponse->viewData($objectToSave, $this->response->type());
return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => $message)), 'status'=>200, 'type' => 'json'));
} else {
return $this->RestResponse->saveFailResponse('Objects', 'add', false, $id, $this->response->type());
return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'errors' => $error_message)), 'status'=>200, 'type' => 'json'));
}
} else {
$this->MispObject->Event->unpublishEvent($object['Object']['event_id']);
$this->Flash->success('Object saved.');
if (is_numeric($objectToSave)) {
$this->MispObject->Event->unpublishEvent($object['Object']['event_id']);
$this->Flash->success('Object saved.');
} else {
$this->Flash->error($error_message);
}
$this->redirect(array('controller' => 'events', 'action' => 'view', $object['Object']['event_id']));
}
}
@ -601,10 +483,324 @@ class ObjectsController extends AppController
$this->set('action', 'edit');
$this->set('object', $object);
$this->set('update_template_available', $update_template_available);
$this->set('newer_template_version', $newer_template_version);
$this->set('newer_template_version', empty($templateData['newer_template_version']) ? false : $templateData['newer_template_version']);
$this->render('add');
}
// ajax edit - post a single edited field and this method will attempt to save it and return a json with the validation errors if they occur.
public function editField($id)
{
if (Validation::uuid($id)) {
$this->MispObject->recursive = -1;
$temp = $this->MispObject->findByUuid($id);
if ($temp == null) {
throw new NotFoundException(__('Invalid object'));
}
$id = $temp['Object']['id'];
} elseif (!is_numeric($id)) {
throw new NotFoundException(__('Invalid event id.'));
}
if ((!$this->request->is('post') && !$this->request->is('put'))) {
throw new MethodNotAllowedException(__('This function can only be accessed via POST or PUT'));
}
$object = $this->MispObject->find('first', array(
'conditions' => array('Object.id' => $id),
'contain' => 'Event',
'recursive' => -1
));
if (empty($object)) {
return $this->RestResponse->saveFailResponse('Objects', 'edit', false, 'Invalid object');
}
if (!$this->_isSiteAdmin()) {
if ($this->MispObject->data['Event']['orgc_id'] == $this->Auth->user('org_id')
&& (($this->userRole['perm_modify'] && $this->MispObject->data['Event']['user_id'] != $this->Auth->user('id'))
|| $this->userRole['perm_modify_org'])) {
// Allow the edit
} else {
return $this->RestResponse->saveFailResponse('Objects', 'edit', false, 'Invalid attribute');
}
}
$validFields = array('comment', 'distribution', 'first_seen', 'last_seen');
$changed = false;
if (empty($this->request->data['Object'])) {
$this->request->data = array('Object' => $this->request->data);
if (empty($this->request->data['Object'])) {
throw new MethodNotAllowedException('Invalid input.');
}
}
$seen_changed = false;
foreach ($this->request->data['Object'] as $changedKey => $changedField) {
if (!in_array($changedKey, $validFields)) {
throw new MethodNotAllowedException('Invalid field.');
}
if ($object['Object'][$changedKey] == $changedField) {
$this->autoRender = false;
return $this->RestResponse->saveSuccessResponse('Objects', 'edit', $id, false, 'nochange');
}
$seen_changed = $changedKey == 'first_seen' || $changedKey == 'last_seen';
$object['Object'][$changedKey] = $changedField;
$changed = true;
}
$forcedSeenOnElements = array();
if (!$changed) {
return $this->RestResponse->saveSuccessResponse('Objects', 'edit', $id, false, 'nochange');
} elseif ($seen_changed) {
$forcedSeenOnElements[$changedKey] = $changedField;
}
$date = new DateTime();
$object['Object']['timestamp'] = $date->getTimestamp();
$object = $this->MispObject->syncObjectAndAttributeSeen($object, $forcedSeenOnElements);
if ($this->MispObject->save($object)) {
$event = $this->MispObject->Event->find('first', array(
'recursive' => -1,
'fields' => array('id', 'published', 'timestamp', 'info', 'uuid'),
'conditions' => array(
'id' => $object['Object']['event_id'],
)));
$event['Event']['timestamp'] = $date->getTimestamp();
$event['Event']['published'] = 0;
if ($seen_changed) {
$this->MispObject->Attribute->saveAttributes($object['Attribute']);
}
$this->MispObject->Event->save($event, array('fieldList' => array('published', 'timestamp', 'info')));
return $this->RestResponse->saveSuccessResponse('Objects', 'edit', $id, false, 'Field updated');
} else {
return $this->RestResponse->saveFailResponse('Objects', 'edit', false, $this->MispObject->validationErrors);
}
}
public function fetchViewValue($id, $field = null)
{
$validFields = array('timestamp', 'comment', 'distribution', 'first_seen', 'last_seen');
if (!isset($field) || !in_array($field, $validFields)) {
throw new MethodNotAllowedException('Invalid field requested.');
}
if (!$this->request->is('ajax')) {
throw new MethodNotAllowedException('This function can only be accessed via AJAX.');
}
$params = array(
'conditions' => array('Object.id' => $id),
'fields' => array('id', 'distribution', 'event_id', $field),
'contain' => array(
'Event' => array(
'fields' => array('distribution', 'id', 'org_id'),
)
),
'flatten' => 1
);
$object = $this->MispObject->fetchObjectSimple($this->Auth->user(), $params);
if (empty($object)) {
throw new NotFoundException(__('Invalid object'));
}
$object = $object[0];
$result = $object['Object'][$field];
if ($field == 'distribution') {
$result=$this->MispObject->shortDist[$result];
}
$this->set('value', $result);
$this->layout = 'ajax';
$this->render('ajax/objectViewFieldForm');
}
public function fetchEditForm($id, $field = null)
{
$validFields = array('distribution', 'comment', 'first_seen', 'last_seen');
if (!isset($field) || !in_array($field, $validFields)) {
throw new MethodNotAllowedException('Invalid field requested.');
}
if (!$this->request->is('ajax')) {
throw new MethodNotAllowedException('This function can only be accessed via AJAX.');
}
$fields = array('id', 'distribution', 'event_id');
$fields[] = $field;
$params = array(
'conditions' => array('Object.id' => $id),
'fields' => $fields,
'contain' => array(
'Event' => array(
'fields' => array('distribution', 'id', 'user_id', 'orgc_id', 'org_id'),
)
)
);
$object = $this->MispObject->fetchObjectSimple($this->Auth->user(), $params);
if (empty($object)) {
throw new NotFoundException(__('Invalid attribute'));
}
$object = $object[0];
if (!$this->_isSiteAdmin()) {
if ($object['Event']['orgc_id'] == $this->Auth->user('org_id')
&& (($this->userRole['perm_modify'] && $object['Event']['user_id'] != $this->Auth->user('id'))
|| $this->userRole['perm_modify_org'])) {
// Allow the edit
} else {
throw new NotFoundException(__('Invalid object'));
}
}
$this->layout = 'ajax';
if ($field == 'distribution') {
$distributionLevels = $this->MispObject->shortDist;
unset($distributionLevels[4]);
$this->set('distributionLevels', $distributionLevels);
}
$this->set('object', $object['Object']);
$fieldURL = ucfirst($field);
$this->render('ajax/objectEdit' . $fieldURL . 'Form');
}
// Construct a template with valid object attributes to add to an object
public function quickFetchTemplateWithValidObjectAttributes($id) {
$this->MispObject->id = $id;
if (!$this->MispObject->exists()) {
if ($this->request->is('ajax')) {
return $this->RestResponse->saveFailResponse('Objects', 'add', false, 'Invalid object', $this->response->type());
} else {
throw new NotFoundException(__('Invalid object'));
}
}
$fields = array('template_uuid', 'template_version', 'id');
$params = array(
'conditions' => array('Object.id' => $id),
'fields' => $fields,
'flatten' => 1,
);
// fetchObjects restrict access based on user
$object = $this->MispObject->fetchObjects($this->Auth->user(), $params);
if (empty($object)) {
if ($this->request->is('ajax')) {
return $this->RestResponse->saveFailResponse('Objects', 'add', false, 'Invalid object', $this->response->type());
} else {
throw new NotFoundException(__('Invalid object'));
}
} else {
$object = $object[0];
}
// get object attributes already set
$objectRelation = array();
foreach($object['Attribute'] as $attr) {
$objectRelation[$attr['object_relation']] = 1;
}
$objectRelation = array_keys($objectRelation);
// get object attribute defined in the object's template
$template = $this->MispObject->ObjectTemplate->find('first', array(
'conditions' => array(
'ObjectTemplate.uuid' => $object['Object']['template_uuid'],
'ObjectTemplate.version' => $object['Object']['template_version'],
),
'recursive' => -1,
'flatten' => 1,
'contain' => 'ObjectTemplateElement'
));
if (empty($template)) {
if ($this->request->is('ajax')) {
return $this->RestResponse->saveFailResponse('Objects', 'add', false, 'Invalid template', $this->response->type());
} else {
throw new NotFoundException(__('Invalid template'));
}
}
// unset object invalid object attribute
foreach($template['ObjectTemplateElement'] as $i => $objAttr) {
if (in_array($objAttr['object_relation'], $objectRelation) && !$objAttr['multiple']) {
unset($template['ObjectTemplateElement'][$i]);
}
}
if ($this->request->is('get') || $this->request->is('post')) {
$this->set('template', $template);
$this->set('objectId', $object['Object']['id']);
$items = array();
foreach ($template['ObjectTemplateElement'] as $objectAttribute) {
$name = sprintf('%s :: %s', $objectAttribute['object_relation'], $objectAttribute['type']);
$items[] = array(
'name' => $name,
'value' => '/objects/quickAddAttributeForm/' . $object['Object']['id'] . '/' . $objectAttribute['object_relation'],
'template' => array(
'name' => $name,
'infoExtra' => $objectAttribute['description'],
)
);
}
$this->set('options', array(
'flag_redraw_chosen' => true
));
$this->set('items', $items);
$this->render('/Elements/generic_picker');
} else {
return $template;
}
}
/**
* GET: Returns a form allowing to add a valid object attribute to an object
* POST/PUT: Add the attribute to the object
*/
public function quickAddAttributeForm($id, $fieldName = null) {
if ($this->request->is('GET')) {
if (!isset($fieldName)) {
throw new MethodNotAllowedException('No field requested.');
}
$this->MispObject->id = $id;
if (!$this->MispObject->exists()) {
throw new NotFoundException(__('Invalid object'));
}
$fields = array('template_uuid', 'template_version', 'id', 'event_id');
$params = array(
'conditions' => array('Object.id' => $id),
'fields' => $fields,
'flatten' => 1,
);
// fetchObjects restrict access based on user
$object = $this->MispObject->fetchObjects($this->Auth->user(), $params);
if (empty($object)) {
throw new NotFoundException(__('Invalid object'));
} else {
$object = $object[0];
}
$template = $this->MispObject->ObjectTemplate->find('first', array(
'conditions' => array(
'ObjectTemplate.uuid' => $object['Object']['template_uuid'],
'ObjectTemplate.version' => $object['Object']['template_version'],
),
'recursive' => -1,
'flatten' => 1,
'contain' => array(
'ObjectTemplateElement' => array('conditions' => array(
'object_relation' => $fieldName
))
)
));
if (empty($template)) {
throw new NotFoundException(__('Invalid object'));
}
if (empty($template['ObjectTemplateElement'])) {
throw new NotFoundException(__('Invalid fields') . ' `' . h($fieldName) . '`');
}
// check if fields can be added
foreach($object['Attribute'] as $i => $objAttr) {
$objectAttrFromTemplate = $template['ObjectTemplateElement'][0];
if ($objAttr['object_relation'] == $fieldName && !$objectAttrFromTemplate['multiple']) {
throw new NotFoundException(__('Invalid field'));
}
}
$template = $this->MispObject->prepareTemplate($template, $object);
$this->layout = 'ajax';
$this->set('object', $object['Object']);
$template_element = $template['ObjectTemplateElement'][0];
unset($template_element['value']); // avoid filling if multiple
$this->set('template_element', $template_element);
$distributionData = $this->MispObject->Event->Attribute->fetchDistributionData($this->Auth->user());
$this->set('distributionData', $distributionData);
$info = array();
foreach ($distributionData['levels'] as $key => $value) {
$info['distribution'][$key] = array('key' => $value, 'desc' => $this->MispObject->Event->Attribute->distributionDescriptions[$key]['formdesc']);
}
$this->set('info', $info);
$this->render('ajax/quickAddAttributeForm');
} else if ($this->request->is('post') || $this->request->is('put')) {
return $this->edit($this->request->data['Object']['id'], false, true);
}
}
public function delete($id, $hard = false)
{
$id = $this->Toolbox->findIdByUuid($this->MispObject, $id);

8
app/Controller/ServersController.php
View File

@ -394,6 +394,7 @@ class ServersController extends AppController
}
$this->set('allTags', $allTags);
$this->set('host_org_id', Configure::read('MISP.host_org_id'));
$this->render('edit');
}
}
@ -1135,6 +1136,9 @@ class ServersController extends AppController
$this->set('phpversion', phpversion());
$this->set('phpmin', $this->phpmin);
$this->set('phprec', $this->phprec);
$this->set('pythonmin', $this->pythonmin);
$this->set('pythonrec', $this->pythonrec);
$this->set('pymisp', $this->pymisp);
}
}
@ -2221,6 +2225,7 @@ misp.direct_call(relative_path, body)
} else {
$this->set('checkedTableColumn', $dbSchemaDiagnostics['checked_table_column']);
$this->set('dbSchemaDiagnostics', $dbSchemaDiagnostics['diagnostic']);
$this->set('dbIndexDiagnostics', $dbSchemaDiagnostics['diagnostic_index']);
$this->set('expectedDbVersion', $dbSchemaDiagnostics['expected_db_version']);
$this->set('actualDbVersion', $dbSchemaDiagnostics['actual_db_version']);
$this->set('error', $dbSchemaDiagnostics['error']);
@ -2228,9 +2233,10 @@ misp.direct_call(relative_path, body)
$this->set('updateFailNumberReached', $dbSchemaDiagnostics['update_fail_number_reached']);
$this->set('updateLocked', $dbSchemaDiagnostics['update_locked']);
$this->set('dataSource', $dbSchemaDiagnostics['dataSource']);
$this->set('columnPerTable', $dbSchemaDiagnostics['columnPerTable']);
$this->set('indexes', $dbSchemaDiagnostics['indexes']);
$this->render('/Elements/healthElements/db_schema_diagnostic');
}
}
public function viewDeprecatedFunctionUse()

18
app/Controller/ShadowAttributesController.php
View File

@ -96,7 +96,7 @@ class ShadowAttributesController extends AppController
$this->Attribute->delete($activeAttribute['Attribute']['id']);
} else {
// Update the live attribute with the shadow data
$fieldsToUpdate = array('value1', 'value2', 'value', 'type', 'category', 'comment', 'to_ids');
$fieldsToUpdate = array('value1', 'value2', 'value', 'type', 'category', 'comment', 'to_ids', 'first_seen', 'last_seen');
foreach ($fieldsToUpdate as $f) {
$activeAttribute['Attribute'][$f] = $shadow[$f];
}
@ -415,7 +415,7 @@ class ShadowAttributesController extends AppController
array(
'conditions' => array('ShadowAttribute.id' => $this->ShadowAttribute->id),
'recursive' => -1,
'fields' => array('id', 'old_id', 'event_id', 'type', 'category', 'value', 'comment','to_ids', 'uuid', 'event_org_id', 'email', 'deleted', 'timestamp')
'fields' => array('id', 'old_id', 'event_id', 'type', 'category', 'value', 'comment','to_ids', 'uuid', 'event_org_id', 'email', 'deleted', 'timestamp', 'first_seen', 'last_seen')
)
);
$this->set('ShadowAttribute', $sa['ShadowAttribute']);
@ -689,12 +689,12 @@ class ShadowAttributesController extends AppController
if ($attachment) {
$fields = array(
'static' => array('old_id' => 'Attribute.id', 'uuid' => 'Attribute.uuid', 'event_id' => 'Attribute.event_id', 'event_uuid' => 'Event.uuid', 'event_org_id' => 'Event.orgc_id', 'category' => 'Attribute.category', 'type' => 'Attribute.type'),
'optional' => array('value', 'to_ids', 'comment')
'optional' => array('value', 'to_ids', 'comment', 'first_seen', 'last_seen')
);
} else {
$fields = array(
'static' => array('old_id' => 'Attribute.id', 'uuid' => 'Attribute.uuid', 'event_id' => 'Attribute.event_id', 'event_uuid' => 'Event.uuid', 'event_org_id' => 'Event.orgc_id'),
'optional' => array('category', 'type', 'value', 'to_ids', 'comment')
'optional' => array('category', 'type', 'value', 'to_ids', 'comment', 'first_seen', 'last_seen')
);
if ($existingAttribute['Attribute']['object_id']) {
unset($fields['optional']['type']);
@ -731,7 +731,7 @@ class ShadowAttributesController extends AppController
array(
'conditions' => array('ShadowAttribute.id' => $this->ShadowAttribute->id),
'recursive' => -1,
'fields' => array('id', 'old_id', 'event_id', 'type', 'category', 'value', 'comment','to_ids', 'uuid', 'event_org_id', 'email', 'deleted', 'timestamp')
'fields' => array('id', 'old_id', 'event_id', 'type', 'category', 'value', 'comment','to_ids', 'uuid', 'event_org_id', 'email', 'deleted', 'timestamp', 'first_seen', 'last_seen')
)
);
$this->set('ShadowAttribute', $sa['ShadowAttribute']);
@ -833,6 +833,8 @@ class ShadowAttributesController extends AppController
'type' => $existingAttribute['Attribute']['type'],
'to_ids' => $existingAttribute['Attribute']['to_ids'],
'value' => $existingAttribute['Attribute']['value'],
'first_seen' => $existingAttribute['Attribute']['first_seen'],
'last_seen' => $existingAttribute['Attribute']['last_seen'],
'email' => $this->Auth->user('email'),
'org_id' => $this->Auth->user('org_id'),
'proposal_to_delete' => true,
@ -873,7 +875,7 @@ class ShadowAttributesController extends AppController
'recursive' => -1,
'contain' => 'Event',
'fields' => array(
'ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id',
'ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.first_seen', 'ShadowAttribute.last_seen',
'Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.distribution', 'Event.uuid'
),
'conditions' => array('AND' => array('ShadowAttribute.id' => $id, $distConditions, 'ShadowAttribute.deleted' => 0))
@ -944,7 +946,7 @@ class ShadowAttributesController extends AppController
}
$params = array(
'conditions' => $conditions,
'fields' => array('ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.timestamp'),
'fields' => array('ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.timestamp', 'ShadowAttribute.first_seen', 'ShadowAttribute.last_seen'),
'contain' => array(
'Event' => array(
'fields' => array('id', 'org_id', 'info', 'orgc_id', 'uuid'),
@ -1124,7 +1126,7 @@ class ShadowAttributesController extends AppController
}
}
$keys = array_flip(array('uuid', 'event_id', 'value', 'type', 'category', 'to_ids'));
$keys = array_flip(array('uuid', 'event_id', 'value', 'type', 'category', 'to_ids', 'first_seen', 'last_seen'));
$proposal = array_intersect_key($attribute['Attribute'], $keys);
$proposal['email'] = $this->Auth->user('email');

3
app/Lib/Export/StixExport.php
View File

@ -90,7 +90,8 @@ class StixExport
$decoded = json_decode($result, true);
if (!isset($decoded['success']) || !$decoded['success']) {
$this->__delete_temporary_files($f);
return 'Error while processing your query: ' . $decoded['error'];
$error = $decoded && !empty($decoded['error']) ? $decoded['error'] : $result;
return 'Error while processing your query: ' . $error;
}
$file = new File($this->__tmp_dir . $filename . '.out');
$stix_event = ($this->__return_type == 'stix') ? $file->read() : substr($file->read(), 1, -1);

8
app/Lib/Tools/EventGraphTool.php
View File

@ -232,12 +232,12 @@
$event = $this->__get_filtered_event($id);
$this->__json['items'] = array();
$this->__json['relations'] = array();
$this->__json['existing_object_relation'] = array();
if (empty($event)) {
return $this->__json;
}
if (!empty($event['Object'])) {
$object = $event['Object'];
} else {
@ -312,7 +312,7 @@
if (empty($event)) {
return $this->__json;
}
if (!empty($event['Object'])) {
$object = $event['Object'];
} else {
@ -419,7 +419,7 @@
if (empty($event)) {
return $this->__json;
}
if (!empty($event['Object'])) {
$object = $event['Object'];
} else {

138
app/Lib/Tools/EventTimelineTool.php Normal file
View File

@ -0,0 +1,138 @@
<?php
class EventTimelineTool
{
private $__lookupTables = array();
private $__user = false;
private $__json = array();
private $__eventModel = false;
private $__refModel = false;
# Will be use latter on
private $__related_events = array();
private $__related_attributes = array();
public function construct($eventModel, $user, $filterRules, $extended_view=0)
{
$this->__eventModel = $eventModel;
$this->__objectTemplateModel = $eventModel->Object->ObjectTemplate;
$this->__user = $user;
$this->__filterRules = $filterRules;
$this->__json = array();
$this->__extended_view = $extended_view;
$this->__lookupTables = array(
'analysisLevels' => $this->__eventModel->analysisLevels,
'distributionLevels' => $this->__eventModel->Attribute->distributionLevels
);
return true;
}
public function construct_for_ref($refModel, $user)
{
$this->__refModel = $refModel;
$this->__user = $user;
$this->__json = array();
return true;
}
private function __get_event($id)
{
$fullevent = $this->__eventModel->fetchEvent($this->__user, array('eventid' => $id, 'flatten' => 0, 'includeTagRelations' => 1, 'extended' => $this->__extended_view));
$event = array();
if (empty($fullevent)) {
return $event;
}
if (!empty($fullevent[0]['Object'])) {
$event['Object'] = $fullevent[0]['Object'];
} else {
$event['Object'] = array();
}
if (!empty($fullevent[0]['Attribute'])) {
$event['Attribute'] = $fullevent[0]['Attribute'];
} else {
$event['Attribute'] = array();
}
return $event;
}
public function get_timeline($id)
{
$event = $this->__get_event($id);
$this->__json['items'] = array();
if (empty($event)) {
return $this->__json;
}
if (!empty($event['Object'])) {
$object = $event['Object'];
} else {
$object = array();
}
if (!empty($event['Attribute'])) {
$attribute = $event['Attribute'];
} else {
$attribute = array();
}
// extract links and node type
foreach ($attribute as $attr) {
$toPush = array(
'id' => $attr['id'],
'uuid' => $attr['uuid'],
'content' => $attr['value'],
'event_id' => $attr['event_id'],
'group' => 'attribute',
'timestamp' => $attr['timestamp'],
'first_seen' => $attr['first_seen'],
'last_seen' => $attr['last_seen'],
);
$this->__json['items'][] = $toPush;
}
foreach ($object as $obj) {
$toPush_obj = array(
'id' => $obj['id'],
'uuid' => $obj['uuid'],
'content' => $obj['name'],
'group' => 'object',
'meta-category' => $obj['meta-category'],
'template_uuid' => $obj['template_uuid'],
'event_id' => $obj['event_id'],
'timestamp' => $obj['timestamp'],
'Attribute' => array(),
);
$toPush_obj['first_seen'] = $obj['first_seen'];
$toPush_obj['last_seen'] = $obj['last_seen'];
$toPush_obj['first_seen_overwrite'] = false;
$toPush_obj['last_seen_overwrite'] = false;
foreach ($obj['Attribute'] as $obj_attr) {
// replaced *_seen based on object attribute
if ($obj_attr['object_relation'] == 'first-seen' && is_null($toPush_obj['first_seen'])) {
$toPush_obj['first_seen'] = $obj_attr['value']; // replace first_seen of the object to seen of the element
$toPush_obj['first_seen_overwrite'] = true;
} elseif ($obj_attr['object_relation'] == 'last-seen' && is_null($toPush_obj['last_seen'])) {
$toPush_obj['last_seen'] = $obj_attr['value']; // replace last_seen of the object to seen of the element
$toPush_obj['last_seen_overwrite'] = true;
}
$toPush_attr = array(
'id' => $obj_attr['id'],
'uuid' => $obj_attr['uuid'],
'content' => $obj_attr['value'],
'contentType' => $obj_attr['object_relation'],
'event_id' => $obj_attr['event_id'],
'group' => 'object_attribute',
'timestamp' => $obj_attr['timestamp'],
);
$toPush_obj['Attribute'][] = $toPush_attr;
}
$this->__json['items'][] = $toPush_obj;
}
return $this->__json;
}
}

10
app/Model/AdminSetting.php
View File

@ -21,12 +21,9 @@ class AdminSetting extends AppModel
$setting_object = $this->find('first', array(
'conditions' => array('setting' => $setting)
));
if (!empty($setting_object)) {
$setting_object['AdminSetting']['value'] = $value;
} else {
$this->create();
$setting_object['AdminSetting'] = array('setting' => $setting, 'value' => $value);
}
$this->deleteAll(array('setting' => $setting));
$this->create();
$setting_object['AdminSetting'] = array('setting' => $setting, 'value' => $value);
if ($this->save($setting_object)) {
return true;
} else {
@ -34,6 +31,7 @@ class AdminSetting extends AppModel
}
}
public function getSetting($setting)
{
$setting_object = $this->find('first', array(

139
app/Model/AppModel.php
View File

@ -77,10 +77,21 @@ class AppModel extends Model
27 => false, 28 => false, 29 => false, 30 => false, 31 => false, 32 => false,
33 => false, 34 => false, 35 => false, 36 => false, 37 => false, 38 => false,
39 => false, 40 => false, 41 => false, 42 => false, 43 => false, 44 => false,
45 => false
45 => false, 46 => false
);
public $advanced_updates_description = array(
'seenOnAttributeAndObject' => array(
'title' => 'First seen/Last seen Attribute table',
'description' => 'Update the Attribute table to support first_seen and last_seen feature, with a microsecond resolution.',
'liveOff' => true, # should the instance be offline for users other than site_admin
'recommendBackup' => true, # should the update recommend backup
'exitOnError' => false, # should the update exit on error
'requirements' => 'MySQL version must be >= 5.6', # message stating the requirements necessary for the update
'record' => false, # should the update success be saved in the admin_table
// 'preUpdate' => 'seenOnAttributeAndObjectPreUpdate', # Function to execute before the update. If it throws an error, it cancels the update
'url' => '/servers/updateDatabase/seenOnAttributeAndObject/' # url pointing to the funcion performing the update
),
);
public $actions_description = array(
'verifyGnuPGkeys' => array(
@ -223,6 +234,9 @@ class AppModel extends Model
$dbUpdateSuccess = $this->updateDatabase($command);
$this->__addServerPriority();
break;
case 46:
$dbUpdateSuccess = $this->updateDatabase('seenOnAttributeAndObject');
break;
default:
$dbUpdateSuccess = $this->updateDatabase($command);
break;
@ -269,8 +283,7 @@ class AppModel extends Model
public function updateDatabase($command)
{
$this->Log = ClassRegistry::init('Log');
$this->__resetUpdateProgress();
$liveOff = false;
$exitOnError = false;
if (isset($this->advanced_updates_description[$command])) {
@ -1301,7 +1314,6 @@ class AppModel extends Model
break;
case 44:
$sqlArray[] = "ALTER TABLE object_template_elements CHANGE `disable_correlation` `disable_correlation` tinyint(1);";
break;
case 45:
$sqlArray[] = "ALTER TABLE `events` ADD `sighting_timestamp` int(11) NOT NULL DEFAULT 0 AFTER `publish_timestamp`;";
@ -1323,6 +1335,58 @@ class AppModel extends Model
$sqlArray[] = 'ALTER TABLE `threads` DROP `org`;';
$sqlArray[] = 'ALTER TABLE `users` DROP `org`;';
break;
case 'seenOnAttributeAndObject':
$sqlArray[] =
"ALTER TABLE `attributes`
DROP INDEX uuid,
DROP INDEX event_id,
DROP INDEX sharing_group_id,
DROP INDEX type,
DROP INDEX category,
DROP INDEX value1,
DROP INDEX value2,
DROP INDEX object_id,
DROP INDEX object_relation;
";
$sqlArray[] = "ALTER TABLE `attributes` DROP INDEX deleted"; // deleted index may not be present
$sqlArray[] = "ALTER TABLE `attributes` DROP INDEX comment"; // for replayability
$sqlArray[] = "ALTER TABLE `attributes` DROP INDEX first_seen"; // for replayability
$sqlArray[] = "ALTER TABLE `attributes` DROP INDEX last_seen"; // for replayability
$sqlArray[] =
"ALTER TABLE `attributes`
ADD COLUMN `first_seen` BIGINT(20) NULL DEFAULT NULL,
ADD COLUMN `last_seen` BIGINT(20) NULL DEFAULT NULL,
MODIFY comment TEXT COLLATE utf8_unicode_ci
;";
$indexArray[] = array('attributes', 'uuid');
$indexArray[] = array('attributes', 'event_id');
$indexArray[] = array('attributes', 'sharing_group_id');
$indexArray[] = array('attributes', 'type');
$indexArray[] = array('attributes', 'category');
$indexArray[] = array('attributes', 'value1', 255);
$indexArray[] = array('attributes', 'value2', 255);
$indexArray[] = array('attributes', 'object_id');
$indexArray[] = array('attributes', 'object_relation');
$indexArray[] = array('attributes', 'deleted');
$indexArray[] = array('attributes', 'first_seen');
$indexArray[] = array('attributes', 'last_seen');
$sqlArray[] = "
ALTER TABLE `objects`
ADD `first_seen` BIGINT(20) NULL DEFAULT NULL,
ADD `last_seen` BIGINT(20) NULL DEFAULT NULL,
MODIFY comment TEXT COLLATE utf8_unicode_ci
;";
$indexArray[] = array('objects', 'first_seen');
$indexArray[] = array('objects', 'last_seen');
$sqlArray[] = "
ALTER TABLE `shadow_attributes`
ADD `first_seen` BIGINT(20) NULL DEFAULT NULL,
ADD `last_seen` BIGINT(20) NULL DEFAULT NULL,
MODIFY comment TEXT COLLATE utf8_unicode_ci
;";
$indexArray[] = array('shadow_attributes', 'first_seen');
$indexArray[] = array('shadow_attributes', 'last_seen');
break;
default:
return false;
break;
@ -1341,7 +1405,7 @@ class AppModel extends Model
$this->__setUpdateProgress(0, $total_update_count, $command);
$str_index_array = array();
foreach($indexArray as $toIndex) {
$str_index_array[] = __('Indexing ') . implode($toIndex, '->');
$str_index_array[] = __('Indexing ') . sprintf('%s -> %s', $toIndex[0], $toIndex[1]);
}
$this->__setUpdateCmdMessages(array_merge($sqlArray, $str_index_array));
$flagStop = false;
@ -1417,11 +1481,21 @@ class AppModel extends Model
foreach ($indexArray as $i => $iA) {
$this->__setUpdateProgress(count($sqlArray)+$i, false);
if (isset($iA[2])) {
$this->__addIndex($iA[0], $iA[1], $iA[2]);
$indexSuccess = $this->__addIndex($iA[0], $iA[1], $iA[2]);
} else {
$this->__addIndex($iA[0], $iA[1]);
$indexSuccess = $this->__addIndex($iA[0], $iA[1]);
}
if ($indexSuccess['success']) {
$this->__setUpdateResMessages(count($sqlArray)+$i, __('Successfuly indexed ') . sprintf('%s -> %s', $iA[0], $iA[1]));
} else {
$this->__setUpdateResMessages(count($sqlArray)+$i, sprintf('%s %s %s %s',
__('Failed to add index'),
sprintf('%s -> %s', $iA[0], $iA[1]),
__('The returned error is:') . PHP_EOL,
$indexSuccess['errorMessage']
));
$this->__setUpdateError(count($sqlArray)+$i);
}
$this->__setUpdateResMessages(count($sqlArray)+$i, __('Successfuly indexed ') . implode($iA, '->'));
}
}
$this->__setUpdateProgress(count($sqlArray)+count($indexArray), false);
@ -1517,10 +1591,12 @@ class AppModel extends Model
}
$result = true;
$duplicate = false;
$errorMessage = '';
try {
$this->query($addIndex);
} catch (Exception $e) {
$duplicate = (strpos($e->getMessage(), '1061') !== false);
$errorMessage = $e->getMessage();
$result = false;
}
$this->Log->create();
@ -1531,9 +1607,14 @@ class AppModel extends Model
'email' => 'SYSTEM',
'action' => 'update_database',
'user_id' => 0,
'title' => ($result ? 'Added index ' : 'Failed to add index ') . $field . ' to ' . $table . ($duplicate ? ' (index already set)' : ''),
'change' => ($result ? 'Added index ' : 'Failed to add index ') . $field . ' to ' . $table . ($duplicate ? ' (index already set)' : ''),
'title' => ($result ? 'Added index ' : 'Failed to add index ') . $field . ' to ' . $table . ($duplicate ? ' (index already set)' : $errorMessage),
'change' => ($result ? 'Added index ' : 'Failed to add index ') . $field . ' to ' . $table . ($duplicate ? ' (index already set)' : $errorMessage),
));
$additionResult = array('success' => $result || $duplicate);
if (!$result) {
$additionResult['errorMessage'] = $errorMessage;
}
return $additionResult;
}
public function cleanCacheFiles()
@ -1624,6 +1705,29 @@ class AppModel extends Model
return true;
}
// Try to create a table with a BIGINT(20)
public function seenOnAttributeAndObjectPreUpdate() {
$sqlArray[] = "CREATE TABLE IF NOT EXISTS testtable (
`testfield` BIGINT(6) NULL DEFAULT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;";
try {
foreach($sqlArray as $i => $sql) {
$this->query($sql);
}
} catch (Exception $e) {
throw new Exception('Pre update test failed: ' . PHP_EOL . $sql . PHP_EOL . ' The returned error is: ' . $e->getMessage());
}
// clean up
$sqlArray[] = "DROP TABLE testtable;";
foreach($sqlArray as $i => $sql) {
$this->query($sql);
}
}
public function failingPreUpdate() {
throw new Exception('Yolo fail');
}
public function runUpdates($verbose = false, $useWorker = true, $processId = false)
{
$this->AdminSetting = ClassRegistry::init('AdminSetting');
@ -1653,7 +1757,6 @@ class AppModel extends Model
$job = $this->Job->find('first', array(
'conditions' => array('Job.id' => $processId)
));
if (!empty($updates)) {
// Exit if updates are locked.
// This is not as reliable as a real lock implementation
@ -1738,6 +1841,7 @@ class AppModel extends Model
return true;
}
$this->changeLockState(time());
$this->__resetUpdateProgress();
$update_done = 0;
foreach ($updates as $update => $temp) {
@ -1818,9 +1922,9 @@ class AppModel extends Model
$this->__saveUpdateProgress($updateProgress);
}
private function __resetUpdateProgress()
private function __getEmptyUpdateMessage()
{
$updateProgress = array(
return array(
'commands' => array(),
'results' => array(),
'time' => array('started' => array(), 'elapsed' => array()),
@ -1829,6 +1933,11 @@ class AppModel extends Model
'failed_num' => array(),
'toward_db_version' => ''
);
}
private function __resetUpdateProgress()
{
$updateProgress = $this->__getEmptyUpdateMessage();
$this->__saveUpdateProgress($updateProgress);
}
@ -1858,9 +1967,7 @@ class AppModel extends Model
if ($updateProgress !== false) {
$updateProgress = json_decode($updateProgress, true);
} else {
$this->__resetUpdateProgress();
$updateProgress = $this->AdminSetting->getSetting('update_progress');
$updateProgress = json_decode($updateProgress, true);
$updateProgress = $this->__getEmptyUpdateMessage();
}
foreach($updateProgress as $setting => $value) {
if (!is_array($value)) {

198
app/Model/Attribute.php
View File

@ -35,7 +35,7 @@ class Attribute extends AppModel
);
public $defaultFields = array(
'id', 'event_id', 'object_id', 'object_relation', 'category', 'type', 'value', 'to_ids', 'uuid', 'timestamp', 'distribution', 'sharing_group_id', 'comment', 'deleted', 'disable_correlation'
'id', 'event_id', 'object_id', 'object_relation', 'category', 'type', 'value', 'to_ids', 'uuid', 'timestamp', 'distribution', 'sharing_group_id', 'comment', 'deleted', 'disable_correlation', 'first_seen', 'last_seen'
);
public $distributionDescriptions = array(
@ -89,16 +89,16 @@ class Attribute extends AppModel
'Payload delivery' => array(
'desc' => __('Information about how the malware is delivered'),
'formdesc' => __('Information about the way the malware payload is initially delivered, for example information about the email or web-site, vulnerability used, originating IP etc. Malware sample itself should be attached here.'),
'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|authentihash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash','filename|impfuzzy', 'filename|pehash', 'mac-address', 'mac-eui-64', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'hostname', 'domain', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'email-body', 'url', 'user-agent', 'AS', 'pattern-in-file', 'pattern-in-traffic', 'stix2-pattern', 'yara', 'sigma', 'mime-type', 'attachment', 'malware-sample', 'link', 'malware-type', 'comment', 'text', 'hex', 'vulnerability', 'weakness', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'other', 'hostname|port', 'email-dst-display-name', 'email-src-display-name', 'email-header', 'email-reply-to', 'email-x-mailer', 'email-mime-boundary', 'email-thread-index', 'email-message-id', 'mobile-application-id', 'whois-registrant-email', 'anonymised')
'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|authentihash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash','filename|impfuzzy', 'filename|pehash', 'mac-address', 'mac-eui-64', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'hostname', 'domain', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'email-body', 'url', 'user-agent', 'AS', 'pattern-in-file', 'pattern-in-traffic', 'stix2-pattern', 'yara', 'sigma', 'mime-type', 'attachment', 'malware-sample', 'link', 'malware-type', 'comment', 'text', 'hex', 'vulnerability', 'weakness', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'other', 'hostname|port', 'email-dst-display-name', 'email-src-display-name', 'email-header', 'email-reply-to', 'email-x-mailer', 'email-mime-boundary', 'email-thread-index', 'email-message-id', 'mobile-application-id', 'chrome-extension-id', 'whois-registrant-email', 'anonymised')
),
'Artifacts dropped' => array(
'desc' => __('Any artifact (files, registry keys etc.) dropped by the malware or other modifications to the system'),
'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|authentihash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy','filename|pehash', 'regkey', 'regkey|value', 'pattern-in-file', 'pattern-in-memory','pdb', 'stix2-pattern', 'yara', 'sigma', 'attachment', 'malware-sample', 'named pipe', 'mutex', 'windows-scheduled-task', 'windows-service-name', 'windows-service-displayname', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'cookie', 'gene', 'mime-type', 'anonymised')
'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|authentihash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy','filename|pehash', 'regkey', 'regkey|value', 'pattern-in-file', 'pattern-in-memory','pdb', 'stix2-pattern', 'yara', 'sigma', 'attachment', 'malware-sample', 'named pipe', 'mutex', 'windows-scheduled-task', 'windows-service-name', 'windows-service-displayname', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'cookie', 'gene', 'kusto-query', 'mime-type', 'anonymised')
),
'Payload installation' => array(
'desc' => __('Info on where the malware gets installed in the system'),
'formdesc' => __('Location where the payload was placed in the system and the way it was installed. For example, a filename|md5 type attribute can be added here like this: c:\\windows\\system32\\malicious.exe|41d8cd98f00b204e9800998ecf8427e.'),
'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|authentihash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy', 'filename|pehash', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'stix2-pattern', 'yara', 'sigma', 'vulnerability', 'weakness', 'attachment', 'malware-sample', 'malware-type', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'mobile-application-id', 'other', 'mime-type', 'anonymised')
'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|authentihash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy', 'filename|pehash', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'stix2-pattern', 'yara', 'sigma', 'vulnerability', 'weakness', 'attachment', 'malware-sample', 'malware-type', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'mobile-application-id', 'chrome-extension-id', 'other', 'mime-type', 'anonymised')
),
'Persistence mechanism' => array(
'desc' => __('Mechanisms used by the malware to start at boot'),
@ -191,6 +191,7 @@ class Attribute extends AppModel
'stix2-pattern' => array('desc' => __('STIX 2 pattern'), 'default_category' => 'Payload installation', 'to_ids' => 1),
'sigma' => array('desc' => __('Sigma - Generic Signature Format for SIEM Systems'), 'default_category' => 'Payload installation', 'to_ids' => 1),
'gene' => array('desc' => __('GENE - Go Evtx sigNature Engine'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
'kusto-query' => array('desc' => __('Kusto query - Kusto from Microsoft Azure is a service for storing and running interactive analytics over Big Data.'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
'mime-type' => array('desc' => __('A media type (also MIME type and content type) is a two-part identifier for file formats and format contents transmitted on the Internet'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
'identity-card-number' => array('desc' => __('Identity card number'), 'default_category' => 'Person', 'to_ids' => 0),
'cookie' => array('desc' => __('HTTP cookie as often stored on the user web client. This can include authentication cookie or session cookie.'), 'default_category' => 'Network activity', 'to_ids' => 0),
@ -317,6 +318,7 @@ class Attribute extends AppModel
'place-port-of-onward-foreign-destination' => array('desc' => __('A Port where the passenger is transiting to'), 'default_category' => 'Person', 'to_ids' => 0),
'passenger-name-record-locator-number' => array('desc' => __('The Passenger Name Record Locator is a key under which the reservation for a trip is stored in the system. The PNR contains, among other data, the name, flight segments and address of the passenger. It is defined by a combination of five or six letters and numbers.'), 'default_category' => 'Person', 'to_ids' => 0),
'mobile-application-id' => array('desc' => __('The application id of a mobile application'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'chrome-extension-id' => array('desc' => __('Chrome extension id'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'cortex' => array('desc' => __('Cortex analysis result'), 'default_category' => 'External analysis', 'to_ids' => 0),
'boolean' => array('desc' => __('Boolean value - to be used in objects'), 'default_category' => 'Other', 'to_ids' => 0),
'anonymised' => array('desc' => __('Anonymised value - described with the anonymisation object via a relationship'), 'formdesc' => __('Anonymised value - described with the anonymisation object via a relationship.'), 'default_category' => 'Other', 'to_ids' => 0)
@ -374,7 +376,9 @@ class Attribute extends AppModel
'deleted',
'disable_correlation',
'object_id',
'object_relation'
'object_relation',
'first_seen',
'last_seen'
);
public $searchResponseTypes = array(
@ -526,6 +530,16 @@ class Attribute extends AppModel
'rule' => array('inList', array('0', '1', '2', '3', '4', '5')),
'message' => 'Options: Your organisation only, This community only, Connected communities, All communities, Sharing group, Inherit event',
'required' => true
),
'first_seen' => array(
'rule' => array('datetimeOrNull'),
'required' => false,
'message' => array('Invalid ISO 8601 format')
),
'last_seen' => array(
'rule' => array('datetimeOrNull'),
'required' => false,
'message' => array('Invalid ISO 8601 format')
)
);
@ -604,6 +618,7 @@ class Attribute extends AppModel
if (isset($v['Attribute']['object_relation']) && $v['Attribute']['object_relation'] === null) {
$results[$k]['Attribute']['object_relation'] = '';
}
$results[$k] = $this->UTCToISODatetime($results[$k], $this->alias);
}
return $results;
}
@ -635,6 +650,13 @@ class Attribute extends AppModel
$this->data['Attribute']['value2'] = '';
}
}
$this->data = $this->ISODatetimeToUTC($this->data, $this->alias);
// update correlation... (only needed here if there's an update)
if ($this->id || !empty($this->data['Attribute']['id'])) {
$this->__beforeSaveCorrelation($this->data['Attribute']);
}
// always return true after a beforeSave()
return true;
}
@ -853,6 +875,16 @@ class Attribute extends AppModel
$date = new DateTime();
$this->data['Attribute']['timestamp'] = $date->getTimestamp();
}
// parse first_seen different formats
if (isset($this->data['Attribute']['first_seen'])) {
$this->data['Attribute']['first_seen'] = $this->data['Attribute']['first_seen'] === '' ? null : $this->data['Attribute']['first_seen'];
}
// parse last_seen different formats
if (isset($this->data['Attribute']['last_seen'])) {
$this->data['Attribute']['last_seen'] = $this->data['Attribute']['last_seen'] === '' ? null : $this->data['Attribute']['last_seen'];
}
// TODO: add explanatory comment
// TODO: i18n?
$result = $this->runRegexp($this->data['Attribute']['type'], $this->data['Attribute']['value']);
@ -985,6 +1017,20 @@ class Attribute extends AppModel
return $this->runValidation($value, $this->data['Attribute']['type']);
}
// check whether the variable is null or datetime
public function datetimeOrNull($fields)
{
$k = array_keys($fields)[0];
$seen = $fields[$k];
try {
new DateTime($seen);
$returnValue = true;
} catch (Exception $e) {
$returnValue = false;
}
return $returnValue || is_null($seen);
}
private $__hexHashLengths = array(
'authentihash' => 64,
'md5' => 32,
@ -1262,6 +1308,7 @@ class Attribute extends AppModel
case 'stix2-pattern':
case 'sigma':
case 'gene':
case 'kusto-query':
case 'mime-type':
case 'identity-card-number':
case 'cookie':
@ -1345,6 +1392,7 @@ class Attribute extends AppModel
case 'github-organisation':
case 'cpe':
case 'twitter-id':
case 'chrome-extension-id':
case 'mobile-application-id':
// no newline
if (!preg_match("#\n#", $value)) {
@ -2137,6 +2185,50 @@ class Attribute extends AppModel
return $fails;
}
public function ISODatetimeToUTC($data, $alias)
{
// convert into utc and micro sec
if (!empty($data[$alias]['first_seen'])) {
$d = new DateTime($data[$alias]['first_seen']);
$d->setTimezone(new DateTimeZone('GMT'));
$fs_sec = $d->format('U');
$fs_micro = $d->format('u');
$fs_micro = str_pad($fs_micro, 6, "0", STR_PAD_LEFT);
$fs = $fs_sec . $fs_micro;
$data[$alias]['first_seen'] = $fs;
}
if (!empty($data[$alias]['last_seen'])) {
$d = new DateTime($data[$alias]['last_seen']);
$d->setTimezone(new DateTimeZone('GMT'));
$ls_sec = $d->format('U');
$ls_micro = $d->format('u');
$ls_micro = str_pad($ls_micro, 6, "0", STR_PAD_LEFT);
$ls = $ls_sec . $ls_micro;
$data[$alias]['last_seen'] = $ls;
}
return $data;
}
public function UTCToISODatetime($data, $alias)
{
if (!empty($data[$alias]['first_seen'])) {
$fs = $data[$alias]['first_seen'];
$fs_sec = intval($fs / 1000000); // $fs is in micro (10^6)
$fs_micro = $fs % 1000000;
$fs_micro = str_pad($fs_micro, 6, "0", STR_PAD_LEFT);
$fs = $fs_sec . '.' . $fs_micro;
$data[$alias]['first_seen'] = DateTime::createFromFormat('U.u', $fs)->format('Y-m-d\TH:i:s.uP');
}
if (!empty($data[$alias]['last_seen'])) {
$ls = $data[$alias]['last_seen'];
$ls_sec = intval($ls / 1000000); // $ls is in micro (10^6)
$ls_micro = $ls % 1000000;
$ls_micro = str_pad($ls_micro, 6, "0", STR_PAD_LEFT);
$ls = $ls_sec . '.' . $ls_micro;
$data[$alias]['last_seen'] = DateTime::createFromFormat('U.u', $ls)->format('Y-m-d\TH:i:s.uP');
}
return $data;
}
public function hids($user, $type, $tags = '', $from = false, $to = false, $last = false, $jobId = false, $enforceWarninglist = false)
{
@ -3526,6 +3618,7 @@ class Attribute extends AppModel
$defaultDistribution = Configure::read('MISP.default_attribute_distribution');
}
}
$saveResult = true;
foreach ($attributes as $k => $attribute) {
if (!empty($attribute['encrypt']) && $attribute['encrypt']) {
$attribute = $this->onDemandEncrypt($attribute);
@ -3535,9 +3628,9 @@ class Attribute extends AppModel
}
unset($attribute['Attachment']);
$this->create();
$this->save($attribute);
$saveResult = $saveResult && $this->save($attribute);
}
return true;
return $saveResult;
}
public function onDemandEncrypt($attribute)
@ -3655,6 +3748,32 @@ class Attribute extends AppModel
return $conditions;
}
public function setTimestampSeenConditions($timestamp, $conditions, $scope = 'Attribute.first_seen', $returnRaw = false)
{
if (is_array($timestamp)) {
$timestamp[0] = intval($this->Event->resolveTimeDelta($timestamp[0])) * 1000000; // seen in stored in micro-seconds in the DB
$timestamp[1] = intval($this->Event->resolveTimeDelta($timestamp[1])) * 1000000; // seen in stored in micro-seconds in the DB
if ($timestamp[0] > $timestamp[1]) {
$temp = $timestamp[0];
$timestamp[0] = $timestamp[1];
$timestamp[1] = $temp;
}
$conditions['AND'][] = array($scope . ' >=' => $timestamp[0]);
$conditions['AND'][] = array($scope . ' <=' => $timestamp[1]);
} else {
$timestamp = intval($this->Event->resolveTimeDelta($timestamp)) * 1000000; // seen in stored in micro-seconds in the DB
if ($scope == 'Attribute.first_seen') {
$conditions['AND'][] = array($scope . ' >=' => $timestamp);
} else {
$conditions['AND'][] = array($scope . ' <=' => $timestamp);
}
}
if ($returnRaw) {
return $timestamp;
}
return $conditions;
}
public function setToIDSConditions($to_ids, $conditions)
{
if ($to_ids === 'exclude') {
@ -3963,7 +4082,8 @@ class Attribute extends AppModel
if (isset($attribute['uuid'])) {
$existingAttribute = $this->find('first', array(
'conditions' => array('Attribute.uuid' => $attribute['uuid']),
'recursive' => -1
'contain' => array('AttributeTag' => 'Tag'),
'recursive' => -1,
));
$this->Log = ClassRegistry::init('Log');
if (count($existingAttribute)) {
@ -3982,7 +4102,7 @@ class Attribute extends AppModel
return true;
}
// If a field is not set in the request, just reuse the old value
$recoverFields = array('value', 'to_ids', 'distribution', 'category', 'type', 'comment', 'sharing_group_id', 'object_id', 'object_relation');
$recoverFields = array('value', 'to_ids', 'distribution', 'category', 'type', 'comment', 'sharing_group_id', 'object_id', 'object_relation', 'first_seen', 'last_seen');
foreach ($recoverFields as $rF) {
if (!isset($attribute[$rF])) {
$attribute[$rF] = $existingAttribute['Attribute'][$rF];
@ -4052,7 +4172,9 @@ class Attribute extends AppModel
'comment',
'sharing_group_id',
'deleted',
'disable_correlation'
'disable_correlation',
'first_seen',
'last_seen'
);
if ($objectId) {
$fieldList[] = 'object_id';
@ -4074,28 +4196,36 @@ class Attribute extends AppModel
));
return $this->validationErrors;
} else {
if (isset($attribute['Tag']) && $user['Role']['perm_tagger']) {
foreach ($attribute['Tag'] as $tag) {
$tag_id = $this->AttributeTag->Tag->captureTag($tag, $user);
if ($tag_id) {
// fix the IDs here
$this->AttributeTag->attachTagToAttribute($this->id, $attribute['event_id'], $tag_id);
} else {
// If we couldn't attach the tag it is most likely because we couldn't create it - which could have many reasons
// However, if a tag couldn't be added, it could also be that the user is a tagger but not a tag editor
// In which case if no matching tag is found, no tag ID is returned. Logging these is pointless as it is the correct behaviour.
if ($user['Role']['perm_tag_editor']) {
$this->Log->create();
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Attrubute',
'model_id' => $this->id,
'email' => $user['email'],
'action' => 'edit',
'user_id' => $user['id'],
'title' => 'Failed create or attach Tag ' . $tag['name'] . ' to the attribute.',
'change' => ''
));
if ($user['Role']['perm_tagger']) {
/*
We should uncomment the line below in the future once we have tag soft-delete
A solution to still keep the behavior for previous instance could be to not soft-delete the Tag if the remote instance
has a version below x
*/
// $this->AttributeTag->pruneOutdatedAttributeTagsFromSync(isset($attribute['Tag']) ? $attribute['Tag'] : array(), $existingAttribute['AttributeTag']);
if (isset($attribute['Tag'])) {
foreach ($attribute['Tag'] as $tag) {
$tag_id = $this->AttributeTag->Tag->captureTag($tag, $user);
if ($tag_id) {
// fix the IDs here
$this->AttributeTag->attachTagToAttribute($this->id, $attribute['event_id'], $tag_id);
} else {
// If we couldn't attach the tag it is most likely because we couldn't create it - which could have many reasons
// However, if a tag couldn't be added, it could also be that the user is a tagger but not a tag editor
// In which case if no matching tag is found, no tag ID is returned. Logging these is pointless as it is the correct behaviour.
if ($user['Role']['perm_tag_editor']) {
$this->Log->create();
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Attrubute',
'model_id' => $this->id,
'email' => $user['email'],
'action' => 'edit',
'user_id' => $user['id'],
'title' => 'Failed create or attach Tag ' . $tag['name'] . ' to the attribute.',
'change' => ''
));
}
}
}
}
@ -4214,6 +4344,8 @@ class Attribute extends AppModel
'deleted' => array('function' => 'set_filter_deleted'),
'timestamp' => array('function' => 'set_filter_timestamp'),
'attribute_timestamp' => array('function' => 'set_filter_timestamp'),
'first_seen' => array('function' => 'set_filter_seen'),
'last_seen' => array('function' => 'set_filter_seen'),
'to_ids' => array('function' => 'set_filter_to_ids'),
'comment' => array('function' => 'set_filter_comment')
),
@ -4367,7 +4499,7 @@ class Attribute extends AppModel
$loop = false;
if (empty($params['limit'])) {
$memory_in_mb = $this->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
$memory_scaling_factor = isset($exportTool->memory_scaling_factor) ? $exportTool->memory_scaling_factor : 100;
$memory_scaling_factor = isset($exportTool->memory_scaling_factor) ? $exportTool->memory_scaling_factor : 80;
$params['limit'] = $memory_in_mb * $memory_scaling_factor;
$loop = true;
$params['page'] = 1;

16
app/Model/AttributeTag.php
View File

@ -103,6 +103,22 @@ class AttributeTag extends AppModel
return true;
}
// This function help mirroring the tags at attribute level. It will delete tags that are not present on the remote attribute
public function pruneOutdatedAttributeTagsFromSync($newerTags, $originalAttributeTags)
{
$newerTagsName = array();
foreach ($newerTags as $tag) {
$newerTagsName[] = strtolower($tag['name']);
}
foreach ($originalAttributeTags as $k => $attributeTag) {
if (!$attributeTag['local']) { //
if (!in_array(strtolower($attributeTag['Tag']['name']), $newerTagsName)) {
$this->softDelete($attributeTag['id']);
}
}
}
}
public function countForTag($tag_id, $user)
{
return $this->find('count', array(

34
app/Model/Event.php
View File

@ -2064,9 +2064,9 @@ class Event extends AppModel
// do not expose all the data ...
$fields = array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.date', 'Event.threat_level_id', 'Event.info', 'Event.published', 'Event.uuid', 'Event.attribute_count', 'Event.analysis', 'Event.timestamp', 'Event.distribution', 'Event.proposal_email_lock', 'Event.user_id', 'Event.locked', 'Event.publish_timestamp', 'Event.sharing_group_id', 'Event.disable_correlation', 'Event.extends_uuid');
$fieldsAtt = array('Attribute.id', 'Attribute.type', 'Attribute.category', 'Attribute.value', 'Attribute.to_ids', 'Attribute.uuid', 'Attribute.event_id', 'Attribute.distribution', 'Attribute.timestamp', 'Attribute.comment', 'Attribute.sharing_group_id', 'Attribute.deleted', 'Attribute.disable_correlation', 'Attribute.object_id', 'Attribute.object_relation');
$fieldsAtt = array('Attribute.id', 'Attribute.type', 'Attribute.category', 'Attribute.value', 'Attribute.to_ids', 'Attribute.uuid', 'Attribute.event_id', 'Attribute.distribution', 'Attribute.timestamp', 'Attribute.comment', 'Attribute.sharing_group_id', 'Attribute.deleted', 'Attribute.disable_correlation', 'Attribute.object_id', 'Attribute.object_relation', 'Attribute.first_seen', 'Attribute.last_seen');
$fieldsObj = array('*');
$fieldsShadowAtt = array('ShadowAttribute.id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.value', 'ShadowAttribute.to_ids', 'ShadowAttribute.uuid', 'ShadowAttribute.event_uuid', 'ShadowAttribute.event_id', 'ShadowAttribute.old_id', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.proposal_to_delete', 'ShadowAttribute.timestamp');
$fieldsShadowAtt = array('ShadowAttribute.id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.value', 'ShadowAttribute.to_ids', 'ShadowAttribute.uuid', 'ShadowAttribute.event_uuid', 'ShadowAttribute.event_id', 'ShadowAttribute.old_id', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.proposal_to_delete', 'ShadowAttribute.timestamp', 'ShadowAttribute.first_seen', 'ShadowAttribute.last_seen');
$fieldsOrg = array('id', 'name', 'uuid', 'local');
$fieldsServer = array('id', 'url', 'name');
if (!$options['includeAllTags']) {
@ -2702,6 +2702,13 @@ class Event extends AppModel
return $conditions;
}
public function set_filter_seen(&$params, $conditions, $options)
{
$f = $options['scope'] . '.' . $options['filter'];
$conditions = $this->Attribute->setTimestampSeenConditions($params[$options['filter']], $conditions, $f);
return $conditions;
}
public function set_filter_timestamp(&$params, $conditions, $options)
{
if ($options['filter'] == 'from') {
@ -5269,11 +5276,26 @@ class Event extends AppModel
private function __fillAttribute($attribute, $defaultDistribution)
{
if (!isset($attribute['category'])) {
$attribute['category'] = $this->Event->Attribute->typeDefinitions[$attribute['type']]['default_category'];
if (is_array($attribute['type'])) {
$attribute_type = $attribute['type'][0];
if (empty($attribute['category'])) {
$categories = array();
foreach ($attribute['type'] as $type) {
$category = $this->Attribute->typeDefinitions[$type]['default_category'];
if (!in_array($category, $categories)) {
$categories[] = $category;
}
}
$attribute['category'] = count($categories) === 1 ? $categories[0] : $categories;
}
} else {
$attribute_type = $attribute['type'];
if (empty($attribute['category'])) {
$attribute['category'] = $this->Attribute->typedefinitions[$attribute_type]['default_category'];
}
}
if (!isset($attribute['to_ids'])) {
$attribute['to_ids'] = $this->Event->Attribute->typeDefinitions[$attribute['type']]['to_ids'];
$attribute['to_ids'] = $this->Attribute->typeDefinitions[$attribute_type]['to_ids'];
}
$attribute['value'] = $this->Attribute->runRegexp($attribute['type'], $attribute['value']);
$attribute['distribution'] = (isset($attribute['distribution']) ? (int)$attribute['distribution'] : $defaultDistribution);
@ -5844,7 +5866,7 @@ class Event extends AppModel
unset($data[$dataType . 'Tag'][$k]);
continue;
}
$dataTag['Tag']['local'] = $dataTag['local'];
$dataTag['Tag']['local'] = empty($dataTag['local']) ? 0 : 1;
if (!isset($excludeGalaxy) || !$excludeGalaxy) {
if (substr($dataTag['Tag']['name'], 0, strlen('misp-galaxy:')) === 'misp-galaxy:') {
$cluster = $this->GalaxyCluster->getCluster($dataTag['Tag']['name']);

8
app/Model/Galaxy.php
View File

@ -378,8 +378,12 @@ class Galaxy extends AppModel
{
$galaxy = $this->find('first', array(
'recursive' => -1,
'fields' => 'id',
'conditions' => array('Galaxy.type' => $type, 'Galaxy.namespace' => $namespace),
'fields' => array('MAX(Galaxy.version) as latest_version', 'id'),
'conditions' => array(
'Galaxy.type' => $type,
'Galaxy.namespace' => $namespace
),
'group' => array('name', 'id')
));
return empty($galaxy) ? 0 : $galaxy['Galaxy']['id'];
}

407
app/Model/MispObject.php
View File

@ -46,6 +46,8 @@ class MispObject extends AppModel
),
);
public $shortDist = array(0 => 'Organisation', 1 => 'Community', 2 => 'Connected', 3 => 'All', 4 => ' Sharing Group', 5 => 'Inherit');
public $validate = array(
'uuid' => array(
'uuid' => array(
@ -60,6 +62,18 @@ class MispObject extends AppModel
)
);
public function afterFind($results, $primary = false)
{
foreach ($results as $k => $v) {
$results[$k] = $this->Attribute->UTCToISODatetime($results[$k], $this->alias);
}
return $results;
}
public function beforeSave($options = array()) {
$this->data = $this->Attribute->ISODatetimeToUTC($this->data, $this->alias);
}
public function beforeValidate($options = array())
{
parent::beforeValidate();
@ -75,6 +89,14 @@ class MispObject extends AppModel
$date = new DateTime();
$this->data[$this->alias]['timestamp'] = $date->getTimestamp();
}
// parse first_seen different formats
if (isset($this->data[$this->alias]['first_seen'])) {
$this->data[$this->alias]['first_seen'] = $this->data[$this->alias]['first_seen'] === '' ? null : $this->data[$this->alias]['first_seen'];
}
// parse last_seen different formats
if (isset($this->data[$this->alias]['last_seen'])) {
$this->data[$this->alias]['last_seen'] = $this->data[$this->alias]['last_seen'] === '' ? null : $this->data[$this->alias]['last_seen'];
}
if (empty($this->data[$this->alias]['template_version'])) {
$this->data[$this->alias]['template_version'] = 1;
}
@ -184,6 +206,18 @@ class MispObject extends AppModel
$result = $this->id;
foreach ($object['Attribute'] as $k => $attribute) {
$object['Attribute'][$k]['object_id'] = $this->id;
if (
(!array_key_exists('first_seen', $object['Attribute'][$k]) || is_null($object['Attribute'][$k]['first_seen'])) &&
(array_key_exists('first_seen', $object['Object']) && !is_null($object['Object']['first_seen']))
) {
$object['Attribute'][$k]['first_seen'] = $object['Object']['first_seen'];
}
if (
(!array_key_exists('last_seen', $object['Attribute'][$k]) || is_null($object['Attribute'][$k]['last_seen'])) &&
(array_key_exists('last_seen', $object['Object']) && !is_null($object['Object']['last_seen']))
) {
$object['Attribute'][$k]['last_seen'] = $object['Object']['last_seen'];
}
}
$this->Attribute->saveAttributes($object['Attribute']);
} else {
@ -254,6 +288,28 @@ class MispObject extends AppModel
return $conditions;
}
public function fetchObjectSimple($user, $options = array())
{
$params = array(
'conditions' => $this->buildConditions($user),
'fields' => array(),
'recursive' => -1
);
if (isset($options['conditions'])) {
$params['conditions']['AND'][] = $options['conditions'];
}
if (isset($options['fields'])) {
$params['fields'] = $options['fields'];
}
$results = $this->find('all', array(
'conditions' => $params['conditions'],
'recursive' => -1,
'fields' => $params['fields'],
'contain' => array('Event' => array('distribution', 'id', 'user_id', 'orgc_id', 'org_id')),
'sort' => false
));
return $results;
}
// Method that fetches all objects
// very flexible, it's basically a replacement for find, with the addition that it restricts access based on user
@ -489,12 +545,57 @@ class MispObject extends AppModel
$tmpfile->delete();
$tmpfile->close();
}
if (!isset($attributes['Attribute'][$k]['first_seen'])) {
$attributes['Attribute'][$k]['first_seen'] = null;
}
if (!isset($attributes['Attribute'][$k]['last_seen'])) {
$attributes['Attribute'][$k]['last_seen'] = null;
}
unset($attributes['Attribute'][$k]['save']);
}
return $attributes;
}
public function deltaMerge($object, $objectToSave)
// Set Object's *-seen (and ObjectAttribute's *-seen and ObjectAttribute's value if requested) to the provided *-seen value
// Therefore, synchronizing the 3 values
public function syncObjectAndAttributeSeen($object, $forcedSeenOnElements, $applyOnAttribute=True) {
if (empty($forcedSeenOnElements)) {
return $object;
}
if (isset($forcedSeenOnElements['first_seen'])) {
$object['Object']['first_seen'] = $forcedSeenOnElements['first_seen'];
}
if (isset($forcedSeenOnElements['last_seen'])) {
$object['Object']['last_seen'] = $forcedSeenOnElements['last_seen'];
}
if ($applyOnAttribute) {
if (isset($object['Attribute'])) {
$attributes = $object['Attribute'];
} else {
$attributes = $this->find('first', array(
'conditions' => array('id' => $object['Object']['id']),
'contain' => array('Attribute')
))['Attribute'];
}
foreach($attributes as $i => $attribute) {
if (isset($forcedSeenOnElements['first_seen'])) {
$attributes[$i]['first_seen'] = $forcedSeenOnElements['first_seen'];
if ($attribute['object_relation'] == 'first-seen') {
$attributes[$i]['value'] = $forcedSeenOnElements['first_seen'];
}
} elseif (isset($forcedSeenOnElements['last_seen'])) {
$attributes[$i]['last_seen'] = $forcedSeenOnElements['last_seen'];
if ($attribute['object_relation'] == 'last-seen') {
$attributes[$i]['value'] = $forcedSeenOnElements['last_seen'];
}
}
}
$object['Attribute'] = $attributes;
}
return $object;
}
public function deltaMerge($object, $objectToSave, $onlyAddNewAttribute=false)
{
if (!isset($objectToSave['Object'])) {
$dataToBackup = array('ObjectReferences', 'Attribute', 'ShadowAttribute');
@ -513,69 +614,141 @@ class MispObject extends AppModel
}
unset($dataToBackup);
}
$object['Object']['comment'] = $objectToSave['Object']['comment'];
$object['Object']['distribution'] = $objectToSave['Object']['distribution'];
if ($object['Object']['distribution'] == 4) {
$object['Object']['sharing_group_id'] = $objectToSave['Object']['sharing_group_id'];
if (isset($objectToSave['Object']['comment'])) {
$object['Object']['comment'] = $objectToSave['Object']['comment'];
}
if (isset($objectToSave['Object']['distribution'])) {
$object['Object']['distribution'] = $objectToSave['Object']['distribution'];
if ($object['Object']['distribution'] == 4) {
$object['Object']['sharing_group_id'] = $objectToSave['Object']['sharing_group_id'];
}
}
$date = new DateTime();
$object['Object']['timestamp'] = $date->getTimestamp();
$forcedSeenOnElements = array();
if (isset($objectToSave['Object']['first_seen'])) {
$forcedSeenOnElements['first_seen'] = $objectToSave['Object']['first_seen'];
}
if (isset($objectToSave['Object']['last_seen'])) {
$forcedSeenOnElements['last_seen'] = $objectToSave['Object']['last_seen'];
}
$object = $this->syncObjectAndAttributeSeen($object, $forcedSeenOnElements, false);
$this->save($object);
$checkFields = array('category', 'value', 'to_ids', 'distribution', 'sharing_group_id', 'comment', 'disable_correlation');
if (!empty($objectToSave['Attribute'])) {
foreach ($objectToSave['Attribute'] as $newKey => $newAttribute) {
foreach ($object['Attribute'] as $origKey => $originalAttribute) {
if (!empty($newAttribute['uuid'])) {
if ($newAttribute['uuid'] == $originalAttribute['uuid']) {
$different = false;
foreach ($checkFields as $f) {
if ($f == 'sharing_group_id' && empty($newAttribute[$f])) {
$newAttribute[$f] = 0;
if (!$onlyAddNewAttribute) {
$checkFields = array('category', 'value', 'to_ids', 'distribution', 'sharing_group_id', 'comment', 'disable_correlation', 'first_seen', 'last_seen');
if (!empty($objectToSave['Attribute'])) {
foreach ($objectToSave['Attribute'] as $newKey => $newAttribute) {
foreach ($object['Attribute'] as $origKey => $originalAttribute) {
if (!empty($newAttribute['uuid'])) {
if ($newAttribute['uuid'] == $originalAttribute['uuid']) {
$different = false;
foreach ($checkFields as $f) {
if ($f == 'sharing_group_id' && empty($newAttribute[$f])) {
$newAttribute[$f] = 0;
}
if ($newAttribute[$f] != $originalAttribute[$f]) {
$different = true;
}
// Set seen of object at attribute level
if (isset($forcedSeenOnElements['first_seen'])) {
$newAttribute['first_seen'] = $forcedSeenOnElements['first_seen'];
if ($newAttribute['object_relation'] == 'first-seen') {
// $newAttribute['value'] = $forcedSeenOnElements['first_seen'];
}
$different = true;
}
if (isset($forcedSeenOnElements['last_seen'])) {
$newAttribute['last_seen'] = $forcedSeenOnElements['last_seen'];
if ($newAttribute['object_relation'] == 'last-seen') {
// $newAttribute['value'] = $forcedSeenOnElements['last_seen'];
}
$different = true;
}
}
if ($newAttribute[$f] != $originalAttribute[$f]) {
$different = true;
if ($different) {
$newAttribute['id'] = $originalAttribute['id'];
$newAttribute['event_id'] = $object['Object']['event_id'];
$newAttribute['object_id'] = $object['Object']['id'];
$newAttribute['timestamp'] = $date->getTimestamp();
$result = $this->Event->Attribute->save(array('Attribute' => $newAttribute), array(
'category',
'value',
'to_ids',
'distribution',
'sharing_group_id',
'comment',
'timestamp',
'object_id',
'event_id',
'disable_correlation',
'first_seen',
'last_seen'
));
}
unset($object['Attribute'][$origKey]);
continue 2;
}
if ($different) {
$newAttribute['id'] = $originalAttribute['id'];
$newAttribute['event_id'] = $object['Object']['event_id'];
$newAttribute['object_id'] = $object['Object']['id'];
$newAttribute['timestamp'] = $date->getTimestamp();
$result = $this->Event->Attribute->save(array('Attribute' => $newAttribute), array(
'category',
'value',
'to_ids',
'distribution',
'sharing_group_id',
'comment',
'timestamp',
'object_id',
'event_id',
'disable_correlation'
));
}
unset($object['Attribute'][$origKey]);
continue 2;
}
}
}
$this->Event->Attribute->create();
$newAttribute['event_id'] = $object['Object']['event_id'];
$newAttribute['object_id'] = $object['Object']['id'];
if (!isset($newAttribute['timestamp'])) {
$newAttribute['distribution'] = Configure::read('MISP.default_attribute_distribution');
if ($newAttribute['distribution'] == 'event') {
$newAttribute['distribution'] = 5;
$this->Event->Attribute->create();
$newAttribute['event_id'] = $object['Object']['event_id'];
$newAttribute['object_id'] = $object['Object']['id'];
// Set seen of object at attribute level
if (isset($forcedSeenOnElements['first_seen'])) {
$newAttribute['first_seen'] = $forcedSeenOnElements['first_seen'];
if ($newAttribute['object_relation'] == 'first-seen') {
$newAttribute['value'] = $forcedSeenOnElements['first_seen'];
}
}
if (isset($forcedSeenOnElements['last_seen'])) {
$newAttribute['last_seen'] = $forcedSeenOnElements['last_seen'];
if ($newAttribute['object_relation'] == 'last-seen') {
$newAttribute['value'] = $forcedSeenOnElements['last_seen'];
}
}
if (!isset($newAttribute['timestamp'])) {
$newAttribute['distribution'] = Configure::read('MISP.default_attribute_distribution');
if ($newAttribute['distribution'] == 'event') {
$newAttribute['distribution'] = 5;
}
}
$this->Event->Attribute->save($newAttribute);
$attributeArrays['add'][] = $newAttribute;
unset($objectToSave['Attribute'][$newKey]);
}
foreach ($object['Attribute'] as $origKey => $originalAttribute) {
$originalAttribute['deleted'] = 1;
$this->Event->Attribute->save($originalAttribute);
}
$this->Event->Attribute->save($newAttribute);
$attributeArrays['add'][] = $newAttribute;
unset($objectToSave['Attribute'][$newKey]);
}
foreach ($object['Attribute'] as $origKey => $originalAttribute) {
$originalAttribute['deleted'] = 1;
$this->Event->Attribute->save($originalAttribute);
} else { // we only add the new attribute
$newAttribute = $objectToSave['Attribute'][0];
$this->Event->Attribute->create();
$newAttribute['event_id'] = $object['Object']['event_id'];
$newAttribute['object_id'] = $object['Object']['id'];
// Set seen of object at attribute level
if (
(!array_key_exists('first_seen', $newAttribute) || is_null($newAttribute['first_seen'])) &&
(!array_key_exists('first_seen', $object['Object']) && !is_null($object['Object']['first_seen']))
) {
$newAttribute['first_seen'] = $object['Object']['first_seen'];
}
if (
(!array_key_exists('last_seen', $newAttribute) || is_null($newAttribute['last_seen'])) &&
(!array_key_exists('last_seen', $object['Object']) && !is_null($object['Object']['last_seen']))
) {
$newAttribute['last_seen'] = $object['Object']['last_seen'];
$different = true;
}
if (!isset($newAttribute['timestamp'])) {
$newAttribute['distribution'] = Configure::read('MISP.default_attribute_distribution');
if ($newAttribute['distribution'] == 'event') {
$newAttribute['distribution'] = 5;
}
}
$saveAttributeResult = $this->Attribute->saveAttributes(array($newAttribute));
return $saveAttributeResult ? $this->id : $this->validationErrors;
}
return $this->id;
}
@ -692,8 +865,6 @@ class MispObject extends AppModel
'change' => 'Validation errors: ' . json_encode($this->validationErrors) . ' Full Object: ' . json_encode($attribute),
));
return $this->validationErrors;
} else {
$this->Event->unpublishEvent($eventId);
}
if (!empty($object['Attribute'])) {
foreach ($object['Attribute'] as $attribute) {
@ -703,14 +874,14 @@ class MispObject extends AppModel
return true;
}
public function updateTimestamp($id)
public function updateTimestamp($id, $timestamp = false)
{
$date = new DateTime();
$object = $this->find('first', array(
'recursive' => -1,
'conditions' => array('Object.id' => $id)
));
$object['Object']['timestamp'] = $date->getTimestamp();
$object['Object']['timestamp'] = $timestamp == false ? $date->getTimestamp() : $timestamp;
$object['Object']['skip_zmq'] = 1;
$object['Object']['skip_kafka'] = 1;
$result = $this->save($object);
@ -839,4 +1010,130 @@ class MispObject extends AppModel
return $saved_object['Object']['id'];
}
public function resolveUpdatedTemplate($template, $object, $update_template_available = false)
{
$toReturn = array(
'updateable_attribute' => false,
'not_updateable_attribute' => false,
'newer_template_version' => false,
'template' => $template
);
if (!empty($template)) {
$newer_template = $this->ObjectTemplate->find('first', array(
'conditions' => array(
'ObjectTemplate.uuid' => $object['Object']['template_uuid'],
'ObjectTemplate.version >' => $object['Object']['template_version'],
),
'recursive' => -1,
'contain' => array(
'ObjectTemplateElement'
),
'order' => array('ObjectTemplate.version DESC')
));
if (!empty($newer_template)) {
$toReturn['newer_template_version'] = $newer_template['ObjectTemplate']['version'];
// ignore IDs for comparison
$cur_template_temp = Hash::remove(Hash::remove($template['ObjectTemplateElement'], '{n}.id'), '{n}.object_template_id');
$newer_template_temp = Hash::remove(Hash::remove($newer_template['ObjectTemplateElement'], '{n}.id'), '{n}.object_template_id');
$template_difference = array();
// check how current template is included in the newer
foreach ($cur_template_temp as $cur_obj_rel) {
$flag_sim = false;
foreach ($newer_template_temp as $newer_obj_rel) {
$tmp = Hash::diff($cur_obj_rel, $newer_obj_rel);
if (count($tmp) == 0) {
$flag_sim = true;
break;
}
}
if (!$flag_sim) {
$template_difference[] = $cur_obj_rel;
}
}
$toReturn['updateable_attribute'] = $object['Attribute'];
$toReturn['not_updateable_attribute'] = array();
} else {
$toReturn['newer_template_version'] = false;
}
if (!empty($template_difference)) { // older template not completely embeded in newer
foreach ($template_difference as $temp_diff_element) {
foreach ($object['Attribute'] as $i => $attribute) {
if (
$attribute['object_relation'] == $temp_diff_element['object_relation']
&& $attribute['type'] == $temp_diff_element['type']
) { // This attribute cannot be merged automatically
$attribute['merge-possible'] = false;
$toReturn['not_updateable_attribute'][] = $attribute;
unset($toReturn['updateable_attribute'][$i]);
}
}
}
}
if ($update_template_available) { // template version bump requested
$toReturn['template'] = $newer_template; // bump the template version
}
}
return $toReturn;
}
public function reviseObject($revised_object, $object) {
$revised_object = json_decode(base64_decode($revised_object), true);
$revised_object_both = array('mergeable' => array(), 'notMergeable' => array());
// Loop through attributes to inject and perform the correct action
// (inject, duplicate, add warnings, ...) when applicable
foreach ($revised_object['Attribute'] as $attribute_to_inject) {
$flag_no_collision = true;
foreach ($object['Attribute'] as $attribute) {
if (
$attribute['object_relation'] == $attribute_to_inject['object_relation']
&& $attribute['type'] == $attribute_to_inject['type']
&& $attribute['value'] !== $attribute_to_inject['value']
) { // Collision on value
$multiple = !empty(Hash::extract($template['ObjectTemplateElement'], sprintf('{n}[object_relation=%s][type=%s][multiple=true]', $attribute['object_relation'], $attribute['type'])));
if ($multiple) { // if multiple is set, check if an entry exists already
$flag_entry_exists = false;
foreach ($object['Attribute'] as $attr) {
if (
$attr['object_relation'] == $attribute_to_inject['object_relation']
&& $attr['type'] == $attribute_to_inject['type']
&& $attr['value'] === $attribute_to_inject['value']
) {
$flag_entry_exists = true;
break;
}
}
if (!$flag_entry_exists) { // entry does no exists, can be duplicated
$attribute_to_inject['is_multiple'] = true;
$revised_object_both['mergeable'][] = $attribute_to_inject;
$object['Attribute'][] = $attribute_to_inject;
}
} else { // Collision on value, multiple not set => propose overwrite
$attribute_to_inject['current_value'] = $attribute['value'];
$attribute_to_inject['merge-possible'] = true; // the user can still swap value
$revised_object_both['notMergeable'][] = $attribute_to_inject;
}
$flag_no_collision = false;
} else if (
$attribute['object_relation'] == $attribute_to_inject['object_relation']
&& $attribute['type'] == $attribute_to_inject['type']
&& $attribute['value'] === $attribute_to_inject['value']
) { // all good, they are basically the same, do nothing
$revised_object_both['mergeable'][] = $attribute_to_inject;
$flag_no_collision = false;
}
}
if ($flag_no_collision) { // no collision, nor equalities => inject it straight away
$revised_object_both['mergeable'][] = $attribute_to_inject;
$object['Attribute'][] = $attribute_to_inject;
}
}
return array(
'object' => $object,
'revised_object_both' => $revised_object_both
);
}
}

90
app/Model/Server.php
View File

@ -2552,23 +2552,29 @@ class Server extends AppModel
return $final;
}
$filter_rules = json_decode($filter_rules, true);
$url_params = null;
foreach ($filter_rules as $field => $rules) {
$temp = array();
foreach ($rules as $operator => $elements) {
foreach ($elements as $k => $element) {
if ($operator === 'NOT') {
$element = '!' . $element;
}
if (!empty($element)) {
$temp[] = $element;
if ($field === 'url_params') {
$url_params = json_decode($rules, true);
} else {
foreach ($rules as $operator => $elements) {
foreach ($elements as $k => $element) {
if ($operator === 'NOT') {
$element = '!' . $element;
}
if (!empty($element)) {
$temp[] = $element;
}
}
}
}
if (!empty($temp)) {
$temp = implode('|', $temp);
$final[substr($field, 0, strlen($field) -1)] = $temp;
if (!empty($temp)) {
$temp = implode('|', $temp);
$final[substr($field, 0, strlen($field) -1)] = $temp;
}
}
}
$final = array_merge_recursive($final, $url_params);
return $final;
}
@ -4351,20 +4357,30 @@ class Server extends AppModel
'actual_db_version' => $actualDbVersion,
'checked_table_column' => array(),
'diagnostic' => array(),
'diagnostic_index' => array(),
'expected_db_version' => '?',
'error' => '',
'update_locked' => $this->isUpdateLocked(),
'remaining_lock_time' => $this->getLockRemainingTime(),
'update_fail_number_reached' => $this->UpdateFailNumberReached()
'update_fail_number_reached' => $this->UpdateFailNumberReached(),
'indexes' => array()
);
if ($dataSource == 'Database/Mysql') {
$dbActualSchema = $this->getActualDBSchema();
$dbExpectedSchema = $this->getExpectedDBSchema();
if ($dbExpectedSchema !== false) {
$db_schema_comparison = $this->compareDBSchema($dbActualSchema['schema'], $dbExpectedSchema['schema']);
$db_indexes_comparison = $this->compareDBIndexes($dbActualSchema['indexes'], $dbExpectedSchema['indexes']);
$schemaDiagnostic['checked_table_column'] = $dbActualSchema['column'];
$schemaDiagnostic['diagnostic'] = $db_schema_comparison;
$schemaDiagnostic['diagnostic_index'] = $db_indexes_comparison;
$schemaDiagnostic['expected_db_version'] = $dbExpectedSchema['db_version'];
foreach($dbActualSchema['schema'] as $tableName => $tableMetas) {
foreach($tableMetas as $tableMeta) {
$schemaDiagnostic['columnPerTable'][$tableName][] = $tableMeta['column_name'];
}
}
$schemaDiagnostic['indexes'] = $dbActualSchema['indexes'];
} else {
$schemaDiagnostic['error'] = sprintf('Diagnostic not available as the expected schema file could not be loaded');
}
@ -4501,6 +4517,7 @@ class Server extends AppModel
)
){
$dbActualSchema = array();
$dbActualIndexes = array();
$dataSource = $this->getDataSource()->config['datasource'];
if ($dataSource == 'Database/Mysql') {
$sqlGetTable = sprintf('SELECT TABLE_NAME FROM information_schema.tables WHERE table_schema = %s;', "'" . $this->getDataSource()->config['database'] . "'");
@ -4515,12 +4532,13 @@ class Server extends AppModel
foreach ($sqlResult as $column_schema) {
$dbActualSchema[$table][] = $column_schema['columns'];
}
$dbActualIndexes[$table] = $this->getDatabaseIndexes($this->getDataSource()->config['database'], $table);
}
}
else if ($dataSource == 'Database/Postgres') {
return array('Database/Postgres' => array('description' => __('Can\'t check database schema for Postgres database type')));
}
return array('schema' => $dbActualSchema, 'column' => $tableColumnNames);
return array('schema' => $dbActualSchema, 'column' => $tableColumnNames, 'indexes' => $dbActualIndexes);
}
public function compareDBSchema($dbActualSchema, $dbExpectedSchema)
@ -4582,7 +4600,7 @@ class Server extends AppModel
if ($colElementDiff == 'column_default') {
$expectedValue = $column['column_default'];
$actualValue = $keyedActualColumn[$columnName]['column_default'];
if (preg_match(sprintf('/(\'|")+%s(\1)+/', $expectedValue), $actualValue)) { // some version of mysql quote the default value
if (preg_match(sprintf('@(\'|")+%s(\1)+@', $expectedValue), $actualValue) || (empty($expectedValue) && $actualValue === 'NULL')) { // some version of mysql quote the default value
continue;
} else {
$isCritical = true;
@ -4627,6 +4645,42 @@ class Server extends AppModel
return $dbDiff;
}
public function compareDBIndexes($actualIndex, $expectedIndex)
{
$indexDiff = array();
foreach($expectedIndex as $tableName => $indexes) {
if (!array_key_exists($tableName, $actualIndex)) {
// If table does not exists, it is covered by the schema diagnostic
} else {
$tableIndexDiff = array_diff($indexes, $actualIndex[$tableName]); // check for missing indexes
if (count($tableIndexDiff) > 0) {
foreach($tableIndexDiff as $columnDiff) {
$indexDiff[$tableName][$columnDiff] = sprintf(__('Column `%s` should be indexed'), $columnDiff);
}
}
$tableIndexDiff = array_diff($actualIndex[$tableName], $indexes); // check for additional indexes
if (count($tableIndexDiff) > 0) {
foreach($tableIndexDiff as $columnDiff) {
$indexDiff[$tableName][$columnDiff] = sprintf(__('Column `%s` is indexed but should not'), $columnDiff);
}
}
}
}
return $indexDiff;
}
public function getDatabaseIndexes($database, $table)
{
$sqlTableIndex = sprintf(
"SELECT DISTINCT TABLE_NAME, COLUMN_NAME FROM information_schema.statistics WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s';",
$database,
$table
);
$sqlTableIndexResult = $this->query($sqlTableIndex);
$tableIndex = Hash::extract($sqlTableIndexResult, '{n}.statistics.COLUMN_NAME');
return $tableIndex;
}
public function writeableDirsDiagnostics(&$diagnostic_errors)
{
App::uses('File', 'Utility');
@ -4918,11 +4972,17 @@ class Server extends AppModel
if (!$alive || !$correct_user) {
$ok = false;
$workerIssueCount++;
$worker_array[$entry]['ok'] = false;
}
$worker_array[$entry]['workers'][] = array('pid' => $pid, 'user' => $worker['user'], 'alive' => $alive, 'correct_user' => $correct_user, 'ok' => $ok);
}
foreach ($worker_array as $k => $queue) {
if (isset($worker_array[$k]['workers'])) {
foreach($worker_array[$k]['workers'] as $worker) {
if ($worker['ok']) {
$worker_array[$k]['ok'] = true; // If at least one worker is up, the queue can be considered working
}
}
}
if ($k != 'scheduler') {
$worker_array[$k]['jobCount'] = CakeResque::getQueueSize($k);
}

35
app/Model/ShadowAttribute.php
View File

@ -133,6 +133,16 @@ class ShadowAttribute extends AppModel
'rule' => array('boolean'),
),
),
'first_seen' => array(
'rule' => array('datetimeOrNull'),
'required' => false,
'message' => array('Invalid ISO 8601 format')
),
'last_seen' => array(
'rule' => array('datetimeOrNull'),
'required' => false,
'message' => array('Invalid ISO 8601 format')
)
);
public function __construct($id = false, $table = null, $ds = null)
@ -177,6 +187,9 @@ class ShadowAttribute extends AppModel
if ($this->data['ShadowAttribute']['deleted']) {
$this->__beforeDeleteCorrelation($this->data['ShadowAttribute']);
}
// convert into utc and micro sec
$this->data = $this->Attribute->ISODatetimeToUTC($this->data, $this->alias);
return true;
}
@ -358,6 +371,14 @@ class ShadowAttribute extends AppModel
return true;
}
public function afterFind($results, $primary = false)
{
foreach ($results as $k => $v) {
$results[$k] = $this->Attribute->UTCToISODatetime($results[$k], $this->alias);
}
return $results;
}
public function validateTypeValue($fields)
{
$category = $this->data['ShadowAttribute']['category'];
@ -474,6 +495,20 @@ class ShadowAttribute extends AppModel
return $fails;
}
// check whether the variable is null or datetime
public function datetimeOrNull($fields)
{
$k = array_keys($fields)[0];
$seen = $fields[$k];
try {
new DateTime($seen);
$returnValue = true;
} catch (Exception $e) {
$returnValue = false;
}
return $returnValue || is_null($seen);
}
public function setDeleted($id)
{
$this->Behaviors->detach('SysLogLogable.SysLogLogable');

14
app/View/Attributes/add.ctp
View File

@ -65,6 +65,16 @@
'field' => 'disable_correlation',
'type' => 'checkbox'
),
array(
'field' => 'first_seen',
'type' => 'text',
'hidden' => true
),
array(
'field' => 'last_seen',
'type' => 'text',
'hidden' => true
),
'<div id="extended_event_preview" style="width:446px;"></div>'
),
'submit' => array(
@ -74,6 +84,9 @@
"'" . ($action == 'add' ? h($event_id) : h($attribute['Attribute']['id'])) . "'",
"'" . h($action) . "'"
)
),
'metaFields' => array(
'<div id="bothSeenSliderContainer" style="height: 170px;"></div>'
)
)
));
@ -130,4 +143,5 @@
checkSharingGroup('Attribute');
});
</script>
<?php echo $this->element('form_seen_input'); ?>
<?php echo $this->Js->writeBuffer(); // Write cached scripts

17
app/View/Attributes/ajax/attributeEditFirst_seenForm.ctp Normal file
View File

@ -0,0 +1,17 @@
<?php
echo $this->Form->create('Attribute', array(
'id' => 'Attribute' . '_' . $object['id'] . '_first_seen_form',
'url' => '/attributes/editField/' . $object['id']
));
?>
<?php
echo $this->Form->input('first_seen', array(
'label' => false,
'type' => 'text',
'value' => 0,
'id' => 'Attribute' . '_' . $object['id'] . '_first_seen_field',
'div' => false
));
echo $this->Form->end();
?>
</div>

15
app/View/Attributes/ajax/attributeEditLast_seenForm.ctp Normal file
View File

@ -0,0 +1,15 @@
<?php
echo $this->Form->create('Attribute', array(
'id' => 'Attribute' . '_' . $object['id'] . '_last_seen_form',
'url' => '/attributes/editField/' . $object['id']
));
echo $this->Form->input('last_seen', array(
'label' => false,
'type' => 'text',
'value' => 0,
'id' => 'Attribute' . '_' . $object['id'] . '_last_seen_field',
'div' => false
));
echo $this->Form->end();
?>
</div>

22
app/View/Attributes/search.ctp
View File

@ -32,13 +32,27 @@
'type' => 'checkbox',
'label' => __('Alternate Search Result (Events)')
));
echo $this->Form->input('first_seen', array(
'type' => 'text',
'div' => 'input hidden',
'required' => false,
));
echo $this->Form->input('last_seen', array(
'type' => 'text',
'div' => 'input hidden',
'required' => false,
));
?>
<div class="clear">
<h3><?php echo __('First seen and Last seen.'); ?></h3>
<p><?php echo __('Attributes not having first seen or last seen set might not appear in the search'); ?></p>
</div>
</fieldset>
<?php
echo $this->Form->button(__('Search'), array('class' => 'btn btn-primary'));
echo $this->Form->end();
?>
<?php echo $this->Form->end(); ?>
<div id="bothSeenSliderContainer"></div>
<button onclick="$('#AttributeSearchForm').submit();" class="btn btn-primary">Submit</button>
</div>
<?php echo $this->element('form_seen_input'); ?>
<script type="text/javascript">
//
// Generate Category / Type filtering array

3
app/View/Elements/Events/View/row_attribute.ctp
View File

@ -54,6 +54,9 @@
<td class="short context hidden">
<?php echo h($object['uuid']); ?>
</td>
<td class="short context hidden">
<?php echo $this->element('/Events/View/seen_field', array('object' => $object)); ?>
</td>
<td class="short">
<?php echo date('Y-m-d', $object['timestamp']); ?>
</td>

17
app/View/Elements/Events/View/row_object.ctp
View File

@ -1,7 +1,7 @@
<?php
$tr_class = '';
$linkClass = 'white';
$currentType = 'denyForm';
$currentType = 'Object';
$tr_class = 'tableHighlightBorderTop borderBlue';
if ($event['Event']['id'] != $object['event_id']) {
if (!$isSiteAdmin && $event['extensionEvents'][$object['event_id']]['Orgc']['id'] != $me['org_id']) {
@ -36,6 +36,9 @@
<td class="short context hidden">
<?php echo h($object['uuid']); ?>
</td>
<td class="short context hidden">
<?php echo $this->element('/Events/View/seen_field', array('object' => $object)); ?>
</td>
<td class="short">
<?php echo date('Y-m-d', $object['timestamp']); ?>
</td>
@ -85,18 +88,21 @@
}
?>
</td>
<td class="shortish">
<?php echo h($object['comment']); ?>
<td class="showspaces bitwider" onmouseenter="quickEditHover(this, 'Object', '<?php echo $object['id']; ?>', 'comment', <?php echo $event['Event']['id'];?>);">
<div id = "Object_<?php echo $object['id']; ?>_comment_placeholder" class = "inline-field-placeholder"></div>
<div id = "Object_<?php echo $object['id']; ?>_comment_solid" class="inline-field-solid">
<?php echo nl2br(h($object['comment'])); ?>&nbsp;
</div>
</td>
<td colspan="4">&nbsp;
</td>
<td class="shortish">
<td class="shortish" onmouseenter="quickEditHover(this, 'Object', '<?php echo $object['id']; ?>', 'distribution', <?php echo $event['Event']['id'];?>);">
<?php
$turnRed = '';
if ($object['objectType'] == 0 && $object['distribution'] == 0) $turnRed = 'style="color:red"';
?>
<div id = "<?php echo $currentType . '_' . $object['id'] . '_distribution_placeholder'; ?>" class = "inline-field-placeholder"></div>
<div id = "<?php echo $currentType . '_' . $object['id'] . '_distribution_solid'; ?>" <?php echo $turnRed; ?> class="inline-field-solid" ondblclick="activateField('<?php echo $currentType; ?>', '<?php echo $object['id']; ?>', 'distribution', <?php echo $event['Event']['id'];?>);">
<div id = "<?php echo $currentType . '_' . $object['id'] . '_distribution_solid'; ?>" <?php echo $turnRed; ?> class="inline-field-solid">
<?php
if ($object['objectType'] == 0) {
if ($object['distribution'] == 4):
@ -155,5 +161,6 @@
'child' => $attrKey == $lastElement ? 'last' : true
));
}
echo '<tr class="objectAddFieldTr"><td><span class="fa fa-plus-circle objectAddField" title="' . __('Add an Object Attribute') .'" onclick="popoverPopup(this, ' . h($object['id']) . ', \'objects\', \'quickFetchTemplateWithValidObjectAttributes\')"></span></td></tr>';
}
?>

3
app/View/Elements/Events/View/row_proposal.ctp
View File

@ -49,6 +49,9 @@
<td class="short context hidden">
<?php echo $object['objectType'] == 0 ? h($object['uuid']) : '&nbsp;'; ?>
</td>
<td class="short context hidden">
<?php echo $this->element('/Events/View/seen_field', array('object' => $object)); ?>
</td>
<td class="short">
<div id = "<?php echo $currentType . '_' . $object['id'] . '_timestamp_solid'; ?>">
<?php

3
app/View/Elements/Events/View/row_proposal_delete.ctp
View File

@ -50,6 +50,9 @@
<td class="short context hidden">
<?php echo h($object['uuid']); ?>
</td>
<td class="short context hidden">
<?php echo $this->element('/Events/View/seen_field', array('object' => $object)); ?>
</td>
<td style="font-weight:bold;text-align:left;">DELETE</td>
<?php
if ($extended):

9
app/View/Elements/Events/View/seen_field.ctp Normal file
View File

@ -0,0 +1,9 @@
<?php if ($object['first_seen'] != null || $object['last_seen'] != null): ?>
<div>
<div><?php echo $object['first_seen'] != null ? h($object['first_seen']) : '<span style="display: block; text-align:center;">_</span>'; ?></div>
<i style="display: block; text-align: center;" class="fas fa-arrow-down"></i>
<div><?php echo $object['last_seen'] != null ? h($object['last_seen']) : '<span style="display: block; text-align:center;">_</span>'; ?></div>
</div>
<?php else: ?>
<div></div>
<?php endif; ?>

3
app/View/Elements/Feeds/View/row_attribute.ctp
View File

@ -22,6 +22,9 @@
<td class="short">
<?php echo date('Y-m-d', $object['timestamp']); ?>
</td>
<td class="short">
<?php echo $this->element('/Servers/View/seen_field', array('object' => $object)); ?>
</td>
<td class="short">
<div id="Attribute_<?php echo $object['uuid']; ?>_category_solid" class="inline-field-solid">
<?php echo h($object['category']); ?>

3
app/View/Elements/Feeds/View/row_object.ctp
View File

@ -12,6 +12,9 @@
<td class="short">
<?php echo date('Y-m-d', $object['timestamp']); ?>
</td>
<td class="short">
<?php echo $this->element('/Servers/View/seen_field', array('object' => $object)); ?>
</td>
<td colspan="<?php echo $fieldCount -2;?>">
<span class="bold"><?php echo __('Name: ');?></span><?php echo h($object['name']);?>
<span class="fa fa-expand useCursorPointer" title="<?php echo __('Expand or Collapse');?>" role="button" tabindex="0" aria-label="<?php echo __('Expand or Collapse');?>" data-toggle="collapse" data-target="#Object_<?php echo h($object['uuid']); ?>_collapsible"></span>

9
app/View/Elements/Feeds/View/seen_field.ctp Normal file
View File

@ -0,0 +1,9 @@
<?php if (array_key_exists('first_seen', $object) && ($object['first_seen'] != null || $object['last_seen'] != null)): ?>
<div>
<div><?php echo $object['first_seen'] != null ? h($object['first_seen']) : '<span style="display: block; text-align:center;">_</span>'; ?></div>
<i style="display: block; text-align: center;" class="fas fa-arrow-down"></i>
<div><?php echo $object['last_seen'] != null ? h($object['last_seen']) : '<span style="display: block; text-align:center;">_</span>'; ?></div>
</div>
<?php else: ?>
<div></div>
<?php endif; ?>

1
app/View/Elements/Feeds/eventattribute.ctp
View File

@ -39,6 +39,7 @@
<table class="table table-striped table-condensed">
<tr>
<th><?php echo $this->Paginator->sort('timestamp', __('Date'));?></th>
<th><?php echo __('First seen') ?> <i class="fas fa-arrow-right"></i> <?php echo __('Last seen') ?></th>
<th><?php echo $this->Paginator->sort('category');?></th>
<th><?php echo $this->Paginator->sort('type');?></th>
<th><?php echo $this->Paginator->sort('value');?></th>

3
app/View/Elements/Servers/View/row_attribute.ctp
View File

@ -22,6 +22,9 @@
<td class="short">
<?php echo date('Y-m-d', $object['timestamp']); ?>
</td>
<td class="short">
<?php echo $this->element('/Servers/View/seen_field', array('object' => $object)); ?>
</td>
<td class="short">
<div id = "Attribute_<?php echo $object['id']; ?>_category_placeholder" class = "inline-field-placeholder"></div>
<div id = "Attribute_<?php echo $object['id']; ?>_category_solid" class="inline-field-solid" ondblclick="activateField('Attribute', '<?php echo $object['id']; ?>', 'category', <?php echo $event['Event']['id'];?>);">

3
app/View/Elements/Servers/View/row_object.ctp
View File

@ -12,6 +12,9 @@
<td class="short">
<?php echo date('Y-m-d', $object['timestamp']); ?>
</td>
<td class="short">
<?php echo $this->element('/Servers/View/seen_field', array('object' => $object)); ?>
</td>
<td colspan="<?php echo $fieldCount -2;?>">
<span class="bold"><?php echo __('Name');?>: </span><?php echo h($object['name']);?>
<span class="fa fa-expand useCursorPointer" title="<?php echo __('Expand or Collapse');?>" role="button" tabindex="0" aria-label="<?php echo __('Expand or Collapse');?>" data-toggle="collapse" data-target="#Object_<?php echo h($object['id']); ?>_collapsible"></span>

9
app/View/Elements/Servers/View/seen_field.ctp Normal file
View File

@ -0,0 +1,9 @@
<?php if (array_key_exists('first_seen', $object) && ($object['first_seen'] != null || $object['last_seen'] != null)): ?>
<div>
<div><?php echo $object['first_seen'] != null ? h($object['first_seen']) : '<span style="display: block; text-align:center;">_</span>'; ?></div>
<i style="display: block; text-align: center;" class="fas fa-arrow-down"></i>
<div><?php echo $object['last_seen'] != null ? h($object['last_seen']) : '<span style="display: block; text-align:center;">_</span>'; ?></div>
</div>
<?php else: ?>
<div></div>
<?php endif; ?>

1
app/View/Elements/Servers/eventattribute.ctp
View File

@ -39,6 +39,7 @@
<table class="table table-striped table-condensed">
<tr>
<th><?php echo $this->Paginator->sort('timestamp', __('Date'));?></th>
<th><?php echo __('First seen') ?> <i class="fas fa-arrow-right"></i> <?php echo __('Last seen') ?></th>
<th><?php echo $this->Paginator->sort('category');?></th>
<th><?php echo $this->Paginator->sort('type');?></th>
<th><?php echo $this->Paginator->sort('value');?></th>

1
app/View/Elements/eventattribute.ctp
View File

@ -135,6 +135,7 @@
?>
<th class="context hidden"><?php echo $this->Paginator->sort('id');?></th>
<th class="context hidden">UUID</th>
<th class="context hidden"><?php echo __('First seen') ?> <i class="fas fa-arrow-right"></i> <?php echo __('Last seen') ?></th>
<th><?php echo $this->Paginator->sort('timestamp', __('Date'), array('direction' => 'desc'));?></th>
<?php
if ($extended):

90
app/View/Elements/form_seen_input.ctp Normal file
View File

@ -0,0 +1,90 @@
<?php echo $this->Html->script('moment-with-locales'); ?>
<script>
<?php
$temp = explode('_', $this->params->controller);
if (count($temp) > 1) {
$temp = array_map(function($i, $str) {
return $i > 0 ? substr(ucfirst($str), 0, -1) : ucfirst($str);
}, array_keys($temp), $temp);
$temp = implode('', $temp);
} else {
$temp = substr(ucfirst($this->params->controller), 0, -1);
}
?>
var controller = "<?php echo $temp; ?>"; // get current controller name so that we can access all form fields
function reflect_change_on_form() {
var first_seen = '';
if ($('#date_fs').val() !== '') {
first_seen += $('#date_fs').val();
if ($("#time_fs").val() !== '') {
first_seen += 'T' + $('#time_fs').val();
}
}
var last_seen = '';
if ($('#date_ls').val() !== '') {
last_seen += $('#date_ls').val();
if ($("#time_ls").val() !== '') {
last_seen += 'T' + $('#time_ls').val();
}
}
$('#'+controller+'FirstSeen').val(first_seen);
$('#'+controller+'LastSeen').val(last_seen);
}
function extractDatetimePart(text) {
try {
var split = text.split('T')
return {
date: split[0],
time: split[1]
}
} catch (error) {
return { date: '', time: ''}
}
}
$(document).ready(function() {
var sliders_container = "#bothSeenSliderContainer"
var inputs_container = $('<div class="input-group input-daterange"></div>');
// create separate date and time input
var date_div_fs = $('<div class="input clear larger-input-field" style="margin-left: 10px;"></div>').append(
$('<label><?php echo __('First seen date') . '<span class="fas fa-calendar label-icon"></span>'; ?><input id="date_fs" type="text" style="width: 240px;"></input></label>')
);
$(inputs_container).append(date_div_fs);
var date_div_ls = $('<div class="input text larger-input-field"></div>').append(
$('<label><?php echo __('Last seen date') . '<span class="fas fa-calendar label-icon"></span>'; ?><input id="date_ls" type="text" style="width: 240px;"></input></label>')
);
$(inputs_container).append(date_div_ls);
$(sliders_container).append(inputs_container);
var time_div_fs = $('<div class="input clear larger-input-field" style="margin-left: 10px;"></div>').append(
$('<label><?php echo __('First seen time') . '<span class="fas fa-clock label-icon"></span>'; ?><input id="time_fs" type="text" style="width: 240px; text-align: center; margin-bottom: 0px" placeholder="HH:MM:SS.ssssss+TT:TT"></input></label>'),
$('<span class="apply_css_arrow"></span>').text('<?php echo __('Expected format: HH:MM:SS.ssssss+TT:TT') ?>')
);
$(sliders_container).append(time_div_fs);
var time_div_ls = $('<div class="input larger-input-field"></div>').append(
$('<label><?php echo __('Last seen time') . '<span class="fas fa-clock label-icon"></span>'; ?><input id="time_ls" type="text" style="width: 240px; text-align: center; margin-bottom: 0px" placeholder="HH:MM:SS.ssssss+TT:TT"></input></label>'),
$('<span class="apply_css_arrow"></span>').text('<?php echo __('Expected format: HH:MM:SS.ssssss+TT:TT') ?>')
);
$(sliders_container).append(time_div_ls);
$('#'+controller+'FirstSeen').closest('form').submit(function( event ) {
reflect_change_on_form();
});
var d1 = extractDatetimePart($('#'+controller+'FirstSeen').val());
var d2 = extractDatetimePart($('#'+controller+'LastSeen').val());
$('#date_fs').val(d1.date);
$('#time_fs').val(d1.time);
$('#date_ls').val(d2.date);
$('#time_ls').val(d2.time);
$('.input-daterange').datepicker({
preventMultipleSet: true,
format: 'yyyy-mm-dd',
todayHighlight: true
})
});
</script>

18
app/View/Elements/genericElements/IndexTable/Fields/actions.ctp
View File

@ -10,9 +10,25 @@
* - postLinkConfirm: As the user to confirm the POST before submission with the given message
* - onClick: custom onClick action instead of a simple GET/POST request
* - icon: FA icon (added using the helper, knowing the fa domain is not needed, just add the short name such as "edit")
*/
* - requirement evaluates to true/false
* - complex_requirement - add complex requirements via lambda functions:
* - function($row, $options): the lambda function. $row contain the row data
* - options: array of options. datapaths described in the datapath keyname will be extracted and replaced with the actual row value
*/
echo '<td class="short action-links">';
foreach ($actions as $action) {
if (isset($action['complex_requirement'])) {
if (isset($action['complex_requirement']['options']['datapath'])) {
foreach ($action['complex_requirement']['options']['datapath'] as $name => $path) {
$action['complex_requirement']['options']['datapath'][$name] = Hash::extract($row, $path)[0];
}
}
$options = isset($action['complex_requirement']['options']) ? $action['complex_requirement']['options'] : array();
$requirementMet = $action['complex_requirement']['function']($row, $options);
if (!$requirementMet) {
continue;
}
}
$url_param_data_paths = '';
$url = empty($action['url']) ? '#' : h($action['url']);
if (!empty($action['url_params_data_paths'])) {

62
app/View/Elements/healthElements/db_indexes_diagnostic.ctp Normal file
View File

@ -0,0 +1,62 @@
<div>
<label for="toggleTableDBIndexes" style="display: inline-block;">
<input type="checkbox" id="toggleTableDBIndexes" class="form-input" checked></input>
<?php echo __('Show database indexes') ?>
</label>
</div>
<div id="containerDBIndexes" class="" style="max-height: 800px; overflow-y: auto; padding: 5px;">
<?php if(empty($diagnostic)): ?>
<span class="label label-success"><?php echo __('Index diagnostic:'); ?><i class="fa fa-check"></i></span>
<?php else: ?>
<div class="alert alert-warning">
<strong><?php echo __('Notice'); ?></strong>
<?php echo __('The highlighted issues may be benign. if you are unsure, please open an issue and ask for clarification.'); ?>
</div>
<table id="tableDBIndexes" class="table table-condensed table-bordered">
<thead>
<tr>
<th>Table name</th>
<th>Column name</th>
<th>Indexed</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<?php foreach($columnPerTable as $tableName => $columnArray): ?>
<?php
$columnCount = 0;
$rowHtml = '';
?>
<?php foreach($columnArray as $columnName): ?>
<?php
$columnIndexed = !empty($indexes[$tableName]) && in_array($columnName, $indexes[$tableName]);
$warning = isset($diagnostic[$tableName][$columnName]);
if ($warning) {
$columnCount++;
}
$rowHtml .= sprintf('%s%s%s%s%s',
sprintf('<tr class="%s">', $warning ? 'error' : 'indexInfo hidden'),
sprintf('<td>%s</td>', h($columnName)),
sprintf('<td><i class="bold fa %s"></i></td>', $columnIndexed ? 'green fa-check' : 'red fa-times'),
sprintf('<td>%s</td>', $warning ? h($diagnostic[$tableName][$columnName]) : ''),
'</tr>'
);
?>
<?php endforeach; ?>
<?php if ($columnCount > 0): ?>
<?php echo sprintf('<tr><td rowspan="%s" colspan="0" class="bold">%s</td></tr>', $columnCount+1, h($tableName)); ?>
<?php echo $rowHtml; ?>
<?php endif; ?>
<?php endforeach; ?>
</tbody>
</table>
<?php endif; ?>
</div>
<script>
$(document).ready(function() {
$('#toggleTableDBIndexes').change(function() {
$('#containerDBIndexes').toggle();
})
})
</script>

8
app/View/Elements/healthElements/db_schema_diagnostic.ctp
View File

@ -24,7 +24,6 @@
*/
function highlightAndSanitize($dirty, $toHighlight, $colorType = 'success')
{
if (is_array($dirty)) {
@ -168,7 +167,12 @@
__('DataSource: ') . h($dataSource),
__('DataSource: ') . h($dataSource),
$dataSource != 'Database/Mysql' ? 'times' : 'check'
)
);
echo $this->element('/healthElements/db_indexes_diagnostic', array(
'columnPerTable' => $columnPerTable,
'diagnostic' => $dbIndexDiagnostics,
'indexes' => $indexes
));
?>
<script>
var dbSchemaDiagnostics = <?php echo json_encode($dbSchemaDiagnostics); ?>;

9
app/View/Elements/healthElements/diagnostics.ctp
View File

@ -66,7 +66,7 @@
</span><br />
<pre class="hidden green bold" id="gitResult"></pre>
<button title="<?php echo __('Pull the latest MISP version from github');?>" class="btn btn-inverse" style="padding-top:1px;padding-bottom:1px;" onClick = "updateMISP();"><?php echo __('Update MISP');?></button>
<a title="<?php echo __('Click the following button to go to the update progress page. This page lists all updates that are currently queued and executed.'); ?>" class="btn btn-inverse" style="padding-top:1px;padding-bottom:1px;" href="<?php echo $baseurl; ?>/servers/updateProgress/"><?php echo __('Update Progress');?></a>
<a title="<?php echo __('Click the following button to go to the update progress page. This page lists all updates that are currently queued and executed.'); ?>" style="margin-left: 5px;" href="<?php echo $baseurl; ?>/servers/updateProgress/"><i class="fas fa-tasks"></i> <?php echo __('View Update Progress');?></a>
</div>
<h3><?php echo __('Submodules version');?>
<it id="refreshSubmoduleStatus" class="fas fa-sync useCursorPointer" style="font-size: small; margin-left: 5px;" title="<?php echo __('Refresh submodules version.'); ?>"></it>
@ -159,7 +159,7 @@
<p><span class="bold"><?php echo __('PHP ini path');?></span>:… <span class="green"><?php echo h($php_ini); ?></span><br />
<span class="bold"><?php echo __('PHP Version');?> (><?php echo $phprec; ?> <?php echo __('recommended');?>): </span><span class="<?php echo $phpversions['web']['phpcolour']; ?>"><?php echo h($phpversions['web']['phpversion']) . ' (' . $phpversions['web']['phptext'] . ')';?></span><br />
<span class="bold"><?php echo __('PHP CLI Version');?> (><?php echo $phprec; ?> <?php echo __('recommended');?>): </span><span class="<?php echo $phpversions['cli']['phpcolour']; ?>"><?php echo h($phpversions['cli']['phpversion']) . ' (' . $phpversions['cli']['phptext'] . ')';?></span></p>
<p class="red bold"><?php echo __('Please note that the we will be dropping support for Python 2.7 and PHP 7.1 as of 2020-01-01 and are henceforth considered deprecated (but supported until the end of 2019). Both of these versions will by then reached End of Life and will become a liability. Furthermore, by dropping support for these outdated versions of the languages, we\'ll be able to phase out support for legacy code that exists solely to support them. Make sure that you plan ahead accordingly. More info: ');?><a href="https://secure.php.net/supported-versions.php">PHP</a>, <a href="https://www.python.org/dev/peps/pep-0373">Python</a>.</p>
<p class="red bold"><?php echo __('Please note that the support for Python versions below 3.6 and below PHP 7.2 has been dropped as of 2020-01-01 and are henceforth considered unsupported. More info: ');?><a href="https://secure.php.net/supported-versions.php">PHP</a>, <a href="https://www.python.org/dev/peps/pep-0373">Python</a>.</p>
<p><?php echo __('The following settings might have a negative impact on certain functionalities of MISP with their current and recommended minimum settings. You can adjust these in your php.ini. Keep in mind that the recommendations are not requirements, just recommendations. Depending on usage you might want to go beyond the recommended values.');?></p>
<?php
foreach ($phpSettings as $settingName => &$phpSetting):
@ -237,7 +237,10 @@
'remainingLockTime' => $dbSchemaDiagnostics['remaining_lock_time'],
'updateFailNumberReached' => $dbSchemaDiagnostics['update_fail_number_reached'],
'updateLocked' => $dbSchemaDiagnostics['update_locked'],
'dataSource' => $dbSchemaDiagnostics['dataSource']
'dataSource' => $dbSchemaDiagnostics['dataSource'],
'columnPerTable' => $dbSchemaDiagnostics['columnPerTable'],
'dbIndexDiagnostics' => $dbSchemaDiagnostics['diagnostic_index'],
'indexes' => $dbSchemaDiagnostics['indexes'],
)); ?>
</div>
<h3><?= __("Redis info") ?></h3>

142
app/View/Elements/serverRuleElements/pull.ctp
View File

@ -2,75 +2,79 @@
<div class="confirmation">
<legend><?php echo __('Set pull rules');?></legend>
<div style="padding-left:5px;padding-right:5px;padding-bottom:5px;">
<div style="padding:10px;">
<table>
<tr>
<td style="width:120px;">
<p style="color:green;font-weight:bold;"><?php echo __('Allowed Tags (OR)');?></p>
<select id="tagspullLeftValues" size="5" multiple style="width:185px;">
</select>
</td>
<td style="width:50px;text-align:center;">
<span title="<?php echo __('Move tag to the list of tags to allow');?>" role="button" tabindex="0" aria-label="<?php echo __('Move tag to the list of tags to allow');?>" class="btn btn-inverse" id="tagspullLeftLeft" style="padding:2px;" onClick="serverRuleMoveFilter('pull', 'tags', 'Middle', 'Left');">&lt;&lt;</span>
<span title="<?php echo __('Remove tag from the list of tags to allow');?>" role="button" tabindex="0" aria-label="<?php echo __('Remove tag from the list of tags to allow');?>" class="btn btn-inverse" id="tagspullLeftRight" style="padding:2px;" onClick="serverRuleMoveFilter('pull', 'tags', 'Left', 'Middle');">&gt;&gt;</span>
</td>
<td style="width:120px;">
<input id="tagspullNewValue" style="width:180px;">
</td>
<td style="width:50px;text-align:center;">
<span title="<?php echo __('Remove tag from the list of tags to block');?>" role="button" tabindex="0" aria-label="<?php echo __('Remove tag from the list of tags to block');?>" class="btn btn-inverse" id="tagspullRightLeft" style="padding:2px;" onClick="serverRuleMoveFilter('pull', 'tags', 'Right', 'Middle');">&lt;&lt;</span>
<span title="<?php echo __('Move tag to the list of tags to block');?>" role="button" tabindex="0" aria-label="<?php echo __('Move tag to the list of tags to block');?>" class="btn btn-inverse" id="tagspullRightRight" style="padding:2px;" onClick="serverRuleMoveFilter('pull', 'tags', 'Middle', 'Right');">&gt;&gt;</span>
</td>
<td style="width:120px;">
<p style="color:red;font-weight:bold;"><?php echo __('Blocked Tags (AND NOT)');?></p>
<select id="tagspullRightValues" size="5" multiple style="width:185px;"></select>
</td>
</tr>
<tr>
<td class="bold green center" style="padding-bottom:15px;padding-top:15px;"><?php echo __('AND');?></td>
<td>&nbsp;</td>
<td>&nbsp;</td>
<td>&nbsp;</td>
<td class="bold red center" style="padding-bottom:15px;padding-top:15px;"><?php echo __('AND NOT');?></td>
</tr>
<tr>
<td style="width:120px;">
<p style="color:green;font-weight:bold;"><?php echo __('Allowed Orgs (OR)');?></p>
<select id="orgspullLeftValues" size="5" multiple style="width:185px;">
</select>
</td>
<td style="width:50px;text-align:center;">
<span title="<?php echo __('Move organisation to the list of tags to allow');?>" role="button" tabindex="0" aria-label="<?php echo __('Move organisation to the list of organisations to allow');?>" class="btn btn-inverse" id="orgspullLeftLeft" style="padding:2px;" onClick="serverRuleMoveFilter('pull', 'orgs', 'Middle', 'Left');">&lt;&lt;</span>
<span title="<?php echo __('Remove organisation to the list of tags to allow');?>" role="button" tabindex="0" aria-label="<?php echo __('Remove organisation form the list of organisations to allow');?>" class="btn btn-inverse" id="orgspullLeftRight" style="padding:2px;" onClick="serverRuleMoveFilter('pull', 'orgs', 'Left', 'Middle');">&gt;&gt;</span>
</td>
<td style="width:120px;">
<input id="orgspullNewValue" style="width:180px;">
</td>
<td style="width:50px;text-align:center;">
<span title="<?php echo __('Remove organisation from the list of tags to allow');?>" role="button" tabindex="0" aria-label="<?php echo __('Remove organisation from the list of organisations to block');?>" class="btn btn-inverse" id="orgspullRightLeft" style="padding:2px;" onClick="serverRuleMoveFilter('pull', 'orgs', 'Right', 'Middle');">&lt;&lt;</span>
<span title="<?php echo __('Move organisation to the list of tags to block');?>" role="button" tabindex="0" aria-label="<?php echo __('Move organisation to the list of organisations to block');?>" class="btn btn-inverse" id="orgspullRightRight" style="padding:2px;" onClick="serverRuleMoveFilter('pull', 'orgs', 'Middle', 'Right');">&gt;&gt;</span>
</td>
<td style="width:120px;">
<p style="color:red;font-weight:bold;"><?php echo __('Blocked Orgs (AND NOT)');?></p>
<select id="orgspullRightValues" size="5" multiple style="width:185px;"></select>
</td>
</tr>
</table>
</div>
<table>
<tr>
<td style="vertical-align:top">
<span title="<?php echo __('Accept changes');?>" role="button" tabindex="0" aria-label="<?php echo __('Accept changes');?>" id="PromptYesButton" class="btn btn-primary" onClick="submitServerRulePopulateTagPicklistValues('pull');"><?php echo __('Update');?></span>
</td>
<td style="width:540px;">
</td>
<td style="vertical-align:top;">
<span title="<?php echo __('Cancel');?>" role="button" tabindex="0" aria-label="<?php echo __('Cancel');?>" class="btn btn-inverse" id="PromptNoButton" onClick="serverRuleCancel();"><?php echo __('Cancel');?></span>
</td>
</tr>
</table>
<div style="padding:10px;">
<table>
<tr>
<td style="width:120px;">
<p style="color:green;font-weight:bold;"><?php echo __('Allowed Tags (OR)');?></p>
<select id="tagspullLeftValues" size="5" multiple style="width:185px;">
</select>
</td>
<td style="width:50px;text-align:center;">
<span title="<?php echo __('Move tag to the list of tags to allow');?>" role="button" tabindex="0" aria-label="<?php echo __('Move tag to the list of tags to allow');?>" class="btn btn-inverse" id="tagspullLeftLeft" style="padding:2px;" onClick="serverRuleMoveFilter('pull', 'tags', 'Middle', 'Left');">&lt;&lt;</span>
<span title="<?php echo __('Remove tag from the list of tags to allow');?>" role="button" tabindex="0" aria-label="<?php echo __('Remove tag from the list of tags to allow');?>" class="btn btn-inverse" id="tagspullLeftRight" style="padding:2px;" onClick="serverRuleMoveFilter('pull', 'tags', 'Left', 'Middle');">&gt;&gt;</span>
</td>
<td style="width:120px;">
<input id="tagspullNewValue" style="width:180px;">
</td>
<td style="width:50px;text-align:center;">
<span title="<?php echo __('Remove tag from the list of tags to block');?>" role="button" tabindex="0" aria-label="<?php echo __('Remove tag from the list of tags to block');?>" class="btn btn-inverse" id="tagspullRightLeft" style="padding:2px;" onClick="serverRuleMoveFilter('pull', 'tags', 'Right', 'Middle');">&lt;&lt;</span>
<span title="<?php echo __('Move tag to the list of tags to block');?>" role="button" tabindex="0" aria-label="<?php echo __('Move tag to the list of tags to block');?>" class="btn btn-inverse" id="tagspullRightRight" style="padding:2px;" onClick="serverRuleMoveFilter('pull', 'tags', 'Middle', 'Right');">&gt;&gt;</span>
</td>
<td style="width:120px;">
<p style="color:red;font-weight:bold;"><?php echo __('Blocked Tags (AND NOT)');?></p>
<select id="tagspullRightValues" size="5" multiple style="width:185px;"></select>
</td>
</tr>
<tr>
<td class="bold green center" style="padding-bottom:15px;padding-top:15px;"><?php echo __('AND');?></td>
<td>&nbsp;</td>
<td>&nbsp;</td>
<td>&nbsp;</td>
<td class="bold red center" style="padding-bottom:15px;padding-top:15px;"><?php echo __('AND NOT');?></td>
</tr>
<tr>
<td style="width:120px;">
<p style="color:green;font-weight:bold;"><?php echo __('Allowed Orgs (OR)');?></p>
<select id="orgspullLeftValues" size="5" multiple style="width:185px;">
</select>
</td>
<td style="width:50px;text-align:center;">
<span title="<?php echo __('Move organisation to the list of tags to allow');?>" role="button" tabindex="0" aria-label="<?php echo __('Move organisation to the list of organisations to allow');?>" class="btn btn-inverse" id="orgspullLeftLeft" style="padding:2px;" onClick="serverRuleMoveFilter('pull', 'orgs', 'Middle', 'Left');">&lt;&lt;</span>
<span title="<?php echo __('Remove organisation to the list of tags to allow');?>" role="button" tabindex="0" aria-label="<?php echo __('Remove organisation form the list of organisations to allow');?>" class="btn btn-inverse" id="orgspullLeftRight" style="padding:2px;" onClick="serverRuleMoveFilter('pull', 'orgs', 'Left', 'Middle');">&gt;&gt;</span>
</td>
<td style="width:120px;">
<input id="orgspullNewValue" style="width:180px;">
</td>
<td style="width:50px;text-align:center;">
<span title="<?php echo __('Remove organisation from the list of tags to allow');?>" role="button" tabindex="0" aria-label="<?php echo __('Remove organisation from the list of organisations to block');?>" class="btn btn-inverse" id="orgspullRightLeft" style="padding:2px;" onClick="serverRuleMoveFilter('pull', 'orgs', 'Right', 'Middle');">&lt;&lt;</span>
<span title="<?php echo __('Move organisation to the list of tags to block');?>" role="button" tabindex="0" aria-label="<?php echo __('Move organisation to the list of organisations to block');?>" class="btn btn-inverse" id="orgspullRightRight" style="padding:2px;" onClick="serverRuleMoveFilter('pull', 'orgs', 'Middle', 'Right');">&gt;&gt;</span>
</td>
<td style="width:120px;">
<p style="color:red;font-weight:bold;"><?php echo __('Blocked Orgs (AND NOT)');?></p>
<select id="orgspullRightValues" size="5" multiple style="width:185px;"></select>
</td>
</tr>
</table>
<div>
<p style="color:green;font-weight:bold;"><?php echo __('Additional sync parameters (based on the event index filters)');?></p>
<input style="width:650px;" placeholder='{"timestamp": "30d"}' maxlength="40" type="text" value="" id="urlParams" required="required" data-original-title="" title="">
</div>
</div>
<div>
<table style="width:100%;">
<tr>
<td style="vertical-align:top">
<span title="<?php echo __('Accept changes');?>" role="button" tabindex="0" aria-label="<?php echo __('Accept changes');?>" id="PromptYesButton" class="btn btn-primary" onClick="submitServerRulePopulateTagPicklistValues('pull');"><?php echo __('Update');?></span>
</td>
<td style="width:540px;">
</td>
<td style="vertical-align:top;">
<span title="<?php echo __('Cancel');?>" role="button" tabindex="0" aria-label="<?php echo __('Cancel');?>" class="btn btn-inverse" id="PromptNoButton" onClick="serverRuleCancel();"><?php echo __('Cancel');?></span>
</td>
</tr>
</table>
</div>
</div>
</div>
</div>

33
app/View/Elements/view_timeline.ctp Normal file
View File

@ -0,0 +1,33 @@
<?php
$mayModify = (($isAclModify && $event['Event']['user_id'] == $me['id'] && $event['Orgc']['id'] == $me['org_id']) || ($isAclModifyOrg && $event['Orgc']['id'] == $me['org_id']));
$mayPublish = ($isAclPublish && $event['Orgc']['id'] == $me['org_id']);
?>
<div>
<div id="timeline-header" class="eventgraph_header">
<label id="timeline-scope" class="btn center-in-network-header network-control-btn">
<span class="useCursorPointer fa fa-object-group" style="margin-right: 3px;"></span><?php echo __('Time scope')?>
</label>
<label id="timeline-display" class="btn center-in-network-header network-control-btn">
<span class="useCursorPointer fa fa-list-alt" style="margin-right: 3px;"></span><?php echo __('Display')?>
<span id="timeline-display-badge" class="badge"></span>
</label>
<select id="timeline-typeahead" class="center-in-network-header network-typeahead flushright position-absolute max-width-400" style="display:none" data-provide="typeahead" size="20" placeholder="Search for an item">
</select>
</div>
<div id="event_timeline" style="min-height: 100px;" data-user-manipulation="<?php echo $mayModify || $isSiteAdmin ? 'true' : 'false'; ?>" data-extended="<?php echo $extended; ?>">
<div class="loadingTimeline">
<div class="spinner"></div>
<div class="loadingText"><?php echo __('Loading');?></div>
</div>
</div>
<span id="fullscreen-btn-timeline" class="fullscreen-btn-timeline btn btn-xs btn-primary" data-toggle="tooltip" data-placement="top" data-title="<?php echo __('Toggle fullscreen');?>"><span class="fa fa-desktop"></span></span>
</div>
<?php
echo $this->Html->script('moment-with-locales');
echo $this->Html->script('event-timeline');
echo $this->Html->css('event-timeline');
?>

2
app/View/Events/resolved_misp_format.ctp
View File

@ -305,7 +305,7 @@
if (is_array($attribute[$field])) {
echo '<td class="short" style="width:40px;text-align:center;"><select ' . $field_header . 'Select" style="padding:0px;height:20px;margin-bottom:0px;">';
foreach ($attribute[$field] as $v => $value) {
echo '<option value="' . h($value) . '" ' . ($v ? '' : 'selected="selected"') . '>' . h($value) . '</option>';
echo '<option value="' . h($value) . '">' . h($value) . '</option>';
}
echo '</select></td>';
} else {

6
app/View/Events/view.ctp
View File

@ -508,6 +508,9 @@
<button class="btn btn-inverse toggle qet galaxy-toggle-button" id="eventgraph_toggle" data-toggle-type="eventgraph" onclick="enable_interactive_graph();">
<span class="icon-plus icon-white" title="<?php echo __('Toggle Event graph');?>" role="button" tabindex="0" aria-label="<?php echo __('Toggle Event graph');?>" style="vertical-align:top;"></span><?php echo __('Event graph');?>
</button>
<button class="btn btn-inverse toggle qet galaxy-toggle-button" id="eventtimeline_toggle" data-toggle-type="eventtimeline" onclick="enable_timeline();">
<span class="icon-plus icon-white" title="<?php echo __('Toggle Event timeline');?>" role="button" tabindex="0" aria-label="<?php echo __('Toggle Event timeline');?>" style="vertical-align:top;"></span><?php echo __('Event timeline');?>
</button>
<button class="btn btn-inverse toggle qet galaxy-toggle-button" id="correlationgraph_toggle" data-toggle-type="correlationgraph" onclick="enable_correlation_graph();">
<span class="icon-plus icon-white" title="<?php echo __('Toggle Correlation graph');?>" role="button" tabindex="0" aria-label="<?php echo __('Toggle Correlation graph');?>" style="vertical-align:top;"></span><?php echo __('Correlation graph');?>
</button>
@ -533,6 +536,9 @@
<div id="eventgraph_div" class="info_container_eventgraph_network" style="display: none;" data-fullscreen="false">
<?php echo $this->element('view_event_graph'); ?>
</div>
<div id="eventtimeline_div" class="info_container_eventtimeline" style="display: none;" data-fullscreen="false">
<?php echo $this->element('view_timeline'); ?>
</div>
<div id="correlationgraph_div" class="info_container_eventgraph_network" style="display: none;" data-fullscreen="false">
</div>
<div id="attackmatrix_div" class="info_container_eventgraph_network" style="display: none;" data-fullscreen="false" data-mitre-attack-galaxy-id="<?php echo h($mitreAttackGalaxyId)?>">

5
app/View/Helper/CommandHelper.php
View File

@ -67,11 +67,12 @@ App::uses('AppHelper', 'View/Helper');
if (!is_numeric($data) && ($trigger == 'event' || $trigger == 'thread')) {
$replacement = '%MALFORMED URL%';
} else {
$url = str_replace('$1', $data, $this->__replacement[$trigger]['url']);
if (filter_var(str_replace('$1', $data, $this->__replacement[$trigger]['url']), FILTER_VALIDATE_URL)) {
if (substr($data, 0, 7) === 'http://' || substr($data, 0, 8) === 'https://') {
if (substr($url, 0, 7) === 'http://' || substr($url, 0, 8) === 'https://') {
$replacement = $this->Html->link(
str_replace('$1', $data, $this->__replacement[$trigger]['text']),
str_replace('$1', $data, $this->__replacement[$trigger]['url'])
$url
);
} else {
$replacement = '%MALFORMED URL%';

16
app/View/Objects/add.ctp
View File

@ -5,7 +5,7 @@
echo $this->Form->create('Object', array('id', 'url' => $url, 'enctype' => 'multipart/form-data'));
?>
<h3><?php echo ucfirst($action) . ' ' . Inflector::humanize(h($template['ObjectTemplate']['name'])) . __(' Object'); ?></h3>
<div class="row-fluid" style="margin-bottom:10px;">
<div id="meta-div" class="row-fluid" style="margin-bottom:10px;">
<dl class="span8">
<dt><?php echo __('Object Template');?></dt>
<dd>
@ -77,6 +77,19 @@
));
?>
</dd>
<?php
echo $this->Form->input('first_seen', array(
'type' => 'text',
'div' => 'input hidden',
'required' => false,
));
echo $this->Form->input('last_seen', array(
'type' => 'text',
'div' => 'input hidden',
'required' => false,
));
?>
<div id="bothSeenSliderContainer"></div>
</dl>
</div>
<?php
@ -309,6 +322,7 @@
<?php
echo $this->element('form_seen_input');
if (!$ajax) {
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event', 'menuItem' => 'addObject', 'event' => $event));
}

22
app/View/Objects/ajax/objectEditCommentForm.ctp Normal file
View File

@ -0,0 +1,22 @@
<?php
echo $this->Form->create('Object', array('class' => 'inline-form inline-field-form', 'id' => 'Object_' . $object['id'] . '_comment_form', 'url' => '/objects/editField/' . $object['id']));
?>
<div class='inline-input inline-input-container'>
<div class="inline-input-accept inline-input-button inline-input-passive"><span class = "icon-ok" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
<div class="inline-input-decline inline-input-button inline-input-passive"><span class = "icon-remove" role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
<?php
echo $this->Form->input('comment', array(
'type' => 'textarea',
'label' => false,
'value' => $object['comment'],
'error' => array('escape' => false),
'class' => 'inline-input',
'id' => 'Object' . '_' . $object['id'] . '_comment_field',
'div' => false
));
echo $this->Form->end();
?>
</div>
<?php
echo $this->Form->end();
?>

19
app/View/Objects/ajax/objectEditDistributionForm.ctp Normal file
View File

@ -0,0 +1,19 @@
<?php
echo $this->Form->create('Object', array('class' => 'inline-form inline-field-form', 'id' => 'Object_' . $object['id'] . '_distribution_form', 'url' => '/objects/editField/' . $object['id']));
?>
<div class='inline-input inline-input-container'>
<div class="inline-input-accept inline-input-button inline-input-passive"><span class = "icon-ok" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
<div class="inline-input-decline inline-input-button inline-input-passive"><span class = "icon-remove" role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
<?php
echo $this->Form->input('distribution', array(
'options' => array($distributionLevels),
'label' => false,
'selected' => $object['distribution'],
'error' => array('escape' => false),
'class' => 'inline-input',
'id' => 'Object_' . $object['id'] . '_distribution_field',
'div' => false
));
echo $this->Form->end();
?>
</div>

14
app/View/Objects/ajax/objectEditFirst_seenForm.ctp Normal file
View File

@ -0,0 +1,14 @@
<?php
echo $this->Form->create('Object', array('id' => 'Object' . '_' . $object['id'] . '_first_seen_form', 'url' => '/objects/editField/' . $object['id']));
?>
<?php
echo $this->Form->input('first_seen', array(
'label' => false,
'type' => 'text',
'value' => 0,
'id' => 'Object' . '_' . $object['id'] . '_first_seen_field',
'div' => false
));
echo $this->Form->end();
?>
</div>

14
app/View/Objects/ajax/objectEditLast_seenForm.ctp Normal file
View File

@ -0,0 +1,14 @@
<?php
echo $this->Form->create('Object', array('id' => 'Object' . '_' . $object['id'] . '_last_seen_form', 'url' => '/objects/editField/' . $object['id']));
?>
<?php
echo $this->Form->input('last_seen', array(
'label' => false,
'type' => 'text',
'value' => 0,
'id' => 'Object' . '_' . $object['id'] . '_last_seen_field',
'div' => false
));
echo $this->Form->end();
?>
</div>

2
app/View/Objects/ajax/objectViewFieldForm.ctp Normal file
View File

@ -0,0 +1,2 @@
<?php
echo nl2br(h($value)) . '&nbsp;';

152
app/View/Objects/ajax/quickAddAttributeForm.ctp Normal file
View File

@ -0,0 +1,152 @@
<div class="generic-picker-embeded-block">
<?php
$url = '/objects/quickAddAttributeForm/' . $object['id'];
$element = $template_element;
$k = 0;
$action = 'add';
echo $this->Form->create('Object', array(
'id' => 'Object_' . $object['id'] . '_quick_add_attribute_form',
'url' => $url,
'class' => 'allDiv'
));
?>
<fieldset>
<legend><?php echo __('Add Object attribute'); ?></legend>
<div class="add_attribute_fields">
<?php
$formSettings = array(
'type' => 'hidden',
'value' => $element['object_relation'],
'label' => false,
'div' => false
);
echo $this->Form->input('Attribute.' . $k . '.object_relation', $formSettings);
$formSettings['value'] = $object['id'];
echo $this->Form->input('Object.id', $formSettings);
$formSettings['value'] = $object['event_id'];
echo $this->Form->input('Object.event_id', $formSettings);
$formSettings['value'] = $element['type'];
echo '<div style="margin-bottom: 5px; font-size: 14px;">';
echo $this->Form->input('Attribute.' . $k . '.type', $formSettings);
echo '<span class="bold">' . Inflector::humanize(h($element['object_relation'])) . '</span>';
echo ' :: ' . h($element['type']) . '';
echo '<br>';
echo '<span class="immutableAttributeDescription">' . h($element['description']) . '</span>';
echo '</div>';
?>
<?php
$formSettings = array(
'options' => array_combine($element['categories'], $element['categories']),
'default' => $element['default_category'],
'div' => true
);
echo $this->Form->input('Attribute.' . $k . '.category', $formSettings);
?>
<div class='input'>
<?php
echo $this->Form->input('Attribute.' . $k . '.distribution', array(
'class' => 'Attribute_distribution_select',
'options' => $distributionData['levels'],
'default' => !empty($element['distribution']) ? $element['distribution'] : $distributionData['initial'],
'div' => false,
'label' => __('Distribution ') . $this->element('formInfo', array('type' => 'distribution')),
));
?>
</div>
<div class='input'>
<div id="SGContainer" style="display:none;">
<?php
if (!empty($distributionData['sgs'])) {
echo $this->Form->input('Attribute.' . $k . '.sharing_group_id', array(
'options' => $distributionData['sgs'],
'label' => __('Sharing Group')
));
}
?>
</div>
</div>
<div class="clear">
<?php
echo '<label for="Attribute' . $k . '.value">' . __('Value') . '</label>';
echo $this->element(
'Objects/object_value_field',
array(
'element' => $element,
'k' => $k,
'action' => $action
)
);
?>
</div>
<?php
echo $this->Form->input('Attribute.' . $k . '.to_ids', array(
'type' => 'checkbox',
'checked' => $element['to_ids'],
));
?>
<?php
echo $this->Form->input('Attribute.' . $k . '.disable_correlation', array(
'type' => 'checkbox',
'checked' => $element['disable_correlation'],
));
?>
<?php
echo $this->Form->input('Attribute.' . $k . '.comment', array(
'type' => 'textarea',
'required' => false,
'allowEmpty' => true,
'div' => 'input clear',
'class' => 'input-xxlarge'
));
?>
</div>
</fieldset>
<div class="overlay_spacing">
<?php if ($ajax): ?>
<span id="submitButton" class="btn btn-primary" style="margin-bottom:5px;float:left;" title="<?php echo __('Submit'); ?>" role="button" tabindex="0" aria-label="<?php echo __('Submit'); ?>" onClick="submitPopoverForm('<?php echo h($object['id']); ?>', 'quickAddAttributeForm', <?php echo h($object['event_id']); ?>, 0, $(this).closest('div.popover').attr('data-dismissid'))"><?php echo __('Submit'); ?></span>
<?php else:
echo $this->Form->button('Submit', array('class' => 'btn btn-primary'));
endif;
?>
</div>
<?php
echo $this->Form->end();
?>
</div>
<script>
<?php
$formInfoTypes = array('distribution' => 'Distribution');
echo 'var formInfoFields = ' . json_encode($formInfoTypes) . PHP_EOL;
foreach ($formInfoTypes as $formInfoType => $humanisedName) {
echo 'var ' . $formInfoType . 'FormInfoValues = {' . PHP_EOL;
foreach ($info[$formInfoType] as $key => $formInfoData) {
echo '"' . $key . '": "<span class=\"blue bold\">' . h($formInfoData['key']) . '</span>: ' . h($formInfoData['desc']) . '<br />",' . PHP_EOL;
}
echo '}' . PHP_EOL;
}
?>
$(document).ready(function() {
initPopoverContent('Attribute0');
$('#Attribute0Distribution').change(function() {
initPopoverContent('Attribute0');
if ($('#Attribute0Distribution').val() == 4) $('#SGContainer').show();
else $('#SGContainer').hide();
});
});
</script>

7
app/View/Objects/revise_object.ctp
View File

@ -49,6 +49,13 @@
<td class="bold"><?php echo __('Comment');?></td>
<td><?php echo h($data['Object']['comment']); ?></td>
</tr>
<td class="bold"><?php echo __('First seen');?></td>
<td><?php echo h($data['Object']['first_seen']); ?></td>
</tr>
<tr>
<td class="bold"><?php echo __('Last seen');?></td>
<td><?php echo h($data['Object']['last_seen']); ?></td>
</tr>
<tr>
<table id="attribute_table" class="table table-condensed table-striped">
<thead>

77
app/View/Servers/edit.ctp
View File

@ -1,29 +1,25 @@
<div class="servers form">
<?php echo $this->Form->create('Server', array('type' => 'file', 'novalidate'=>true)); ?>
<fieldset>
<legend><?php echo __('Edit Server');?></legend>
<?php
echo '<h4 class="input clear">' . __('Instance identification') . '</h4>';
echo $this->Form->input('url', array(
'label' => __('Base URL'),
));
echo $this->Form->input('name', array(
'label' => __('Instance name'),
));
if (!empty($host_org_id) && $this->request->data['Server']['remote_org_id'] == $host_org_id):
?>
<div id="InternalDiv" class = "input clear" style="width:100%;">
<hr />
<p class="red" style="width:50%;"><?php echo __('You can set this instance up as an internal instance by checking the checkbox below. This means that any synchronisation between this instance and the remote will not be automatically degraded as it would in a normal synchronisation scenario. Please make sure that you own both instances and that you are OK with this otherwise dangerous change. This also requires that the current instance\'s host organisation and the remote sync organisation are the same.');?></p>
<?php
echo $this->Form->input('internal', array(
'label' => __('Internal instance'),
'type' => 'checkbox',
));
?>
</div>
<?php
endif;
<?php
echo $this->Form->create('Server', array('type' => 'file', 'novalidate'=>true));
echo '<fieldset>';
echo sprintf('<legend>%s</legend>', $this->action === 'add' ? __('Add Server') : __('Edit Server'));
echo '<h4 class="input clear">' . __('Instance identification') . '</h4>';
echo $this->Form->input('url', array(
'label' => __('Base URL'),
));
echo $this->Form->input('name', array(
'label' => __('Instance name'),
));
if (!empty($host_org_id) && !empty($this->request->data['Server']) && $this->request->data['Server']['remote_org_id'] == $host_org_id) {
echo sprintf(
'<div id="InternalDiv" class = "input clear" style="width:100%;"><hr /><p class="red" style="width:50%;">%s</p>%s</div>',
__('You can set this instance up as an internal instance by checking the checkbox below. This means that any synchronisation between this instance and the remote will not be automatically degraded as it would in a normal synchronisation scenario. Please make sure that you own both instances and that you are OK with this otherwise dangerous change. This also requires that the current instance\'s host organisation and the remote sync organisation are the same.'),
$this->Form->input('internal', array(
'label' => __('Internal instance'),
'type' => 'checkbox',
))
);
}
?>
<div class="input clear"></div>
<div class="input clear" style="width:100%;">
@ -33,11 +29,14 @@
</div>
<div class = "input clear"></div>
<?php
echo $this->Form->input('organisation_type', array(
'label' => __('Organisation Type'),
'options' => $organisationOptions,
'default' => $oldRemoteSetting
));
$org_type_form = array(
'label' => __('Organisation Type'),
'options' => $organisationOptions
);
if (!empty($oldRemoteSetting)) {
$org_type_form['default'] = $oldRemoteSetting;
}
echo $this->Form->input('organisation_type', $org_type_form);
?>
<div id="ServerExternalContainer" class="input select hiddenField" style="display:none;">
<label for="ServerExternal"><?php echo __('External Organisation');?></label>
@ -153,6 +152,7 @@
<span id="pull_tags_NOT" style="display:none;"><?php echo __('Events with the following tags blocked: ');?><span id="pull_tags_NOT_text" style="color:red;"></span><br /></span>
<span id="pull_orgs_OR" style="display:none;"><?php echo __('Events with the following organisations allowed: ');?><span id="pull_orgs_OR_text" style="color:green;"></span><br /></span>
<span id="pull_orgs_NOT" style="display:none;"><?php echo __('Events with the following organisations blocked: ');?><span id="pull_orgs_NOT_text" style="color:red;"></span><br /></span>
<span id="pull_url_params" style="display:none;"><?php echo __('Additional parameters: ');?><span id="pull_url_params_text" style="color:green;"></span><br /></span>
<span id="pull_modify" class="btn btn-inverse" style="line-height:10px; padding: 4px 4px;"><?php echo __('Modify');?></span><br /><br />
<?php
echo $this->Form->input('push_rules', array('style' => 'display:none;', 'label' => false, 'div' => false));
@ -162,7 +162,7 @@
echo $this->Form->checkbox('delete_client_cert', array('style' => 'display:none;', 'label' => false, 'div' => false));
?>
</fieldset>
<span role="button" tabindex="0" aria-label="<?php echo __('Submit');?>" title="<?php echo __('Submit');?>" class="btn btn-primary" onClick="serverSubmitForm('Edit');"><?php echo __('Submit');?></span>
<span role="button" tabindex="0" aria-label="<?php echo __('Submit');?>" title="<?php echo __('Submit');?>" class="btn btn-primary" onClick="serverSubmitForm('<?php echo Inflector::humanize($this->action);?>');"><?php echo __('Submit');?></span>
<?php
echo $this->Form->end();
?>
@ -172,7 +172,7 @@
<?php echo $this->element('serverRuleElements/pull'); ?>
</div>
<?php
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'sync', 'menuItem' => 'edit'));
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'sync', 'menuItem' => $this->action));
?>
@ -192,7 +192,17 @@ var formInfoValues = {
'ServerSelfSigned' : "<?php echo __('Click this, if you would like to allow a connection despite the other instance using a self-signed certificate (not recommended).');?>"
};
var rules = {"push": {"tags": {"OR":[], "NOT":[]}, "orgs": {"OR":[], "NOT":[]}}, "pull": {"tags": {"OR":[], "NOT":[]}, "orgs": {"OR":[], "NOT":[]}}};
var rules = {
"push": {
"tags": {"OR":[], "NOT":[]},
"orgs": {"OR":[], "NOT":[]}
},
"pull": {
"tags": {"OR":[], "NOT":[]},
"orgs": {"OR":[], "NOT":[]},
"url_params": ""
}
};
var validOptions = ['pull', 'push'];
var validFields = ['tags', 'orgs'];
var tags = <?php echo json_encode($allTags); ?>;
@ -201,7 +211,6 @@ var delete_cert = false;
var delete_client_cert = false;
var host_org_id = "<?php echo h($host_org_id); ?>";
var modelContext = 'Server';
$(document).ready(function() {
serverOrgTypeChange();
$('#ServerOrganisationType').change(function() {

12
app/View/Servers/index.ctp
View File

@ -63,6 +63,18 @@ foreach ($servers as $row_pos => $server):
}
}
}
if ($syncOption === 'pull') {
if (!empty($rules['pull']['url_params'])) {
$ruleDescription[$syncOption] .= sprintf(
'%s: %s',
sprintf("<span class='bold'>%s</span>", __('URL params')),
sprintf(
"<pre class='jsonify'>%s</pre>",
h(json_encode(json_decode($rules['pull']['url_params']), JSON_PRETTY_PRINT))
)
);
}
}
}
$arrows = '';
foreach (['up', 'down'] as $direction) {

2
app/View/Servers/update_progress.ctp
View File

@ -5,7 +5,7 @@ $towardDbVersion = $updateProgress['toward_db_version'];
$updateLocked = $updateProgress['update_locked'];
$lockRemainingTime = $updateProgress['lock_remaining_time'];
$updateFailNumberReached = $updateProgress['update_fail_number_reached'];
if ($updateProgress['total'] !== 0 ) {
if (!empty($updateProgress['total'])) {
$percentageFail = floor(count($updateProgress['failed_num']) / $updateProgress['total']*100);
$percentage = floor(($updateProgress['current']) / $updateProgress['total']*100);
} else {

13
app/View/ShadowAttributes/add.ctp
View File

@ -40,7 +40,18 @@
echo $this->Form->input('batch_import', array(
'type' => 'checkbox',
));
echo $this->Form->input('first_seen', array(
'type' => 'text',
'div' => 'input hidden',
'required' => false,
));
echo $this->Form->input('last_seen', array(
'type' => 'text',
'div' => 'input hidden',
'required' => false,
));
?>
<div id="bothSeenSliderContainer"></div>
</div>
</fieldset>
<p style="color:red;font-weight:bold;display:none;<?php if ($ajax) echo 'text-align:center;'; ?>" id="warning-message"><?php echo __('Warning: You are about to share data that is of a classified nature (Attribution / targeting data). Make sure that you are authorised to share this.');?></p>
@ -72,6 +83,8 @@
if (!$ajax) {
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event', 'menuItem' => 'proposeAttribute', 'event' => $event));
}
echo $this->element('form_seen_input');
?>
<script type="text/javascript">
<?php

13
app/View/ShadowAttributes/edit.ctp
View File

@ -40,7 +40,18 @@
echo $this->Form->input('to_ids', array(
'label' => __('IDS Signature?'),
));
echo $this->Form->input('first_seen', array(
'type' => 'text',
'div' => 'input hidden',
'required' => false,
));
echo $this->Form->input('last_seen', array(
'type' => 'text',
'div' => 'input hidden',
'required' => false,
));
?>
<div id="bothSeenSliderContainer"></div>
</fieldset>
<p style="color:red;font-weight:bold;display:none;<?php if (isset($ajax) && $ajax) echo "text-align:center;"?>" id="warning-message"><?php echo __('Warning: You are about to share data that is of a sensitive nature (Attribution / targeting data). Make sure that you are authorised to share this.');?></p>
<?php if (isset($ajax) && $ajax): ?>
@ -69,6 +80,8 @@
<?php
$event['Event']['id'] = $this->request->data['ShadowAttribute']['event_id'];
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event', 'menuItem' => 'proposeAttribute', 'event' => $event));
echo $this->element('form_seen_input');
?>
<script type="text/javascript">

6
app/View/Users/login.ctp
View File

@ -43,6 +43,12 @@
<?php
echo $this->Form->button(__('Login'), array('class' => 'btn btn-primary'));
echo $this->Form->end();
if (true == Configure::read('ApacheShibbAuth')):
?>
<div class="clear"></div>
<a class="btn btn-info" href="/Shibboleth.sso/Login">Login with SAML</a>
<?php
endif;
?>
</td>
<td style="width:250px;padding-left:50px">

83
app/files/feed-metadata/defaults.json
View File

@ -404,39 +404,6 @@
"hide_tag": false
}
},
{
"Feed": {
"id": "24",
"name": "booterblacklist.com Latest",
"provider": "booterblacklist.com",
"url": "http://booterblacklist.com/data/booterlist_latest.txt",
"rules": "",
"enabled": true,
"distribution": "0",
"sharing_group_id": "0",
"tag_id": "615",
"default": false,
"source_format": "freetext",
"fixed_event": true,
"delta_merge": true,
"event_id": "6230",
"publish": true,
"override_ids": false,
"settings": "{\"csv\":{\"value\":\"\",\"delimiter\":\",\"},\"common\":{\"excluderegex\":\"\"}}",
"input_source": "network",
"delete_local_file": false,
"lookup_visible": false,
"cache_timestamp": "1495871995"
},
"Tag": {
"id": "615",
"name": "osint:source-type=\"block-or-filter-list\"",
"colour": "#004f89",
"exportable": true,
"org_id": "0",
"hide_tag": false
}
},
{
"Feed": {
"id": "27",
@ -470,31 +437,6 @@
"hide_tag": false
}
},
{
"Feed": {
"id": "29",
"name": "Ransomware Tracker CSV Feed",
"provider": "Ransomware Tracker abuse.ch",
"url": "https://ransomwaretracker.abuse.ch/feeds/csv/",
"rules": "{\"tags\":{\"OR\":[],\"NOT\":[]},\"orgs\":{\"OR\":[],\"NOT\":[]}}",
"enabled": true,
"distribution": "0",
"sharing_group_id": "0",
"tag_id": "0",
"default": false,
"source_format": "csv",
"fixed_event": true,
"delta_merge": false,
"event_id": "0",
"publish": false,
"override_ids": false,
"settings": "{\"csv\":{\"value\":\"4,5,8\",\"delimiter\":\",\"},\"common\":{\"excluderegex\":\"\"}}",
"input_source": "network",
"delete_local_file": false,
"lookup_visible": true,
"cache_timestamp": "1495872683"
}
},
{
"Feed": {
"id": "30",
@ -1166,31 +1108,6 @@
"cache_timestamp": "1514750889"
}
},
{
"Feed": {
"id": "61",
"name": "conficker all domains generated",
"provider": "cert.at",
"url": "https://www.cert.at/static/downloads/data/conficker/all_domains.txt",
"rules": "",
"enabled": true,
"distribution": "3",
"sharing_group_id": "0",
"tag_id": "0",
"default": false,
"source_format": "csv",
"fixed_event": false,
"delta_merge": false,
"event_id": "0",
"publish": false,
"override_ids": false,
"settings": "{\"csv\":{\"value\":\"1\",\"delimiter\":\",\"},\"common\":{\"excluderegex\":\"\"}}",
"input_source": "network",
"delete_local_file": false,
"lookup_visible": true,
"cache_timestamp": "1514750904"
}
},
{
"Feed": {
"id": "64",

2
app/files/misp-decaying-models

@ -1 +1 @@
Subproject commit 0bf4d8bafc71f44cd65c7ad291bc0b4270dcecb8
Subproject commit f1b974790f0969fab4251046f292eedc59a272ca

2
app/files/misp-galaxy

@ -1 +1 @@
Subproject commit 47e0d00555aef510fcac854f7d5d2fa18f502adb
Subproject commit dbaab413b6b4680ae458a7d7a8ac6e1917fcc357

2
app/files/misp-objects

@ -1 +1 @@
Subproject commit 33a7d6b574b7354ba8243ba461bfb30db0528023
Subproject commit fa634803911d211f993049242d41eebaf342a9c4

2
app/files/taxonomies

@ -1 +1 @@
Subproject commit 6179f6bb4adb02c99c9a0b133d2b0756758d0585
Subproject commit d5cc5db3d736e5acede93d514070636834f385d4

2
app/files/warninglists

@ -1 +1 @@
Subproject commit 260171d89b494f4e8c84fa80ac263d296b43a2f9
Subproject commit eee124de3401f9f957bb30a5a4989d422324eca8

1
app/webroot/css/contextual_menu.css
View File

@ -26,6 +26,7 @@
.contextual-menu > label {
font-weight: bold;
margin-right: 3px;
user-select: none;
}
.contextual-menu > div > label {

151
app/webroot/css/event-timeline.css Normal file
View File

@ -0,0 +1,151 @@
.info_container_eventtimeline {
position: relative;
margin-top:10px;
border: 1px solid #0088cc;
border-radius: 7px;
box-shadow: 0px 0px 6px #B2B2B2;
width: 100%;
min-height: 100px;
padding: 1px;
}
.vis-timeline {
position: relative;
border: none;
overflow: hidden;
padding: 0;
margin: 0;
box-sizing: border-box;
}
.fullscreen-btn-timeline {
float: right;
display: inline-block;
bottom: 33px;
position: relative;
right: 3px;
opacity: 0.7;
z-index: 1;
cursor: nesw-resize;
}
.loadingTimeline {
display: none;
position: absolute;
margin-top: 10px;
width: 100%;
text-align: center;
}
.loadingTimeline > div.loadingText {
left: 0px;
}
.timestamp-obj {
box-shadow: #ff0000 0px 0px 0px 5px;
}
.timestamp-attr {
box-shadow: #ff0000 0px 0px 0px 5px;
padding-top: 4px;
padding-bottom: 4px;
}
.vis-item.attribute {
background-color: orange;
border-color: black;
}
.vis-item.attribute.vis-selected {
box-shadow: 0 0 20px rgba(82, 168, 236, 1);`
}
.vis-item.object {
background-color: #3465a4;
border-color: black;
color: white;
}
.vis-item.object.vis-selected {
box-shadow: 0 0 20px rgba(82, 168, 236, 1);`
}
.vis-item.object_attribute {
background-color: greenyellow;
border-color: green;
}
.timeline-objectName {
text-align:left;
border-bottom: white solid 1px;
}
.timeline-objectAttrType {
text-align:left;
line-height: 14px;
font-style: italic;
}
.timeline-objectAttrVal {
text-align:left;
line-height: 14px;
padding-left: 5px;
}
.vis-delete {
top: -8px !important;
padding: 3px !important;
height: 22px !important;
width: 20px !important;
}
.vis-expand-action {
position: absolute;
padding: 3px 2px !important;
height: 22px !important;
width: 22px !important;
top: 15px;
right: -25px;
box-sizing: border-box;
cursor: pointer;
-webkit-transition: background 0.2s linear;
-moz-transition: background 0.2s linear;
-ms-transition: background 0.2s linear;
-o-transition: background 0.2s linear;
transition: background 0.2s linear;
}
.vis-item .vis-expand-action:after {
color: green;
font-family: arial, sans-serif;
font-size: 22px;
font-weight: bold;
-webkit-transition: color 0.2s linear;
-moz-transition: color 0.2s linear;
-ms-transition: color 0.2s linear;
-o-transition: color 0.2s linear;
transition: color 0.2s linear;
}
.expand-btn:after {
content: "\21d4";
}
.collapse-btn:after {
content: "\21b5";
}
.vis-item .vis-expand-action:hover {
background: green;
}
.vis-item .vis-expand-action:hover:after {
color: white;
}
.vis-onUpdateTime-tooltip {
background: #7b7b7b !important;
padding: 0px !important;
border-radius: 5px !important;
border: solid 1px #000 !important;
}
#timeline-typeahead {
width: 400px;
}

78
app/webroot/css/main.css
View File

@ -25,6 +25,10 @@ button .full-width {
margin-bottom:10px;
}
form button.btn[disabled] {
cursor: not-allowed;
}
/*.close{
float:none;
}*/
@ -301,6 +305,39 @@ td.highlight3 {
}
table.table-striped > tbody > tr.objectAddFieldTr > td {
position: absolute;
background-color: #ffffff00;
}
table.table-striped > tbody > tr > td > span.objectAddField {
padding: 0px;
line-height: normal;
display: inline-block;
position: relative;
top: -19px;
left: -10px;
color: white;
background-color: #1369a0;
border: 0px;
border-radius: 15px;
height: 16px;
width: 16px;
text-align: center;
}
table.table-striped > tbody > tr > td > span.objectAddField:before {
left: 2px;
height: 12px;
width: 12px;
line-height: 16px;
}
.table-striped tbody > tr > td > span.objectAddField:hover {
transform: scale(1.7);
cursor: pointer;
}
tr.highlightBlueSides {
border-left:2px solid #0088cc;
border-right:2px solid #0088cc;
@ -880,6 +917,20 @@ a.proposal_link_red:hover {
border: 0;
}
.immutableAttributeDescription:before {
background: rgba(255, 255, 255, 0);
content: '';
margin-right: 5px;
margin-left: 2px;
border-left: 1px solid;
border-bottom: 1px solid;
height: 10px;
width: 5px;
position: relative;
display: block;
float: left;
}
.modal-body-long {
max-height: 600px;
}
@ -1043,6 +1094,16 @@ a.pill-pre-picker {
position: absolute;
}
div.generic-picker-wrapper > div.generic-picker-embeded-block {
margin-left: 15px;
margin-top: -12px;
display: inline-block;
}
div.generic-picker-embeded-block legend {
margin-bottom: 0px;
}
.generic-picker-item-element-check {
float: right;
border: 1px solid #999;
@ -2307,6 +2368,23 @@ table tr:hover .down-expand-button {
position: relative;
}
.nanosec-div {
font-weight:bold;
position: relative;
width: 100%;
text-align: center;
top: -5px;
}
.larger-input-field {
width: 270px !important;
}
.precision-tool {
margin-left: 10px;
display: inline-block;
}
.fa-as-icon {
font-size: 20px;
padding-left: 2px;

11
app/webroot/js/bootstrap-datepicker.js
View File

@ -1461,6 +1461,7 @@
return i.jquery ? i[0] : i;
});
delete options.inputs;
this.preventMultipleSet = options.preventMultipleSet;
datepickerPlugin.call($(this.inputs), options)
.on('changeDate', $.proxy(this.dateUpdated, this));
@ -1507,10 +1508,12 @@
if (i === -1)
return;
$.each(this.pickers, function(i, p){
if (!p.getUTCDate())
p.setUTCDate(new_date);
});
if (!this.preventMultipleSet) {
$.each(this.pickers, function(i, p){
if (!p.getUTCDate())
p.setUTCDate(new_date);
});
}
if (new_date < this.dates[j]){
// Date being moved earlier/left

18
app/webroot/js/contextual_menu.js
View File

@ -143,6 +143,7 @@ class ContextualMenu {
__create_menu_div_bootstrap_popover() {
var div = document.createElement('div');
div.classList.add("contextual-menu");
div.style.display = 'none';
this.container.appendChild(div);
var that = this;
this.trigger_container.tabIndex = 0; // required for the popover focus feature
@ -151,19 +152,20 @@ class ContextualMenu {
container: 'body',
html: true,
placement: "bottom",
content: function () {return $(that.menu); }, // return contextual menu html
content: function () { var html=$(that.menu); html.css('display', 'inline-block'); return html;}, // return contextual menu html
trigger: "manual",
template: '<div class="popover" id="popover-contextual-menu-'+this.trigger_container.id+'" role="tooltip" style="'+additional_styling+'"><div class="arrow"></div></h3><div class="popover-content"></div></div>'
})
// Overwrite the default popover behavior: hidding cause the popover to be detached from the DOM, making impossible to fetch input values in the form
$(this.trigger_container).click (function(e) {
if (that.has_been_shown_once) {
$('#popover-contextual-menu-'+this.id).toggle();
} else {
that.has_been_shown_once = true;
$(this).popover('show');
}
$(this).popover('toggle');
// if (that.has_been_shown_once) {
// $('#popover-contextual-menu-'+this.id).toggle();
// } else {
// that.has_been_shown_once = true;
// $(this).popover('show');
// }
});
return div;
}
@ -371,7 +373,7 @@ class ContextualMenu {
this.menu.appendChild(label);
this.menu.appendChild(div);
if(options.event !== undefined) {
button.addEventListener("click", function(evt) {
button.addEventListener("click", function(evt) {
var corresponding_select_id = evt.target.dataset.correspondingId;
var selected_value = $('#'+corresponding_select_id).val();
options.event(selected_value);

661
app/webroot/js/event-timeline.js Normal file
View File

@ -0,0 +1,661 @@
var max_displayed_char_timeline = 64;
var eventTimeline;
var items_timeline;
var items_backup;
var use_local_timezone = true;
var mapping_text_to_id = new Map();
var user_manipulation = $('#event_timeline').data('user-manipulation');
var extended_text = $('#event_timeline').data('extended') == 1 ? "extended:1/" : "";
var container_timeline = document.getElementById('event_timeline');
var hardThreshold = 1000;
var softThreshold = 200;
var timeline_disabled = false;
var default_editable = {
add: false, // add new items by double tapping
updateTime: true, // drag items horizontally
remove: true
};
var relationship_type_mapping = {
'followed-by': 'after',
'preceding-by': 'before',
}
var options = {
template: function (item, element, data) {
switch(item.group) {
case "attribute":
return build_attr_template(item);
case "object":
return build_object_template(item);
case "object_attribute":
console.log('Error');
break;
default:
break;
}
},
moment: function(date) {
if (use_local_timezone) {
return vis.moment(date);
} else {
return vis.moment(date).utc();
}
},
verticalScroll: true,
zoomKey: 'altKey',
maxHeight: 400,
minHeight: 400,
multiselect: true,
editable: user_manipulation ? default_editable : false,
tooltipOnItemUpdateTime: true,
onRemove: function(item, callback) { // clear timestamps
update_seen(item, 'first', null, false, function() {
update_seen(item, 'last', null, false, function() { reflect_change(true); });
});
eventTimeline.setSelection([]);
$('.timelineSelectionTooltip').remove()
},
onMove: function(item, callback) {
var newStart = moment(item.start.toISOString());
var newEnd = (item.end !== undefined && item.end !== null) ? moment(item.end.toISOString()) : null;
var c1 = item.first_seen !== null ? !item.first_seen.isSame(newStart) : true;
var c2 = item.last_seen !== null ? !item.last_seen.isSame(newEnd) && item.seen_enabled : true;
if (c1) {
if (item.first_seen === null) {
if (!c2) {
update_seen(item, 'first', newStart, true, undefined);
} else {
update_seen(
item,
'first',
newStart,
false,
function() {
update_seen(
item,
'last',
newEnd,
true,
function() {
reflect_change(true);
}
);
}
);
}
} else {
update_seen(item, 'first', newStart, false, function() {
if (c2) {
update_seen(item, 'last', newEnd, true, undefined);
}
});
}
}
if (c2 && !c1) {
update_seen(item, 'last', newEnd, true, undefined);
}
}
};
function isDefined(element) {
return element !== undefined && element !== null;
}
function generate_timeline_tooltip(itemID, target) {
var item = items_timeline.get(itemID);
if (
item.first_seen === undefined
|| item.first_seen === null
|| item.first_seen_overwrite
) { // do not generate if first_seen not set
return;
}
var closest = $(target.closest(".vis-selected.vis-editable"));
var btn_type = item.last_seen !== null ? 'collapse-btn' : 'expand-btn';
var fct_type = item.last_seen !== null ? 'collapseItem' : 'expandItem';
var btn = $('<div class="timelineSelectionTooltip vis-expand-action '+btn_type+'" data-itemid="'+item.id+'"></div>')
if (item.last_seen !== null) {
btn.click(collapseItem);
} else {
btn.click(expandItem);
}
closest.append(btn);
}
/* UTIL */
function collapseItem() {
var itemID = $(this).data('itemid');
var item = items_timeline.get(itemID);
update_seen(item, 'last', null, true, undefined);
}
function expandItem() {
var itemID = $(this).data('itemid');
var item = items_timeline.get(itemID);
var newEnd = get_next_step(item.first_seen);
update_seen(item, 'last', newEnd, true, undefined);
}
function get_next_step(mom) {
var scale = adapt_scale(eventTimeline.timeAxis.step.scale);
var momAhead = mom.clone();
momAhead.add(1, scale);
return momAhead;
}
function adapt_scale(scale) {
first_letter = scale.charAt(0);
if (first_letter !== 'm' && first_letter !== 'w') {
return first_letter;
} else {
switch (scale) {
case 'millisecond':
return 'ms';
case 'minute':
return 'm';
case 'month':
return 'M';
case 'week':
return 'w';
case 'weekday':
return 'd';
default:
return scale;
}
}
}
function build_attr_template(attr) {
var span = $('<span data-itemID="'+attr.id+'">');
if (!attr.seen_enabled) {
span.addClass('timestamp-attr');
}
span.text(attr.content);
span.data('seen_enabled', attr.seen_enabled);
var html = span[0].outerHTML;
return html;
}
function build_object_template(obj) {
var table = $('<table>');
table.data('seen_enabled', obj.seen_enabled);
if (!obj.seen_enabled) {
table.addClass('timestamp-obj');
}
var bolt_html = obj.overwrite_enabled ? " <i class=\"fa fa-bolt\" style=\"color: yellow; font-size: large;\" title=\"The Object is overwritten by its attributes\">" : "";
table.append($('<tr class="timeline-objectName"><th>'+obj.content+bolt_html+'</th><th></th></tr>'));
for (var attr of obj.Attribute) {
var overwritten = obj.overwrite_enabled && (attr.contentType == "first-seen" || attr.contentType == "last-seen") ? " <i class=\"fa fa-bolt\" style=\"color: yellow;\" title=\"Overwrite object "+attr.contentType+"\"></i>" : "";
table.append(
$('<tr>').append(
$('<td class="timeline-objectAttrType">' + attr.contentType + '</td>'
+'<td class="timeline-objectAttrVal">' + attr.content+overwritten + '</td>'
)
)
)
}
var html = table[0].outerHTML;
return html;
}
function contain_seen_attribute(obj) {
if (obj['Attribute'] === undefined) {
return false;
}
for (var i = 0; i < obj['Attribute'].length; i++) {
var attribute = obj['Attribute'][i];
if (attribute['contentType'] == 'first-seen' || attribute['contentType'] == 'last-seen') {
return true;
}
}
return false;
}
function reflect_change(onIndex, itemType, itemId, item) {
if (onIndex) {
updateIndex(scope_id, 'event'); // MISP function
} else { // reflect change on item only
quick_fetch_seens(itemType, item.orig_id, function(firstSeen, lastSeen) {
var updatedItem = items_timeline.get(itemId);
updatedItem.first_seen = firstSeen;
updatedItem.last_seen = lastSeen;
updatedItem.first_seen_overwrite = false;
updatedItem.last_seen_overwrite = false;
if (user_manipulation) {
var e = $.extend({}, default_editable);
e.remove = true;
updatedItem.editable = e;
}
set_spanned_time(updatedItem);
items_timeline.remove(updatedItem.id);
items_timeline.add(updatedItem);
});
}
}
function quick_fetch_seens(itemType, itemId, callback) {
var url = "/" + itemType + "/" + "fetchViewValue" + "/" + itemId + "/";
var dfs = $.ajax({
dataType: "html",
cache: false,
success: function(data, textStatus) {
return data;
},
url: url+"first_seen"
});
var dls = $.ajax({
dataType: "html",
cache: false,
success: function(data, textStatus) {
return data;
},
url: url+"last_seen"
});
$.when( dfs, dls).done(function(a1, a2) {
firstSeen = a1[0].replace('&nbsp;', '');
firstSeen = firstSeen == '' ? null : firstSeen;
lastSeen = a2[0].replace('&nbsp;', '');
lastSeen = lastSeen == '' ? null : lastSeen;
callback(firstSeen, lastSeen);
});
}
function update_seen(item, seenType, value, reflect, callback) {
var itemType = item.group + 's';
var momentISO = value !== null ? value.toISOString() : null;
fetch_form_and_submit(itemType, item, seenType, momentISO, reflect, callback);
}
function fetch_form_and_submit(itemType, item, seenType, value, reflect, callback) {
var url = "/" + itemType + "/fetchEditForm/" + item.orig_id + "/" + seenType+"_seen";
$.ajax({
beforeSend: function (XMLHttpRequest) {
$(".loadingTimeline").show();
},
dataType:"html",
cache: false,
success: function (data, textStatus) {
var form = $(data);
$(container_timeline).append(form);
form.css({display: 'none'});
var field = form.find('input[name*="' + seenType + '_seen"]');
field.val(value);
// submit the form
$.ajax({
data: form.serialize(),
cache: false,
success:function (data, textStatus) {
if (reflect) {
if (contain_seen_attribute(item)) {
reflect_change(true, itemType, item.id, item);
} else {
reflect_change(false, itemType, item.id, item);
}
}
form.remove()
},
error:function() {
console.log('fail', 'Request failed for an unknown reason.');
},
complete: function () {
$(".loadingTimeline").hide();
if (callback !== undefined) {
callback();
}
},
type:"post",
url: form.attr('action')
});
},
error: function() {
console.log('Feature not supported.');
},
url: url,
});
}
function timestampToMoment(timestamp) {
var factor = 1000;
var d = moment(timestamp*factor);
return d;
}
function set_spanned_time(item) {
var timestamp = item.timestamp;
var fs = item.first_seen == null ? null : moment(item.first_seen);
var ls = item.last_seen == null ? null : moment(item.last_seen);
item.first_seen = fs;
item.last_seen = ls;
item.seen_enabled = false;
item.overwrite_enabled = false;
if (fs===null && ls===null) {
item.start = timestampToMoment(timestamp);
item.type = 'box';
} else if (fs===null && ls!==null) {
item.start = timestampToMoment(timestamp);
item.type = 'box';
} else if (ls===null && fs!==null) {
item.start = fs;
item.seen_enabled = true;
delete item.end;
item.type = 'box';
} else { // fs and ls are defined
item.start = fs;
item.end = ls;
item.seen_enabled = true;
if (fs == ls) {
item.type = 'box';
} else {
item.type = 'range';
}
}
if (item.first_seen_overwrite === true || item.last_seen_overwrite === true) {
if (user_manipulation) {
var e = $.extend({}, default_editable);
e.remove = false;
item.editable = e;
}
item.overwrite_enabled = true;
}
}
function map_scope(val) {
switch(val) {
case 'First seen/Last seen':
return 'seen';
case 'Object relationship':
return 'relationship';
default:
return 'seen';
}
}
function timelinePopupCallback(state) {
if (eventTimeline === undefined) {
return;
}
reload_timeline();
}
function adjust_text_length(elem) {
var maxChar = $('#slider_timeline_display_max_char_num').val();
maxChar = maxChar === undefined ? 64 : maxChar;
elem.content = elem.content.substring(0, maxChar) + (elem.content.length < maxChar ? "" : "[...]");
}
function update_badge() {
if (use_local_timezone) {
$("#timeline-display-badge").text("Timezone: " + ": " + moment().format('Z'));
} else {
$("#timeline-display-badge").text("Timezone: " + ": " + moment().utc().format('Z (z)'));
}
}
function reload_timeline() {
update_badge();
var payload = {scope: map_scope($('#select_timeline_scope').val())};
$.ajax({
url: "/events/"+"getEventTimeline"+"/"+scope_id+"/"+extended_text+"event.json",
dataType: 'json',
type: 'post',
contentType: 'application/json',
data: JSON.stringify( payload ),
processData: false,
beforeSend: function (XMLHttpRequest) {
$(".loadingTimeline").show();
},
success: function( data, textStatus, jQxhr ){
items_timeline.clear();
for (var item of data.items) {
item.className = item.group;
item.orig_id = item.id;
item.id = item.uuid;
set_spanned_time(item);
if (item.group == 'object') {
for (var attr of item.Attribute) {
mapping_text_to_id.set(attr.contentType+': '+attr.content+' ('+item.orig_id+')', item.id);
adjust_text_length(attr);
}
} else {
mapping_text_to_id.set(item.content+' ('+item.orig_id+')', item.id);
adjust_text_length(item);
}
}
items_timeline.add(data.items);
handle_not_seen_enabled($('#checkbox_timeline_display_hide_not_seen_enabled').prop('checked'), false)
},
error: function( jqXhr, textStatus, errorThrown ){
console.log( errorThrown );
},
complete: function() {
$(".loadingTimeline").hide();
}
});
}
function enable_timeline() {
if (eventTimeline !== undefined) {
return;
}
init_popover();
var chosen_options_timeline = {
max_shown_results: 20,
inherit_select_classes: true
};
var payload = {scope: map_scope($('#select_timeline_scope').val())};
$.ajax({
url: "/events/"+"getEventTimeline"+"/"+scope_id+"/"+extended_text+"event.json",
dataType: 'json',
type: 'post',
contentType: 'application/json',
data: JSON.stringify( payload ),
processData: false,
beforeSend: function (XMLHttpRequest) {
$(".loadingTimeline").show();
},
success: function( data, textStatus, jQxhr ){
if (data.items.length > hardThreshold) {
$('#eventtimeline_div').html('<div class="alert alert-danger" style="margin: 10px;">Timeline: To much data to show</div>');
timeline_disabled = true;
return;
} else if (data.items.length > softThreshold) {
var res = confirm('You are about to draw a lot ('+data.items.length+') of items in the timeline. Do you wish to continue?');
if (!res) {
$('#eventtimeline_div').html('<div class="alert alert-danger" style="margin: 10px;">Timeline: To much data to show</div>');
timeline_disabled = true;
return;
}
}
for (var item of data.items) {
item.className = item.group;
item.orig_id = item.id;
item.id = item.uuid;
set_spanned_time(item);
if (item.group == 'object') {
for (var attr of item.Attribute) {
mapping_text_to_id.set(attr.contentType+': '+attr.content+' ('+item.orig_id+')', item.id);
adjust_text_length(attr);
}
} else {
mapping_text_to_id.set(item.content+' ('+item.orig_id+')', item.id);
adjust_text_length(item);
}
}
items_timeline = new vis.DataSet(data.items);
eventTimeline = new vis.Timeline(container_timeline, items_timeline, options);
update_badge();
eventTimeline.on('select', handle_selection);
eventTimeline.on('doubleClick', handle_doubleClick);
items_timeline.on('update', function(eventname, data) {
handle_selection({
event: { target: $('span[data-itemID="'+data.items[0]+'"]')},
items: data.items
});
});
var $selectTypeahead = $('#timeline-typeahead');
Array.from(mapping_text_to_id.keys()).forEach(function(element) {
var value = mapping_text_to_id[element];
var $option = $('<option></option>');
$option.text(element);
$option.attr('value', value);
$selectTypeahead.append($option);
});
$selectTypeahead.css('display', '').chosen(chosen_options_timeline).on('change', function(evt, params) {
var value = params.selected;
var id = mapping_text_to_id.get(value);
eventTimeline.focus(id);
$("#timeline-typeahead").blur();
});
},
error: function( jqXhr, textStatus, errorThrown ){
console.log( errorThrown );
},
complete: function() {
$(".loadingTimeline").hide();
}
});
}
function handle_selection(data) {
var event = data.event;
var target = event.target;
var items = data.items;
if (items.length == 0) {
$('.timelineSelectionTooltip').remove()
} else {
for (var itemID of items) {
generate_timeline_tooltip(itemID, target);
}
}
}
function edit_item(id, callback) {
var item = items_timeline.get(id);
var group = item.group;
if (group == 'attribute') {
simplePopup('/attributes/edit/'+item.orig_id);
} else if (group == 'object') {
window.location = '/objects/edit/'+item.orig_id;
}
}
function handle_doubleClick(data) {
// should be replaced by keyboard shortcut: SHIFT+E ?
//edit_item(data.item);
}
function handle_not_seen_enabled(hide, include_hidden) {
include_hidden = include_hidden !== undefined ? include_hidden : true;
if (hide) {
var hidden = items_timeline.get({
filter: function(item) {
return !item.seen_enabled;
}
});
var hidden_ids = [];
items_timeline.forEach(function(item) {
hidden_ids.push(item.id);
});
items_timeline.remove(hidden)
items_backup = hidden;
} else if (include_hidden) {
if (items_backup !== undefined) {
items_timeline.add(items_backup);
}
}
}
$('#fullscreen-btn-timeline').click(function() {
var timeline_div = $('#eventtimeline_div');
var fullscreen_enabled = !timeline_div.data('fullscreen');
timeline_div.data('fullscreen', fullscreen_enabled);
var height_val = fullscreen_enabled == true ? "calc(100vh - 42px - 42px - 10px)" : "400px";
timeline_div.css("max-height", height_val);
setTimeout(function() { // timeline takes time to be drawn
timeline_div[0].scrollIntoView({
behavior: "smooth",
});
}, 1);
eventTimeline.setOptions({maxHeight: height_val});
});
// init_scope_menu
var menu_scope_timeline, menu_display_timeline;
function init_popover() {
if (timeline_disabled) return;
menu_scope_timeline = new ContextualMenu({
trigger_container: document.getElementById("timeline-scope"),
bootstrap_popover: true,
style: "z-index: 1",
container: document.getElementById("eventtimeline_div")
});
menu_scope_timeline.add_select({
id: "select_timeline_scope",
label: "Scope",
tooltip: "The time scope represented by the timeline",
event: function(value) {
if (value == "First seen/Last seen") {
reload_timeline();
}
},
options: ["First seen/Last seen"],
default: "First seen/Last seen"
});
menu_display_timeline = new ContextualMenu({
trigger_container: document.getElementById("timeline-display"),
bootstrap_popover: true,
style: "z-index: 1",
container: document.getElementById("eventtimeline_div")
});
menu_display_timeline.add_slider({
id: 'slider_timeline_display_max_char_num',
label: "Charater to show",
title: "Maximum number of charater to display in the label",
min: 8,
max: 2048,
value: max_displayed_char_timeline,
step: 8,
applyButton: true,
event: function(value) {
$("#slider_timeline__display_max_char_num").parent().find("span").text(value);
},
eventApply: function(value) {
reload_timeline();
}
});
menu_display_timeline.add_checkbox({
id: 'checkbox_timeline_display_hide_not_seen_enabled',
label: "Hide first seen not set",
title: "Hide items that does not have first seen sets",
event: function(value) {
handle_not_seen_enabled(value)
}
});
menu_display_timeline.add_checkbox({
id: 'checkbox_timeline_display_gmt',
label: "Display with current timezone",
title: "Set the dates relative to the browser timezone. Otherwise, keep dates in GMT",
event: function(value) {
use_local_timezone = value;
reload_timeline()
},
checked: true
});
}

117
app/webroot/js/misp.js
View File

@ -419,6 +419,11 @@ function updateIndex(id, context, newPage) {
} else {
console.log("genericPopupCallback function not defined");
}
if (typeof timelinePopupCallback !== "undefined") {
timelinePopupCallback("success");
} else {
console.log("timelinepopupcallback function not defined");
}
},
url: url,
});
@ -447,15 +452,42 @@ function updateAttributeFieldOnSuccess(name, type, id, field, event) {
});
}
function updateObjectFieldOnSuccess(name, type, id, field, event) {
$.ajax({
beforeSend: function (XMLHttpRequest) {
if (field != 'timestamp') {
$(".loading").show();
}
},
dataType:"html",
cache: false,
success:function (data, textStatus) {
if (field != 'timestamp') {
$(".loading").hide();
$(name + '_solid').html(data);
$(name + '_placeholder').empty();
$(name + '_solid').show();
} else {
$('#' + type + '_' + id + '_' + 'timestamp_solid').html(data);
}
},
url:"/objects/fetchViewValue/" + id + "/" + field,
});
}
function activateField(type, id, field, event) {
resetForms();
if (type == 'denyForm') return;
var objectType = 'attributes';
var containerName = 'Attribute';
if (type == 'ShadowAttribute') {
objectType = 'shadow_attributes';
} else if (type == 'Object') {
objectType = 'objects';
containerName = 'Object';
}
var name = '#' + type + '_' + id + '_' + field;
var container_name = '#Attribute_' + id + '_' + field;
var container_name = '#' + containerName + '_' + id + '_' + field;
$.ajax({
beforeSend: function (XMLHttpRequest) {
$(".loading").show();
@ -593,6 +625,8 @@ function submitForm(type, id, field, context) {
var name = '#' + type + '_' + id + '_' + field;
if (type == 'ShadowAttribute') {
object_type = 'shadow_attributes';
} else if (type == 'Object') {
object_type = 'objects';
}
$.ajax({
data: $(name + '_field').closest("form").serialize(),
@ -748,7 +782,8 @@ function handleAjaxEditResponse(data, name, type, id, field, event) {
responseArray = data;
if (type == 'Attribute') {
if (responseArray.saved) {
showMessage('success', responseArray.success);
var msg = responseArray.success !== undefined ? responseArray.success : responseArray.message;
showMessage('success', msg);
updateAttributeFieldOnSuccess(name, type, id, field, event);
updateAttributeFieldOnSuccess(name, type, id, 'timestamp', event);
eventUnpublish();
@ -759,6 +794,17 @@ function handleAjaxEditResponse(data, name, type, id, field, event) {
}
if (type == 'ShadowAttribute') {
updateIndex(event, 'event');
} else if (type == 'Object') {
if (responseArray.saved) {
var msg = responseArray.success !== undefined ? responseArray.success : responseArray.message;
showMessage('success', msg);
updateObjectFieldOnSuccess(name, type, id, field, event);
updateObjectFieldOnSuccess(name, type, id, 'timestamp', event);
eventUnpublish();
} else {
showMessage('fail', 'Validation failed: ' + responseArray.errors.value);
updateObjectFieldOnSuccess(name, type, id, field, event);
}
}
if (responseArray.hasOwnProperty('check_publish')) {
checkAndSetPublishedInfo();
@ -1139,7 +1185,7 @@ function openGenericModal(url) {
});
}
function submitPopoverForm(context_id, referer, update_context_id, modal) {
function submitPopoverForm(context_id, referer, update_context_id, modal, popover_dissmis_id_to_close) {
var url = null;
var context = 'event';
var contextNamingConvention = 'Attribute';
@ -1175,6 +1221,12 @@ function submitPopoverForm(context_id, referer, update_context_id, modal) {
case 'addSighting':
closePopover = false;
break;
case 'addObjectReference':
url = "/objectReferences/add/" + context_id;
break;
case 'quickAddAttributeForm':
url = "/objects/quickAddAttributeForm/" + context_id;
break;
}
if ($("#submitButton").parent().hasClass('modal-footer')) {
var $form = $("#submitButton").parent().parent().find('.modal-body form');
@ -1193,6 +1245,9 @@ function submitPopoverForm(context_id, referer, update_context_id, modal) {
if (closePopover) {
$("#gray_out").fadeOut();
$("#popover_form").fadeOut();
if (popover_dissmis_id_to_close !== undefined) {
$('[data-dismissid="' + popover_dissmis_id_to_close + '"]').popover('destroy');
}
$(".loading").show();
}
}
@ -1217,7 +1272,14 @@ function submitPopoverForm(context_id, referer, update_context_id, modal) {
$('#sightingsListAllToggle').removeClass('btn-inverse');
$('#sightingsListAllToggle').addClass('btn-primary');
}
if (context == 'event' && (referer == 'add' || referer == 'massEdit' || referer == 'replaceAttributes' || referer == 'addObjectReference')) eventUnpublish();
if (
(
context == 'event' &&
(referer == 'add' || referer == 'massEdit' || referer == 'replaceAttributes' || referer == 'addObjectReference' || referer == 'quickAddAttributeForm')
)
){
eventUnpublish();
}
},
error: function (jqXHR, textStatus, errorThrown) {
showMessage('fail', textStatus + ": " + errorThrown);
@ -1260,6 +1322,11 @@ function handleAjaxModalResponse(response, context_id, url, referer, context, co
result = "fail";
}
recoverValuesFromPersistance(savedArray);
},
error: function (jqXHR, textStatus, errorThrown) {
showMessage('fail', textStatus + ": " + errorThrown);
},
complete: function () {
$(".loading").hide();
},
url:url
@ -1718,8 +1785,22 @@ function popoverPopup(clicked, id, context, target, admin) {
$clicked.popover('show');
}
},
error:function() {
popover.options.content = '<div class="alert alert-error" style="margin-bottom: 0px;">Something went wrong - the queried function returned an exception. Contact your administrator for further details (the exception has been logged).</div>';
error:function(jqXHR, textStatus, errorThrown ) {
var errorJSON = '';
try {
errorJSON = JSON.parse(jqXHR.responseText);
errorJSON = errorJSON['errors'];
if (errorJSON === undefined) {
errorJSON = '';
}
} catch (SyntaxError) {
// no error provided
}
var errorText = '<div class="alert alert-error" style="margin-bottom: 3px;">Something went wrong - the queried function returned an exception. Contact your administrator for further details (the exception has been logged).</div>';
if (errorJSON !== '') {
errorText += '<div class="well"><strong>Returned error:</strong> ' + $('<span/>').text(errorJSON).html() + '</div>';
}
popover.options.content = errorText;
$clicked.popover('show');
},
url: url
@ -3328,8 +3409,15 @@ function zeroMQServerAction(action) {
function convertServerFilterRules(rules) {
validOptions.forEach(function (type) {
container = "#"+ modelContext + type.ucfirst() + "Rules";
if ($(container).val() != '' && $(container).val() != '[]') rules[type] = JSON.parse($(container).val());
else {rules[type] = {"tags": {"OR": [], "NOT": []}, "orgs": {"OR": [], "NOT": []}}};
if ($(container).val() != '' && $(container).val() != '[]') {
rules[type] = JSON.parse($(container).val());
} else {
if (type === 'pull') {
rules[type] = {"tags": {"OR": [], "NOT": []}, "orgs": {"OR": [], "NOT": []}, "url_params": ""}
} else {
rules[type] = {"tags": {"OR": [], "NOT": []}, "orgs": {"OR": [], "NOT": []}}
}
};
});
serverRuleUpdate();
return rules;
@ -3360,6 +3448,14 @@ function serverRuleUpdate() {
}
});
});
if (type === 'pull') {
if (rules[type]['url_params']) {
$("#pull_url_params").show();
$("#pull_url_params_text").text(rules[type]['url_params']);
} else {
$("#pull_url_params").hide();
}
}
});
serverRuleGenerateJSON();
}
@ -3415,6 +3511,7 @@ function serverRulePopulateTagPicklist() {
}));
});
});
$('#urlParams').val(rules["pull"]["url_params"]);
}
function submitServerRulePopulateTagPicklistValues(context) {
@ -3428,7 +3525,9 @@ function submitServerRulePopulateTagPicklistValues(context) {
rules[context][field]["NOT"].push($(this).val());
});
});
if (context === 'pull') {
rules[context]["url_params"] = $('#urlParams').val();
}
$('#server_' + context + '_rule_popover').fadeOut();
$('#gray_out').fadeOut();
serverRuleUpdate();

2
cti-python-stix2

@ -1 +1 @@
Subproject commit e2a4129ad343eb8d98bd8bda36411486fe3a8d63
Subproject commit 96946d956d41b4c9c9b24683add88f457181ee75

5723
db_schema.json
View File

File diff suppressed because one or more lines are too long

6
docs/INSTALL.rhel7.md
View File

@ -240,6 +240,9 @@ installCoreRHEL () {
cd $PATH_TO_MISP/app/files/scripts/mixbox
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
# FIXME: Remove once stix-fixed
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
# install STIX2.0 library to support STIX 2.0 export:
cd $PATH_TO_MISP/cti-python-stix2
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
@ -317,8 +320,9 @@ installCake_RHEL ()
sudo chown $WWW_USER:$WWW_USER /usr/share/httpd/.composer
cd $PATH_TO_MISP/app
# Update composer.phar (optional)
#EXPECTED_SIGNATURE="$(wget -q -O - https://composer.github.io/installer.sig)"
#$SUDO_WWW $RUN_PHP -- php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
#$SUDO_WWW $RUN_PHP -- php -r "if (hash_file('SHA384', 'composer-setup.php') === '48e3236262b34d30969dca3c37281b3b4bbe3221bda826ac6a9a62d6444cdb0dcd0615698a5cbe587c3f0fe57a54d8f5') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
#$SUDO_WWW $RUN_PHP -- php -r "if (hash_file('SHA384', 'composer-setup.php') === '$EXPECTED_SIGNATURE') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
#$SUDO_WWW $RUN_PHP "php composer-setup.php"
#$SUDO_WWW $RUN_PHP -- php -r "unlink('composer-setup.php');"
$SUDO_WWW $RUN_PHP "php composer.phar install"

5
docs/INSTALL.rhel8.md
View File

@ -222,6 +222,9 @@ installCoreRHEL () {
cd $PATH_TO_MISP/app/files/scripts/mixbox
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
# FIXME: Remove once stix-fixed
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
# install STIX2.0 library to support STIX 2.0 export:
cd $PATH_TO_MISP/cti-python-stix2
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install .
@ -304,7 +307,7 @@ installCake_RHEL ()
cd $PATH_TO_MISP/app
# Update composer.phar (optional)
$SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === 'a5c698ffe4b8e849a443b120cd5ba38043260d5c4023dbf93e1558871f1f07f58274fc6f4c93bcfd858c6bd0775cd8d1') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === 'baf1608c33254d00611ac1705c1d9958c817a1a33bce370c0595974b342601bd80b92a3f46067da89e3b06bff421f182') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
$SUDO_WWW php composer-setup.php
$SUDO_WWW php -r "unlink('composer-setup.php');"
$SUDO_WWW php composer.phar install

4
docs/INSTALL.ubuntu1804.md
View File

@ -167,6 +167,8 @@ installCore () {
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
cd $PATH_TO_MISP/app/files/scripts/python-maec
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
# FIXME: Remove once stix-fixed
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -I antlr4-python3-runtime==4.7.2
# install STIX2.0 library to support STIX 2.0 export:
cd ${PATH_TO_MISP}/cti-python-stix2
$SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install .
@ -309,7 +311,7 @@ Now configure your Apache webserver with the DocumentRoot ${PATH_TO_MISP}/app/we
```bash
# <snippet-begin 1_apacheConfig.sh>
apacheConfig () {
debug "Generating Apache config"
debug "Generating Apache config, if this hangs, make sure you have enough entropy (install: haveged or wait)"
sudo cp ${PATH_TO_MISP}/INSTALL/apache.24.misp.ssl /etc/apache2/sites-available/misp-ssl.conf
if [[ ! -z ${MISP_BASEURL} ]] && [[ "$(echo $MISP_BASEURL|cut -f 1 -d :)" == "http" || "$(echo $MISP_BASEURL|cut -f 1 -d :)" == "https" ]]; then

4
docs/generic/globalVariables.md
View File

@ -85,9 +85,9 @@ MISPvars () {
# sudo config to run $LUSER commands
if [[ "$(groups ${MISP_USER} |grep -o 'staff')" == "staff" ]]; then
SUDO_USER="sudo -H -u ${MISP_USER} -g staff"
SUDO_CMD="sudo -H -u ${MISP_USER} -g staff"
else
SUDO_USER="sudo -H -u ${MISP_USER}"
SUDO_CMD="sudo -H -u ${MISP_USER}"
fi
SUDO_WWW="sudo -H -u ${WWW_USER} "

Some files were not shown because too many files have changed in this diff Show More