MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch '2.4' into feature/tags_deletion

pull/5428/head
Tom King 2020-01-14 10:03:44 +00:00
parent 9008b565dd 59ff6d9f6e
commit 0a4fbc80c5
94 changed files with 1932 additions and 2178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.travis.yml
View File

@ -113,7 +113,7 @@ install:
- sudo -E su $USER -c 'app/Console/cake Admin updateTaxonomies'
- sudo -E su $USER -c 'app/Console/cake Admin updateWarningLists'
- sudo -E su $USER -c 'app/Console/cake Admin updateNoticeLists'
- sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates "1337"'
- sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates 1'
- sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" true'
- sudo -E su $USER -c 'app/Console/cake Live 1'
- sudo chmod 777 ./key.txt
@ -164,7 +164,6 @@ script:
- git submodule update
- pipenv install -d
- pipenv run python tests/testlive_comprehensive.py
- pipenv run python tests/test.py
- pipenv run python tests/test_mispevent.py
- popd
- cp PyMISP/tests/keys.py PyMISP/examples/events/

82
INSTALL/INSTALL.sh
View File

@ -148,9 +148,9 @@ MISPvars () {
# sudo config to run $LUSER commands
if [[ "$(groups ${MISP_USER} |grep -o 'staff')" == "staff" ]]; then
SUDO_USER="sudo -H -u ${MISP_USER} -g staff"
SUDO_CMD="sudo -H -u ${MISP_USER} -g staff"
else
SUDO_USER="sudo -H -u ${MISP_USER}"
SUDO_CMD="sudo -H -u ${MISP_USER}"
fi
SUDO_WWW="sudo -H -u ${WWW_USER} "
@ -544,12 +544,12 @@ checkID () {
sudo adduser $MISP_USER $WWW_USER
fi
# FIXME: the below SUDO_USER check is a duplicate from global variables, try to have just one check
# FIXME: the below SUDO_CMD check is a duplicate from global variables, try to have just one check
# sudo config to run $LUSER commands
if [[ "$(groups ${MISP_USER} |grep -o 'staff')" == "staff" ]]; then
SUDO_USER="sudo -H -u ${MISP_USER} -g staff"
SUDO_CMD="sudo -H -u ${MISP_USER} -g staff"
else
SUDO_USER="sudo -H -u ${MISP_USER}"
SUDO_CMD="sudo -H -u ${MISP_USER}"
fi
}
@ -953,7 +953,8 @@ composer73 () {
# Update composer.phar
# If hash changes, check here: https://getcomposer.org/download/ and replace with the correct one
# Current Sum for: v1.8.3
SHA384_SUM='48e3236262b34d30969dca3c37281b3b4bbe3221bda826ac6a9a62d6444cdb0dcd0615698a5cbe587c3f0fe57a54d8f5'
SHA384_SUM="$(wget -q -O - https://composer.github.io/installer.sig)"
php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
$SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === '$SHA384_SUM') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); exit(137); } echo PHP_EOL;"
checkFail "composer.phar checksum failed, please investigate manually. " $?
@ -1008,11 +1009,12 @@ nuke () {
# Final function to let the user know what happened
theEnd () {
space
echo "Admin (root) DB Password: $DBPASSWORD_ADMIN" |$SUDO_USER tee /home/${MISP_USER}/mysql.txt
echo "User (misp) DB Password: $DBPASSWORD_MISP" |$SUDO_USER tee -a /home/${MISP_USER}/mysql.txt
echo "Authkey: $AUTH_KEY" |$SUDO_USER tee -a /home/${MISP_USER}/MISP-authkey.txt
echo "Admin (root) DB Password: $DBPASSWORD_ADMIN" |$SUDO_CMD tee /home/${MISP_USER}/mysql.txt
echo "User (misp) DB Password: $DBPASSWORD_MISP" |$SUDO_CMD tee -a /home/${MISP_USER}/mysql.txt
echo "Authkey: $AUTH_KEY" |$SUDO_CMD tee -a /home/${MISP_USER}/MISP-authkey.txt
clear
# Commenting out, see: https://github.com/MISP/MISP/issues/5368
# clear -x
space
echo -e "${LBLUE}MISP${NC} Installed, access here: ${MISP_BASEURL}"
echo
@ -1607,7 +1609,7 @@ mispmodules () {
cd /usr/local/src/
## TODO: checkUsrLocalSrc in main doc
debug "Cloning misp-modules"
$SUDO_USER git clone https://github.com/MISP/misp-modules.git
$SUDO_CMD git clone https://github.com/MISP/misp-modules.git
cd misp-modules
# some misp-modules dependencies
sudo apt install libpq5 libjpeg-dev tesseract-ocr libpoppler-cpp-dev imagemagick libopencv-dev zbar-tools libzbar0 libzbar-dev libfuzzy-dev -y
@ -1762,39 +1764,39 @@ mail2misp () {
debug "Installing Mail2${LBLUE}MISP${NC}"
cd /usr/local/src/
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
$SUDO_USER git clone https://github.com/MISP/mail_to_misp.git
$SUDO_USER git clone git://github.com/stricaud/faup.git faup
$SUDO_USER git clone git://github.com/stricaud/gtcaca.git gtcaca
$SUDO_CMD git clone https://github.com/MISP/mail_to_misp.git
$SUDO_CMD git clone git://github.com/stricaud/faup.git faup
$SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
sudo chown -R ${MISP_USER}:${MISP_USER} faup mail_to_misp gtcaca
cd gtcaca
$SUDO_USER mkdir -p build
$SUDO_CMD mkdir -p build
cd build
$SUDO_USER cmake .. && $SUDO_USER make
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
$SUDO_USER mkdir -p build
$SUDO_CMD mkdir -p build
cd build
$SUDO_USER cmake .. && $SUDO_USER make
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
cd ../../mail_to_misp
$SUDO_USER virtualenv -p python3 venv
$SUDO_USER ./venv/bin/pip install https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
$SUDO_USER ./venv/bin/pip install -r requirements.txt
$SUDO_USER cp mail_to_misp_config.py-example mail_to_misp_config.py
$SUDO_CMD virtualenv -p python3 venv
$SUDO_CMD ./venv/bin/pip install https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
$SUDO_CMD ./venv/bin/pip install -r requirements.txt
$SUDO_CMD cp mail_to_misp_config.py-example mail_to_misp_config.py
##$SUDO cp mail_to_misp_config.py-example mail_to_misp_config.py
$SUDO_USER sed -i "s/^misp_url\ =\ 'YOUR_MISP_URL'/misp_url\ =\ 'https:\/\/localhost'/g" /usr/local/src/mail_to_misp/mail_to_misp_config.py
$SUDO_USER sed -i "s/^misp_key\ =\ 'YOUR_KEY_HERE'/misp_key\ =\ '${AUTH_KEY}'/g" /usr/local/src/mail_to_misp/mail_to_misp_config.py
$SUDO_CMD sed -i "s/^misp_url\ =\ 'YOUR_MISP_URL'/misp_url\ =\ 'https:\/\/localhost'/g" /usr/local/src/mail_to_misp/mail_to_misp_config.py
$SUDO_CMD sed -i "s/^misp_key\ =\ 'YOUR_KEY_HERE'/misp_key\ =\ '${AUTH_KEY}'/g" /usr/local/src/mail_to_misp/mail_to_misp_config.py
}
ssdeep () {
debug "Install ssdeep 2.14.1"
cd /usr/local/src
$SUDO_USER wget https://github.com/ssdeep-project/ssdeep/releases/download/release-2.14.1/ssdeep-2.14.1.tar.gz
$SUDO_USER tar zxvf ssdeep-2.14.1.tar.gz
$SUDO_CMD wget https://github.com/ssdeep-project/ssdeep/releases/download/release-2.14.1/ssdeep-2.14.1.tar.gz
$SUDO_CMD tar zxvf ssdeep-2.14.1.tar.gz
cd ssdeep-2.14.1
$SUDO_USER ./configure --datadir=/usr --prefix=/usr --localstatedir=/var --sysconfdir=/etc
$SUDO_USER make
$SUDO_CMD ./configure --datadir=/usr --prefix=/usr --localstatedir=/var --sysconfdir=/etc
$SUDO_CMD make
sudo make install
#installing ssdeep_php
@ -1822,25 +1824,25 @@ viper () {
fi
fi
echo "Cloning Viper"
$SUDO_USER git clone https://github.com/viper-framework/viper.git
$SUDO_USER git clone https://github.com/viper-framework/viper-web.git
$SUDO_CMD git clone https://github.com/viper-framework/viper.git
$SUDO_CMD git clone https://github.com/viper-framework/viper-web.git
sudo chown -R $MISP_USER:$MISP_USER viper
sudo chown -R $MISP_USER:$MISP_USER viper-web
cd viper
echo "Creating virtualenv"
$SUDO_USER virtualenv -p python3 venv
$SUDO_CMD virtualenv -p python3 venv
echo "Submodule update"
# TODO: Check for current user install permissions
$SUDO_USER git submodule update --init --recursive
$SUDO_CMD git submodule update --init --recursive
echo "pip install deps"
$SUDO_USER ./venv/bin/pip install pefile olefile jbxapi Crypto pypdns pypssl r2pipe pdftools virustotal-api SQLAlchemy PrettyTable python-magic scrapy https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
$SUDO_USER ./venv/bin/pip install .
$SUDO_CMD ./venv/bin/pip install pefile olefile jbxapi Crypto pypdns pypssl r2pipe pdftools virustotal-api SQLAlchemy PrettyTable python-magic scrapy https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
$SUDO_CMD ./venv/bin/pip install .
echo 'update-modules' |/usr/local/src/viper/venv/bin/viper
cd /usr/local/src/viper-web
$SUDO_USER sed -i '1 s/^.*$/\#!\/usr\/local\/src\/viper\/venv\/bin\/python/' viper-web
$SUDO_USER /usr/local/src/viper/venv/bin/pip install -r requirements.txt
$SUDO_CMD sed -i '1 s/^.*$/\#!\/usr\/local\/src\/viper\/venv\/bin\/python/' viper-web
$SUDO_CMD /usr/local/src/viper/venv/bin/pip install -r requirements.txt
echo "Launching viper-web"
$SUDO_USER /usr/local/src/viper-web/viper-web -p 8888 -H 0.0.0.0 &
$SUDO_CMD /usr/local/src/viper-web/viper-web -p 8888 -H 0.0.0.0 &
echo 'PATH="/home/misp/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/usr/local/src/viper:/var/www/MISP/app/Console"' |sudo tee /etc/environment
echo ". /etc/environment" >> /home/${MISP_USER}/.profile
@ -1852,8 +1854,8 @@ viper () {
fi
echo "Setting misp_url/misp_key"
$SUDO_USER sed -i "s/^misp_url\ =/misp_url\ =\ http:\/\/localhost/g" ${VIPER_HOME}/viper.conf
$SUDO_USER sed -i "s/^misp_key\ =/misp_key\ =\ $AUTH_KEY/g" ${VIPER_HOME}/viper.conf
$SUDO_CMD sed -i "s/^misp_url\ =/misp_url\ =\ http:\/\/localhost/g" ${VIPER_HOME}/viper.conf
$SUDO_CMD sed -i "s/^misp_key\ =/misp_key\ =\ $AUTH_KEY/g" ${VIPER_HOME}/viper.conf
# Reset admin password to: admin/Password1234
echo "Fixing admin.db with default password"
VIPER_COUNT=0
@ -2004,7 +2006,7 @@ installSupported () {
# TODO: Double check how the user is added and subsequently used during the install.
# TODO: Work on possibility to install as user X and install MISP for user Y
# TODO: Check if logout needed. (run SUDO_USER in installer)
# TODO: Check if logout needed. (run SUDO_CMD in installer)
# <snippet-begin add-user.sh>
# TODO: Double check how to properly handle postfix
# <snippet-begin postfix.sh>

2
INSTALL/INSTALL.sh.sha1
View File

@ -1 +1 @@
5bb0ceb0ab45af769c8a3b044f9a494e8733b1cb INSTALL.sh
966b18b8623bd83c9235c8a210d741db25a937d7 INSTALL.sh

2
INSTALL/INSTALL.sh.sha256
View File

@ -1 +1 @@
9402bcf66dd2c8a82b8871c5c414a5710d5faa0b1ad40bb0edec57a8883f52f7 INSTALL.sh
61c7f3242e7eeae16ac3cf65b60eb893c642f90f8b819be0670d25407ffd8b79 INSTALL.sh

2
INSTALL/INSTALL.sh.sha384
View File

@ -1 +1 @@
616975d3ec3ca34c590570f272ac244535ececcf535b66aa765b4e36c68e78649e65e5d719977d18b6d69ff59f709cc0 INSTALL.sh
41d7749f890bb150d3914bae0a986609073b1b7403cb561cd662957c529fc3bf382f7d7a7692e8fe1525cdc49d7b1cad INSTALL.sh

2
INSTALL/INSTALL.sh.sha512
View File

@ -1 +1 @@
5baa423f8306b0b2e16fc91380e1e551550f31ed013a38233b049040aab81579bad4e3c81203c0ec324bc17010bc1063e1d67bddf45a922ec7cec3a551aa49ee INSTALL.sh
f7fd098037a4b57a4b37a254ca4ddc80e418cab557518d4de73ef1ae4e382195729f71919510199e48e5e92af522f8d9bfe7c03ed6ad4423534c75a9016ef40f INSTALL.sh

2
INSTALL/INSTALL.tpl.sh
View File

@ -220,7 +220,7 @@ installSupported () {
# TODO: Double check how the user is added and subsequently used during the install.
# TODO: Work on possibility to install as user X and install MISP for user Y
# TODO: Check if logout needed. (run SUDO_USER in installer)
# TODO: Check if logout needed. (run SUDO_CMD in installer)
# <snippet-begin add-user.sh>
# TODO: Double check how to properly handle postfix
# <snippet-begin postfix.sh>

BIN
INSTALL/screenshots/sh-misp-attack.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 218 KiB

BIN
INSTALL/screenshots/sh-misp-decaying-of-indicators.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 160 KiB

BIN
INSTALL/screenshots/sh-misp-eventgraph.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 248 KiB

BIN
INSTALL/screenshots/sh-misp-threat-actor-attack.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 215 KiB

2
PyMISP

@ -1 +1 @@
Subproject commit c5dfa9b5090de50d782f1903455a4d4b604c072e
Subproject commit 2e7215bbec6c2fa1d527e09be99d4280fdda3fd1

2
VERSION.json
View File

@ -1 +1 @@
{"major":2, "minor":4, "hotfix":118}
{"major":2, "minor":4, "hotfix":119}

1
app/Config/config.default.php
View File

@ -110,7 +110,6 @@ $config = array(
'ApacheShibbAuth' => // Configuration for shibboleth authentication
array(
'apacheEnv' => 'REMOTE_USER', // If proxy variable = HTTP_REMOTE_USER
'ssoAuth' => 'AUTH_TYPE',
'MailTag' => 'EMAIL_TAG',
'OrgTag' => 'FEDERATION_TAG',
'GroupTag' => 'GROUP_TAG',

19
app/Console/Command/AdminShell.php
View File

@ -159,7 +159,7 @@ class AdminShell extends AppShell
}
}
public function updateWarningLists()
public function updateWarningLists()
{
$result = $this->Warninglist->update();
$success = count($result['success']);
@ -182,13 +182,7 @@ class AdminShell extends AppShell
echo 'Usage: ' . APP . '/cake ' . 'Admin updateObjectTemplates [user_id]' . PHP_EOL;
} else {
$userId = $this->args[0];
$user = $this->User->find('first', array(
'recursive' => -1,
'conditions' => array(
'User.id' => $userId,
),
'fields' => array('User.id', 'User.org_id')
));
$user = $this->User->getAuthUser($userId);
# If the user_id passed does not exist, do a global update.
if (empty($user)) {
echo 'User with ID: ' . $userId . ' not found' . PHP_EOL;
@ -315,7 +309,7 @@ class AdminShell extends AppShell
public function runUpdates() {
$whoami = exec('whoami');
if ($whoami === 'httpd' || $whoami === 'www-data' || $whoami === 'apache' || $whoami === 'wwwrun') {
if ($whoami === 'httpd' || $whoami === 'www-data' || $whoami === 'apache' || $whoami === 'wwwrun' || $whoami === 'travis') {
echo 'Executing all updates to bring the database up to date with the current version.' . PHP_EOL;
$processId = $this->args[0];
$this->Server->runUpdates(true, false, $processId);
@ -522,14 +516,15 @@ class AdminShell extends AppShell
public function dumpCurrentDatabaseSchema()
{
$dbActualSchema = $this->Server->getActualDBSchema()['schema'];
$dbActualSchema = $this->Server->getActualDBSchema();
$dbVersion = $this->AdminSetting->find('first', array(
'conditions' => array('setting' => 'db_version')
));
if (!empty($dbVersion) && !empty($dbActualSchema)) {
if (!empty($dbVersion) && !empty($dbActualSchema['schema'])) {
$dbVersion = $dbVersion['AdminSetting']['value'];
$data = array(
'schema' => $dbActualSchema,
'schema' => $dbActualSchema['schema'],
'indexes' => $dbActualSchema['indexes'],
'db_version' => $dbVersion
);
$file = new File(ROOT . DS . 'db_schema.json', true);

6
app/Console/Command/ServerShell.php
View File

@ -130,7 +130,7 @@ class ServerShell extends AppShell
public function fetchFeed() {
if (empty($this->args[0]) || empty($this->args[1])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['fetchFeed'] . PHP_EOL);
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Fetch feeds as local data'] . PHP_EOL);
}
$userId = $this->args[0];
$user = $this->User->getAuthUser($userId);
@ -221,7 +221,7 @@ class ServerShell extends AppShell
$data = array(
'worker' => 'default',
'job_type' => 'cache_servers',
'job_input' => 'Server: ' . $id,
'job_input' => 'Server: ' . $scopeid,
'status' => 0,
'retries' => 0,
'org' => $user['Organisation']['name'],
@ -256,7 +256,7 @@ class ServerShell extends AppShell
public function cacheFeed() {
if (empty($this->args[0]) || empty($this->args[1])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['cacheFeed'] . PHP_EOL);
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Cache feeds for quick lookups'] . PHP_EOL);
}
$userId = $this->args[0];
$user = $this->User->getAuthUser($userId);

121
app/Controller/AppController.php
View File

@ -46,8 +46,8 @@ class AppController extends Controller
public $helpers = array('Utility', 'OrgImg', 'FontAwesome', 'UserName');
private $__queryVersion = '92';
public $pyMispVersion = '2.4.117';
private $__queryVersion = '94';
public $pyMispVersion = '2.4.119';
public $phpmin = '7.0';
public $phprec = '7.2';
public $isApiAuthed = false;
@ -62,6 +62,8 @@ class AppController extends Controller
'attributes' => array('text', 'downloadAttachment', 'returnAttributes', 'restSearch', 'rpz', 'bro'),
);
protected $_legacyParams = array();
public function __construct($id = false, $table = null, $ds = null)
{
parent::__construct($id, $table, $ds);
@ -92,7 +94,8 @@ class AppController extends Controller
'Toolbox',
'RateLimit',
'IndexFilter',
'Deprecation'
'Deprecation',
'RestSearch'
//,'DebugKit.Toolbar'
);
@ -704,9 +707,8 @@ class AppController extends Controller
}
// generic function to standardise on the collection of parameters. Accepts posted request objects, url params, named url params
protected function _harvestParameters($options, &$exception)
protected function _harvestParameters($options, &$exception, $data = array())
{
$data = array();
if (!empty($options['request']->is('post'))) {
if (empty($options['request']->data)) {
$exception = $this->RestResponse->throwException(
@ -717,12 +719,25 @@ class AppController extends Controller
return false;
} else {
if (isset($options['request']->data['request'])) {
$data = $options['request']->data['request'];
$data = array_merge($data, $options['request']->data['request']);
} else {
$data = $options['request']->data;
$data = array_merge($data, $options['request']->data);
}
}
}
/*
* If we simply capture ordered URL params with func_get_args(), reassociate them.
* We can easily detect this by having ordered_url_params passed as a list instead of a dict.
*/
if (isset($options['ordered_url_params'][0])) {
$temp = array();
foreach ($options['ordered_url_params'] as $k => $url_param) {
if (!empty($options['paramArray'][$k])) {
$temp[$options['paramArray'][$k]] = $url_param;
}
}
$options['ordered_url_params'] = $temp;
}
if (!empty($options['paramArray'])) {
foreach ($options['paramArray'] as $p) {
if (
@ -1100,4 +1115,96 @@ class AppController extends Controller
}
}
protected function _legacyAPIRemap($options = array())
{
$ordered_url_params = array();
foreach ($options['paramArray'] as $k => $param) {
if (isset($options['ordered_url_params'][$k])) {
$ordered_url_params[$param] = $options['ordered_url_params'][$k];
} else {
$ordered_url_params[$param] = false;
}
}
$filterData = array(
'request' => $options['request'],
'named_params' => $options['named_params'],
'paramArray' => $options['paramArray'],
'ordered_url_params' => $ordered_url_params
);
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
if (!empty($options['injectedParams'])) {
foreach ($options['injectedParams'] as $injectedParam => $injectedValue) {
$filters[$injectedParam] = $injectedValue;
}
}
if (!empty($options['alias'])) {
foreach ($options['alias'] as $from => $to) {
if (!empty($filters[$from])) {
$filters[$to] = $filters[$from];
}
}
}
$this->_legacyParams = $filters;
return true;
}
public function restSearch()
{
$ordered_url_params = func_get_args();
if (empty($this->RestSearch->paramArray[$this->modelClass])) {
throw new NotFoundException(__('RestSearch is not implemented (yet) for this scope.'));
}
$scope = empty($this->scopeOverride) ? $this->modelClass : $this->scopeOverride;
if (!isset($this->$scope)) {
$this->loadModel($scope);
}
$filterData = array(
'request' => $this->request,
'named_params' => $this->params['named'],
'paramArray' => $this->RestSearch->paramArray[$scope],
'ordered_url_params' => func_get_args()
);
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception, $this->_legacyParams);
if (empty($filters['returnFormat'])) {
$filters['returnFormat'] = 'json';
}
unset($filterData);
if ($filters === false) {
return $exception;
}
$list = array();
$key = empty($filters['key']) ? $filters['returnFormat'] : $filters['key'];
$user = $this->_getApiAuthUser($key, $exception);
if ($user === false) {
return $exception;
}
if (isset($filters['returnFormat'])) {
$returnFormat = $filters['returnFormat'];
} else {
$returnFormat = 'json';
}
if ($returnFormat === 'download') {
$returnFormat = 'json';
}
if ($returnFormat === 'stix' && $this->_isJson()) {
$returnFormat = 'stix-json';
}
$elementCounter = 0;
$renderView = false;
$final = $this->$scope->restSearch($user, $returnFormat, $filters, false, false, $elementCounter, $renderView);
if (!empty($renderView) && !empty($final)) {
$this->layout = false;
$final = json_decode($final, true);
foreach ($final as $key => $data) {
$this->set($key, $data);
}
$this->render('/Events/module_views/' . $renderView);
} else {
$responseType = $this->$scope->validFormats[$returnFormat][0];
$filename = $this->RestSearch->getFilename($filters, $scope, $responseType);
return $this->RestResponse->viewData($final, $responseType, false, true, $filename, array('X-Result-Count' => $elementCounter, 'X-Export-Module-Used' => $returnFormat, 'X-Response-Format' => $responseType));
}
}
}

313
app/Controller/AttributesController.php
View File

@ -239,13 +239,8 @@ class AttributesController extends AppController
$failed = 1;
$message = sprintf('Attributes saved, however, %s attributes could not be saved. Click %s for more info', count($fails), '$flashErrorMessage');
} else {
if (!empty($fails["attribute_0"])) {
$failed = 1;
$message = '0: ' . $v[0];
} else {
$failed = 1;
$message = 'Attribute could not be saved.';
}
$failed = 1;
$message = 'Attribute could not be saved.';
}
}
if (!empty($failKeys)) {
@ -267,8 +262,14 @@ class AttributesController extends AppController
$flashErrorMessage = implode('<br />', $flashErrorMessage);
$this->Session->write('flashErrorMessage', $flashErrorMessage);
}
if (empty($failed)) {
$this->Flash->success($message);
} else {
$this->Flash->error($message);
}
if ($this->request->is('ajax')) {
$this->autoRender = false;
$this->layout = false;
$errors = ($attributeCount > 1) ? $message : $this->Attribute->validationErrors;
if (!empty($successes)) {
return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => $message)),'status' => 200, 'type' => 'json'));
@ -276,11 +277,6 @@ class AttributesController extends AppController
return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => $errors)),'status' => 200, 'type' => 'json'));
}
} else {
if (empty($failed)) {
$this->Flash->success($message);
} else {
$this->Flash->error($message);
}
if ($successes > 0) {
$this->redirect(array('controller' => 'events', 'action' => 'view', $eventId));
}
@ -311,25 +307,31 @@ class AttributesController extends AppController
$this->loadModel('SharingGroup');
$sgs = $this->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1);
$this->set('sharingGroups', $sgs);
$info = array();
$initialDistribution = 5;
$configuredDistribution = Configure::check('MISP.default_attribute_distribution');
if ($configuredDistribution != null && $configuredDistribution != 'event') {
$initialDistribution = $configuredDistribution;
}
$this->set('initialDistribution', $initialDistribution);
$fieldDesc = array();
$distributionLevels = $this->Attribute->distributionLevels;
if (empty($sgs)) {
unset($distributionLevels[4]);
}
$this->set('distributionLevels', $distributionLevels);
foreach ($distributionLevels as $key => $value) {
$fieldDesc['distribution'][$key] = $this->Attribute->distributionDescriptions[$key]['formdesc'];
}
foreach ($this->Attribute->categoryDefinitions as $key => $value) {
$info['category'][$key] = array('key' => $key, 'desc' => isset($value['formdesc'])? $value['formdesc'] : $value['desc']);
$fieldDesc['category'][$key] = isset($value['formdesc']) ? $value['formdesc'] : $value['desc'];
}
foreach ($this->Attribute->typeDefinitions as $key => $value) {
$info['type'][$key] = array('key' => $key, 'desc' => isset($value['formdesc'])? $value['formdesc'] : $value['desc']);
}
foreach ($distributionLevels as $key => $value) {
$info['distribution'][$key] = array('key' => $value, 'desc' => $this->Attribute->distributionDescriptions[$key]['formdesc']);
$fieldDesc['type'][$key] = isset($value['formdesc']) ? $value['formdesc'] : $value['desc'];
}
$this->loadModel('Noticelist');
$notice_list_triggers = $this->Noticelist->getTriggerData();
$this->set('notice_list_triggers', json_encode($notice_list_triggers, true));
$this->set('info', $info);
$this->set('fieldDesc', $fieldDesc);
$this->set('typeDefinitions', $this->Attribute->typeDefinitions);
$this->set('categoryDefinitions', $this->Attribute->categoryDefinitions);
$this->set('published', $events['Event']['published']);
@ -1758,34 +1760,26 @@ class AttributesController extends AppController
private function __searchUI($attributes)
{
$sightingsData = array();
$sgids = $this->Attribute->Event->cacheSgids($this->Auth->user(), true);
$this->Feed = ClassRegistry::init('Feed');
if (!empty($options['overrideLimit'])) {
$overrideLimit = true;
} else {
$overrideLimit = false;
}
$this->loadModel('GalaxyCluster');
$cluster_names = $this->GalaxyCluster->find('list', array('fields' => array('GalaxyCluster.tag_name'), 'group' => array('GalaxyCluster.tag_name', 'GalaxyCluster.id')));
$this->loadModel('Sighting');
$user = $this->Auth->user();
foreach ($attributes as $k => $attribute) {
$attributeId = $attribute['Attribute']['id'];
$attributes[$k]['Attribute']['AttributeTag'] = $attributes[$k]['AttributeTag'];
$attributes[$k]['Attribute'] = $this->Attribute->Event->massageTags($attributes[$k]['Attribute'], 'Attribute');
$attributes[$k]['Attribute'] = $this->Attribute->Event->massageTags($attributes[$k]['Attribute'], 'Attribute', $excludeGalaxy = false, $cullGalaxyTags = true);
unset($attributes[$k]['AttributeTag']);
foreach ($attributes[$k]['Attribute']['AttributeTag'] as $k2 => $attributeTag) {
if (in_array($attributeTag['Tag']['name'], $cluster_names)) {
unset($attributes[$k]['Attribute']['AttributeTag'][$k2]);
}
}
$sightingsData = array_merge(
$sightingsData,
$this->Sighting->attachToEvent($attribute, $this->Auth->user(), $attributes[$k]['Attribute']['id'], $extraConditions = false)
$this->Sighting->attachToEvent($attribute, $user, $attributeId, $extraConditions = false)
);
$correlations = $this->Attribute->Event->getRelatedAttributes($this->Auth->user(), $attributes[$k]['Attribute']['id'], false, false, 'attribute');
$correlations = $this->Attribute->Event->getRelatedAttributes($user, $attributeId, false, false, 'attribute');
if (!empty($correlations)) {
$attributes[$k]['Attribute']['RelatedAttribute'] = $correlations[$attributes[$k]['Attribute']['id']];
$attributes[$k]['Attribute']['RelatedAttribute'] = $correlations[$attributeId];
}
$temp = $this->Feed->attachFeedCorrelations(array($attributes[$k]['Attribute']), $this->Auth->user, $attributes[$k]['Event'], $overrideLimit);
$temp = $this->Feed->attachFeedCorrelations(array($attributes[$k]['Attribute']), $user, $attributes[$k]['Event']);
if (!empty($temp)) {
$attributes[$k]['Attribute'] = $temp[0];
}
@ -1861,62 +1855,6 @@ class AttributesController extends AppController
$this->set('fails', $this->Attribute->checkComposites());
}
public function restSearch(
$returnFormat = false, $value = false, $type = false, $category = false, $org = false, $tags = false, $from = false,
$to = false, $last = false, $eventid = false, $withAttachments = false, $uuid = false, $publish_timestamp = false, $published = false,
$timestamp = false, $enforceWarninglist = false, $to_ids = false, $deleted = false, $includeEventUuid = false, $event_timestamp = false,
$threat_level_id = false
)
{
$paramArray = array(
'value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp',
'timestamp', 'attribute_timestamp','enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags',
'includeProposals', 'returnFormat', 'published', 'limit', 'page', 'requested_attributes', 'includeContext', 'headerless',
'includeWarninglistHits', 'attackGalaxy', 'object_relation', 'includeSightings', 'includeCorrelations', 'includeDecayScore',
'decayingModel', 'excludeDecayed', 'modelOverrides', 'includeFullModel', 'score'
);
$filterData = array(
'request' => $this->request,
'named_params' => $this->params['named'],
'paramArray' => $paramArray,
'ordered_url_params' => @compact($paramArray)
);
$validFormats = $this->Attribute->validFormats;
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
unset($filterData);
if ($filters === false) {
return $exception;
}
$list = array();
$user = $this->_getApiAuthUser($returnFormat, $exception);
if ($user === false) {
return $exception;
}
if (isset($filters['returnFormat'])) {
$returnFormat = $filters['returnFormat'];
} else {
$returnFormat = 'json';
}
if ($returnFormat === 'download') {
$returnFormat = 'json';
}
$elementCounter = 0;
$renderView = '';
$final = $this->Attribute->restSearch($user, $returnFormat, $filters, false, false, $elementCounter, $renderView);
if (!empty($renderView) && !empty($final)) {
$this->layout = false;
$final = json_decode($final, true);
foreach ($final as $key => $data) {
$this->set($key, $data);
}
$this->render('/Events/module_views/' . $renderView);
} else {
$responseType = $this->Attribute->validFormats[$returnFormat][0];
return $this->RestResponse->viewData($final, $responseType, false, true, false, array('X-Result-Count' => $elementCounter, 'X-Export-Module-Used' => $returnFormat, 'X-Response-Format' => $responseType));
}
}
// returns an XML with attributes that belong to an event. The type of attributes to be returned can be restricted by type using the 3rd parameter.
// Similar to the restSearch, this parameter can be chained with '&&' and negations are accepted too. For example filename&&!filename|md5 would return all filenames that don't have an md5
// The usage of returnAttributes is the following: [MISP-url]/attributes/returnAttributes/<API-key>/<type>/<signature flag>
@ -2062,165 +2000,50 @@ class AttributesController extends AppController
$this->__downloadAttachment($this->Attribute->data['Attribute']);
}
public function text($key='download', $type = 'all', $tags = false, $eventId = false, $allowNonIDS = false, $from = false, $to = false, $last = false, $enforceWarninglist = false, $allowNotPublished = false)
public function text()
{
$simpleFalse = array('eventId', 'allowNonIDS', 'tags', 'from', 'to', 'last', 'enforceWarninglist', 'allowNotPublished');
foreach ($simpleFalse as $sF) {
if (!is_array(${$sF}) && (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false')) {
${$sF} = false;
}
$this->_legacyAPIRemap(array(
'paramArray' => array(
'key', 'type', 'tags', 'eventId', 'allowNonIDS', 'from', 'to', 'last', 'enforceWarninglist', 'allowNotPublished'
),
'request' => $this->request,
'named_params' => $this->params['named'],
'ordered_url_params' => func_get_args(),
'injectedParams' => array(
'returnFormat' => 'text'
),
'alias' => array(
'eventId' => 'eventid'
)
));
if (!empty($this->_legacyParams['allowNonIDS'])) {
$this->_legacyParams['to_ids'] = [0,1];
}
if ($type === 'null' || $type === '0' || $type === 'false') {
$type = 'all';
if (!empty($this->_legacyParams['allowNotPublished'])) {
$this->_legacyParams['published'] = [0,1];
}
if ($this->request->is('post')) {
$params = array('type', 'tags', 'eventId', 'allowNonIDS', 'from', 'to', 'last', 'enforceWarninglist', 'allowNotPublished');
foreach ($params as $param) {
if (isset($this->request->data[$param])) {
${$param} = $this->request->data[$param];
}
}
if (!empty($this->_legacyParams['type']) && $this->_legacyParams['type'] === 'all') {
unset($this->_legacyParams['type']);
}
if ($from) {
$from = $this->Attribute->Event->dateFieldCheck($from);
}
if ($to) {
$to = $this->Attribute->Event->dateFieldCheck($to);
}
if ($last) {
$last = $this->Attribute->Event->resolveTimeDelta($last);
}
if ($key != 'download') {
// check if the key is valid -> search for users based on key
$user = $this->checkAuthUser($key);
if (!$user) {
throw new UnauthorizedException(__('This authentication key is not authorized to be used for exports. Contact your administrator.'));
}
} else {
if (!$this->Auth->user('id')) {
throw new UnauthorizedException(__('You have to be logged in to do that.'));
}
}
$this->response->type('txt'); // set the content type
$this->header('Content-Disposition: download; filename="misp.' . (is_array($type) ? 'multi' : $type) . '.txt"');
$this->layout = 'text/default';
$attributes = $this->Attribute->text($this->Auth->user(), $type, $tags, $eventId, $allowNonIDS, $from, $to, $last, $enforceWarninglist, $allowNotPublished);
$this->loadModel('Whitelist');
$attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true);
$this->set('attributes', $attributes);
$this->render('/Attributes/text');
return $this->restSearch();
}
public function rpz($key='download', $tags=false, $eventId=false, $from=false, $to=false, $policy=false, $walled_garden = false, $ns = false, $email = false, $serial = false, $refresh = false, $retry = false, $expiry = false, $minimum_ttl = false, $ttl = false, $enforceWarninglist = false, $ns_alt = false)
public function rpz()
{
// request handler for POSTed queries. If the request is a post, the parameters (apart from the key) will be ignored and replaced by the terms defined in the posted json or xml object.
// The correct format for both is a "request" root element, as shown by the examples below:
// For Json: {"request":{"policy": "Local-Data","walled_garden":"my.stop.page.net"}}
// For XML: <request><policy>Local-Data</policy><walled_garden>my.stop.page.net</walled_garden></request>
// the response type is used to determine the parsing method (xml/json)
if ($this->request->is('post')) {
if ($this->request->input('json_decode', true)) {
$data = $this->request->input('json_decode', true);
} else {
$data = $this->request->data;
}
if (empty($data)) {
throw new BadRequestException(__('Either specify the search terms in the url, or POST a json array / xml (with the root element being "request" and specify the correct headers based on content type).'));
}
$paramArray = array('eventId', 'tags', 'from', 'to', 'policy', 'walled_garden', 'ns', 'email', 'serial', 'refresh', 'retry', 'expiry', 'minimum_ttl', 'ttl', 'enforceWarninglist', 'ns_alt');
foreach ($paramArray as $p) {
if (isset($data['request'][$p])) {
${$p} = $data['request'][$p];
} else {
${$p} = false;
}
}
}
$simpleFalse = array('eventId', 'tags', 'from', 'to', 'policy', 'walled_garden', 'ns', 'email', 'serial', 'refresh', 'retry', 'expiry', 'minimum_ttl', 'ttl', 'enforceWarninglist', 'ns_alt');
foreach ($simpleFalse as $sF) {
if (!is_array(${$sF}) && (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false')) {
${$sF} = false;
}
}
if (!in_array($policy, array('NXDOMAIN', 'NODATA', 'DROP', 'Local-Data', 'PASSTHRU', 'TCP-only'))) {
$policy = false;
}
App::uses('RPZExport', 'Export');
$rpzExport = new RPZExport();
if ($policy) {
$policy = $rpzExport->getIdByPolicy($policy);
}
$this->loadModel('Server');
$rpzSettings = array();
$lookupData = array('policy', 'walled_garden', 'ns', 'ns_alt', 'email', 'serial', 'refresh', 'retry', 'expiry', 'minimum_ttl', 'ttl');
foreach ($lookupData as $v) {
if (${$v} !== false) {
$rpzSettings[$v] = ${$v};
} else {
$tempSetting = Configure::read('Plugin.RPZ_' . $v);
if (isset($tempSetting)) {
$rpzSettings[$v] = Configure::read('Plugin.RPZ_' . $v);
} else {
$rpzSettings[$v] = $this->Server->serverSettings['Plugin']['RPZ_' . $v]['value'];
}
}
}
if ($from) {
$from = $this->Attribute->Event->dateFieldCheck($from);
}
if ($to) {
$to = $this->Attribute->Event->dateFieldCheck($to);
}
if ($key != 'download') {
// check if the key is valid -> search for users based on key
$user = $this->checkAuthUser($key);
if (!$user) {
throw new UnauthorizedException(__('This authentication key is not authorized to be used for exports. Contact your administrator.'));
}
} else {
if (!$this->Auth->user('id')) {
throw new UnauthorizedException(__('You have to be logged in to do that.'));
}
}
if (false === $eventId || $eventId === null) {
$eventIds = $this->Attribute->Event->fetchEventIds($this->Auth->user(), false, false, false, true);
} elseif (is_numeric($eventId)) {
$eventIds = array($eventId);
} else {
throw new MethodNotAllowedException(__('Invalid event ID format.'));
}
$values = array();
foreach ($eventIds as $k => $eventId) {
$values = array_merge_recursive($values, $this->Attribute->rpz($this->Auth->user(), $tags, $eventId, $from, $to, $enforceWarninglist));
}
$this->response->type('txt'); // set the content type
$file = '';
if ($tags) {
$file = 'filtered.';
}
if ($eventId) {
$file .= 'event-' . $eventId . '.';
}
if ($from) {
$file .= 'from-' . $from . '.';
}
if ($to) {
$file .= 'to-' . $to . '.';
}
if ($file == '') {
$file = 'all.';
}
$this->header('Content-Disposition: download; filename="misp.rpz.' . $file . 'txt"');
$this->layout = 'text/default';
$this->loadModel('Whitelist');
foreach ($values as $key => $value) {
$values[$key] = $this->Whitelist->removeWhitelistedValuesFromArray($value);
}
$this->set('values', $values);
$this->set('rpzSettings', $rpzSettings);
$this->render('/Attributes/rpz');
$this->_legacyAPIRemap(array(
'paramArray' => array(
'key', 'tags', 'eventid', 'from', 'to', 'policy', 'walled_garden', 'ns',
'email', 'serial', 'refresh', 'retry', 'expiry', 'minimum_ttl', 'ttl',
'enforceWarninglist', 'ns_alt'
),
'request' => $this->request,
'named_params' => $this->params['named'],
'ordered_url_params' => func_get_args(),
'injectedParams' => array(
'returnFormat' => 'rpz'
)
));
return $this->restSearch();
}
public function bro($key = 'download', $type = 'all', $tags = false, $eventId = false, $from = false, $to = false, $last = false, $enforceWarninglist = false)

4
app/Controller/Component/ACLComponent.php
View File

@ -25,6 +25,7 @@ class ACLComponent extends Component
'pruneDuplicateUUIDs' => array(),
'queryACL' => array(),
'removeDuplicateEvents' => array(),
'restSearch' => array('*'),
'updateDatabase' => array(),
'upgrade2324' => array(),
),
@ -122,8 +123,6 @@ class ACLComponent extends Component
'checkPublishedStatus' => array('*'),
'checkuuid' => array('perm_sync'),
'contact' => array('*'),
'create_dummy_event' => array(),
'create_massive_dummy_events' => array(),
'csv' => array('*'),
'cullEmptyEvents' => array(),
'delegation_index' => array('*'),
@ -132,7 +131,6 @@ class ACLComponent extends Component
'dot' => array(),
'downloadExport' => array('*'),
'downloadOpenIOCEvent' => array('*'),
'downloadSearchResult' => array('*'),
'edit' => array('perm_add'),
'enrichEvent' => array('perm_add'),
'export' => array('*'),

46
app/Controller/Component/RestSearchComponent.php Normal file
View File

@ -0,0 +1,46 @@
<?php
App::uses('Component', 'Controller');
class RestSearchComponent extends Component
{
public $paramArray = array(
'Attribute' => array(
'returnFormat', 'value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp',
'published', 'timestamp','enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags',
'includeProposals', 'returnFormat', 'limit', 'page', 'requested_attributes', 'includeContext', 'headerless',
'includeWarninglistHits', 'attackGalaxy', 'object_relation', 'includeSightings', 'includeCorrelations', 'includeDecayScore',
'decayingModel', 'excludeDecayed', 'modelOverrides', 'includeFullModel', 'score', 'attribute_timestamp'
),
'Event' => array(
'returnFormat', 'value', 'type', 'category', 'org', 'tags', 'searchall', 'from', 'to', 'last', 'eventid', 'withAttachments',
'metadata', 'uuid', 'publish_timestamp', 'timestamp', 'published', 'enforceWarninglist', 'sgReferenceOnly',
'limit', 'page', 'requested_attributes', 'includeContext', 'headerless', 'includeWarninglistHits', 'attackGalaxy', 'deleted',
'excludeLocalTags', 'date', 'includeSightingdb', 'tag', 'object_relation'
),
'Sighting' => array(
'context', 'returnFormat', 'id', 'type', 'from', 'to', 'last', 'org_id', 'source', 'includeAttribute', 'includeEvent'
)
);
public function getFilename($filters, $scope, $responseType)
{
$filename = false;
if ($scope === 'Event') {
$filename = 'misp.event.';
if (!empty($filters['eventid']) && !is_array($filters['eventid'])) {
if (Validation::uuid(trim($filters['eventid']))) {
$filename .= trim($filters['eventid']);
} else if (!empty(intval(trim($filters['eventid'])))) {
$filename .= intval(trim($filters['eventid']));
}
} else {
$filename .= 'list';
}
}
if ($filename !== false) {
$filename .= '.' . $responseType;
}
return $filename;
}
}

3
app/Controller/DecayingModelController.php
View File

@ -591,7 +591,8 @@ class DecayingModelController extends AppController
$filters['uuid'] = $filters['id'];
} else {
$attributes = $this->User->Event->Attribute->fetchAttributes($this->Auth->user(), array(
'conditions' => array('Attribute.id' => $filters['id'])
'conditions' => array('Attribute.id' => $filters['id']),
'flatten' => 1
));
if (!empty($attributes)) {
$filters['uuid'] = $attributes[0]['Attribute']['uuid'];

818
app/Controller/EventsController.php
View File

@ -1253,7 +1253,7 @@ class EventsController extends AppController
if (isset($filters['deleted'])) {
$deleted = $filters['deleted'] == 2 ? 0 : 1;
}
$this->set('includeSightingdb', (!empty($filters['includeSightingdb'] && Configure::read('Plugin.Sightings_sighting_db_enable'))));
$this->set('includeSightingdb', (!empty($filters['includeSightingdb']) && Configure::read('Plugin.Sightings_sighting_db_enable')));
$this->set('deleted', $deleted);
$this->set('typeGroups', array_keys($this->Event->Attribute->typeGroupings));
$this->set('attributeFilter', isset($filters['attributeFilter']) ? $filters['attributeFilter'] : 'all');
@ -1309,7 +1309,7 @@ class EventsController extends AppController
foreach ($event['Object'] as $k => $object) {
if (!empty($object['Attribute'])) {
foreach ($object['Attribute'] as $attribute) {
if ($oldest_timestamp == false || $oldest_timestamp < $attribute['timestamp']) {
if ($oldest_timestamp == false || $oldest_timestamp > $attribute['timestamp']) {
$oldest_timestamp = $attribute['timestamp'];
}
}
@ -1407,7 +1407,7 @@ class EventsController extends AppController
$startDate = null;
$modificationMap = array();
foreach ($event['Attribute'] as $k => $attribute) {
if ($oldest_timestamp == false || $oldest_timestamp < $attribute['timestamp']) {
if ($oldest_timestamp == false || $oldest_timestamp > $attribute['timestamp']) {
$oldest_timestamp = $attribute['timestamp'];
}
if ($startDate === null || $attribute['timestamp'] < $startDate) {
@ -2147,7 +2147,6 @@ class EventsController extends AppController
throw new UnauthorizedException(__('You do not have permission to do that.'));
}
if ($this->request->is('post')) {
$original_file = !empty($this->data['Event']['original_file']) ? $this->data['Event']['stix']['name'] : '';
if ($this->_isRest()) {
$randomFileName = $this->Event->generateRandomFileName();
$tmpDir = APP . "files" . DS . "scripts" . DS . "tmp";
@ -2158,8 +2157,8 @@ class EventsController extends AppController
$this->Auth->user(),
$randomFileName,
$stix_version,
$original_file,
$this->data['Event']['publish']
'uploaded_stix_file.' . ($stix_version == '1' ? 'xml' : 'json'),
false
);
if (is_array($result)) {
return $this->RestResponse->saveSuccessResponse('Events', 'upload_stix', false, $this->response->type(), 'STIX document imported, event\'s created: ' . implode(', ', $result) . '.');
@ -2174,6 +2173,7 @@ class EventsController extends AppController
return $this->RestResponse->saveFailResponse('Events', 'upload_stix', false, $result, $this->response->type());
}
} else {
$original_file = !empty($this->data['Event']['original_file']) ? $this->data['Event']['stix']['name'] : '';
if (isset($this->data['Event']['stix']) && $this->data['Event']['stix']['size'] > 0 && is_uploaded_file($this->data['Event']['stix']['tmp_name'])) {
$randomFileName = $this->Event->generateRandomFileName();
$tmpDir = APP . "files" . DS . "scripts" . DS . "tmp";
@ -2419,7 +2419,6 @@ class EventsController extends AppController
$fieldDesc['analysis'][$key] = $this->Event->analysisDescriptions[$key]['formdesc'];
}
$this->set('analysisLevels', $analysisLevels);
$this->set('fieldDesc', $fieldDesc);
$this->set('eventDescriptions', $this->Event->fieldDescriptions);
$this->set('event', $this->Event->data);
@ -3002,105 +3001,18 @@ class EventsController extends AppController
public function xml($key, $eventid = false, $withAttachment = false, $tags = false, $from = false, $to = false, $last = false)
{
App::uses('XMLConverterTool', 'Tools');
$converter = new XMLConverterTool();
$this->loadModel('Whitelist');
// request handler for POSTed queries. If the request is a post, the parameters (apart from the key) will be ignored and replaced by the terms defined in the posted xml object.
// The correct format for a posted xml is a "request" root element, as shown by the examples below:
// For XML: <request><value>7.7.7.7&amp;&amp;1.1.1.1</value><type>ip-src</type></request>
if ($this->request->is('post')) {
if (empty($this->request->data)) {
throw new BadRequestException(__('Either specify the search terms in the url, or POST an xml (with the root element being "request").'));
} else {
$data = $this->request->data;
}
$paramArray = array('eventid', 'withAttachment', 'tags', 'from', 'to', 'last');
foreach ($paramArray as $p) {
if (isset($data['request'][$p])) {
${$p} = $data['request'][$p];
} else {
${$p} = null;
}
}
}
$simpleFalse = array('tags', 'eventid', 'withAttachment', 'from', 'to', 'last');
foreach ($simpleFalse as $sF) {
if (!is_array(${$sF}) && (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false')) {
${$sF} = false;
}
}
if ($from) {
$from = $this->Event->dateFieldCheck($from);
}
if ($to) {
$to = $this->Event->dateFieldCheck($to);
}
if ($tags) {
$tags = str_replace(';', ':', $tags);
}
if ($last) {
$last = $this->Event->resolveTimeDelta($last);
}
$eventIdArray = array();
if ($eventid) {
if (!is_numeric($eventid)) {
throw new MethodNotAllowedException(__('Invalid Event ID.'));
}
$eventIdArray[] = $eventid;
}
if ($key != 'download') {
// check if the key is valid -> search for users based on key
$user = $this->checkAuthUser($key);
if (!$user) {
throw new UnauthorizedException(__('This authentication key is not authorized to be used for exports. Contact your administrator.'));
}
} else {
if (!$this->Auth->user('id')) {
throw new UnauthorizedException(__('You have to be logged in to do that.'));
}
$user = $this->Auth->user();
}
if ($eventid) {
$final_filename='misp.event' . $eventid . '.export.xml';
} else {
$final_filename='misp.export.xml';
}
$final = "";
$final .= '<?xml version="1.0" encoding="UTF-8"?>' . PHP_EOL . '<response>' . PHP_EOL;
$validEvents = 0;
if (!$eventid) {
$eventIdArray = $this->Event->fetchEventIds($user, $from, $to, $last, true);
}
foreach ($eventIdArray as $currentEventId) {
$result = $this->Event->fetchEvent($user, array('eventid' => $currentEventId, 'tags' => $tags, 'from' => $from, 'to' => $to, 'last' => $last));
if (empty($result)) {
continue;
}
$validEvents++;
if ($withAttachment) {
foreach ($result[0]['Attribute'] as $key => $attribute) {
if ($this->Event->Attribute->typeIsAttachment($attribute['type'])) {
$encodedFile = $this->Event->Attribute->base64EncodeAttachment($attribute);
$result[0]['Attribute'][$key]['data'] = $encodedFile;
}
}
}
$result = $this->Whitelist->removeWhitelistedFromArray($result, false);
$final .= $converter->convert($result[0]) . PHP_EOL;
}
if ($validEvents == 0) {
throw new NotFoundException(__('No events found that match the passed parameters.'));
}
$final .= '</response>' . PHP_EOL;
$this->response->body($final);
$this->response->type('xml');
$this->response->download($final_filename);
return $this->response;
$this->_legacyAPIRemap(array(
'paramArray' => array(
'key', 'eventid', 'withAttachment', 'tags', 'from', 'to', 'last'
),
'request' => $this->request,
'named_params' => $this->params['named'],
'ordered_url_params' => func_get_args(),
'injectedParams' => array(
'returnFormat' => 'xml'
)
));
return $this->restSearch();
}
// Grab an event or a list of events for the event view or any of the XML exports. The returned object includes an array of events (or an array that only includes a single event if an ID was given)
@ -3115,200 +3027,66 @@ class EventsController extends AppController
return $results;
}
public function nids($format = 'suricata', $key = 'download', $id = false, $continue = false, $tags = false, $from = false, $to = false, $last = false, $type = false, $enforceWarninglist = false, $includeAllTags = false, $eventid = false)
public function nids()
{
if ($this->request->is('post')) {
if (empty($this->request->data)) {
throw new BadRequestException(__('Either specify the search terms in the url, or POST a json or xml with the filter parameters. Valid filters: id (event ID), tags (list of tags), from (from date in YYYY-MM-DD format), to (to date in YYYY-MM-DD format), last (events with a published timestamp newer than - valid options are in time + unit format such as 6d or 2w, etc)'));
} else {
$data = $this->request->data;
}
$paramArray = array('id', 'continue', 'tags', 'from', 'to', 'last', 'type', 'enforceWarninglist', 'eventid');
if (!isset($data['request'])) {
$data = array('request' => $data);
}
foreach ($paramArray as $p) {
if (isset($data['request'][$p])) {
${$p} = $data['request'][$p];
} else {
${$p} = null;
}
}
}
$simpleFalse = array('id', 'continue', 'tags', 'from', 'to', 'last', 'type', 'enforceWarninglist', 'includeAllTags', 'eventid');
foreach ($simpleFalse as $sF) {
if (!is_array(${$sF}) && (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false')) {
${$sF} = false;
}
}
if (!empty($eventid)) {
$id = $eventid;
}
if ($from) {
$from = $this->Event->dateFieldCheck($from);
}
if ($to) {
$to = $this->Event->dateFieldCheck($to);
}
if ($tags) {
$tags = str_replace(';', ':', $tags);
}
if ($last) {
$last = $this->Event->resolveTimeDelta($last);
}
// backwards compatibility, swap key and format
if ($format != 'snort' && $format != 'suricata') {
$format = 'suricata'; // default format
}
$this->response->type('txt'); // set the content type
$filename = 'misp.' . $format . '.rules';
if ($id) {
$filename = 'misp.' . $format . '.event' . $id . '.rules';
}
$this->header('Content-Disposition: download; filename="' . $filename . '"');
$this->layout = 'text/default';
if ($key != 'download') {
// check if the key is valid -> search for users based on key
$user = $this->checkAuthUser($key);
if (!$user) {
throw new UnauthorizedException(__('This authentication key is not authorized to be used for exports. Contact your administrator.'));
}
} else {
// check if there's a user logged in or not
if (!$this->Auth->user('id')) {
throw new UnauthorizedException(__('You have to be logged in to do that.'));
}
$user = $this->Auth->user();
}
// display the full snort rulebase
$this->loadModel('Attribute');
$rules = $this->Attribute->nids($user, $format, $id, $continue, $tags, $from, $to, $last, $type, $enforceWarninglist, $includeAllTags);
$this->set('rules', $rules);
$this->render('/Events/nids');
}
public function hids($type, $key = 'download', $tags = false, $from = false, $to = false, $last = false, $enforceWarninglist = false)
{
$simpleFalse = array('tags', 'from', 'to', 'last', 'enforceWarninglist');
if ($this->request->is('post')) {
if (empty($this->request->data)) {
throw new BadRequestException(__('Either specify the search terms in the url, or POST a json or xml with the filter parameters.'));
} else {
$data = $this->request->data;
}
if (!isset($data['request'])) {
$data = array('request' => $data);
}
foreach ($simpleFalse as $sF) {
if (isset($data['request'][$sF])) {
${$sF} = $data['request'][$sF];
}
}
}
foreach ($simpleFalse as $sF) {
if (!is_array(${$sF}) && (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false')) {
${$sF} = false;
}
}
if (!in_array($type, array('md5', 'sha1', 'sha256'))) {
throw new MethodNotAllowedException(__('Invalid hash type.'));
}
if ($from) {
$from = $this->Event->dateFieldCheck($from);
}
if ($to) {
$to = $this->Event->dateFieldCheck($to);
}
if ($tags) {
$tags = str_replace(';', ':', $tags);
}
if ($last) {
$last = $this->Event->resolveTimeDelta($last);
}
$this->response->type('txt'); // set the content type
$this->header('Content-Disposition: download; filename="misp.' . $type . '.rules"');
$this->layout = 'text/default';
if ($key != 'download') {
// check if the key is valid -> search for users based on key
$user = $this->checkAuthUser($key);
if (!$user) {
throw new UnauthorizedException(__('This authentication key is not authorized to be used for exports. Contact your administrator.'));
}
} else {
// check if there's a user logged in or not
if (!$this->Auth->user('id')) {
throw new UnauthorizedException(__('You have to be logged in to do that.'));
}
}
$this->loadModel('Attribute');
$rules = $this->Attribute->hids($this->Auth->user(), $type, $tags, $from, $to, $last, false, $enforceWarninglist);
return new CakeResponse(array('body'=> implode(PHP_EOL, $rules), 'status' => 200, 'type' => 'txt'));
}
// csv function (DEPCRECATED)
// Usage: csv($key, $eventid) - key can be a valid auth key or the string 'download'. Download requires the user to be logged in interactively and will generate a .csv file
// $eventid can be one of 3 options: left empty it will get all the visible to_ids attributes,
// $ignore is a flag that allows the export tool to ignore the ids flag. 0 = only IDS signatures, 1 = everything.
public function csv($key, $eventid = false, $ignore = false, $tags = false, $category = false, $type = false, $includeContext = false, $from = false, $to = false, $last = false, $headerless = false, $enforceWarninglist = false, $value = false, $timestamp = false)
{
$paramArray = array('eventid', 'ignore', 'tags', 'category', 'type', 'includeContext', 'from', 'to', 'last', 'headerless', 'enforceWarninglist', 'value', 'timestamp');
$filterData = array(
$this->_legacyAPIRemap(array(
'paramArray' => array(
'format', 'key', 'id', 'continue', 'tags', 'from', 'to', 'last',
'type', 'enforceWarninglist', 'includeAllTags', 'eventid'
),
'request' => $this->request,
'named_params' => $this->params['named'],
'paramArray' => $paramArray,
'ordered_url_params' => @compact($paramArray)
'ordered_url_params' => func_get_args()
));
if (empty($this->_legacyParams['returnFormat'])) {
$this->_legacyParams['returnFormat'] = 'suricata';
}
return $this->restSearch();
}
public function hids($type)
{
$typeMappings = array(
'md5' => array('malware-sample', 'md5', 'filename|md5'),
'sha1' => array('sha1', 'filename|sha1'),
'sha256' => array('sha256', 'filename|sha256')
);
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
if ($filters === false) {
return $exception;
}
$list = array();
$user = $this->_getApiAuthUser($key, $exception);
if ($user === false) {
return $exception;
}
if (!empty($eventid) && !is_array($eventid)) {
$filename = 'misp.csv.event' . $eventid . '.csv';
} else {
$filename = 'misp.csv.filtered_results.csv';
}
if (!isset($filters['ignore'])) {
$filters['ignore'] = 0;
}
$final = $this->Event->restSearch($user, 'csv', $filters);
// if it's a search, grab the attributeIDList from the session and get the IDs from it. Use those as the condition
// We don't need to look out for permissions since that's filtered by the search itself
// We just want all the attributes found by the search
if (!empty($params['eventid']) && $params['eventid'] === 'search') {
$ioc = $this->Session->read('paginate_conditions_ioc');
$paginateConditions = $this->Session->read('paginate_conditions');
unset($paginateConditions['contain']['Event']['Orgc']);
$attributes = $this->Event->Attribute->find('all', array(
'conditions' => $paginateConditions['conditions'],
'contain' => $paginateConditions['contain'],
));
$orgs = $this->Event->Orgc->find('list', array(
'fields' => array('Orgc.id', 'Orgc.name'),
'recursive' => -1
));
if ($ioc) {
$this->loadModel('Whitelist');
$attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true);
}
$list = array();
foreach ($attributes as $attribute) {
$attribute['Event']['Orgc'] = array(
'id' => $attribute['Event']['org_id'],
'name' => $orgs[$attribute['Event']['org_id']]
);
$list[] = $attribute['Attribute']['id'];
}
}
$responseType = 'csv';
return $this->RestResponse->viewData($final, $responseType, false, true, $filename);
$ordered_url_params = func_get_args();
unset($ordered_url_params[0]);
$ordered_url_params = array_values($ordered_url_params);
$this->scopeOverride = 'Attribute';
$this->_legacyAPIRemap(array(
'paramArray' => array(
'key', 'id', 'withAttachment', 'tags', 'from', 'to', 'last'
),
'request' => $this->request,
'named_params' => $this->params['named'],
'ordered_url_params' => $ordered_url_params,
'injectedParams' => array(
'returnFormat' => 'hashes',
'type' => (isset($typeMappings[$type])) ? $typeMappings[$type] : $type
)
));
return $this->restSearch();
}
// DEPRECATED - use restSearch with "returnFormat":"csv"
public function csv($key)
{
$this->_legacyAPIRemap(array(
'paramArray' => array(
'key', 'eventid', 'ignore', 'tags', 'category', 'type', 'includeContext',
'from', 'to', 'last', 'headerless', 'enforceWarninglist', 'value', 'timestamp'
),
'key' => $key,
'request' => $this->request,
'named_params' => $this->params['named'],
'ordered_url_params' => func_get_args(),
'injectedParams' => array(
'returnFormat' => 'csv'
)
));
return $this->restSearch();
}
public function _addIOCFile($id)
@ -3477,127 +3255,6 @@ class EventsController extends AppController
return $toReturn;
}
public function downloadSearchResult()
{
$ioc = $this->Session->read('paginate_conditions_ioc');
$paginateConditions = $this->Session->read('paginate_conditions');
$attributes = $this->Event->Attribute->fetchAttributes($this->Auth->user(), array(
'conditions' => $paginateConditions['conditions'],
'contain' => $paginateConditions['contain'],
));
if ($ioc) {
$this->loadModel('Whitelist');
$attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true);
}
$idList = array();
foreach ($attributes as $attribute) {
if (!in_array($attribute['Attribute']['event_id'], $idList)) {
$idList[] = $attribute['Attribute']['event_id'];
}
}
$results = $this->__fetchEvent(null, $idList);
$this->set('results', $results);
if ($this->response->type() === 'application/json') {
$type = 'json';
} else {
$type = 'xml';
}
App::uses(strtoupper($type) . 'ConverterTool', 'Tools');
$tool = strtoupper($type) . 'ConverterTool';
$converter = new $tool();
$body = $converter->eventCollection2Format($results);
$body = $converter->frameCollection($body, $this->mispVersion);
$this->response->body($body);
$this->response->download('misp.search.results.' . $type);
return $this->response;
}
// Use the REST interface to search for attributes or events. Usage:
// MISP-base-url/events/restSearch/[api-key]/[value]/[type]/[category]/[orgc]
// value, type, category, orgc are optional
// target can be either "event" or "attribute"
// the last 4 fields accept the following operators:
// && - you can use && between two search values to put a logical OR between them. for value, 1.1.1.1&&2.2.2.2 would find attributes with the value being either of the two.
// ! - you can negate a search term. For example: google.com&&!mail would search for all attributes with value google.com but not ones that include mail. www.google.com would get returned, mail.google.com wouldn't.
public function restSearch(
$returnFormat = false,
$value = false,
$type = false,
$category = false,
$org = false,
$tags = false,
$searchall = false,
$from = false,
$to = false,
$last = false,
$eventid = false,
$withAttachments = false,
$metadata = false,
$uuid = false,
$publish_timestamp = false,
$timestamp = false,
$published = false,
$enforceWarninglist = false,
$sgReferenceOnly = false
) {
$paramArray = array(
'value', 'type', 'category', 'object_relation', 'org', 'tag', 'tags', 'searchall', 'from', 'to', 'last', 'eventid', 'withAttachments',
'metadata', 'uuid', 'published', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'sgReferenceOnly', 'returnFormat',
'limit', 'page', 'requested_attributes', 'includeContext', 'headerless', 'includeWarninglistHits', 'attackGalaxy', 'deleted',
'excludeLocalTags', 'date', 'includeSightingdb'
);
$filterData = array(
'request' => $this->request,
'named_params' => $this->params['named'],
'paramArray' => $paramArray,
'ordered_url_params' => @compact($paramArray)
);
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
unset($filterData);
if ($filters === false) {
return $exception;
}
$list = array();
$user = $this->_getApiAuthUser($returnFormat, $exception);
if ($user === false) {
return $exception;
}
if (isset($filters['returnFormat'])) {
$returnFormat = $filters['returnFormat'];
} else {
$returnFormat = 'json';
}
if ($returnFormat === 'download') {
$returnFormat = 'json';
}
$elementCounter = 0;
$renderView = false;
$final = $this->Event->restSearch($user, $returnFormat, $filters, false, false, $elementCounter, $renderView);
if (!empty($renderView) && !empty($final)) {
$this->layout = false;
$final = json_decode($final, true);
foreach ($final as $key => $data) {
$this->set($key, $data);
}
$this->render('/Events/module_views/' . $renderView);
} else {
$responseType = $this->Event->validFormats[$returnFormat][0];
$filename = 'misp.event.';
if (!empty($filters['eventid']) && !is_array($filters['eventid'])) {
if (Validation::uuid(trim($filters['eventid']))) {
$filename .= trim($filters['eventid']);
} else if (!empty(intval(trim($filters['eventid'])))) {
$filename .= intval(trim($filters['eventid']));
}
} else {
$filename .= 'list';
}
$filename .= '.' . $responseType;
return $this->RestResponse->viewData($final, $responseType, false, true, $filename, array('X-Result-Count' => $elementCounter, 'X-Export-Module-Used' => $returnFormat, 'X-Response-Format' => $responseType));
}
}
public function downloadOpenIOCEvent($key, $eventid, $enforceWarninglist = false)
{
// return a downloadable text file called misp.openIOC.<eventId>.ioc for individual events
@ -3643,125 +3300,6 @@ class EventsController extends AppController
return $this->response;
}
public function create_dummy_event()
{
if (!$this->_isSiteAdmin() || !$this->request->is('post')) {
throw new MethodNotAllowedException(__('You don\'t have the privileges to access this.'));
}
$data['Event']['info'] = 'Test event showing every category-type combination';
$data['Event']['date'] = '2013-10-09';
$data['Event']['threat_level_id'] = 4; //'Undefined'
$data['Event']['analysis'] = '0';
$data['Event']['distribution'] = '0';
$defaultValues = array(
'md5' => '098f6bcd4621d373cade4e832627b4f6',
'sha1' => 'a7645200866fd00bde529733ceac8506ab1f5518',
'sha256' => '0f58957831a9cf0b768451ee6b236555f519c04f0da5a5ea87538fd0990b29d1',
'filename' => 'test.exe',
'filename|md5' => 'test.exe|8886be8e4e862189a68d27e8fc7a6144',
'filename|sha1' => 'test.exe|a7645200866fd00bde529733ceac8506ab1f5518',
'filename|sha256' => 'test.exe|0f58957831a9cf0b768451ee6b236555f519c04f0da5a5ea87538fd0990b29d1',
'ip-src' => '1.1.1.1',
'ip-dst' => '2.2.2.2',
'hostname' => 'www.futuremark.com',
'domain' => 'evildomain.org',
'email-src' => 'bla@bla.com',
'email-dst' => 'hmm@hmm.com',
'email-subject' => 'Some made-up email subject',
'email-attachment' => 'filename.exe',
'url' => 'http://www.evilsite.com/test',
'http-method' => 'POST',
'user-agent' => 'Microsoft Internet Explorer',
'regkey' => 'HKLM\Software\Microsoft\Windows\CurrentVersion\Run\fishy',
'regkey|value' => 'HKLM\Software\Microsoft\Windows\CurrentVersion\Run\fishy|%ProgramFiles%\Malicios\malware.exe',
'AS' => '45566',
'snort' => 'alert ip 1.1.1.1 any -> $HOME_NET any (msg: "MISP e1 Incoming From IP: 1.1.1.1"; classtype:trojan-activity; sid:21; rev:1; priority:1; reference:url,http://localhost:8888/events/view/1;)',
'pattern-in-file' => 'Somestringinfile',
'pattern-in-traffic' => 'Somestringintraffic',
'pattern-in-memory' => 'Somestringinmemory',
'yara' => 'rule silent_banker : banker{meta:description = "This is just an example" thread_level = 3 in_the_wild = true strings: $a = {6A 40 68 00 30 00 00 6A 14 8D 91} $b = {8D 4D B0 2B C1 83 C0 27 99 6A 4E 59 F7 F9} $c = "UVODFRYSIHLNWPEJXQZAKCBGMT" condition:}',
'vulnerability' => 'CVE-2011-0001',
'attachment' => 'file.txt',
'malware-sample' => 'test.exe|8886be8e4e862189a68d27e8fc7a6144',
'link' => 'http://www.somesite.com/',
'comment' => 'Comment',
'text' => 'Any text',
'other' => 'Could be anything',
'named pipe' => '\\.\pipe\PipeName',
'mutex' => 'mutexstring',
'target-user' => 'user1',
'target-email' => 'someone@something.com',
'target-machine' => 'machinename',
'target-org' => 'EA games',
'target-location' => 'Hell',
'target-external' => 'some target'
);
$this->loadModel('Attribute');
foreach ($this->Attribute->categoryDefinitions as $category => $v) {
foreach ($v['types'] as $k => $type) {
$data['Attribute'][] = array(
'category' => $category,
'type' => $type,
'value' => $defaultValues[$type],
'to_ids' => '0',
'distribution' => '0',
);
}
}
$this->Event->_add($data, false, $this->Auth->user());
}
// for load testing, it's slow, execution time is set at 1 hour maximum
public function create_massive_dummy_events()
{
if (!$this->_isSiteAdmin() || !$this->request->is('post')) {
throw new MethodNotAllowedException(__('You don\'t have the privileges to access this.'));
}
ini_set('max_execution_time', 3600);
$this->Event->Behaviors->unload('SysLogLogable.SysLogLogable');
$date = new DateTime();
$ts = $date->getTimestamp();
$default = array('Event' => array(
'info' => 'A junk event for load testing',
'date' => '2014-09-01',
'threat_level_id' => 4,
'distribution' => 0,
'analysis' => 0,
'org_id' => $this->Auth->user('org_id'),
'orgc_id' => $this->Auth->user('org_id'),
'timestamp' => $ts,
'user_id' => $this->Auth->user('id'),
));
$default['Event']['info'] = 'A junk event for load testing';
$default['Event']['date'] = '2013-10-09';
$default['Event']['threat_level_id'] = 4; //'Undefined'
$default['Event']['analysis'] = '0';
$default['Event']['distribution'] = '0';
for ($i = 0; $i < 50; $i++) {
$data = $default;
$data['Event']['uuid'] = CakeText::uuid();
for ($j = 0; $j < 3000; $j++) {
$value = mt_rand();
$data['Attribute'][] = array(
'category' => 'Other',
'type' => 'text',
'value' => $value,
'to_ids' => '0',
'distribution' => '0',
'value1' => $value,
'value2' => '',
'comment' => '',
'uuid' => CakeText::uuid(),
'timestamp' => $ts,
'disable_correlation' => 1
);
}
$this->Event->create();
$this->Event->saveAssociated($data, array('validate' => false));
}
}
public function proposalEventIndex()
{
$this->loadModel('ShadowAttribute');
@ -4284,147 +3822,42 @@ class EventsController extends AppController
}
}
public function stix2($key, $id = false, $withAttachments = false, $tags = false, $from = false, $to = false, $last = false)
public function stix2()
{
if ($key != 'download') {
// check if the key is valid -> search for users based on key
$user = $this->checkAuthUser($key);
if (!$user) {
throw new UnauthorizedException(__('This authentication key is not authorized to be used for exports. Contact your administrator.'));
}
} else {
if (!$this->Auth->user('id')) {
throw new UnauthorizedException(__('You have to be logged in to do that.'));
}
}
if ($this->request->is('post')) {
if (empty($this->request->data)) {
throw new BadRequestException(__('Either specify the search terms in the url, or POST an xml (with the root element being "request").'));
} else {
$data = $this->request->data;
}
$paramArray = array('id', 'withAttachments', 'tags', 'from', 'to', 'last');
foreach ($paramArray as $p) {
if (isset($data['request'][$p])) {
${$p} = $data['request'][$p];
} else {
${$p} = null;
}
}
}
$simpleFalse = array('id', 'withAttachments', 'tags', 'from', 'to', 'last');
foreach ($simpleFalse as $sF) {
if (!is_array(${$sF}) && (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false')) {
${$sF} = false;
}
}
if ($from) {
$from = $this->Event->dateFieldCheck($from);
}
if ($to) {
$to = $this->Event->dateFieldCheck($to);
}
if ($last) {
$last = $this->Event->resolveTimeDelta($last);
}
// set null if a null string is passed
$numeric = false;
if (is_numeric($id)) {
$numeric = true;
}
$result = $this->Event->stix2($id, $tags, $withAttachments, $this->Auth->user(), 'json', $from, $to, $last);
if ($result['success'] == 1) {
if ($numeric) {
$filename = 'misp.stix2.event' . $id . '.json';
} else {
$filename = 'misp.stix2.event.collection.json';
}
$this->header('Content-Disposition: download; filename="' . $filename . '"');
return $this->RestResponse->viewData($result['data'], 'application/json', false, true, $filename);
} else {
throw new Exception(h($result['message']));
}
$this->_legacyAPIRemap(array(
'paramArray' => array(
'key', 'id', 'withAttachment', 'tags', 'from', 'to', 'last'
),
'request' => $this->request,
'named_params' => $this->params['named'],
'ordered_url_params' => func_get_args(),
'injectedParams' => array(
'returnFormat' => 'stix2'
),
'alias' => array(
'id' => 'eventid'
)
));
return $this->restSearch();
}
public function stix($key, $id = false, $withAttachments = false, $tags = false, $from = false, $to = false, $last = false)
public function stix()
{
if ($key != 'download') {
// check if the key is valid -> search for users based on key
$user = $this->checkAuthUser($key);
if (!$user) {
throw new UnauthorizedException(__('This authentication key is not authorized to be used for exports. Contact your administrator.'));
}
} else {
if (!$this->Auth->user('id')) {
throw new UnauthorizedException(__('You have to be logged in to do that.'));
}
}
// request handler for POSTed queries. If the request is a post, the parameters (apart from the key) will be ignored and replaced by the terms defined in the posted xml object.
// The correct format for a posted xml is a "request" root element, as shown by the examples below:
// For XML: <request><id>!3&amp;!4</id><tags>OSINT</tags></request>
// This would return all OSINT tagged events except for event #3 and #4
if ($this->request->is('post')) {
if (empty($this->request->data)) {
throw new BadRequestException(__('Either specify the search terms in the url, or POST an xml (with the root element being "request").'));
} else {
$data = $this->request->data;
}
if (!isset($data['request'])) {
$data = array('request' => $data);
}
$paramArray = array('id', 'withAttachment', 'tags', 'from', 'to', 'last');
foreach ($paramArray as $p) {
if (isset($data['request'][$p])) {
${$p} = $data['request'][$p];
} else {
${$p} = null;
}
}
}
$simpleFalse = array('id', 'withAttachments', 'tags', 'from', 'to', 'last');
foreach ($simpleFalse as $sF) {
if (!is_array(${$sF}) && (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false')) {
${$sF} = false;
}
}
if ($from) {
$from = $this->Event->dateFieldCheck($from);
}
if ($to) {
$to = $this->Event->dateFieldCheck($to);
}
if ($last) {
$last = $this->Event->resolveTimeDelta($last);
}
// set null if a null string is passed
$numeric = false;
if (is_numeric($id)) {
$numeric = true;
}
// set the export type based on the request
if ($this->response->type() === 'application/json') {
$returnType = 'json';
} else {
$returnType = 'xml';
$this->response->type('xml'); // set the content type
$this->layout = 'xml/default';
}
$result = $this->Event->stix($id, $tags, $withAttachments, $this->Auth->user(), $returnType, $from, $to, $last);
if ($result['success'] == 1) {
// read the output file and pass it to the view
if (!$numeric) {
$this->header('Content-Disposition: download; filename="misp.stix.event.collection.' . $returnType . '"');
} else {
$this->header('Content-Disposition: download; filename="misp.stix.event' . $id . '.' . $returnType . '"');
}
$this->set('data', $result['data']);
} else {
throw new BadRequestException(h($result['message']));
}
$this->_legacyAPIRemap(array(
'paramArray' => array(
'key', 'id', 'withAttachment', 'tags', 'from', 'to', 'last'
),
'request' => $this->request,
'named_params' => $this->params['named'],
'ordered_url_params' => func_get_args(),
'injectedParams' => array(
'returnFormat' => 'stix'
),
'alias' => array(
'id' => 'eventid'
)
));
return $this->restSearch();
}
public function filterEventIdsForPush()
@ -5232,22 +4665,24 @@ class EventsController extends AppController
throw new Exception("Invalid options.");
}
$event = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $eventId, 'metadata' => true));
if (empty($event)) {
throw new NotFoundException(__('Event not found or you are not authorised to view it.'));
if ($scope !== 'tag_collection') {
$event = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $eventId, 'metadata' => true));
if (empty($event)) {
throw new NotFoundException(__('Event not found or you are not authorised to view it.'));
}
$scoresDataAttr = $this->Event->Attribute->AttributeTag->getTagScores($this->Auth->user(), $eventId, $matrixTags);
$scoresDataEvent = $this->Event->EventTag->getTagScores($eventId, $matrixTags);
$maxScore = 0;
$scoresData = array();
foreach (array_keys($scoresDataAttr['scores'] + $scoresDataEvent['scores']) as $key) {
$sum = (isset($scoresDataAttr['scores'][$key]) ? $scoresDataAttr['scores'][$key] : 0) + (isset($scoresDataEvent['scores'][$key]) ? $scoresDataEvent['scores'][$key] : 0);
$scoresData[$key] = $sum;
$maxScore = max($maxScore, $sum);
}
$scores = $scoresData;
} else {
$scores = $scoresData = array();
}
$scoresDataAttr = $this->Event->Attribute->AttributeTag->getTagScores($this->Auth->user(), $eventId, $matrixTags);
$scoresDataEvent = $this->Event->EventTag->getTagScores($eventId, $matrixTags);
$maxScore = 0;
$scoresData = array();
foreach (array_keys($scoresDataAttr['scores'] + $scoresDataEvent['scores']) as $key) {
$sum = (isset($scoresDataAttr['scores'][$key]) ? $scoresDataAttr['scores'][$key] : 0) + (isset($scoresDataEvent['scores'][$key]) ? $scoresDataEvent['scores'][$key] : 0);
$scoresData[$key] = $sum;
$maxScore = max($maxScore, $sum);
}
$scores = $scoresData;
// FIXME: temporary fix: add the score of deprecated mitre galaxies to the new one (for the stats)
if ($matrixData['galaxy']['id'] == $mitreAttackGalaxyId) {
$mergedScore = array();
@ -5407,7 +4842,7 @@ class EventsController extends AppController
$options = array();
foreach ($enabledModules['modules'] as $temp) {
if ($temp['name'] == $module) {
$format = (isset($temp['mispattributes']['format']) ? $temp['mispattributes']['format'] : 'simplified');
$format = (!empty($temp['mispattributes']['format']) ? $temp['mispattributes']['format'] : 'simplified');
if (isset($temp['meta']['config'])) {
foreach ($temp['meta']['config'] as $conf) {
$options[$conf] = Configure::read('Plugin.' . $type . '_' . $module . '_' . $conf);
@ -5671,7 +5106,7 @@ class EventsController extends AppController
throw new Exception($result);
}
$importComment = !empty($result['comment']) ? $result['comment'] : 'Enriched via the ' . $module['name'] . ' module';
if (isset($module['mispattributes']['format']) && $module['mispattributes']['format'] === 'misp_standard') {
if (!empty($module['mispattributes']['format']) && $module['mispattributes']['format'] === 'misp_standard') {
$event = $this->Event->handleMispFormatFromModuleResult($result);
$event['Event'] = array('id' => $eventId);
if ($this->_isRest()) {
@ -5724,8 +5159,9 @@ class EventsController extends AppController
$this->set('importComment', $importComment);
$this->render($render_name);
}
} else {
$this->Flash->error($fail);
}
$this->Flash->error($fail);
}
$this->set('configTypes', $this->Module->configTypes);
$this->set('module', $module);

14
app/Controller/GalaxiesController.php
View File

@ -111,17 +111,19 @@ class GalaxiesController extends AppController
{
$mitreAttackGalaxyId = $this->Galaxy->getMitreAttackGalaxyId();
$local = !empty($this->params['named']['local']) ? $this->params['named']['local'] : '0';
$conditions = $namespace == '0' ? array() : array('namespace' => $namespace);
$conditions = $namespace === '0' ? array() : array('namespace' => $namespace);
$galaxies = $this->Galaxy->find('all', array(
'recursive' => -1,
'fields' => array('MAX(Galaxy.version) as latest_version', 'id', 'kill_chain_order', 'name', 'icon', 'description'),
'conditions' => $conditions,
'group' => array('name', 'id', 'kill_chain_order', 'icon', 'description'),
'order' => array('name asc')
));
$items = array();
$items[] = array(
'name' => __('All clusters'),
'value' => "/galaxies/selectCluster/" . h($target_id) . '/' . h($target_type) . '/0'. '/local:' . $local
$items = array(
array(
'name' => __('All clusters'),
'value' => "/galaxies/selectCluster/" . h($target_id) . '/' . h($target_type) . '/0'. '/local:' . $local
)
);
foreach ($galaxies as $galaxy) {
if (!isset($galaxy['Galaxy']['kill_chain_order'])) {

2
app/Controller/LogsController.php
View File

@ -416,7 +416,7 @@ class LogsController extends AppController
$this->set('actions', $actions);
// combobox for models
$models = array('Attribute', 'Event', 'EventBlacklist', 'EventTag', 'DecayingModel', 'MispObject', 'Organisation', 'Post', 'Regexp', 'Role', 'Server', 'ShadowAttribute', 'SharingGroup', 'Tag', 'Task', 'Taxonomy', 'Template', 'Thread', 'User', 'Whitelist');
$models = array('Attribute', 'Event', 'EventBlacklist', 'EventTag', 'Feed', 'DecayingModel', 'MispObject', 'Organisation', 'Post', 'Regexp', 'Role', 'Server', 'ShadowAttribute', 'SharingGroup', 'Tag', 'Task', 'Taxonomy', 'Template', 'Thread', 'User', 'Whitelist');
$models = array('' => 'ALL') + $this->_arrayToValuesIndexArray($models);
$this->set('models', $models);
$this->set('actionDefinitions', $this->{$this->defaultModel}->actionDefinitions);

31
app/Controller/ServersController.php
View File

@ -1503,6 +1503,10 @@ class ServersController extends AppController
if (isset($version['perm_sync'])) {
$perm_sync = $version['perm_sync'];
}
$perm_sighting = false;
if (isset($version['perm_sighting'])) {
$perm_sighting = $version['perm_sighting'];
}
App::uses('Folder', 'Utility');
$file = new File(ROOT . DS . 'VERSION.json', true);
$local_version = json_decode($file->read(), true);
@ -1529,10 +1533,14 @@ class ServersController extends AppController
if (!$mismatch && $version[2] < 111) {
$mismatch = 'proposal';
}
if (!$perm_sync) {
if (!$perm_sync && !$perm_sighting) {
$result['status'] = 7;
return new CakeResponse(array('body'=> json_encode($result), 'type' => 'json'));
}
if (!$perm_sync && $perm_sighting) {
$result['status'] = 8;
return new CakeResponse(array('body'=> json_encode($result), 'type' => 'json'));
}
return new CakeResponse(
array(
'body'=> json_encode(
@ -1630,7 +1638,7 @@ class ServersController extends AppController
throw new MethodNotAllowedException('This action requires API access.');
}
$versionArray = $this->Server->checkMISPVersion();
$this->set('response', array('version' => $versionArray['major'] . '.' . $versionArray['minor'] . '.' . $versionArray['hotfix'], 'perm_sync' => $this->userRole['perm_sync']));
$this->set('response', array('version' => $versionArray['major'] . '.' . $versionArray['minor'] . '.' . $versionArray['hotfix'], 'perm_sync' => $this->userRole['perm_sync'], 'perm_sighting' => $this->userRole['perm_sighting']));
$this->set('_serialize', 'response');
}
@ -2207,7 +2215,24 @@ misp.direct_call(relative_path, body)
if (!$this->_isSiteAdmin()) {
throw new MethodNotAllowedException(__('Only site admin accounts get the DB schema diagnostic.'));
}
return $this->RestResponse->viewData($this->Server->dbSchemaDiagnostic(), $this->response->type());
$dbSchemaDiagnostics = $this->Server->dbSchemaDiagnostic();
if ($this->_isRest()) {
return $this->RestResponse->viewData($dbSchemaDiagnostics, $this->response->type());
} else {
$this->set('checkedTableColumn', $dbSchemaDiagnostics['checked_table_column']);
$this->set('dbSchemaDiagnostics', $dbSchemaDiagnostics['diagnostic']);
$this->set('dbIndexDiagnostics', $dbSchemaDiagnostics['diagnostic_index']);
$this->set('expectedDbVersion', $dbSchemaDiagnostics['expected_db_version']);
$this->set('actualDbVersion', $dbSchemaDiagnostics['actual_db_version']);
$this->set('error', $dbSchemaDiagnostics['error']);
$this->set('remainingLockTime', $dbSchemaDiagnostics['remaining_lock_time']);
$this->set('updateFailNumberReached', $dbSchemaDiagnostics['update_fail_number_reached']);
$this->set('updateLocked', $dbSchemaDiagnostics['update_locked']);
$this->set('dataSource', $dbSchemaDiagnostics['dataSource']);
$this->set('columnPerTable', $dbSchemaDiagnostics['columnPerTable']);
$this->set('indexes', $dbSchemaDiagnostics['indexes']);
$this->render('/Elements/healthElements/db_schema_diagnostic');
}
}
public function viewDeprecatedFunctionUse()

18
app/Controller/ShadowAttributesController.php
View File

@ -197,24 +197,6 @@ class ShadowAttributesController extends AppController
}
}
// If we accept a proposed attachment, then the attachment itself needs to be moved from files/eventId/shadow/shadowId to files/eventId/attributeId
private function _moveFile($shadowId, $newId, $eventId)
{
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$attachments_dir = $this->ShadowAttribute->getDefaultAttachments_dir();
}
$pathOld = $attachments_dir . DS . 'shadow' . DS . $shadowId;
$pathNew = $attachments_dir . DS . $newId;
if (rename($pathOld, $pathNew)) {
return true;
} else {
$this->Flash->error(__('Moving of the file that this attachment references failed.', true), 'default', array());
$this->redirect(array('controller' => 'events', 'action' => 'view', $eventId));
}
}
private function __discard($id)
{
$sa = $this->ShadowAttribute->find(

39
app/Controller/SightingsController.php
View File

@ -276,45 +276,6 @@ class SightingsController extends AppController
return $this->RestResponse->viewData($sightings);
}
public function restSearch($context = false)
{
$allowedContext = array(false, 'event', 'attribute');
$paramArray = array('returnFormat', 'id', 'type', 'from', 'to', 'last', 'org_id', 'source', 'includeAttribute', 'includeEvent');
$filterData = array(
'request' => $this->request,
'named_params' => $this->params['named'],
'paramArray' => $paramArray,
'ordered_url_params' => compact($paramArray)
);
$filters = $this->_harvestParameters($filterData, $exception);
// validate context
if (!in_array($context, $allowedContext, true)) {
throw new MethodNotAllowedException(_('Invalid context.'));
}
// ensure that an id is provided if context is set
if ($context !== false && !isset($filters['id'])) {
throw new MethodNotAllowedException(_('An id must be provided if the context is set.'));
}
$filters['context'] = $context;
if (!isset($returnFormat)) {
$returnFormat = 'json';
}
if (isset($filters['returnFormat'])) {
$returnFormat = $filters['returnFormat'];
}
if ($returnFormat === 'download') {
$returnFormat = 'json';
}
$sightings = $this->Sighting->restSearch($this->Auth->user(), $returnFormat, $filters);
$validFormats = $this->Sighting->validFormats;
$responseType = $validFormats[$returnFormat][0];
return $this->RestResponse->viewData($sightings, $responseType, false, true);
}
public function listSightings($id = false, $context = 'attribute', $org_id = false)
{
$rawId = $id;

73
app/Controller/TagsController.php
View File

@ -860,41 +860,56 @@ class TagsController extends AppController
$this->render('/Servers/json/simple');
}
private function __findObjectByUuid($object_uuid, &$type)
private function __findObjectByUuid($object_uuid, &$type, $scope = 'modify')
{
$this->loadModel('Event');
$object = $this->Event->find('first', array(
'conditions' => array(
'Event.uuid' => $object_uuid,
),
'fields' => array('Event.orgc_id', 'Event.id'),
'recursive' => -1
if (!$this->userRole['perm_tagger']) {
throw new MethodNotAllowedException(__('This functionality requires tagging permission.'));
}
$object = $this->Event->fetchEvent($this->Auth->user(), array(
'event_uuid' => $object_uuid,
'metadata' => 1
));
$type = 'Event';
if (!empty($object)) {
$object = $object[0];
if (
$scope !== 'view' &&
!$this->_isSiteAdmin() &&
!$this->userRole['perm_tagger'] &&
$object['Event']['orgc_id'] != $this->Auth->user('org_id')
) {
throw new MethodNotAllowedException('Invalid Target.');
$message = __('Cannot alter the tags of this data, only the organisation that has created the data (orgc) can modify global tags.');
if ($this->Auth->user('org_id') === Configure::read('MISP.host_org_id')) {
$message .= ' ' . __('Please consider using local tags if you are in the host organisation of the instance.');
}
throw new MethodNotAllowedException($message);
}
} else {
$type = 'Attribute';
$object = $this->Event->Attribute->find('first', array(
'conditions' => array(
'Attribute.uuid' => $object_uuid,
),
'fields' => array('Attribute.id'),
'recursive' => -1,
'contain' => array('Event.orgc_id')
));
$object = $this->Event->Attribute->fetchAttributes(
$this->Auth->user(),
array(
'conditions' => array(
'Attribute.uuid' => $object_uuid
),
'flatten' => 1
)
);
if (!empty($object)) {
if (!$this->_isSiteAdmin() && !$this->userRole['perm_tagger'] && $object['Event']['orgc_id'] != $this->Auth->user('org_id')) {
throw new MethodNotAllowedException('Invalid Target.');
$object = $object[0];
if (
$scope !== 'view' &&
!$this->_isSiteAdmin() &&
$object['Event']['orgc_id'] != $this->Auth->user('org_id')
) {
$message = __('Cannot alter the tags of this data, only the organisation that has created the data (orgc) can modify global tags.');
if ($this->Auth->user('org_id') === Configure::read('MISP.host_org_id')) {
$message .= ' ' . __('Please consider using local tags if you are in the host organisation of the instance.');
}
throw new MethodNotAllowedException($message);
}
} else {
throw new MethodNotAllowedException('Invalid Target.');
throw new MethodNotAllowedException(__('Invalid Target.'));
}
}
return $object;
@ -932,11 +947,11 @@ class TagsController extends AppController
$local = $this->request->data['local'];
}
}
if (!is_bool($local)) {
throw new InvalidArgumentException('Invalid local flag');
if (!empty($local) && $this->Auth->user('org_id') != Configure::read('MISP.host_org_id')) {
throw new MethodNotAllowedException(__('Local tags can only be added by users of the host organisation.'));
}
$objectType = '';
$object = $this->__findObjectByUuid($uuid, $objectType);
$object = $this->__findObjectByUuid($uuid, $objectType, $local ? 'view' : 'modify');
$existingTag = $this->Tag->find('first', array('conditions' => $conditions, 'recursive' => -1));
if (empty($existingTag)) {
if (!is_numeric($tag)) {
@ -1038,9 +1053,9 @@ class TagsController extends AppController
throw new MethodNotAllowedException('Invalid Tag.');
}
$objectType = '';
$object = $this->__findObjectByUuid($uuid, $objectType);
$object = $this->__findObjectByUuid($uuid, $objectType, 'view');
if (empty($object)) {
throw new MethodNotAllowedException('Invalid Target.');
throw new MethodNotAllowedException(__('Invalid Target.'));
}
$connectorObject = $objectType . 'Tag';
$this->loadModel($objectType);
@ -1052,6 +1067,14 @@ class TagsController extends AppController
));
if (empty($existingAssociation)) {
throw new MethodNotAllowedException('Could not remove tag as it is not attached to the target ' . $objectType);
} else {
if (empty($existingAssociation[$objectType . 'Tag']['local'])) {
$object = $this->__findObjectByUuid($uuid, $objectType);
} else {
if ($object['Event']['orgc_id'] !== $this->Auth->user('org_id') && $this->Auth->user('org_id') != Configure::read('MISP.host_org_id')) {
throw new MethodNotAllowedException(__('Insufficient privileges to remove local tags from events you do not own.'));
}
}
}
$result = $this->$objectType->$connectorObject->delete($existingAssociation[$connectorObject]['id']);
if ($result) {

106
app/Controller/UsersController.php
View File

@ -74,16 +74,25 @@ class UsersController extends AppController
$temp[$v['setting']] = $v['value'];
}
$user['UserSetting'] = $temp;
return $this->RestResponse->viewData(array(
'User' => $user['User'],
'Role' => $user['Role'],
'UserSetting' => $user['UserSetting']
), $this->response->type());
return $this->RestResponse->viewData($this->__massageUserObject($user), $this->response->type());
} else {
$this->set('user', $user);
}
}
private function __massageUserObject($user)
{
unset($user['User']['server_id']);
$user['User']['password'] = '*****';
$objectsToInclude = array('User', 'Role', 'UserSetting', 'Organisation');
foreach ($objectsToInclude as $objectToInclude) {
if (isset($user[$objectToInclude])) {
$temp[$objectToInclude] = $user[$objectToInclude];
}
}
return $temp;
}
public function request_API()
{
if (Configure::read('MISP.disable_emailing')) {
@ -108,14 +117,20 @@ class UsersController extends AppController
if (!$this->_isAdmin() && Configure::read('MISP.disableUserSelfManagement')) {
throw new MethodNotAllowedException('User self-management has been disabled on this instance.');
}
$id = $this->Auth->user('id');
$this->User->read(null, $id);
if (!$this->User->exists()) {
$currentUser = $this->User->find('first', array(
'conditions' => array('User.id' => $this->Auth->user('id')),
'recursive' => -1
));
if (empty($currentUser)) {
throw new NotFoundException('Something went wrong. Your user account could not be accessed.');
}
$id = $currentUser['User']['id'];
if ($this->request->is('post') || $this->request->is('put')) {
if (empty($this->request->data['User'])) {
$this->request->data = array('User' => $this->request->data);
}
$abortPost = false;
if (!$this->_isSiteAdmin() && !empty($this->request->data['User']['email'])) {
if (!empty($this->request->data['User']['email']) && !$this->_isSiteAdmin()) {
$organisation = $this->User->Organisation->find('first', array(
'conditions' => array('Organisation.id' => $this->Auth->user('org_id')),
'recursive' => -1
@ -132,7 +147,7 @@ class UsersController extends AppController
}
}
if ($abortPost) {
$this->Flash->error(__('Invalid e-mail domain. Your user is restricted to creating users for the following domain(s): ') . implode(', ', $organisation['Organisation']['restricted_to_domain']));
$message = __('Invalid e-mail domain. Your user is restricted to creating users for the following domain(s): ') . implode(', ', $organisation['Organisation']['restricted_to_domain']);
}
}
}
@ -154,17 +169,49 @@ class UsersController extends AppController
if (!$abortPost) {
// What fields should be saved (allowed to be saved)
$fieldList = array('email', 'autoalert', 'gpgkey', 'certif_public', 'nids_sid', 'contactalert', 'disabled');
if ("" != $this->request->data['User']['password']) {
if (!empty($this->request->data['User']['password'])) {
$fieldList[] = 'password';
$fieldList[] = 'confirm_password';
}
foreach ($this->request->data['User'] as $k => $v) {
$currentUser['User'][$k] = $v;
}
// Save the data
if ($this->User->save($this->request->data, true, $fieldList)) {
$this->Flash->success(__('The profile has been updated'));
$this->_refreshAuth();
$this->redirect(array('action' => 'view', $id));
} else {
$this->Flash->error(__('The profile could not be updated. Please, try again.'));
if ($this->_isRest()) {
if (!empty($this->request->data['User']['password'])) {
if ($this->request->data['User']['password'] === '*****') {
unset($this->request->data['User']['password']);
} else {
$currentUser['User']['confirm_password'] = $this->request->data['User']['password'];
}
}
}
if ($this->User->save($currentUser, true, $fieldList)) {
if ($this->_isRest()) {
$user = $this->User->find('first', array(
'conditions' => array('User.id' => $id),
'recursive' => -1,
'contain' => array(
'Organisation',
'Role',
'UserSetting'
)
));
return $this->RestResponse->viewData($this->__massageUserObject($user), $this->response->type());
} else {
$this->Flash->success(__('The profile has been updated'));
$this->_refreshAuth();
$this->redirect(array('action' => 'view', $id));
}
} else {
$message = __('The profile could not be updated. Please, try again.');
$abortPost = true;
}
}
if ($abortPost) {
return $this->RestResponse->saveFailResponse('Users', 'edit', $id, $message, $this->response->type());
} else {
$this->Flash->error($message);
}
} else {
$this->User->set('password', '');
@ -839,9 +886,9 @@ class UsersController extends AppController
continue;
}
if ($field != 'confirm_password') {
array_push($fieldsOldValues, $this->User->field($field));
$fieldsOldValues[$field] = $this->User->field($field);
} else {
array_push($fieldsOldValues, $this->User->field('password'));
$fieldsOldValues[$field] = $this->User->field('password');
}
}
if (
@ -882,31 +929,28 @@ class UsersController extends AppController
}
$cP++;
}
array_push($fieldsNewValues, $newValueStr);
$fieldsNewValues[$field] = $newValueStr;
} else {
array_push($fieldsNewValues, $newValue);
$fieldsNewValues[$field] = $newValue;
}
} else {
array_push($fieldsNewValues, $this->data['User']['password']);
$fieldsNewValues[$field] = $this->data['User']['password'];
}
}
// compare
$fieldsResultStr = '';
$c = 0;
$fieldsResult = array();
foreach ($fields as $field) {
if (isset($fieldsOldValues[$c]) && $fieldsOldValues[$c] != $fieldsNewValues[$c]) {
if (isset($fieldsOldValues[$field]) && $fieldsOldValues[$field] != $fieldsNewValues[$field]) {
if ($field != 'confirm_password' && $field != 'enable_password') {
$fieldsResultStr = $fieldsResultStr . ', ' . $field . ' (' . $fieldsOldValues[$c] . ') => (' . $fieldsNewValues[$c] . ')';
$fieldsResult[$field] = array($fieldsOldValues[$field], $fieldsNewValues[$field]);
}
}
$c++;
}
$fieldsResultStr = substr($fieldsResultStr, 2);
$user = $this->User->find('first', array(
'recursive' => -1,
'conditions' => array('User.id' => $this->User->id)
));
$this->User->extralog($this->Auth->user(), "edit", "user", $fieldsResultStr, $user);
$this->User->extralog($this->Auth->user(), "edit", "user", $fieldsResult, $user);
if ($this->_isRest()) {
$user['User']['password'] = '******';
return $this->RestResponse->viewData($user, $this->response->type());
@ -1757,10 +1801,12 @@ class UsersController extends AppController
'group' => 'Event.orgc_id',
'conditions' => array('Event.orgc_id' => array_keys($orgs)),
'recursive' => -1,
'fields' => array('Event.orgc_id', 'count(*)')
'fields' => array('Event.orgc_id', 'count(*)', 'sum(Event.attribute_count) as attributeCount')
));
foreach ($events as $event) {
$orgs[$event['Event']['orgc_id']]['eventCount'] = $event[0]['count(*)'];
$orgs[$event['Event']['orgc_id']]['attributeCount'] = $event[0]['attributeCount'];
$orgs[$event['Event']['orgc_id']]['orgActivity'] = $this->User->getOrgActivity($event['Event']['orgc_id'], array('event_timestamp' => '365d'));
}
unset($events);
$orgs = Set::combine($orgs, '{n}.name', '{n}');

60
app/Lib/Export/HashesExport.php Normal file
View File

@ -0,0 +1,60 @@
<?php
class HashesExport
{
public $additional_params = array(
'flatten' => 1
);
public $validTypes = array(
'simple' => array(
'md5', 'sha1', 'sha256', 'sha224', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'tlsh',
'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'pehash', 'authentihash',
'impfuzzy'
),
'composite' => array(
'malware-sample', 'filename|md5', 'filename|sha1', 'filename|sha256', 'filename|sha224', 'filename|sha512',
'filename|sha512/224', 'filename|sha512/256', 'filename|ssdeep', 'filename|imphash', 'filename|tlsh',
'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'filename|pehash',
'filename|authentihash', 'filename|impfuzzy'
)
);
public function handler($data, $options = array())
{
if ($options['scope'] === 'Attribute') {
if (in_array($data['Attribute']['type'], $this->validTypes['composite'])) {
return explode('|', $data['Attribute']['value'])[1];
} else if (in_array($data['Attribute']['type'], $this->validTypes['simple'])) {
return $data['Attribute']['value'];
}
}
if ($options['scope'] === 'Event') {
$result = array();
foreach ($data['Attribute'] as $attribute) {
if (in_array($attribute['type'], $this->validTypes['composite'])) {
$result[] = explode('|', $attribute['value'])[1];
} else if (in_array($attribute['type'], $this->validTypes['simple'])) {
$result[] = $attribute['value'];
}
}
return implode($this->separator(), $result);
}
return '';
}
public function header($options = array())
{
return '';
}
public function footer()
{
return "\n";
}
public function separator()
{
return "\n";
}
}

2
app/Locale/no/LC_MESSAGES/default.po
View File

@ -1,7 +1,7 @@
msgid ""
msgstr ""
"Project-Id-Version: misp\n"
"PO-Revision-Date: 2019-09-30 01:23\n"
"PO-Revision-Date: 2019-11-30 07:05\n"
"Last-Translator: Steve Clement (SteveClement)\n"
"Language-Team: Norwegian\n"
"MIME-Version: 1.0\n"

6
app/Model/AppModel.php
View File

@ -1875,12 +1875,18 @@ class AppModel extends Model
private function __saveUpdateProgress($updateProgress)
{
if (!isset($this->AdminSetting)) {
$this->AdminSetting = ClassRegistry::init('AdminSetting');
}
$data = json_encode($updateProgress);
$this->AdminSetting->changeSetting('update_progress', $data);
}
public function changeLockState($locked)
{
if (!isset($this->AdminSetting)) {
$this->AdminSetting = ClassRegistry::init('AdminSetting');
}
$this->AdminSetting->changeSetting('update_locked', $locked);
}

28
app/Model/Attribute.php
View File

@ -93,7 +93,7 @@ class Attribute extends AppModel
),
'Artifacts dropped' => array(
'desc' => __('Any artifact (files, registry keys etc.) dropped by the malware or other modifications to the system'),
'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|authentihash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy','filename|pehash', 'regkey', 'regkey|value', 'pattern-in-file', 'pattern-in-memory','pdb', 'stix2-pattern', 'yara', 'sigma', 'attachment', 'malware-sample', 'named pipe', 'mutex', 'windows-scheduled-task', 'windows-service-name', 'windows-service-displayname', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'cookie', 'gene', 'mime-type', 'anonymised')
'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|authentihash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy','filename|pehash', 'regkey', 'regkey|value', 'pattern-in-file', 'pattern-in-memory','pdb', 'stix2-pattern', 'yara', 'sigma', 'attachment', 'malware-sample', 'named pipe', 'mutex', 'windows-scheduled-task', 'windows-service-name', 'windows-service-displayname', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'cookie', 'gene', 'kusto-query', 'mime-type', 'anonymised')
),
'Payload installation' => array(
'desc' => __('Info on where the malware gets installed in the system'),
@ -107,7 +107,7 @@ class Attribute extends AppModel
),
'Network activity' => array(
'desc' => __('Information about network traffic generated by the malware'),
'types' => array('ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'port', 'hostname', 'domain', 'domain|ip', 'mac-address', 'mac-eui-64', 'email-dst', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'pattern-in-file', 'stix2-pattern', 'pattern-in-traffic', 'attachment', 'comment', 'text', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'other', 'hex', 'cookie', 'hostname|port', 'bro', 'zeek', 'anonymised', 'community-id', 'email-subject')
'types' => array('ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'port', 'hostname', 'domain', 'domain|ip', 'mac-address', 'mac-eui-64', 'email-dst', 'email-src', 'eppn', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'pattern-in-file', 'stix2-pattern', 'pattern-in-traffic', 'attachment', 'comment', 'text', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'other', 'hex', 'cookie', 'hostname|port', 'bro', 'zeek', 'anonymised', 'community-id', 'email-subject')
),
'Payload type' => array(
'desc' => __('Information about the final payload(s)'),
@ -135,7 +135,7 @@ class Attribute extends AppModel
'Social network' => array(
'desc' => __('Social networks and platforms'),
// email-src and email-dst or should we go with a new email type that is neither / both?
'types' => array('github-username', 'github-repository', 'github-organisation', 'jabber-id', 'twitter-id', 'email-src', 'email-dst', 'comment', 'text', 'other', 'whois-registrant-email', 'anonymised')
'types' => array('github-username', 'github-repository', 'github-organisation', 'jabber-id', 'twitter-id', 'email-src', 'email-dst', 'eppn','comment', 'text', 'other', 'whois-registrant-email', 'anonymised')
),
'Person' => array(
'desc' => __('A human being - natural person'),
@ -165,6 +165,7 @@ class Attribute extends AppModel
'domain' => array('desc' => __('A domain name used in the malware'), 'formdesc' => __("A domain name used in the malware. Use this instead of hostname when the upper domain is important or can be used to create links between events."), 'default_category' => 'Network activity', 'to_ids' => 1),
'domain|ip' => array('desc' => __('A domain name and its IP address (as found in DNS lookup) separated by a |'),'formdesc' => __("A domain name and its IP address (as found in DNS lookup) separated by a | (no spaces)"), 'default_category' => 'Network activity', 'to_ids' => 1),
'email-src' => array('desc' => __("The email address used to send the malware."), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'eppn' => array('desc' => __("eduPersonPrincipalName - eppn - the NetId of the person for the purposes of inter-institutional authentication. Should be stored in the form of user@univ.edu, where univ.edu is the name of the local security domain."), 'default_category' => 'Network activity', 'to_ids' => 1),
'email-dst' => array('desc' => __("A recipient email address"), 'formdesc' => __("A recipient email address that is not related to your constituency."), 'default_category' => 'Network activity', 'to_ids' => 1),
'email-subject' => array('desc' => __("The subject of the email"), 'default_category' => 'Payload delivery', 'to_ids' => 0),
'email-attachment' => array('desc' => __("File name of the email attachment."), 'default_category' => 'Payload delivery', 'to_ids' => 1),
@ -190,6 +191,7 @@ class Attribute extends AppModel
'stix2-pattern' => array('desc' => __('STIX 2 pattern'), 'default_category' => 'Payload installation', 'to_ids' => 1),
'sigma' => array('desc' => __('Sigma - Generic Signature Format for SIEM Systems'), 'default_category' => 'Payload installation', 'to_ids' => 1),
'gene' => array('desc' => __('GENE - Go Evtx sigNature Engine'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
'kusto-query' => array('desc' => __('Kusto query - Kusto from Microsoft Azure is a service for storing and running interactive analytics over Big Data.'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
'mime-type' => array('desc' => __('A media type (also MIME type and content type) is a two-part identifier for file formats and format contents transmitted on the Internet'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
'identity-card-number' => array('desc' => __('Identity card number'), 'default_category' => 'Person', 'to_ids' => 0),
'cookie' => array('desc' => __('HTTP cookie as often stored on the user web client. This can include authentication cookie or session cookie.'), 'default_category' => 'Network activity', 'to_ids' => 0),
@ -401,6 +403,7 @@ class Attribute extends AppModel
'suricata' => array('txt', 'NidsSuricataExport', 'rules'),
'snort' => array('txt', 'NidsSnortExport', 'rules'),
'text' => array('txt', 'TextExport', 'txt'),
'hashes' => array('txt', 'HashesExport', 'txt'),
'yara' => array('txt', 'YaraExport', 'yara'),
'yara-json' => array('json', 'YaraExport', 'json'),
'rpz' => array('txt', 'RPZExport', 'rpz'),
@ -437,6 +440,7 @@ class Attribute extends AppModel
'mac-eui-64' => 'Network activity',
'hostname' => 'Network activity',
'domain' => 'Network activity',
'eppn' => 'Network activity',
'url' => 'Network activity',
'ja3-fingerprint-md5' => 'Network activity',
'hassh-md5' => 'Network activity',
@ -1203,6 +1207,7 @@ class Attribute extends AppModel
}
break;
case 'email-src':
case 'eppn':
case 'email-dst':
case 'target-email':
case 'whois-registrant-email':
@ -1258,6 +1263,7 @@ class Attribute extends AppModel
case 'stix2-pattern':
case 'sigma':
case 'gene':
case 'kusto-query':
case 'mime-type':
case 'identity-card-number':
case 'cookie':
@ -2247,7 +2253,7 @@ class Attribute extends AppModel
$rules = array();
foreach ($eventIds as $event) {
$conditions['AND'] = array('Attribute.to_ids' => 1, "Event.published" => 1, 'Attribute.event_id' => $event['Event']['id']);
$valid_types = array('ip-dst', 'ip-src', 'ip-dst|port', 'ip-src|port', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'domain', 'domain|ip', 'hostname', 'url', 'user-agent', 'snort');
$valid_types = array('ip-dst', 'ip-src', 'ip-dst|port', 'ip-src|port', 'eppn', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'domain', 'domain|ip', 'hostname', 'url', 'user-agent', 'snort');
$conditions['AND']['Attribute.type'] = $valid_types;
if (!empty($type)) {
$conditions['AND'][] = array('Attribute.type' => $type);
@ -3254,6 +3260,8 @@ class Attribute extends AppModel
if (empty($at['Tag'])) {
unset($results[$k]['AttributeTag'][$k2]);
$tagCulled = true;
} else {
$results[$k]['AttributeTag'][$k2]['Tag']['local'] = $results[$k]['AttributeTag'][$k2]['local'];
}
}
if ($tagCulled) {
@ -3263,6 +3271,7 @@ class Attribute extends AppModel
if (isset($result['Event']['EventTag'])) {
$results[$k]['Event']['Tag'] = array();
foreach ($result['Event']['EventTag'] as $et) {
$et['Tag']['local'] = $et['local'];
$results[$k]['Event']['Tag'][] = $et['Tag'];
}
unset($results[$k]['Event']['EventTag']);
@ -3581,11 +3590,6 @@ class Attribute extends AppModel
return true;
}
public function convertToOpenIOC($user, $attributes)
{
return $this->IOCExport->buildAll($user, $event);
}
private function __createTagSubQuery($tag_id, $blocked = false, $scope = 'Event', $limitAttributeHitsTo = 'event')
{
$conditionKey = $blocked ? array('NOT' => array('EventTag.tag_id' => $tag_id)) : array('EventTag.tag_id' => $tag_id);
@ -4116,6 +4120,7 @@ class Attribute extends AppModel
$result = $this->fetchAttributes($user, array(
'conditions' => array('Attribute.id' => $id),
'flatten' => 1,
'deleted' => [0,1],
'recursive' => -1,
'contain' => array('Event')
));
@ -4403,8 +4408,9 @@ class Attribute extends AppModel
$temp = '';
foreach ($results as $attribute) {
$elementCounter++;
$temp .= $exportTool->handler($attribute, $exportToolParams);
if ($temp !== '') {
$handlerResult = $exportTool->handler($attribute, $exportToolParams);
$temp .= $handlerResult;
if ($handlerResult !== '') {
if ($i != count($results) -1) {
$temp .= $exportTool->separator($exportToolParams);
}

48
app/Model/DecayingModelsFormulas/Base.php
View File

@ -6,8 +6,30 @@ abstract class DecayingModelBase
return 'BONFIRE LIT';
}
protected function __extractTagBasename($tagName) {
$pieces = array();
if (preg_match('/^[^:="]+:[^:="]+="[^:="]+"$/i', $tagName)) {
$temp = explode(':', $tagName);
$pieces = array_merge(array($temp[0]), explode('=', $temp[1]));
$pieces['complete'] = $tagName;
$pieces['namespace'] = $pieces[0];
$pieces['predicate'] = $pieces[1];
$pieces['2tag'] = sprintf('%s:%s', $pieces[0], $pieces[1]);
$pieces['base'] = sprintf('%s:%s', $pieces[0], $pieces[1]);
} elseif (preg_match('/^[^:="]+:[^:="]+$/i', $tagName)) {
$pieces = explode(':', $tagName);
$pieces['complete'] = $tagName;
$pieces['namespace'] = $pieces[0];
$pieces['predicate'] = $pieces[1];
$pieces['2tag'] = sprintf('%s:%s', $pieces[0], $pieces[1]);
$pieces['base'] = $pieces[0];
}
return $pieces;
}
// Get effective taxonomy ratio based on taxonomies attached to the attribute
// Basically, it adapts the ratio defined in the model to fit the actual attached tags
// Basically, it adapts the ratio defined in the model to fit the actual attached tags
protected function __getRatioScore($model, $tags)
{
$ratioScore = array();
@ -17,21 +39,21 @@ abstract class DecayingModelBase
}
$total_score = 0.0;
foreach ($tags as $tag) {
$namespace_predicate = explode('=', $tag['Tag']['name'])[0];
if (isset($taxonomy_base_ratio[$namespace_predicate]) && is_numeric($tag['Tag']['numerical_value'])) {
$total_score += floatval($taxonomy_base_ratio[$namespace_predicate]);
$tagBaseName = $this->__extractTagBasename($tag['Tag']['name'])['base'];
if (isset($taxonomy_base_ratio[$tagBaseName]) && is_numeric($tag['Tag']['numerical_value'])) {
$total_score += floatval($taxonomy_base_ratio[$tagBaseName]);
}
}
foreach ($tags as $i => $tag) {
$namespace_predicate = explode('=', $tag['Tag']['name'])[0];
if (isset($taxonomy_base_ratio[$namespace_predicate]) && is_numeric($tag['Tag']['numerical_value'])) {
$ratioScore[$namespace_predicate] = floatval($taxonomy_base_ratio[$namespace_predicate]) / $total_score;
$tagBaseName = $this->__extractTagBasename($tag['Tag']['name'])['base'];
if (isset($taxonomy_base_ratio[$tagBaseName]) && is_numeric($tag['Tag']['numerical_value'])) {
$ratioScore[$tagBaseName] = floatval($taxonomy_base_ratio[$tagBaseName]) / $total_score;
}
}
return $ratioScore;
}
// return attribute tag with event tag matching the namespace+predicate overridden
// return attribute tag with event tag matching the tag basename overridden
protected function __getPrioritisedTag($attribute)
{
$tags = array();
@ -40,15 +62,15 @@ abstract class DecayingModelBase
if (isset($attribute['EventTag'])) {
foreach ($attribute['EventTag'] as $i => $tag) {
$tags[] = $tag;
$namespace_predicate = explode('=', $tag['Tag']['name'])[0];
$temp_mapping[$namespace_predicate][] = $i;
$tagBaseName = $this->__extractTagBasename($tag['Tag']['name'])['base'];
$temp_mapping[$tagBaseName][] = $i;
}
}
if (isset($attribute['AttributeTag'])) {
foreach ($attribute['AttributeTag'] as $tag) {
$namespace_predicate = explode('=', $tag['Tag']['name'])[0];
if (!empty($temp_mapping[$namespace_predicate])) { // need to override event tag
foreach ($temp_mapping[$namespace_predicate] as $i => $eventtag_index) {
$tagBaseName = $this->__extractTagBasename($tag['Tag']['name'])['base'];
if (!empty($temp_mapping[$tagBaseName])) { // need to override event tag
foreach ($temp_mapping[$tagBaseName] as $i => $eventtag_index) {
$overridden_tags[] = array(
'EventTag' => $tags[$eventtag_index],
'AttributeTag' => $tag

348
app/Model/Event.php
View File

@ -180,6 +180,7 @@ class Event extends AppModel
'snort' => array('txt', 'NidsSnortExport', 'rules'),
'rpz' => array('txt', 'RPZExport', 'rpz'),
'text' => array('text', 'TextExport', 'txt'),
'hashes' => array('txt', 'HashesExport', 'txt'),
'csv' => array('csv', 'CsvExport', 'csv'),
'stix' => array('xml', 'Stix1Export', 'xml'),
'stix-json' => array('json', 'Stix1Export', 'json'),
@ -2749,140 +2750,6 @@ class Event extends AppModel
return $conditions;
}
public function csv($user, $params, $search = false, &$continue = true)
{
$conditions = array();
$simple_params = array(
'eventid' => array('function' => 'set_filter_eventid'),
'ignore' => array('function' => 'set_filter_ignore'),
'tags' => array('function' => 'set_filter_tags'),
'category' => array('function' => 'set_filter_simple_attribute'),
'type' => array('function' => 'set_filter_simple_attribute'),
'object_relation' => array('function' => 'set_filter_simple_attribute'),
'from' => array('function' => 'set_filter_timestamp'),
'to' => array('function' => 'set_filter_timestamp'),
'last' => array('function' => 'set_filter_timestamp'),
'value' => array('function' => 'set_filter_value'),
'timestamp' => array('function' => 'set_filter_timestamp'),
'attributeIDList' => array('functon' => 'set_filter_attribute_id')
);
foreach ($params as $param => $paramData) {
if (isset($simple_params[$param]) && $params[$param] !== false) {
$options = array(
'filter' => $param,
'scope' => 'Event',
'pop' => !empty($simple_param_scoped[$param]['pop'])
);
$conditions = $this->{$simple_params[$param]['function']}($params, $conditions, $options);
}
}
//$attributeIDList = array(), $includeContext = false, $enforceWarninglist = false
$this->recursive = -1;
if (!empty($params['eventid']) && $params['eventid'] === 'search') {
foreach ($params['attributeIDList'] as $aID) {
$conditions['AND']['OR'][] = array('Attribute.id' => $aID);
}
}
$csv_params = array(
'conditions' => $conditions, //array of conditions
'fields' => array('Attribute.event_id', 'Attribute.distribution', 'Attribute.category', 'Attribute.type', 'Attribute.value', 'Attribute.comment', 'Attribute.uuid', 'Attribute.to_ids', 'Attribute.timestamp', 'Attribute.id', 'Attribute.object_relation'),
'order' => array('Attribute.uuid ASC'),
'flatten' => true
);
// copy over the parameters that have to deal with pagination or additional functionality to be executed
$control_params = array(
'limit', 'page', 'enforceWarninglist'
);
foreach ($control_params as $control_param) {
if (!empty($params[$control_param])) {
$csv_params[$control_param] = $params[$control_param];
}
}
$csv_params = $this->__appendIncludesCSV($csv_params, !empty($params['includeContext']));
$attributes = $this->Attribute->fetchAttributes($user, $csv_params, $continue);
$attributes = $this->__sanitiseCSVAttributes($attributes, !empty($params['includeContext']), !empty($params['ignore']));
return $attributes;
}
private function __appendIncludesCSV($params, $includeContext)
{
if ($includeContext) {
$params['contain'] = array(
'Event' => array(
'fields' => array('id', 'info', 'org_id', 'orgc_id', 'date', 'distribution', 'analysis'),
'SharingGroup' => array('fields' => array('id', 'name')),
'Org' => array('id', 'name'),
'Orgc' => array('id', 'name'),
'ThreatLevel' => array(
'fields' => array('id', 'name'),
),
'EventTag' => array(
'Tag' => array(
'fields' => array('id', 'name')
)
)
),
);
}
$params['contain']['Object'] = array('fields' => array('id', 'uuid', 'name', 'meta-category'));
return $params;
}
private function __sanitiseCSVAttributes($attributes, $includeContext, $ignore)
{
if (!empty($ignore)) {
$this->Whitelist = ClassRegistry::init('Whitelist');
$attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true);
}
foreach ($attributes as &$attribute) {
$this->__escapeCSVField($attribute['Attribute']['value']);
$this->__escapeCSVField($attribute['Attribute']['comment']);
$this->__escapeCSVField($attribute['Attribute']['object_relation']);
$this->__escapeCSVField($attribute['Attribute']['uuid']);
$this->__escapeCSVField($attribute['Attribute']['category']);
$this->__escapeCSVField($attribute['Attribute']['type']);
$attribute['Attribute']['timestamp'] = date('Ymd', $attribute['Attribute']['timestamp']);
if (empty($attribute['Object'])) {
$attribute['Object']['uuid'] = '""';
$attribute['Object']['name'] = '';
$attribute['Object']['meta-category'] = '';
}
$this->__escapeCSVField($attribute['Object']['name']);
$this->__escapeCSVField($attribute['Object']['uuid']);
$this->__escapeCSVField($attribute['Object']['meta-category']);
if ($includeContext) {
$this->__escapeCSVField($attribute['Event']['info']);
$this->__escapeCSVField($attribute['Event']['uuid']);
$this->__escapeCSVField($attribute['Org']['name']);
$this->__escapeCSVField($attribute['Orgc']['name']);
$attribute['Event']['Tag']['name'] = '';
$attribute['attribute_tag'] = '';
if (!empty($attribute['AttributeTag'])) {
$tags = array();
foreach ($attribute['AttributeTag'] as $attributeTag) {
if (!empty($attributeTag['Tag']['name'])) {
$tags[] = $attributeTag['Tag']['name'];
}
}
$attribute['Attribute']['attribute_tag'] = implode(',', $tags);
}
$this->__escapeCSVField($attribute['Attribute']['attribute_tag']);
if (!empty($attribute['Event']['EventTag'])) {
$tags = array();
foreach ($attribute['Event']['EventTag'] as $eventTag) {
if (!empty($eventTag['Tag']['name'])) {
$tags[] = $eventTag['Tag']['name'];
}
}
$attribute['Event']['Tag']['name'] = implode(',', $tags);
}
$this->__escapeCSVField($attribute['Event']['Tag']['name']);
}
}
return $attributes;
}
public function sendAlertEmailRouter($id, $user, $oldpublish = null)
{
if (Configure::read('MISP.block_old_event_alert')) {
@ -4116,7 +3983,11 @@ class Event extends AppModel
$event['Event']['locked'] = 1;
// get a list of the servers
$this->Server = ClassRegistry::init('Server');
$conditions = array('push' => 1);
if ($scope === 'sightings') {
$conditions = array('push_sightings' => 1);
} else {
$conditions = array('push' => 1);
}
if ($passAlong) {
$conditions[] = array('Server.id !=' => $passAlong);
}
@ -4132,14 +4003,18 @@ class Event extends AppModel
$failedServers = array();
App::uses('SyncTool', 'Tools');
foreach ($servers as &$server) {
if (((!isset($server['Server']['internal']) || !$server['Server']['internal']) && $event['Event']['distribution'] < 2) ||
((!isset($server['Server']['push_sightings']) || !$server['Server']['push_sightings'])) && $scope === 'sightings') {
if (
($scope === 'events' &&
(!isset($server['Server']['internal']) || !$server['Server']['internal']) && $event['Event']['distribution'] < 2) ||
($scope === 'sightings' &&
(!isset($server['Server']['push_sightings']) || !$server['Server']['push_sightings']))
) {
continue;
}
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
// Skip servers where the event has come from.
if (($passAlong != $server)) {
if (($passAlong != $server['Server']['id'])) {
$params = array();
if (!empty($server['Server']['push_rules'])) {
$push_rules = json_decode($server['Server']['push_rules'], true);
@ -4564,177 +4439,20 @@ class Event extends AppModel
$localEvents[$e['Event']['uuid']] = array($field => $e['Event'][$field], 'locked' => $e['Event']['locked']);
}
foreach ($uuidsToCheck as $uuid => $eventArrayId) {
if (isset($localEvents[$uuid])
&& ($localEvents[$uuid][$field] >= $eventArray[$eventArrayId][$field]
|| ($scope === 'events' && !$localEvents[$uuid]['locked'])))
{
// remove all events for the sighting sync if the remote is not aware of the new field yet
if (!isset($eventArray[$eventArrayId][$field])) {
unset($eventArray[$eventArrayId]);
} else {
if (isset($localEvents[$uuid])
&& ($localEvents[$uuid][$field] >= $eventArray[$eventArrayId][$field]
|| ($scope === 'events' && !$localEvents[$uuid]['locked'])))
{
unset($eventArray[$eventArrayId]);
}
}
}
}
public function stix2($id, $tags, $attachments, $user, $returnType = 'json', $from = false, $to = false, $last = false, $jobId = false, $returnFile = false)
{
$eventIDs = $this->Attribute->dissectArgs($id);
$tagIDs = $this->Attribute->dissectArgs($tags);
$idList = $this->getAccessibleEventIds($eventIDs[0], $eventIDs[1], $tagIDs[0], $tagIDs[1]);
if (!empty($idList)) {
$event_ids = $this->fetchEventIds($user, $from, $to, $last, true);
$event_ids = array_intersect($event_ids, $idList);
}
$randomFileName = $this->generateRandomFileName();
$scriptDir = APP . "files/scripts/";
$stix2_framing_cmd = $this->getPythonVersion() . ' ' . $scriptDir . 'misp_framing.py stix2 ' . escapeshellarg(CakeText::uuid()) . ' 2>' . APP . 'tmp/logs/exec-errors.log';
$stix2_framing = json_decode(shell_exec($stix2_framing_cmd), true);
if (empty($stix2_framing)) {
return array('success' => 0, 'message' => 'There was an issue generating the STIX 2.0 export.');
}
$separator = $stix2_framing['separator'];
$tmpDir = $scriptDir . "tmp/";
$stixFile = new File($tmpDir . $randomFileName . ".stix");
$stixFile->write($stix2_framing['header']);
if ($jobId) {
$this->Job = ClassRegistry::init('Job');
$this->Job->id = $jobId;
if (!$this->Job->exists()) {
$jobId = false;
}
}
$i = 0;
$eventCount = count($event_ids);
if ($event_ids) {
foreach ($event_ids as $event_id) {
$tempFile = new File($tmpDir . $randomFileName, true, 0644);
$event = $this->fetchEvent($user, array('eventid' => $event_id, 'includeAttachments' => $attachments));
if (empty($event)) {
continue;
}
$event[0]['Tag'] = array();
foreach ($event[0]['EventTag'] as $tag) {
$event[0]['Tag'][] = $tag['Tag'];
}
App::uses('JSONConverterTool', 'Tools');
$converter = new JSONConverterTool();
$event = $converter->convert($event[0]);
$tempFile->write($event);
unset($event);
$scriptFile = $scriptDir . "stix2/misp2stix2.py ";
$result = shell_exec($this->getPythonVersion() . ' ' . $scriptFile . $tempFile->path . ' 2>' . APP . 'tmp/logs/exec-errors.log');
$decoded = json_decode($result, true);
if (isset($decoded['success']) && $decoded['success'] == 1) {
$file = new File($tmpDir . $randomFileName . '.out', true, 0644);
$result = substr($file->read(), 1, -1);
$file->delete();
$stixFile->append($result . (($i + 1) != $eventCount ? $separator : ''));
} else {
return false;
}
$i++;
if ($jobId) {
$this->Job->saveField('message', 'Event ' . $i . '/' . $eventCount);
if ($i % 10 == 0) {
$this->Job->saveField('progress', $i * 80 / $eventCount);
}
}
$tempFile->close();
}
}
$stixFile->append($stix2_framing['footer']);
if ($tempFile) {
$tempFile->delete();
}
if (!$returnFile) {
$data2return = $stixFile->read();
$stixFile->delete();
}
return array('success' => 1, 'data' => $returnFile ? $stixFile->path : $data2return);
}
public function stix($id, $tags, $attachments, $user, $returnType = 'xml', $from = false, $to = false, $last = false, $jobId = false, $returnFile = false)
{
$eventIDs = $this->Attribute->dissectArgs($id);
$tagIDs = $this->Attribute->dissectArgs($tags);
$idList = $this->getAccessibleEventIds($eventIDs[0], $eventIDs[1], $tagIDs[0], $tagIDs[1]);
if (!empty($idList)) {
$event_ids = $this->fetchEventIds($user, $from, $to, $last, true);
$event_ids = array_intersect($event_ids, $idList);
}
$randomFileName = $this->generateRandomFileName();
$tmpDir = APP . "files" . DS . "scripts";
$stix_framing_cmd = $this->getPythonVersion() . ' ' . $tmpDir . DS . 'misp_framing.py stix ' . $this->__getAnnounceBaseurl() . ' ' . escapeshellarg(Configure::read('MISP.org')) . ' ' . escapeshellarg($returnType) . ' 2>' . APP . 'tmp/logs/exec-errors.log';
$stix_framing = json_decode(shell_exec($stix_framing_cmd), true);
if (empty($stix_framing)) {
return array('success' => 0, 'message' => 'There was an issue generating the STIX export.');
}
$separator = $stix_framing['separator'];
$tmpDir = $tmpDir . DS . "tmp";
$stixFile = new File($tmpDir . DS . $randomFileName . ".stix");
$stixFile->write($stix_framing['header']);
$result = array();
if ($jobId) {
$this->Job = ClassRegistry::init('Job');
$this->Job->id = $jobId;
if (!$this->Job->exists()) {
$jobId = false;
}
}
$i = 0;
$eventCount = count($event_ids);
if ($event_ids) {
foreach ($event_ids as $event_id) {
$tempFile = new File($tmpDir . DS . $randomFileName, true, 0644);
$event = $this->fetchEvent($user, array('eventid' => $event_id, 'includeAttachments' => $attachments));
if (empty($event)) {
continue;
}
$event[0]['Tag'] = array();
foreach ($event[0]['EventTag'] as $tag) {
$event[0]['Tag'][] = $tag['Tag'];
}
App::uses('JSONConverterTool', 'Tools');
$converter = new JSONConverterTool();
$event = $converter->convert($event[0]);
$tempFile->write($event);
unset($event);
$scriptFile = APP . "files" . DS . "scripts" . DS . "misp2stix.py";
$result = shell_exec($this->getPythonVersion() . ' ' . $scriptFile . ' ' . $randomFileName . ' ' . escapeshellarg($returnType) . ' ' . $this->__getAnnounceBaseurl() . ' ' . escapeshellarg(Configure::read('MISP.org')) . ' 2>' . APP . 'tmp/logs/exec-errors.log');
// The result of the script will be a returned JSON object with 2 variables: success (boolean) and message
// If success = 1 then the temporary output file was successfully written, otherwise an error message is passed along
$decoded = json_decode($result, true);
if (!isset($decoded['success']) || !$decoded['success']) {
$tempFile->delete();
$stixFile->delete();
return array('success' => 0, 'message' => $decoded['message']);
}
$file = new File(APP . "files" . DS . "scripts" . DS . "tmp" . DS . $randomFileName . ".out");
$stix_event = $file->read();
if (($i + 1) != $eventCount) {
$stix_event .= $separator;
}
$stixFile->append($stix_event);
$file->close();
$file->delete();
$i++;
if ($jobId) {
$this->Job->saveField('message', 'Event ' . $i . '/' . $eventCount);
if ($i % 10 == 0) {
$this->Job->saveField('progress', $i * 80 / $eventCount);
}
}
$tempFile->close();
}
}
$stixFile->append($stix_framing['footer']);
if ($tempFile) {
$tempFile->delete();
}
if (!$returnFile) {
$data = $stixFile->read();
$stixFile->delete();
}
return array('success' => 1, 'data' => $returnFile ? $stixFile->path : $data);
}
public function getAccessibleEventIds($include, $exclude, $includedTags, $excludedTags)
{
$conditions = array();
@ -5557,11 +5275,26 @@ class Event extends AppModel
private function __fillAttribute($attribute, $defaultDistribution)
{
if (!isset($attribute['category'])) {
$attribute['category'] = $this->Event->Attribute->typeDefinitions[$attribute['type']]['default_category'];
if (is_array($attribute['type'])) {
$attribute_type = $attribute['type'][0];
if (empty($attribute['category'])) {
$categories = array();
foreach ($attribute['type'] as $type) {
$category = $this->Attribute->typeDefinitions[$type]['default_category'];
if (!in_array($category, $categories)) {
$categories[] = $category;
}
}
$attribute['category'] = count($categories) === 1 ? $categories[0] : $categories;
}
} else {
$attribute_type = $attribute['type'];
if (empty($attribute['category'])) {
$attribute['category'] = $this->Attribute->typedefinitions[$attribute_type]['default_category'];
}
}
if (!isset($attribute['to_ids'])) {
$attribute['to_ids'] = $this->Event->Attribute->typeDefinitions[$attribute['type']]['to_ids'];
$attribute['to_ids'] = $this->Attribute->typeDefinitions[$attribute_type]['to_ids'];
}
$attribute['value'] = $this->Attribute->runRegexp($attribute['type'], $attribute['value']);
$attribute['distribution'] = (isset($attribute['distribution']) ? (int)$attribute['distribution'] : $defaultDistribution);
@ -6132,6 +5865,7 @@ class Event extends AppModel
unset($data[$dataType . 'Tag'][$k]);
continue;
}
$dataTag['Tag']['local'] = empty($dataTag['local']) ? 0 : 1;
if (!isset($excludeGalaxy) || !$excludeGalaxy) {
if (substr($dataTag['Tag']['name'], 0, strlen('misp-galaxy:')) === 'misp-galaxy:') {
$cluster = $this->GalaxyCluster->getCluster($dataTag['Tag']['name']);

11
app/Model/Feed.php
View File

@ -365,19 +365,12 @@ class Feed extends AppModel
if ($scope === 'Server' || $source[$scope]['source_format'] == 'misp') {
$pipe = $redis->multi(Redis::PIPELINE);
$eventUuidHitPosition = array();
$i = 0;
foreach ($objects as $k => $object) {
if (isset($object[$scope])) {
foreach ($object[$scope] as $currentFeed) {
if ($source[$scope]['id'] == $currentFeed['id']) {
$eventUuidHitPosition[$i] = $k;
$i++;
if (in_array($object['type'], $compositeTypes)) {
$value = explode('|', $object['value']);
$redis->smembers($cachePrefix . 'event_uuid_lookup:' . md5($value[0]));
} else {
$redis->smembers($cachePrefix . 'event_uuid_lookup:' . md5($object['value']));
}
$eventUuidHitPosition[] = $k;
$redis->smembers($cachePrefix . 'event_uuid_lookup:' . $hashTable[$k]);
}
}
}

8
app/Model/Galaxy.php
View File

@ -378,8 +378,12 @@ class Galaxy extends AppModel
{
$galaxy = $this->find('first', array(
'recursive' => -1,
'fields' => 'id',
'conditions' => array('Galaxy.type' => $type, 'Galaxy.namespace' => $namespace),
'fields' => array('MAX(Galaxy.version) as latest_version', 'id'),
'conditions' => array(
'Galaxy.type' => $type,
'Galaxy.namespace' => $namespace
),
'group' => array('name', 'id')
));
return empty($galaxy) ? 0 : $galaxy['Galaxy']['id'];
}

4
app/Model/Log.php
View File

@ -44,6 +44,7 @@ class Log extends AppModel
'merge',
'pruneUpdateLogs',
'publish',
'publish_sightings',
'publish alert',
'pull',
'purge_events',
@ -188,6 +189,7 @@ class Log extends AppModel
* @param int $modelId
* @param string $title
* @param string|array $change
* @return array
* @throws Exception
*/
public function createLogEntry($user, $action, $model, $modelId = 0, $title = '', $change = '')
@ -226,6 +228,8 @@ class Log extends AppModel
if (!$result) {
throw new Exception("Cannot save log because of validation errors: " . json_encode($this->validationErrors));
}
return $result;
}
// to combat a certain bug that causes the upgrade scripts to loop without being able to set the correct version

6
app/Model/MispObject.php
View File

@ -597,8 +597,10 @@ class MispObject extends AppModel
$this->Event->unpublishEvent($eventId);
$objectId = $this->id;
$partialFails = array();
foreach ($object['Object']['Attribute'] as $attribute) {
$this->Attribute->captureAttribute($attribute, $eventId, $user, $objectId, $log);
if (!empty($object['Object']['Attribute'])) {
foreach ($object['Object']['Attribute'] as $attribute) {
$this->Attribute->captureAttribute($attribute, $eventId, $user, $objectId, $log);
}
}
return true;
} else {

3
app/Model/Role.php
View File

@ -220,6 +220,9 @@ class Role extends AppModel
$this->data['Role']['memory_limit'] = '';
}
}
if (empty($this->data['Role']['rate_limit_count'])) {
$this->data['Role']['rate_limit_count'] = 0;
}
return true;
}

331
app/Model/Server.php
View File

@ -1176,6 +1176,15 @@ class Server extends AppModel
),
'Security' => array(
'branch' => 1,
'disable_form_security' => array(
'level' => 0,
'description' => __('Disabling this setting will remove all form tampering protection. Do not set this setting pretty much ever. You were warned.'),
'value' => false,
'errorMessage' => 'This setting leaves your users open to CSRF attacks. Do not please consider disabling this setting.',
'test' => 'testBoolFalse',
'type' => 'boolean',
'null' => true
),
'salt' => array(
'level' => 0,
'description' => __('The salt used for the hashed passwords. You cannot reset this from the GUI, only manually from the settings.php file. Keep in mind, this will invalidate all passwords in the database.'),
@ -2679,7 +2688,7 @@ class Server extends AppModel
$this->read(null, $id);
$url = $this->data['Server']['url'];
$push = $this->checkVersionCompatibility($id, $user);
if (isset($push['canPush']) && !$push['canPush']) {
if (is_array($push) && !$push['canPush'] && !$push['canSight']) {
$push = 'Remote instance is outdated or no permission to push.';
}
if (!is_array($push)) {
@ -2704,111 +2713,119 @@ class Server extends AppModel
}
return $push;
}
if ("full" == $technique) {
$eventid_conditions_key = 'Event.id >';
$eventid_conditions_value = 0;
} elseif ("incremental" == $technique) {
$eventid_conditions_key = 'Event.id >';
$eventid_conditions_value = $this->data['Server']['lastpushedid'];
} elseif (intval($technique) !== 0) {
$eventid_conditions_key = 'Event.id';
$eventid_conditions_value = intval($technique);
} else {
throw new InvalidArgumentException("Technique parameter must be 'full', 'incremental' or event ID.");
}
$sgs = $this->Event->SharingGroup->find('all', array(
'recursive' => -1,
'contain' => array('Organisation', 'SharingGroupOrg' => array('Organisation'), 'SharingGroupServer')
));
$sgIds = array();
foreach ($sgs as $k => $sg) {
if ($this->Event->SharingGroup->checkIfServerInSG($sg, $this->data)) {
$sgIds[] = $sg['SharingGroup']['id'];
// sync events if user is capable
if ($push['canPush']) {
if ("full" == $technique) {
$eventid_conditions_key = 'Event.id >';
$eventid_conditions_value = 0;
} elseif ("incremental" == $technique) {
$eventid_conditions_key = 'Event.id >';
$eventid_conditions_value = $this->data['Server']['lastpushedid'];
} elseif (intval($technique) !== 0) {
$eventid_conditions_key = 'Event.id';
$eventid_conditions_value = intval($technique);
} else {
throw new InvalidArgumentException("Technique parameter must be 'full', 'incremental' or event ID.");
}
}
if (empty($sgIds)) {
$sgIds = array(-1);
}
$findParams = array(
'conditions' => array(
$eventid_conditions_key => $eventid_conditions_value,
'Event.published' => 1,
'Event.attribute_count >' => 0,
'OR' => array(
array(
'AND' => array(
array('Event.distribution >' => 0),
array('Event.distribution <' => 4),
),
),
array(
'AND' => array(
'Event.distribution' => 4,
'Event.sharing_group_id' => $sgIds
$sgs = $this->Event->SharingGroup->find('all', array(
'recursive' => -1,
'contain' => array('Organisation', 'SharingGroupOrg' => array('Organisation'), 'SharingGroupServer')
));
$sgIds = array();
foreach ($sgs as $k => $sg) {
if ($this->Event->SharingGroup->checkIfServerInSG($sg, $this->data)) {
$sgIds[] = $sg['SharingGroup']['id'];
}
}
if (empty($sgIds)) {
$sgIds = array(-1);
}
$findParams = array(
'conditions' => array(
$eventid_conditions_key => $eventid_conditions_value,
'Event.published' => 1,
'Event.attribute_count >' => 0,
'OR' => array(
array(
'AND' => array(
array('Event.distribution >' => 0),
array('Event.distribution <' => 4),
),
),
array(
'AND' => array(
'Event.distribution' => 4,
'Event.sharing_group_id' => $sgIds
),
)
)
)
), // array of conditions
'recursive' => -1, //int
'contain' => array('EventTag' => array('fields' => array('EventTag.tag_id'))),
'fields' => array('Event.id', 'Event.timestamp', 'Event.sighting_timestamp', 'Event.uuid', 'Event.orgc_id'), // array of field names
);
$eventIds = $this->Event->find('all', $findParams);
$eventUUIDsFiltered = $this->getEventIdsForPush($id, $HttpSocket, $eventIds, $user);
if ($eventUUIDsFiltered === false || empty($eventUUIDsFiltered)) {
$pushFailed = true;
}
if (!empty($eventUUIDsFiltered)) {
$eventCount = count($eventUUIDsFiltered);
// now process the $eventIds to push each of the events sequentially
), // array of conditions
'recursive' => -1, //int
'contain' => array('EventTag' => array('fields' => array('EventTag.tag_id'))),
'fields' => array('Event.id', 'Event.timestamp', 'Event.sighting_timestamp', 'Event.uuid', 'Event.orgc_id'), // array of field names
);
$eventIds = $this->Event->find('all', $findParams);
$eventUUIDsFiltered = $this->getEventIdsForPush($id, $HttpSocket, $eventIds, $user);
if ($eventUUIDsFiltered === false || empty($eventUUIDsFiltered)) {
$pushFailed = true;
}
if (!empty($eventUUIDsFiltered)) {
$successes = array();
$fails = array();
$lowestfailedid = null;
foreach ($eventUUIDsFiltered as $k => $eventUuid) {
$params = array();
if (!empty($this->data['Server']['push_rules'])) {
$push_rules = json_decode($this->data['Server']['push_rules'], true);
if (!empty($push_rules['tags']['NOT'])) {
$params['blockedAttributeTags'] = $push_rules['tags']['NOT'];
$eventCount = count($eventUUIDsFiltered);
// now process the $eventIds to push each of the events sequentially
if (!empty($eventUUIDsFiltered)) {
$successes = array();
$fails = array();
$lowestfailedid = null;
foreach ($eventUUIDsFiltered as $k => $eventUuid) {
$params = array();
if (!empty($this->data['Server']['push_rules'])) {
$push_rules = json_decode($this->data['Server']['push_rules'], true);
if (!empty($push_rules['tags']['NOT'])) {
$params['blockedAttributeTags'] = $push_rules['tags']['NOT'];
}
}
$params = array_merge($params, array(
'event_uuid' => $eventUuid,
'includeAttachments' => true,
'includeAllTags' => true,
'deleted' => array(0,1),
'excludeGalaxy' => 1
));
$event = $this->Event->fetchEvent($user, $params);
$event = $event[0];
$event['Event']['locked'] = 1;
$result = $this->Event->uploadEventToServer($event, $this->data, $HttpSocket);
if ('Success' === $result) {
$successes[] = $event['Event']['id'];
} else {
$fails[$event['Event']['id']] = $result;
}
if ($jobId && $k%10 == 0) {
$job->saveField('progress', 100 * $k / $eventCount);
}
}
$params = array_merge($params, array(
'event_uuid' => $eventUuid,
'includeAttachments' => true,
'includeAllTags' => true,
'deleted' => array(0,1),
'excludeGalaxy' => 1
));
$event = $this->Event->fetchEvent($user, $params);
$event = $event[0];
$event['Event']['locked'] = 1;
$result = $this->Event->uploadEventToServer($event, $this->data, $HttpSocket);
if ('Success' === $result) {
$successes[] = $event['Event']['id'];
if (count($fails) > 0) {
// there are fails, take the lowest fail
$lastpushedid = min(array_keys($fails));
} else {
$fails[$event['Event']['id']] = $result;
}
if ($jobId && $k%10 == 0) {
$job->saveField('progress', 100 * $k / $eventCount);
// no fails, take the highest success
$lastpushedid = max($successes);
}
// increment lastid based on the highest ID seen
// Save the entire Server data instead of just a single field, so that the logger can be fed with the extra fields.
$this->data['Server']['lastpushedid'] = $lastpushedid;
$this->save($this->data);
}
if (count($fails) > 0) {
// there are fails, take the lowest fail
$lastpushedid = min(array_keys($fails));
} else {
// no fails, take the highest success
$lastpushedid = max($successes);
}
// increment lastid based on the highest ID seen
// Save the entire Server data instead of just a single field, so that the logger can be fed with the extra fields.
$this->data['Server']['lastpushedid'] = $lastpushedid;
$this->save($this->data);
}
$this->syncProposals($HttpSocket, $this->data, null, null, $this->Event);
}
$this->syncProposals($HttpSocket, $this->data, null, null, $this->Event);
$sightingSuccesses = $this->syncSightings($HttpSocket, $this->data, $user, $this->Event);
if ($push['canPush'] || $push['canSight']) {
$sightingSuccesses = $this->syncSightings($HttpSocket, $this->data, $user, $this->Event);
} else {
$sightingSuccesses = array();
}
if (!isset($successes)) {
$successes = $sightingSuccesses;
@ -4268,7 +4285,7 @@ class Server extends AppModel
$dataSource = $this->getDataSource()->config['datasource'];
if ($dataSource == 'Database/Mysql') {
$sql = sprintf(
'select table_name, sum((data_length+index_length)/1024/1024) AS used, sum(data_free)/1024/1024 reclaimable from information_schema.tables where table_schema = %s group by table_name;',
'select TABLE_NAME, sum((DATA_LENGTH+INDEX_LENGTH)/1024/1024) AS used, sum(DATA_FREE)/1024/1024 AS reclaimable from information_schema.tables where table_schema = %s group by TABLE_NAME;',
"'" . $this->getDataSource()->config['database'] . "'"
);
$sqlResult = $this->query($sql);
@ -4277,14 +4294,14 @@ class Server extends AppModel
foreach ($temp[0] as $k => $v) {
$temp[0][$k] = round($v, 2) . 'MB';
}
$temp[0]['table'] = $temp['tables']['table_name'];
$temp[0]['table'] = $temp['tables']['TABLE_NAME'];
$result[] = $temp[0];
}
return $result;
}
else if ($dataSource == 'Database/Postgres') {
$sql = sprintf(
'select table_name as table, pg_total_relation_size(%s||%s||table_name) as used from information_schema.tables where table_schema = %s group by table_name;',
'select TABLE_NAME as table, pg_total_relation_size(%s||%s||TABLE_NAME) as used from information_schema.tables where table_schema = %s group by TABLE_NAME;',
"'" . $this->getDataSource()->config['database'] . "'",
"'.'",
"'" . $this->getDataSource()->config['database'] . "'"
@ -4330,23 +4347,34 @@ class Server extends AppModel
))['AdminSetting']['value'];
$dataSource = $this->getDataSource()->config['datasource'];
$schemaDiagnostic = array(
'dataSource' => $dataSource,
'actual_db_version' => $actualDbVersion,
'checked_table_column' => array(),
'diagnostic' => array(),
'diagnostic_index' => array(),
'expected_db_version' => '?',
'error' => '',
'update_locked' => $this->isUpdateLocked(),
'remaining_lock_time' => $this->getLockRemainingTime(),
'update_fail_number_reached' => $this->UpdateFailNumberReached()
'update_fail_number_reached' => $this->UpdateFailNumberReached(),
'indexes' => array()
);
if ($dataSource == 'Database/Mysql') {
$dbActualSchema = $this->getActualDBSchema();
$dbExpectedSchema = $this->getExpectedDBSchema();
if ($dbExpectedSchema !== false) {
$db_schema_comparison = $this->compareDBSchema($dbActualSchema['schema'], $dbExpectedSchema['schema']);
$db_indexes_comparison = $this->compareDBIndexes($dbActualSchema['indexes'], $dbExpectedSchema['indexes']);
$schemaDiagnostic['checked_table_column'] = $dbActualSchema['column'];
$schemaDiagnostic['diagnostic'] = $db_schema_comparison;
$schemaDiagnostic['diagnostic_index'] = $db_indexes_comparison;
$schemaDiagnostic['expected_db_version'] = $dbExpectedSchema['db_version'];
foreach($dbActualSchema['schema'] as $tableName => $tableMetas) {
foreach($tableMetas as $tableMeta) {
$schemaDiagnostic['columnPerTable'][$tableName][] = $tableMeta['column_name'];
}
}
$schemaDiagnostic['indexes'] = $dbActualSchema['indexes'];
} else {
$schemaDiagnostic['error'] = sprintf('Diagnostic not available as the expected schema file could not be loaded');
}
@ -4370,7 +4398,7 @@ class Server extends AppModel
*/
private function __attachRecoveryQuery($field, $table)
{
if ($field['is_critical']) {
if (isset($field['error_type'])) {
$length = false;
if (in_array($field['error_type'], array('missing_column', 'column_different'))) {
if ($field['expected']['data_type'] === 'int') {
@ -4387,30 +4415,57 @@ class Server extends AppModel
switch($field['error_type']) {
case 'missing_column':
$field['sql'] = sprintf(
'ALTER TABLE `%s` ADD COLUMN `%s` %s%s %s %s;',
'ALTER TABLE `%s` ADD COLUMN `%s` %s%s %s %s %s;',
$table,
$field['column_name'],
$field['expected']['data_type'],
$length !== null ? sprintf('(%d)', $length) : '',
isset($field['expected']['default']) ? 'DEFAULT "' . $field['expected']['default'] . '"' : '',
isset($field['expected']['column_default']) ? $field['expected']['column_default'] . '"' : '',
$field['expected']['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL',
empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name']
);
break;
case 'column_different':
$field['sql'] = sprintf(
'ALTER TABLE `%s` CHANGE `%s` `%s` %s%s %s %s;',
'ALTER TABLE `%s` MODIFY COLUMN `%s` %s%s %s %s %s;',
$table,
$field['column_name'],
$field['column_name'],
$field['expected']['data_type'],
$length !== null ? sprintf('(%d)', $length) : '',
isset($field['expected']['default']) ? 'DEFAULT "' . $field['expected']['default'] . '"' : '',
isset($field['expected']['column_default']) ? 'DEFAULT "' . $field['expected']['column_default'] . '"' : '',
$field['expected']['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL',
empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name']
);
break;
}
} elseif($field['error_type'] == 'missing_table') {
$allFields = array();
foreach ($field['expected_table'] as $expectedField) {
$length = false;
if ($expectedField['data_type'] === 'int') {
$length = 11;
} elseif ($expectedField['data_type'] === 'tinyint') {
$length = 1;
} elseif ($expectedField['data_type'] === 'varchar') {
$length = $expectedField['character_maximum_length'];
} elseif ($expectedField['data_type'] === 'text') {
$length = null;
}
$fieldSql = sprintf('`%s` %s%s %s %s %s',
$expectedField['column_name'],
$expectedField['data_type'],
$length !== null ? sprintf('(%d)', $length) : '',
isset($expectedField['column_default']) ? 'DEFAULT "' . $expectedField['column_default'] . '"' : '',
$expectedField['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL',
empty($expectedField['collation_name']) ? '' : 'COLLATE ' . $expectedField['collation_name']
);
$allFields[] = $fieldSql;
}
$field['sql'] = __("% The command below is a suggestion and might be incorrect. Please ask if you are not sure what you are doing.") . "</br></br>" . sprintf(
"CREATE TABLE IF NOT EXISTS `%s` ( %s ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;",
$table,
implode(', ', $allFields)
);
}
}
return $field;
@ -4451,30 +4506,33 @@ class Server extends AppModel
'character_maximum_length',
'numeric_precision',
// 'datetime_precision', -- Only available on MySQL 5.6+
'collation_name'
'collation_name',
'column_default'
)
){
$dbActualSchema = array();
$dbActualIndexes = array();
$dataSource = $this->getDataSource()->config['datasource'];
if ($dataSource == 'Database/Mysql') {
$sqlGetTable = sprintf('SELECT table_name FROM information_schema.tables WHERE table_schema = %s;', "'" . $this->getDataSource()->config['database'] . "'");
$sqlGetTable = sprintf('SELECT TABLE_NAME FROM information_schema.tables WHERE table_schema = %s;', "'" . $this->getDataSource()->config['database'] . "'");
$sqlResult = $this->query($sqlGetTable);
$tables = HASH::extract($sqlResult, '{n}.tables.table_name');
$tables = HASH::extract($sqlResult, '{n}.tables.TABLE_NAME');
foreach ($tables as $table) {
$sqlSchema = sprintf(
"SELECT %s
FROM information_schema.columns
WHERE table_schema = '%s' AND table_name = '%s'", implode(',', $tableColumnNames), $this->getDataSource()->config['database'], $table);
WHERE table_schema = '%s' AND TABLE_NAME = '%s'", implode(',', $tableColumnNames), $this->getDataSource()->config['database'], $table);
$sqlResult = $this->query($sqlSchema);
foreach ($sqlResult as $column_schema) {
$dbActualSchema[$table][] = $column_schema['columns'];
}
$dbActualIndexes[$table] = $this->getDatabaseIndexes($this->getDataSource()->config['database'], $table);
}
}
else if ($dataSource == 'Database/Postgres') {
return array('Database/Postgres' => array('description' => __('Can\'t check database schema for Postgres database type')));
}
return array('schema' => $dbActualSchema, 'column' => $tableColumnNames);
return array('schema' => $dbActualSchema, 'column' => $tableColumnNames, 'indexes' => $dbActualIndexes);
}
public function compareDBSchema($dbActualSchema, $dbExpectedSchema)
@ -4491,6 +4549,7 @@ class Server extends AppModel
$dbDiff[$tableName][] = array(
'description' => sprintf(__('Table `%s` does not exist'), $tableName),
'error_type' => 'missing_table',
'expected_table' => $columns,
'column_name' => $tableName,
'is_critical' => true
);
@ -4532,8 +4591,19 @@ class Server extends AppModel
$isCritical = false;
foreach($colElementDiffs as $colElementDiff) {
if(!in_array($colElementDiff, $nonCriticalColumnElements)) {
$isCritical = true;
break;
if ($colElementDiff == 'column_default') {
$expectedValue = $column['column_default'];
$actualValue = $keyedActualColumn[$columnName]['column_default'];
if (preg_match(sprintf('@(\'|")+%s(\1)+@', $expectedValue), $actualValue) || (empty($expectedValue) && $actualValue === 'NULL')) { // some version of mysql quote the default value
continue;
} else {
$isCritical = true;
break;
}
} else {
$isCritical = true;
break;
}
}
}
$dbDiff[$tableName][] = array(
@ -4569,6 +4639,42 @@ class Server extends AppModel
return $dbDiff;
}
public function compareDBIndexes($actualIndex, $expectedIndex)
{
$indexDiff = array();
foreach($expectedIndex as $tableName => $indexes) {
if (!array_key_exists($tableName, $actualIndex)) {
// If table does not exists, it is covered by the schema diagnostic
} else {
$tableIndexDiff = array_diff($indexes, $actualIndex[$tableName]); // check for missing indexes
if (count($tableIndexDiff) > 0) {
foreach($tableIndexDiff as $columnDiff) {
$indexDiff[$tableName][$columnDiff] = sprintf(__('Column `%s` should be indexed'), $columnDiff);
}
}
$tableIndexDiff = array_diff($actualIndex[$tableName], $indexes); // check for additional indexes
if (count($tableIndexDiff) > 0) {
foreach($tableIndexDiff as $columnDiff) {
$indexDiff[$tableName][$columnDiff] = sprintf(__('Column `%s` is indexed but should not'), $columnDiff);
}
}
}
}
return $indexDiff;
}
public function getDatabaseIndexes($database, $table)
{
$sqlTableIndex = sprintf(
"SELECT DISTINCT TABLE_NAME, COLUMN_NAME FROM information_schema.statistics WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s';",
$database,
$table
);
$sqlTableIndexResult = $this->query($sqlTableIndex);
$tableIndex = Hash::extract($sqlTableIndexResult, '{n}.statistics.COLUMN_NAME');
return $tableIndex;
}
public function writeableDirsDiagnostics(&$diagnostic_errors)
{
App::uses('File', 'Utility');
@ -4700,6 +4806,7 @@ class Server extends AppModel
'binary' => Configure::read('GnuPG.binary') ?: '/usr/bin/gpg'
));
} catch (Exception $e) {
$this->logException("Error during initializing GPG.", $e, LOG_NOTICE);
$gpgStatus = 2;
$continue = false;
}
@ -4707,6 +4814,7 @@ class Server extends AppModel
try {
$key = $gpg->addSignKey(Configure::read('GnuPG.email'), Configure::read('GnuPG.password'));
} catch (Exception $e) {
$this->logException("Error during adding GPG signing key.", $e, LOG_NOTICE);
$gpgStatus = 3;
$continue = false;
}
@ -4716,6 +4824,7 @@ class Server extends AppModel
$gpgStatus = 0;
$signed = $gpg->sign('test', Crypt_GPG::SIGN_MODE_CLEAR);
} catch (Exception $e) {
$this->logException("Error during GPG signing.", $e, LOG_NOTICE);
$gpgStatus = 4;
}
}

21
app/Model/Sighting.php
View File

@ -629,6 +629,16 @@ class Sighting extends AppModel
public function restSearch($user, $returnFormat, $filters)
{
$allowedContext = array('event', 'attribute');
// validate context
if (isset($filters['context']) && !in_array($filters['context'], $allowedContext, true)) {
throw new MethodNotAllowedException(_('Invalid context.'));
}
// ensure that an id is provided if context is set
if (!empty($filters['context']) && !isset($filters['id'])) {
throw new MethodNotAllowedException(_('An id must be provided if the context is set.'));
}
if (!isset($this->validFormats[$returnFormat][1])) {
throw new NotFoundException('Invalid output format.');
}
@ -674,10 +684,12 @@ class Sighting extends AppModel
$conditions['Sighting.source'] = $filters['source'];
}
if ($filters['context'] === 'attribute') {
$conditions['Sighting.attribute_id'] = $filters['id'];
} elseif ($filters['context'] === 'event') {
$conditions['Sighting.event_id'] = $filters['id'];
if (!empty($filters['id'])) {
if ($filters['context'] === 'attribute') {
$conditions['Sighting.attribute_id'] = $filters['id'];
} elseif ($filters['context'] === 'event') {
$conditions['Sighting.event_id'] = $filters['id'];
}
}
// fetch sightings matching the query
@ -712,7 +724,6 @@ class Sighting extends AppModel
$filters['requested_attributes'] = array_merge($filters['requested_attributes'], array('event_uuid', 'event_orgc_id', 'event_org_id', 'event_info', 'event_Orgc_name'));
$additional_event_added = true;
}
if (!empty($sight)) {
array_push($allowedSightings, $sight);
}

21
app/Model/Tag.php
View File

@ -413,11 +413,28 @@ class Tag extends AppModel
return ($this->saveAll($tags));
}
public function getTagsByName($tag_names, $containTagConnectors = true)
{
$contain = array('EventTag', 'AttributeTag');
$tag_params = array(
'recursive' => -1,
'conditions' => array('name' => $tag_names)
);
if ($containTagConnectors) {
$tag_params['contain'] = $contain;
}
$tags_temp = $this->find('all', $tag_params);
$tags = array();
foreach ($tags_temp as $temp) {
$tags[strtoupper($temp['Tag']['name'])] = $temp;
}
return $tags;
}
public function getTagsForNamespace($namespace, $containTagConnectors = true)
{
$contain = array('EventTag');
$contain[] = 'AttributeTag';
$contain = array('EventTag', 'AttributeTag');
$tag_params = array(
'recursive' => -1,
'conditions' => array('UPPER(name) LIKE' => strtoupper($namespace) . '%'),

5
app/Model/Taxonomy.php
View File

@ -101,7 +101,7 @@ class Taxonomy extends AppModel
}
$this->deleteAll(array('Taxonomy.namespace' => $current['Taxonomy']['namespace']));
}
$taxonomy['Taxonomy'] = array('namespace' => $vocab['namespace'], 'description' => $vocab['description'], 'version' => $vocab['version'], 'exclusive' => $vocab['exclusive'], 'enabled' => $enabled);
$taxonomy['Taxonomy'] = array('namespace' => $vocab['namespace'], 'description' => $vocab['description'], 'version' => $vocab['version'], 'exclusive' => !empty($vocab['exclusive']), 'enabled' => $enabled);
$predicateLookup = array();
foreach ($vocab['predicates'] as $k => $predicate) {
$taxonomy['Taxonomy']['TaxonomyPredicate'][$k] = $predicate;
@ -283,7 +283,8 @@ class Taxonomy extends AppModel
if (empty($taxonomy)) {
return false;
}
$tags = $this->Tag->getTagsForNamespace($taxonomy['Taxonomy']['namespace'], false);
$tag_names = Hash::extract($taxonomy, 'entries.{n}.tag');
$tags = $this->Tag->getTagsByName($tag_names, false);
if (isset($taxonomy['entries'])) {
foreach ($taxonomy['entries'] as $key => $temp) {
$taxonomy['entries'][$key]['existing_tag'] = isset($tags[strtoupper($temp['tag'])]) ? $tags[strtoupper($temp['tag'])] : false;

68
app/Model/User.php
View File

@ -3,6 +3,9 @@ App::uses('AppModel', 'Model');
App::uses('AuthComponent', 'Controller/Component');
App::uses('RandomTool', 'Tools');
/**
* @property Log $Log
*/
class User extends AppModel
{
public $displayField = 'email';
@ -1419,20 +1422,12 @@ class User extends AppModel
// query
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => $model,
'model_id' => $modelId,
'email' => $user['email'],
'action' => $action,
'title' => $description,
'change' => isset($fieldsResult) ? $fieldsResult : ''));
$result = $this->Log->createLogEntry($user, $action, $model, $modelId, $description, $fieldsResult);
// write to syslogd as well
App::import('Lib', 'SysLog.SysLog');
$syslog = new SysLog();
$syslog->write('notice', $description . ' -- ' . $action . (empty($fieldResult) ? '' : '-- ' . $fieldResult));
$syslog->write('notice', "$description -- $action" . (empty($fieldResult) ? '' : ' -- ' . $result['Log']['change']));
}
/**
@ -1461,4 +1456,57 @@ class User extends AppModel
return new Crypt_GPG($options);
}
public function getOrgActivity($orgId, $params=array())
{
$conditions = array();
$options = array();
foreach($params as $paramName => $value) {
$options['filter'] = $paramName;
$filterParam[$paramName] = $value;
$conditions = $this->Event->set_filter_timestamp($filterParam, $conditions, $options);
}
$conditions['Event.orgc_id'] = $orgId;
$events = $this->Event->find('all', array(
'recursive' => -1,
'fields' => array('Event.orgc_id', 'Event.timestamp', 'Event.attribute_count'),
'conditions' => $conditions,
'order' => 'Event.timestamp'
));
$sparklineData = array();
foreach ($events as $event) {
$date = date("Y-m-d", $event['Event']['timestamp']);
if (!isset($sparklineData[$event['Event']['attribute_count']][$date])) {
$sparklineData[$date] = $event['Event']['attribute_count'];
} else {
$sparklineData[$date] += $event['Event']['attribute_count'];
}
}
// get first and last timestamp
if (isset($params['from'])) {
$startDate = $params['from'];
} else {
$startDate = $this->resolveTimeDelta($params['event_timestamp']);
}
if (isset($params['to'])) {
$endDate = $params['to'];
} else {
$endDate = time();
}
$dates = array();
for ($d=$startDate; $d < $endDate; $d=$d+3600*24) {
$dates[] = date('Y-m-d', $d);
}
$csv = 'Date,Close\n';
foreach ($dates as $date) {
$csv .= sprintf('%s,%s\n', $date, isset($sparklineData[$date]) ? $sparklineData[$date] : 0);
}
$data = array(
'csv' => $csv,
'data' => $sparklineData,
'orgId' => $orgId
);
return $data;
}
}

254
app/View/Attributes/add.ctp
View File

@ -1,166 +1,117 @@
<div class="attributes <?php if (!isset($ajax) || !$ajax) echo 'form';?>">
<?php
$url_params = $action == 'add' ? 'add/' . $event_id : 'edit/' . $attribute['Attribute']['id'];
echo $this->Form->create('Attribute', array('id', 'url' => '/attributes/' . $url_params));
?>
<fieldset>
<legend><?php echo $action == 'add' ? __('Add Attribute') : __('Edit Attribute'); ?></legend>
<div id="formWarning" class="message ajaxMessage"></div>
<div id="compositeWarning" class="message <?php echo !empty($ajax) ? 'ajaxMessage' : '';?>" style="display:none;">Did you consider adding an object instead of a composite attribute?</div>
<div class="add_attribute_fields">
<?php
echo $this->Form->hidden('event_id');
echo $this->Form->input('category', array(
'empty' => __('(choose one)'),
'label' => __('Category ') . $this->element('formInfo', array('type' => 'category')),
));
echo $this->Form->input('type', array(
'empty' => __('(first choose category)'),
'label' => __('Type ') . $this->element('formInfo', array('type' => 'type')),
));
$initialDistribution = 5;
if (Configure::read('MISP.default_attribute_distribution') != null) {
if (Configure::read('MISP.default_attribute_distribution') === 'event') {
$initialDistribution = 5;
} else {
$initialDistribution = Configure::read('MISP.default_attribute_distribution');
}
}
?>
<div class="input clear"></div>
<?php
$distArray = array(
'options' => array($distributionLevels),
'label' => __('Distribution ') . $this->element('formInfo', array('type' => 'distribution')),
);
if ($action == 'add') {
$distArray['selected'] = $initialDistribution;
}
echo $this->Form->input('distribution', $distArray);
?>
<div id="SGContainer" style="display:none;">
<?php
if (!empty($sharingGroups)) {
echo $this->Form->input('sharing_group_id', array(
'options' => array($sharingGroups),
'label' => __('Sharing Group'),
));
}
?>
</div>
<?php
echo $this->Form->input('value', array(
'type' => 'textarea',
'error' => array('escape' => false),
'div' => 'input clear',
'class' => 'input-xxlarge'
));
?>
<div class="input clear"></div>
<?php
echo $this->Form->input('comment', array(
$modelForForm = 'Attribute';
echo $this->element('genericElements/Form/genericForm', array(
'form' => $this->Form,
'data' => array(
'title' => $action === 'add' ? __('Add Attribute') : __('Edit Attribute'),
'model' => $modelForForm,
'fields' => array(
array(
'field' => 'event_id',
'class' => 'org-id-picker-hidden-field',
'type' => 'text',
'label' => __('Contextual Comment'),
'error' => array('escape' => false),
'hidden' => true
),
array(
'field' => 'category',
'class' => 'input',
'empty' => __('(choose one)'),
'options' => $categories,
'stayInLine' => 1
),
array(
'field' => 'type',
'class' => 'input',
'empty' => __('(choose category first)'),
'options' => $types
),
array(
'field' => 'distribution',
'class' => 'input',
'options' => $distributionLevels,
'default' => isset($attribute['Attribute']['distribution']) ? $attribute['Attribute']['distribution'] : $initialDistribution,
'stayInLine' => 1
),
array(
'field' => 'sharing_group_id',
'class' => 'input',
'options' => $sharingGroups,
'label' => __("Sharing Group")
),
array(
'field'=> 'value',
'type' => 'textarea',
'class' => 'input span6',
'div' => 'input clear'
),
array(
'field' => 'comment',
'type' => 'text',
'class' => 'input span6',
'div' => 'input clear',
'class' => 'input-xxlarge'
));
?>
<div class="input clear"></div>
<?php
echo $this->Form->input('to_ids', array(
'label' => __('for Intrusion Detection System'),
'type' => 'checkbox'
));
echo $this->Form->input('batch_import', array(
'label' => __("Contextual Comment")
),
array(
'field' => 'to_ids',
'type' => 'checkbox',
'label' => __("for Intrusion Detection System"),
//'stayInLine' => 1
),
array(
'field' => 'batch_import',
'type' => 'checkbox'
));
echo '<div class="input clear"></div>';
echo $this->Form->input('disable_correlation', array(
),
array(
'field' => 'disable_correlation',
'type' => 'checkbox'
));
?>
</div>
</fieldset>
<p id="notice_message" style="display:none;"></p>
<?php if ($ajax): ?>
<div class="overlay_spacing">
<span id="submitButton" class="btn btn-primary" style="margin-bottom:5px;float:left;" title="<?php echo __('Submit'); ?>" role="button" tabindex="0" aria-label="<?php echo __('Submit'); ?>" onClick="submitPopoverForm('<?php echo $action == 'add' ? $event_id : $attribute['Attribute']['id'];?>', '<?php echo $action; ?>')"><?php echo __('Submit'); ?></span>
<span class="btn btn-inverse" style="float:right;" title="<?php echo __('Cancel'); ?>" role="button" tabindex="0" aria-label="<?php echo __('Cancel'); ?>" id="cancel_attribute_add"><?php echo __('Cancel'); ?></span>
</div>
<?php
else:
?>
<?php
echo $this->Form->button('Submit', array('class' => 'btn btn-primary'));
endif;
echo $this->Form->end();
?>
</div>
<?php
),
'<div id="extended_event_preview" style="width:446px;"></div>'
),
'submit' => array(
'action' => $action,
'ajaxSubmit' => sprintf(
'submitPopoverForm(%s, %s, 0, 1)',
"'" . ($action == 'add' ? h($event_id) : h($attribute['Attribute']['id'])) . "'",
"'" . h($action) . "'"
)
)
)
));
if (!$ajax) {
$event['Event']['id'] = $event_id;
$event['Event']['published'] = $published;
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event', 'menuItem' => 'addAttribute', 'event' => $event));
}
?>
<script type="text/javascript">
var notice_list_triggers = <?php echo $notice_list_triggers; ?>;
var fieldsArray = new Array('AttributeCategory', 'AttributeType', 'AttributeValue', 'AttributeDistribution', 'AttributeComment', 'AttributeToIds', 'AttributeBatchImport', 'AttributeSharingGroupId');
<?php
$formInfoTypes = array('distribution' => 'Distribution', 'category' => 'Category', 'type' => 'Type');
echo 'var formInfoFields = ' . json_encode($formInfoTypes) . PHP_EOL;
foreach ($formInfoTypes as $formInfoType => $humanisedName) {
echo 'var ' . $formInfoType . 'FormInfoValues = {' . PHP_EOL;
foreach ($info[$formInfoType] as $key => $formInfoData) {
echo '"' . $key . '": "<span class=\"blue bold\">' . h($formInfoData['key']) . '</span>: ' . h($formInfoData['desc']) . '<br />",' . PHP_EOL;
}
echo '}' . PHP_EOL;
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event-collection', 'menuItem' => $this->action === 'add' ? 'add' : 'editEvent'));
}
?>
//
//Generate Category / Type filtering array
//
var category_type_mapping = new Array();
<?php
<script type="text/javascript">
var notice_list_triggers = <?php echo $notice_list_triggers; ?>;
var composite_types = <?php echo json_encode($compositeTypes); ?>;
var category_type_mapping = new Array();
<?php
foreach ($categoryDefinitions as $category => $def) {
echo "category_type_mapping['" . addslashes($category) . "'] = {";
$first = true;
foreach ($def['types'] as $type) {
if ($first) $first = false;
else echo ', ';
if ($first) {
$first = false;
} else {
echo ', ';
}
echo "'" . addslashes($type) . "' : '" . addslashes($type) . "'";
}
echo "}; \n";
}
?>
var composite_types = <?php echo json_encode($compositeTypes); ?>;
$(document).ready(function() {
<?php
if ($action == 'edit'):
?>
checkNoticeList('attribute');
<?php
endif;
?>
initPopoverContent('Attribute');
$('#AttributeDistribution').change(function() {
if ($('#AttributeDistribution').val() == 4) $('#SGContainer').show();
else $('#SGContainer').hide();
checkSharingGroup('Attribute');
});
$("#AttributeCategory").on('change', function(e) {
$('#AttributeCategory').change(function() {
formCategoryChanged('Attribute');
if ($(this).val() === 'Internal reference') {
$("#AttributeDistribution").val('0');
$('#SGContainer').hide();
checkSharingGroup('Attribute');
}
});
@ -168,22 +119,15 @@ $(document).ready(function() {
checkNoticeList('attribute');
});
$("#AttributeCategory, #AttributeType, #AttributeDistribution").change(function() {
var start = $("#AttributeType").val();
initPopoverContent('Attribute');
$("#AttributeType").val(start);
if ($.inArray(start, composite_types) > -1) {
$('#compositeWarning').show();
} else {
$('#compositeWarning').hide();
}
$(document).ready(function() {
<?php
if ($action == 'edit'):
?>
checkNoticeList('attribute');
<?php
endif;
?>
checkSharingGroup('Attribute');
});
<?php if ($ajax): ?>
$('#cancel_attribute_add').click(function() {
cancelPopoverForm();
});
<?php endif; ?>
});
</script>
<?php echo $this->Js->writeBuffer(); // Write cached scripts

7
app/View/Elements/Events/View/related_event.ctp
View File

@ -1,5 +1,8 @@
<?php
$href_url = isset($href_url) ? $href_url : $baseurl . '/events';
$href_url = isset($href_url) ? $href_url : $baseurl . '/events/view/' . h($related['id']);
if (isset($from_id)) {
$href_url .= sprintf('/%s/%s', 1, $from_id);
}
$hide = isset($hide) ? $hide : false;
?>
<span class="<?php echo $hide ? 'hidden correlation-expanded-area' : '' ?>">
@ -11,7 +14,7 @@
<?php echo $this->OrgImg->getOrgImg(array('name' => $related['Orgc']['name'], 'id' => $related['Orgc']['id'], 'size' => 24)); ?>
</td>
<td style="line-height: 14px; padding-left: 2px; white-space: nowrap; text-overflow: ellipsis; overflow: hidden; max-width: 430px;">
<a title="<?php echo h($related['info']); ?>" href="<?php echo h($href_url) . '/' . $related['id']?>">
<a title="<?php echo h($related['info']); ?>" href="<?php echo h($href_url)?>">
<span><?php echo h($related['info']) ?>
</a>
</td>

1
app/View/Elements/Feeds/View/row_attribute.ctp
View File

@ -1,5 +1,4 @@
<?php
if ($object['value'] == 'MERGE') debug($object);
$tr_class = '';
$linkClass = 'blue';
$otherColour = 'blue';

1
app/View/Elements/Servers/View/row_attribute.ctp
View File

@ -1,5 +1,4 @@
<?php
if ($object['value'] == 'MERGE') debug($object);
$tr_class = '';
$linkClass = 'blue';
$otherColour = 'blue';

7
app/View/Elements/eventattributetoolbar.ctp
View File

@ -49,10 +49,11 @@
'children' => array(
array(
'id' => 'create-button',
'title' => __('Add attribute'),
'title' => $possibleAction === 'attribute' ? __('Add attribute') : __('Add proposal'),
'fa-icon' => 'plus',
'onClick' => 'clickCreateButton',
'onClickParams' => array($event['Event']['id'], $possibleAction)
//'onClick' => 'clickCreateButton',
'onClick' => 'openGenericModal',
'onClickParams' => array('/' . $possibleAction . 's/add/' . h($event['Event']['id']))
),
array(
'id' => 'multi-edit-button',

59
app/View/Elements/genericElements/Form/genericForm.ctp
View File

@ -16,8 +16,9 @@
h($data['model']);
$fieldsString = '';
$simpleFieldWhitelist = array(
'default', 'type', 'options', 'placeholder', 'label'
'default', 'type', 'options', 'placeholder', 'label', 'empty'
);
$fieldsArrayForPersistence = array();
$formCreate = $this->Form->create($modelForForm);
if (!empty($data['fields'])) {
foreach ($data['fields'] as $fieldData) {
@ -51,6 +52,7 @@
}
}
$temp = $this->Form->input($fieldData['field'], $params);
$fieldsArrayForPersistence []= $modelForForm . Inflector::camelize($fieldData['field']);
if (!empty($fieldData['hidden'])) {
$temp = '<span class="hidden">' . $temp . '</span>';
}
@ -73,18 +75,55 @@
if (!empty($data['submit'])) {
$submitButtonData = array_merge($submitButtonData, $data['submit']);
}
if (!empty($data['ajaxSubmit'])) {
$submitButtonData['ajaxSubmit'] = $ajaxSubmit;
}
$ajaxFlashMessage = '';
if ($ajax) {
$ajaxFlashMessage = sprintf(
'<div id="flashContainer"><div id="main-view-container">%s</div></div>',
$this->Flash->render()
);
}
$formEnd = $this->Form->end();
echo sprintf(
'<div class="form">%s<fieldset><legend>%s</legend>%s</fieldset>%s%s%s</div>',
$formCreate,
empty($data['title']) ? h(Inflector::humanize($this->request->params['action'])) . ' ' . $modelForForm : h($data['title']),
$fieldsString,
$formEnd,
$metaFieldString,
$this->element('genericElements/Form/submitButton', $submitButtonData)
);
if (!empty($ajax)) {
echo sprintf(
'<div id="genericModal" class="modal hide fade" tabindex="-1" role="dialog" aria-labelledby="genericModalLabel" aria-hidden="true">%s%s%s</div>',
sprintf(
'<div class="modal-header"><button type="button" class="close" data-dismiss="modal" aria-hidden="true">×</button><h3 id="genericModalLabel">%s</h3></div>',
empty($data['title']) ? h(Inflector::humanize($this->request->params['action'])) . ' ' . $modelForForm : h($data['title'])
),
sprintf(
'<div class="modal-body modal-body-long">%s</div>',
sprintf(
'%s<fieldset>%s%s</fieldset>%s%s',
$formCreate,
$ajaxFlashMessage,
$fieldsString,
$formEnd,
$metaFieldString
)
),
sprintf(
'<div class="modal-footer">%s</div>',
$this->element('genericElements/Form/submitButton', $submitButtonData)
)
);
} else {
echo sprintf(
'<div class="form">%s<fieldset><legend>%s</legend>%s%s</fieldset>%s%s%s</div>',
$formCreate,
empty($data['title']) ? h(Inflector::humanize($this->request->params['action'])) . ' ' . $modelForForm : h($data['title']),
$ajaxFlashMessage,
$fieldsString,
$formEnd,
$metaFieldString,
$this->element('genericElements/Form/submitButton', $submitButtonData)
);
}
?>
<script type="text/javascript">
var fieldsArray = <?php echo json_encode($fieldsArrayForPersistence); ?>;
$(document).ready(function() {
popoverStartup();
});

36
app/View/Elements/genericElements/Form/submitButton.ctp
View File

@ -1,12 +1,28 @@
<?php
echo sprintf(
'<button onClick="%s" class="btn btn-%s">%s</button>',
sprintf(
"$('#%s%sForm').submit();",
h($model),
h(Inflector::classify($action))
),
empty($type) ? 'primary' : h($type),
empty($text) ? __('Submit') : h($text)
);
if ($ajax) {
echo sprintf(
'%s%s',
sprintf(
'<button id="submitButton" class="btn btn-primary" onClick="%s">%s</button>',
$ajaxSubmit,
__('Submit')
),
sprintf(
'<button class="btn" data-dismiss="modal" aria-hidden="true" onClick="%s">%s</button>',
'cancelPopoverForm();',
__('Cancel')
)
);
} else {
echo sprintf(
'<button onClick="%s" class="btn btn-%s">%s</button>',
sprintf(
"$('#%s%sForm').submit();",
h($model),
h(Inflector::classify($action))
),
empty($type) ? 'primary' : h($type),
empty($text) ? __('Submit') : h($text)
);
}
?>

18
app/View/Elements/genericElements/IndexTable/Fields/actions.ctp
View File

@ -10,9 +10,25 @@
* - postLinkConfirm: As the user to confirm the POST before submission with the given message
* - onClick: custom onClick action instead of a simple GET/POST request
* - icon: FA icon (added using the helper, knowing the fa domain is not needed, just add the short name such as "edit")
*/
* - requirement evaluates to true/false
* - complex_requirement - add complex requirements via lambda functions:
* - function($row, $options): the lambda function. $row contain the row data
* - options: array of options. datapaths described in the datapath keyname will be extracted and replaced with the actual row value
*/
echo '<td class="short action-links">';
foreach ($actions as $action) {
if (isset($action['complex_requirement'])) {
if (isset($action['complex_requirement']['options']['datapath'])) {
foreach ($action['complex_requirement']['options']['datapath'] as $name => $path) {
$action['complex_requirement']['options']['datapath'][$name] = Hash::extract($row, $path)[0];
}
}
$options = isset($action['complex_requirement']['options']) ? $action['complex_requirement']['options'] : array();
$requirementMet = $action['complex_requirement']['function']($row, $options);
if (!$requirementMet) {
continue;
}
}
$url_param_data_paths = '';
$url = empty($action['url']) ? '#' : h($action['url']);
if (!empty($action['url_params_data_paths'])) {

9
app/View/Elements/genericElements/SideMenu/side_menu.ctp
View File

@ -1047,6 +1047,15 @@
'url' => '/feeds/previewEvent/' . h($feed['Feed']['id']) . '/' . h($id),
'text' => __('PreviewEvent')
));
echo $this->element('/genericElements/SideMenu/side_menu_post_link', array(
'url' => sprintf(
'/feeds/getEvent/%s/%s',
h($feed['Feed']['id']),
h($event['Event']['uuid'])
),
'text' => __('Fetch This Event'),
'message' => __('Are you sure you want to fetch and save this event on your instance?')
));
}
}
break;

62
app/View/Elements/healthElements/db_indexes_diagnostic.ctp Normal file
View File

@ -0,0 +1,62 @@
<div>
<label for="toggleTableDBIndexes" style="display: inline-block;">
<input type="checkbox" id="toggleTableDBIndexes" class="form-input" checked></input>
<?php echo __('Show database indexes') ?>
</label>
</div>
<div id="containerDBIndexes" class="" style="max-height: 800px; overflow-y: auto; padding: 5px;">
<?php if(empty($diagnostic)): ?>
<span class="label label-success"><?php echo __('Index diagnostic:'); ?><i class="fa fa-check"></i></span>
<?php else: ?>
<div class="alert alert-warning">
<strong><?php echo __('Notice'); ?></strong>
<?php echo __('The highlighted issues may be benign. if you are unsure, please open an issue and ask for clarification.'); ?>
</div>
<table id="tableDBIndexes" class="table table-condensed table-bordered">
<thead>
<tr>
<th>Table name</th>
<th>Column name</th>
<th>Indexed</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<?php foreach($columnPerTable as $tableName => $columnArray): ?>
<?php
$columnCount = 0;
$rowHtml = '';
?>
<?php foreach($columnArray as $columnName): ?>
<?php
$columnIndexed = !empty($indexes[$tableName]) && in_array($columnName, $indexes[$tableName]);
$warning = isset($diagnostic[$tableName][$columnName]);
if ($warning) {
$columnCount++;
}
$rowHtml .= sprintf('%s%s%s%s%s',
sprintf('<tr class="%s">', $warning ? 'error' : 'indexInfo hidden'),
sprintf('<td>%s</td>', h($columnName)),
sprintf('<td><i class="bold fa %s"></i></td>', $columnIndexed ? 'green fa-check' : 'red fa-times'),
sprintf('<td>%s</td>', $warning ? h($diagnostic[$tableName][$columnName]) : ''),
'</tr>'
);
?>
<?php endforeach; ?>
<?php if ($columnCount > 0): ?>
<?php echo sprintf('<tr><td rowspan="%s" colspan="0" class="bold">%s</td></tr>', $columnCount+1, h($tableName)); ?>
<?php echo $rowHtml; ?>
<?php endif; ?>
<?php endforeach; ?>
</tbody>
</table>
<?php endif; ?>
</div>
<script>
$(document).ready(function() {
$('#toggleTableDBIndexes').change(function() {
$('#containerDBIndexes').toggle();
})
})
</script>

90
app/View/Elements/healthElements/db_schema_status.ctp → app/View/Elements/healthElements/db_schema_diagnostic.ctp
View File

@ -24,7 +24,6 @@
*/
function highlightAndSanitize($dirty, $toHighlight, $colorType = 'success')
{
if (is_array($dirty)) {
@ -46,24 +45,47 @@
?>
<?php
$hasAtLeastOneCriticalWarning = false;
foreach ($dbSchemaDiagnostics as $tableName => $tableDiagnostic) {
foreach ($tableDiagnostic as $i => $columnDiagnostic) {
if ($columnDiagnostic['is_critical']) {
$hasAtLeastOneCriticalWarning = true;
break;
}
}
if ($hasAtLeastOneCriticalWarning) {
break;
}
}
if (count($dbSchemaDiagnostics) > 0) {
echo sprintf('<span style="margin-bottom: 5px;" class="label label-important" title="%s">%s<i style="font-size: larger;" class="fas fa-times"></i></span>',
__('The current database schema does not match the expected format.'),
__('Database schema diagnostic: ')
);
echo sprintf('<div class="alert alert-error"><strong>%s</strong> %s <br/>%s</div>',
__('Critical warning!'),
__('The MISP database seems to be in an inconsistent state. Immediate attention is required.'),
__('⚠ This diagnostic tool is in experimental state - the highlighted issues may be benign. If you are unsure, please open an issue on with the issues identified over at https://github.com/MISP/MISP for clarification.')
);
if ($hasAtLeastOneCriticalWarning) {
echo sprintf('<div class="alert alert-error"><strong>%s</strong> %s <br/>%s</div>',
__('Warning'),
__('The MISP database state does not match the expected schema. Resolving these issues is recommended.'),
__('⚠ This diagnostic tool is in experimental state - the highlighted issues may be benign. If you are unsure, please open an issue on with the issues identified over at https://github.com/MISP/MISP for clarification.')
);
} else {
echo sprintf('<div class="alert alert-warning"><strong>%s</strong> %s <br/>%s</div>',
__('Warning'),
__('The MISP database state does not match the expected schema. Resolving these issues is recommended.'),
__('⚠ This diagnostic tool is in experimental state - the highlighted issues may be benign. If you are unsure, please open an issue on with the issues identified over at https://github.com/MISP/MISP for clarification.')
);
}
echo sprintf('<label class="checkbox"><input id="dbSchemaDiagnosticCheckbox" type="checkbox">%s</label>', __('Reveal benign deltas'));
$table = sprintf('%s%s%s',
'<table class="table table-bordered table-condensed">',
sprintf('<thead><th>%s</th><th>%s</th><th>%s</th><th>%s</th></thead>', __('Table name'), __('Description'), __('Expected schema'), __('Actual schema')),
'<table id="dbSchemaDiagnosticTable" class="table table-bordered table-condensed">',
sprintf('<thead><th>%s</th><th>%s</th><th>%s</th><th>%s</th><th></th></thead>', __('Table name'), __('Description'), __('Expected schema'), __('Actual schema')),
'<tbody>'
);
$rows = '';
foreach ($dbSchemaDiagnostics as $tableName => $tableDiagnostic) {
$rows .= '<tr>';
$rows .= sprintf('<tr data-tablename="%s">', $tableName);
$rows .= sprintf('<td rowspan="%s" colspan="0" class="bold">%s</td>', count($tableDiagnostic)+1, h($tableName));
$rows .= '</tr>';
@ -82,12 +104,21 @@
$saneActual = highlightAndSanitize($columnDiagnostic['actual'], $diffActual, 'important');
$uniqueRow = empty($saneExpected) && empty($saneActual);
$rows .= sprintf('<tr class="%s">', $columnDiagnostic['is_critical'] ? 'error' : '');
$rows .= sprintf('<tr class="%s" data-tablename="%s">', $columnDiagnostic['is_critical'] ? 'error critical' : 'noncritical', $tableName);
$rows .= sprintf('<td %s>%s</td>', $uniqueRow ? 'colspan=3' : '', $saneDescription);
if (!$uniqueRow) {
$rows .= sprintf('<td class="dbColumnDiagnosticRow" data-table="%s" data-index="%s">%s</td>', h($tableName), h($i), implode(' ', $saneExpected));
$rows .= sprintf('<td class="dbColumnDiagnosticRow" data-table="%s" data-index="%s">%s</td>', h($tableName), h($i), implode(' ', $saneActual));
}
$rows .= sprintf('<td class="" data-table="%s" data-index="%s">%s</td>', h($tableName), h($i),
empty($columnDiagnostic['sql']) ? '' :
sprintf(
'<i class="fa fa-wrench useCursorPointer" onclick="quickFixSchema(this, \'%s\')" title="%s" data-query="%s"></i>',
h($columnDiagnostic['sql']),
__('Fix Database schema'),
h($columnDiagnostic['sql'])
)
);
$rows .= '</tr>';
}
}
@ -130,13 +161,44 @@
__('Update are locked due to to many update fails') : sprintf(__('Update unlocked in %s'), h($humanReadableTime)))
: __('Updates are not locked'),
$updateLocked ? 'times' : 'check'
)
);
echo sprintf('<span class="label label-%s" title="%s" style="margin-left: 5px;">%s <i class="fas fa-%s"></i></span>',
$dataSource != 'Database/Mysql' ? 'important' : 'success',
__('DataSource: ') . h($dataSource),
__('DataSource: ') . h($dataSource),
$dataSource != 'Database/Mysql' ? 'times' : 'check'
);
echo $this->element('/healthElements/db_indexes_diagnostic', array(
'columnPerTable' => $columnPerTable,
'diagnostic' => $dbIndexDiagnostics,
'indexes' => $indexes
));
?>
<script>
var dbSchemaDiagnostics = <?php echo json_encode($dbSchemaDiagnostics); ?>;
var dbSchemaDiagnosticsColumns = <?php echo json_encode($checkedTableColumn); ?>;
function adjustRowSpan() {
$('#dbSchemaDiagnosticTable td[rowspan]').each(function() {
var tableindex = $(this).parent().data('tablename');
var childrenCount = $('#dbSchemaDiagnosticTable').find('[data-tablename="' + tableindex + '"]:visible').length;
$(this).attr('rowspan', childrenCount);
})
}
$(document).ready(function() {
// hide non-critical issues
if ($('#dbSchemaDiagnosticCheckbox').prop('checked')) {
$('#dbSchemaDiagnosticTable').find('tr.noncritical').show();
} else {
$('#dbSchemaDiagnosticTable').find('tr.noncritical').hide();
}
adjustRowSpan();
$('#dbSchemaDiagnosticCheckbox').change(function() {
$('#dbSchemaDiagnosticTable').find('tr.noncritical').toggle();
adjustRowSpan();
});
var popoverDiagnostic = $('td.dbColumnDiagnosticRow').popover({
title: '<?php echo __('Column diagnostic'); ?>',
content: function() {
@ -167,4 +229,10 @@ $(document).ready(function() {
trigger: 'hover'
});
});
function quickFixSchema(clicked, sqlQuery) {
var message = "<?php echo sprintf('<div class=\"alert alert-error\" style=\"margin-bottom: 5px;\"><h5>%s</h5> %s</div>', __('Warning'), __('Executing this query might take some time and may harm your database. Please review the query below or backup your database in case of doubt.')) ?>"
message += "<div class=\"well\"><kbd>" + sqlQuery + "</kbd></div>"
openPopover(clicked, message, undefined, 'left');
}
</script>

10
app/View/Elements/healthElements/diagnostics.ctp
View File

@ -227,8 +227,8 @@
echo '</div>';
?>
<h4><?php echo __('Schema status');?></h4>
<div style="width: 70vw; padding-left: 10px;">
<?php echo $this->element('/healthElements/db_schema_status', array(
<div id="schemaStatusDiv" style="width: 70vw; padding-left: 10px;">
<?php echo $this->element('/healthElements/db_schema_diagnostic', array(
'checkedTableColumn' => $dbSchemaDiagnostics['checked_table_column'],
'dbSchemaDiagnostics' => $dbSchemaDiagnostics['diagnostic'],
'expectedDbVersion' => $dbSchemaDiagnostics['expected_db_version'],
@ -236,7 +236,11 @@
'error' => $dbSchemaDiagnostics['error'],
'remainingLockTime' => $dbSchemaDiagnostics['remaining_lock_time'],
'updateFailNumberReached' => $dbSchemaDiagnostics['update_fail_number_reached'],
'updateLocked' => $dbSchemaDiagnostics['update_locked']
'updateLocked' => $dbSchemaDiagnostics['update_locked'],
'dataSource' => $dbSchemaDiagnostics['dataSource'],
'columnPerTable' => $dbSchemaDiagnostics['columnPerTable'],
'dbIndexDiagnostics' => $dbSchemaDiagnostics['diagnostic_index'],
'indexes' => $dbSchemaDiagnostics['indexes'],
)); ?>
</div>
<h3><?= __("Redis info") ?></h3>

13
app/View/Events/add.ctp
View File

@ -1,9 +1,10 @@
<?php
$modelForForm = 'Event';
$action = $this->request->params['action'];
echo $this->element('genericElements/Form/genericForm', array(
'form' => $this->Form,
'data' => array(
'title' => __('Add Event'),
'title' => $action === 'add' ? __('Add Event') : __('Edit Event'),
'model' => $modelForForm,
'fields' => array(
array(
@ -54,15 +55,17 @@
'field' => 'extends_uuid',
'class' => 'input span6',
'placeholder' => __('Event UUID or ID. Leave blank if not applicable.'),
'label' => __("Extends Event")
)
'label' => __("Extends Event"),
'default' => isset($extends_uuid) ? $extends_uuid : ''
),
'<div id="extended_event_preview" style="width:446px;"></div>'
),
'submit' => array(
'action' => $this->request->params['action']
'action' => $action
)
)
));
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event-collection', 'menuItem' => $this->action === 'add' ? 'add' : 'editEvent'));
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event-collection', 'menuItem' => $action === 'add' ? 'add' : 'editEvent'));
?>
<script type="text/javascript">

3
app/View/Events/resolved_misp_format.ctp
View File

@ -103,6 +103,7 @@
<tbody class='MISPObject'>
<tr class='tableHighlightBorderTop borderBlue blueRow' tabindex='0'>
<td colspan="7">
<input type="checkbox" class="ImportMISPObject" checked /> <span class="bold"><?php echo __('Import object');?></span><br />
<?php if(!empty($object['id'])) { ?>
<span class="bold"><?php echo __('ID: ');?></span><span class="ObjectID"><?php echo h($object['id']); ?></span><br />
<?php } ?>
@ -296,7 +297,7 @@
if (is_array($attribute[$field])) {
echo '<td class="short" style="width:40px;text-align:center;"><select ' . $field_header . 'Select" style="padding:0px;height:20px;margin-bottom:0px;">';
foreach ($attribute[$field] as $v => $value) {
echo '<option value="' . h($value) . '" ' . ($v ? '' : 'selected="selected"') . '>' . h($value) . '</option>';
echo '<option value="' . h($value) . '">' . h($value) . '</option>';
}
echo '</select></td>';
} else {

3
app/View/Events/view.ctp
View File

@ -388,7 +388,8 @@
'related' => $relatedEvent['Event'],
'color_red' => $relatedEvent['Event']['orgc_id'] == $me['org_id'],
'hide' => $count > $display_threshold,
'relatedEventCorrelationCount' => $relatedEventCorrelationCount
'relatedEventCorrelationCount' => $relatedEventCorrelationCount,
'from_id' => $event['Event']['id']
));
?>
<?php

10
app/View/Servers/ajax/db_schema_diagnostic.ctp Normal file
View File

@ -0,0 +1,10 @@
<?php echo $this->element('/healthElements/db_schema_diagnostic', array(
'checkedTableColumn' => $dbSchemaDiagnostics['checked_table_column'],
'dbSchemaDiagnostics' => $dbSchemaDiagnostics['diagnostic'],
'expectedDbVersion' => $dbSchemaDiagnostics['expected_db_version'],
'actualDbVersion' => $dbSchemaDiagnostics['actual_db_version'],
'error' => $dbSchemaDiagnostics['error'],
'remainingLockTime' => $dbSchemaDiagnostics['remaining_lock_time'],
'updateFailNumberReached' => $dbSchemaDiagnostics['update_fail_number_reached'],
'updateLocked' => $dbSchemaDiagnostics['update_locked']
)); ?>

2
app/View/Servers/index.ctp
View File

@ -159,7 +159,7 @@ foreach ($servers as $row_pos => $server):
echo sprintf('<a href="%s" title="%s" aria-label="%s" class="%s"></a>', $baseurl . '/servers/pull/' . h($server['Server']['id']) . '/update', __('Pull updates to events that already exist locally'), __('Pull updates'), 'fa fa-sync');
echo sprintf('<a href="%s" title="%s" aria-label="%s" class="%s"></a>', $baseurl . '/servers/pull/' . h($server['Server']['id']) . '/full', __('Pull all'), __('Pull all'), 'fa fa-arrow-circle-down');
}
if ($server['Server']['push']) {
if ($server['Server']['push'] || $server['Server']['push_sightings']) {
echo sprintf('<a href="%s" title="%s" aria-label="%s" class="%s"></a>', $baseurl . '/servers/push/' . h($server['Server']['id']) . '/full', __('Push all'), __('Push all'), 'fa fa-arrow-circle-up');
}
if ($server['Server']['caching_enabled']) {

15
app/View/Servers/view_deprecated_function_use.ctp
View File

@ -9,8 +9,19 @@
'<div class="bold" style="margin-left:8px">%s</div>%s',
h($action),
sprintf(
'<div style="margin-left:16px;"><span class="bold">Total</span>: %s %s</div>',
h($userData['total']),
'<div style="margin-left:16px;"><span class="bold">%s</span>: %s %s</div>',
__('Total'),
sprintf(
'%s Users (%s requests)',
sprintf(
'<span class="bold red">%s</span>',
count($userData) -1
),
sprintf(
'<span class="red">%s</span>',
h($userData['total'])
)
),
sprintf(
'<i class="fas fa-plus-circle" role="button" aria-label="%s" data-toggle="collapse" data-target="#deprecationDetails%s%s"></i>',
__('View details on the usage of %s on the %s controller', h($action), h($controller)),

2
app/View/Tags/index.ctp
View File

@ -72,7 +72,7 @@
<th><?php echo __('Tagged attributes');?></th>
<th><?php echo __('Activity');?></th>
<th><?php echo __('Favourite');?></th>
<?php if ($isAclTagEditor): ?>
<?php if ($isSiteAdmin): ?>
<th class="actions"><?php echo __('Actions');?></th>
<?php endif; ?>
</tr><?php

10
app/View/Users/statistics_orgs.ctp
View File

@ -31,9 +31,11 @@
<th><?php echo __('Name');?></th>
<th><?php echo __('Users');?></th>
<th><?php echo __('Events');?></th>
<th><?php echo __('Attributes');?></th>
<th><?php echo __('Nationality');?></th>
<th><?php echo __('Type');?></th>
<th><?php echo __('Sector');?></th>
<th><?php echo __('Activity (1 year)');?></th>
</tr>
<?php
foreach ($orgs as $data):
@ -47,9 +49,17 @@
<td class="short"><?php echo h($data['name']); ?></td>
<td class="short"><span class="<?php echo isset($data['userCount']) ? 'blue bold' : 'grey'; ?>"><?php echo isset($data['userCount']) ? h($data['userCount']) : '0';?></span></td>
<td class="short"><span class="<?php echo isset($data['eventCount']) ? 'blue bold' : 'grey'; ?>"><?php echo isset($data['eventCount']) ? h($data['eventCount']) : '0';?></span></td>
<td class="short"><span class="<?php echo isset($data['attributeCount']) ? 'blue bold' : 'grey'; ?>"><?php echo isset($data['attributeCount']) ? h($data['attributeCount']) : '0';?></span></td>
<td class="shortish"><?php echo isset($data['nationality']) && $data['nationality'] !== 'Not specified' ? h($data['nationality']) : '&nbsp;'; ?></td>
<td class="shortish"><?php echo isset($data['type']) ? h($data['type']) : '&nbsp;'; ?></td>
<td class="shortish"><?php echo isset($data['sector']) ? h($data['sector']) : '&nbsp;'; ?></td>
<td class="shortish">
<?php
if (isset($data['orgActivity'])) {
echo $this->element('sparkline', array('scope' => 'organisation', 'id' => $data['id'], 'csv' => $data['orgActivity']['csv']));
}
?>
</td>
</tr>
<?php
endforeach;

2
app/files/misp-decaying-models

@ -1 +1 @@
Subproject commit 0bf4d8bafc71f44cd65c7ad291bc0b4270dcecb8
Subproject commit f1b974790f0969fab4251046f292eedc59a272ca

2
app/files/misp-galaxy

@ -1 +1 @@
Subproject commit 8244d5ce622a75d42965d53ce2136ff400dc4b38
Subproject commit 5da0c7bd545ee93cf40786c1c535b9d4897943b1

2
app/files/misp-objects

@ -1 +1 @@
Subproject commit 6df0e18ddf6e2fe789ed339e7466072f8834de43
Subproject commit bce101832597b5f62679ac0299b2b4ef4681a145

74
app/files/scripts/misp2stix.py
View File

@ -89,44 +89,44 @@ class StixBuilder(object):
idgen.set_id_namespace(Namespace(self.baseurl, self.orgname, "MISP"))
self.namespace_prefix = idgen.get_id_namespace_alias()
## MAPPING FOR ATTRIBUTES
self.simple_type_to_method = {"port": self.generate_port_observable, "domain|ip": self.generate_domain_ip_observable,
"named pipe": self.generate_pipe_observable}
self.simple_type_to_method.update(dict.fromkeys(list(hash_type_attributes["single"]) + list(hash_type_attributes["composite"]) + ["filename"], self.resolve_file_observable))
self.simple_type_to_method.update(dict.fromkeys(["ip-src", "ip-dst"], self.generate_ip_observable))
self.simple_type_to_method.update(dict.fromkeys(["ip-src|port", "ip-dst|port", "hostname|port"], self.generate_socket_address_observable))
self.simple_type_to_method.update(dict.fromkeys(["regkey", "regkey|value"], self.generate_regkey_observable))
self.simple_type_to_method.update(dict.fromkeys(["hostname", "domain", "url", "AS", "mutex", "named pipe", "link", "windows-service-name"], self.generate_simple_observable))
self.simple_type_to_method.update(dict.fromkeys(["email-src", "email-dst", "email-subject", "email-reply-to"], self.resolve_email_observable))
self.simple_type_to_method.update(dict.fromkeys(["http-method", "user-agent"], self.resolve_http_observable))
self.simple_type_to_method.update(dict.fromkeys(["pattern-in-file", "pattern-in-traffic", "pattern-in-memory"], self.resolve_pattern_observable))
self.simple_type_to_method.update(dict.fromkeys(["mac-address"], self.resolve_system_observable))
self.simple_type_to_method.update(dict.fromkeys(["attachment"], self.resolve_attachment))
self.simple_type_to_method.update(dict.fromkeys(["email-attachment"], self.generate_email_attachment_observable))
self.simple_type_to_method.update(dict.fromkeys(["malware-sample"], self.resolve_malware_sample))
self.simple_type_to_method = {"port": 'generate_port_observable', "domain|ip": 'generate_domain_ip_observable',
"named pipe": 'generate_pipe_observable'}
self.simple_type_to_method.update(dict.fromkeys(list(hash_type_attributes["single"]) + list(hash_type_attributes["composite"]) + ["filename"], 'resolve_file_observable'))
self.simple_type_to_method.update(dict.fromkeys(["ip-src", "ip-dst"], 'generate_ip_observable'))
self.simple_type_to_method.update(dict.fromkeys(["ip-src|port", "ip-dst|port", "hostname|port"], 'generate_socket_address_observable'))
self.simple_type_to_method.update(dict.fromkeys(["regkey", "regkey|value"], 'generate_regkey_observable'))
self.simple_type_to_method.update(dict.fromkeys(["hostname", "domain", "url", "AS", "mutex", "named pipe", "link", "windows-service-name"], 'generate_simple_observable'))
self.simple_type_to_method.update(dict.fromkeys(["email-src", "email-dst", "email-subject", "email-reply-to"], 'resolve_email_observable'))
self.simple_type_to_method.update(dict.fromkeys(["http-method", "user-agent"], 'resolve_http_observable'))
self.simple_type_to_method.update(dict.fromkeys(["pattern-in-file", "pattern-in-traffic", "pattern-in-memory"], 'resolve_pattern_observable'))
self.simple_type_to_method.update(dict.fromkeys(["mac-address"], 'resolve_system_observable'))
self.simple_type_to_method.update(dict.fromkeys(["attachment"], 'resolve_attachment'))
self.simple_type_to_method.update(dict.fromkeys(["email-attachment"], 'generate_email_attachment_observable'))
self.simple_type_to_method.update(dict.fromkeys(["malware-sample"], 'resolve_malware_sample'))
## MAPPING FOR OBJECTS
self.ttp_names = {'attack-pattern': self.parse_attack_pattern,
'course-of-action': self.parse_course_of_action,
'vulnerability': self.parse_vulnerability,
'weakness': self.parse_weakness}
self.ttp_names = {'attack-pattern': 'parse_attack_pattern',
'course-of-action': 'parse_course_of_action',
'vulnerability': 'parse_vulnerability',
'weakness': 'parse_weakness'}
self.types_mapping = {CourseOfAction: 'add_course_of_action',
ThreatActor: 'add_threat_actor',
TTP: 'add_ttp'}
self.objects_mapping = {"asn": self.parse_asn_object,
"credential": self.parse_credential_object,
"domain-ip": self.parse_domain_ip_object,
"email": self.parse_email_object,
"file": self.parse_file_object,
"ip-port": self.parse_ip_port_object,
"network-connection": self.parse_network_connection_object,
"network-socket": self.parse_network_socket_object,
"pe": self.store_pe,
"pe-section": self.store_pe,
"process": self.parse_process_object,
"registry-key": self.parse_regkey_object,
"url": self.parse_url_object,
"user-account": self.parse_user_account_object,
"whois": self.parse_whois,
"x509": self.parse_x509_object}
self.objects_mapping = {"asn": 'parse_asn_object',
"credential": 'parse_credential_object',
"domain-ip": 'parse_domain_ip_object',
"email": 'parse_email_object',
"file": 'parse_file_object',
"ip-port": 'parse_ip_port_object',
"network-connection": 'parse_network_connection_object',
"network-socket": 'parse_network_socket_object',
"pe": 'store_pe',
"pe-section": 'store_pe',
"process": 'parse_process_object',
"registry-key": 'parse_regkey_object',
"url": 'parse_url_object',
"user-account": 'parse_user_account_object',
"whois": 'parse_whois',
"x509": 'parse_x509_object'}
def loadEvent(self):
pathname = os.path.dirname(self.args[0])
@ -274,11 +274,11 @@ class StixBuilder(object):
if name == 'original-imported-file':
continue
if name in self.ttp_names:
self.ttp_names[name](misp_object)
getattr(self, self.ttp_names[name])(misp_object)
else:
category = misp_object.get('meta-category')
try:
to_ids, observable = self.objects_mapping[name](misp_object)
to_ids, observable = getattr(self, self.objects_mapping[name])(misp_object)
except KeyError:
to_ids, observable = self.create_custom_observable(name, misp_object['Attribute'], misp_object['uuid'])
except TypeError:
@ -525,7 +525,7 @@ class StixBuilder(object):
attribute_type = attribute['type']
attribute_uuid = attribute['uuid']
try:
observable_property = self.simple_type_to_method[attribute_type](attribute)
observable_property = getattr(self, self.simple_type_to_method[attribute_type])(attribute)
except KeyError:
return False
if isinstance(observable_property, Observable):

74
app/files/scripts/stix2/misp2stix2.py
View File

@ -189,43 +189,43 @@ class StixBuilder():
def load_objects_mapping(self):
self.objects_mapping = {
'asn': {'observable': self.resolve_asn_observable,
'pattern': self.resolve_asn_pattern},
'credential': {'observable': self.resolve_credential_observable,
'pattern': self.resolve_credential_pattern},
'domain-ip': {'observable': self.resolve_domain_ip_observable,
'pattern': self.resolve_domain_ip_pattern},
'email': {'observable': self.resolve_email_object_observable,
'pattern': self.resolve_email_object_pattern},
'file': {'observable': self.resolve_file_observable,
'pattern': self.resolve_file_pattern},
'ip-port': {'observable': self.resolve_ip_port_observable,
'pattern': self.resolve_ip_port_pattern},
'network-connection': {'observable': self.resolve_network_connection_observable,
'pattern': self.resolve_network_connection_pattern},
'network-socket': {'observable': self.resolve_network_socket_observable,
'pattern': self.resolve_network_socket_pattern},
'process': {'observable': self.resolve_process_observable,
'pattern': self.resolve_process_pattern},
'registry-key': {'observable': self.resolve_regkey_observable,
'pattern': self.resolve_regkey_pattern},
'stix2': {'pattern': self.resolve_stix2_pattern},
'url': {'observable': self.resolve_url_observable,
'pattern': self.resolve_url_pattern},
'user-account': {'observable': self.resolve_user_account_observable,
'pattern': self.resolve_user_account_pattern},
'x509': {'observable': self.resolve_x509_observable,
'pattern': self.resolve_x509_pattern}
'asn': {'observable': 'resolve_asn_observable',
'pattern': 'resolve_asn_pattern'},
'credential': {'observable': 'resolve_credential_observable',
'pattern': 'resolve_credential_pattern'},
'domain-ip': {'observable': 'resolve_domain_ip_observable',
'pattern': 'resolve_domain_ip_pattern'},
'email': {'observable': 'resolve_email_object_observable',
'pattern': 'resolve_email_object_pattern'},
'file': {'observable': 'resolve_file_observable',
'pattern': 'resolve_file_pattern'},
'ip-port': {'observable': 'resolve_ip_port_observable',
'pattern': 'resolve_ip_port_pattern'},
'network-connection': {'observable': 'resolve_network_connection_observable',
'pattern': 'resolve_network_connection_pattern'},
'network-socket': {'observable': 'resolve_network_socket_observable',
'pattern': 'resolve_network_socket_pattern'},
'process': {'observable': 'resolve_process_observable',
'pattern': 'resolve_process_pattern'},
'registry-key': {'observable': 'resolve_regkey_observable',
'pattern': 'resolve_regkey_pattern'},
'stix2-pattern': {'pattern': 'resolve_stix2_pattern'},
'url': {'observable': 'resolve_url_observable',
'pattern': 'resolve_url_pattern'},
'user-account': {'observable': 'resolve_user_account_observable',
'pattern': 'resolve_user_account_pattern'},
'x509': {'observable': 'resolve_x509_observable',
'pattern': 'resolve_x509_pattern'}
}
def load_galaxy_mapping(self):
self.galaxies_mapping = {'branded-vulnerability': ['vulnerability', self.add_vulnerability_from_galaxy]}
self.galaxies_mapping.update(dict.fromkeys(attack_pattern_galaxies_list, ['attack-pattern', self.add_attack_pattern]))
self.galaxies_mapping.update(dict.fromkeys(course_of_action_galaxies_list, ['course-of-action', self.add_course_of_action]))
self.galaxies_mapping.update(dict.fromkeys(intrusion_set_galaxies_list, ['intrusion-set', self.add_intrusion_set]))
self.galaxies_mapping.update(dict.fromkeys(malware_galaxies_list, ['malware', self.add_malware]))
self.galaxies_mapping.update(dict.fromkeys(threat_actor_galaxies_list, ['threat-actor', self.add_threat_actor]))
self.galaxies_mapping.update(dict.fromkeys(tool_galaxies_list, ['tool', self.add_tool]))
self.galaxies_mapping = {'branded-vulnerability': ['vulnerability', 'add_vulnerability_from_galaxy']}
self.galaxies_mapping.update(dict.fromkeys(attack_pattern_galaxies_list, ['attack-pattern', 'add_attack_pattern']))
self.galaxies_mapping.update(dict.fromkeys(course_of_action_galaxies_list, ['course-of-action', 'add_course_of_action']))
self.galaxies_mapping.update(dict.fromkeys(intrusion_set_galaxies_list, ['intrusion-set', 'add_intrusion_set']))
self.galaxies_mapping.update(dict.fromkeys(malware_galaxies_list, ['malware', 'add_malware']))
self.galaxies_mapping.update(dict.fromkeys(threat_actor_galaxies_list, ['threat-actor', 'add_threat_actor']))
self.galaxies_mapping.update(dict.fromkeys(tool_galaxies_list, ['tool', 'add_tool']))
def get_object_by_uuid(self, uuid):
for _object in self.misp_event['Object']:
@ -373,7 +373,7 @@ class StixBuilder():
except Exception:
return
if galaxy_uuid not in self.galaxies:
to_call(galaxy)
getattr(self, to_call)(galaxy)
self.galaxies.append(galaxy_uuid)
self.relationships['defined'][source_id].append("{}--{}".format(stix_type, galaxy_uuid))
@ -619,7 +619,7 @@ class StixBuilder():
pattern = pattern_arg
else:
name = misp_object['name']
pattern = self.objects_mapping[name]['pattern'](misp_object['Attribute'], indicator_id)
pattern = getattr(self, self.objects_mapping[name]['pattern'])(misp_object['Attribute'], indicator_id)
category = misp_object.get('meta-category')
killchain = self.create_killchain(category)
labels = self.create_object_labels(name, category, True)
@ -638,7 +638,7 @@ class StixBuilder():
observable_objects = observable_arg
else:
name = misp_object['name']
observable_objects = self.objects_mapping[name]['observable'](misp_object['Attribute'], observed_data_id)
observable_objects = getattr(self, self.objects_mapping[name]['observable'])(misp_object['Attribute'], observed_data_id)
category = misp_object.get('meta-category')
labels = self.create_object_labels(name, category, False)
timestamp = self.get_datetime_from_timestamp(misp_object['timestamp'])

235
app/files/scripts/stix2/stix2misp.py
View File

@ -49,13 +49,14 @@ class StixParser():
self.filename = filename
self.stix_version = version
self.event = defaultdict(dict)
self.relationship = defaultdict(list)
self.event['relationship'] = defaultdict(list)
mapping = {'custom_object': self.__load_custom,
'marking-definition': self.__load_marking,
'relationship': self.__load_relationship,
'report': self.__load_report}
mapping.update({object_type: self.__load_usual_object for object_type in ('indicator', 'observed-data', 'vulnerability', 'course-of-action')})
mapping.update({galaxy_type: self.__load_galaxy for galaxy_type in galaxy_types})
mapping = {'custom_object': '_load_custom',
'marking-definition': '_load_marking',
'relationship': '_load_relationship',
'report': '_load_report'}
mapping.update({object_type: '_load_usual_object' for object_type in ('indicator', 'observed-data', 'vulnerability', 'course-of-action')})
mapping.update({galaxy_type: '_load_galaxy' for galaxy_type in galaxy_types})
for parsed_object in event.objects:
try:
object_type = parsed_object._type
@ -64,9 +65,9 @@ class StixParser():
if object_type.startswith('x-misp-object'):
object_type = 'custom_object'
try:
mapping[object_type](parsed_object)
getattr(self, mapping[object_type])(parsed_object)
except KeyError:
self.__load_usual_object(parsed_object)
self._load_usual_object(parsed_object)
try:
event_distribution = args[0]
if not isinstance(event_distribution, int):
@ -84,35 +85,31 @@ class StixParser():
self.misp_event.distribution = event_distribution
self._attribute_distribution = attribute_distribution
def __load_custom(self, parsed_object):
def _load_custom(self, parsed_object):
self.event['custom_object'][parsed_object['id'].split('--')[1]] = parsed_object
def __load_galaxy(self, parsed_object):
def _load_galaxy(self, parsed_object):
try:
self.galaxy[parsed_object['id'].split('--')[1]] = {'object': parsed_object, 'used': False}
except AttributeError:
self.galaxy = {parsed_object['id'].split('--')[1]: {'object': parsed_object, 'used': False}}
def __load_marking(self, parsed_object):
def _load_marking(self, parsed_object):
try:
self.marking_definition[parsed_object['id'].split('--')[1]] = {'object': parsed_object, 'used': False}
except AttributeError:
self.marking_definition = {parsed_object['id'].split('--')[1]: {'object': parsed_object, 'used': False}}
def __load_relationship(self, parsed_object):
try:
self.relationship[parsed_object.source_ref.split('--')[1]].append(parsed_object)
except AttributeError:
self.relationship = defaultdict(list)
self.relationship[parsed_object.source_ref.split('--')[1]].append(parsed_object)
def _load_relationship(self, parsed_object):
self.relationship[parsed_object.source_ref.split('--')[1]].append(parsed_object)
def __load_report(self, parsed_object):
def _load_report(self, parsed_object):
try:
self.report[parsed_object['id'].split('--')[1]] = parsed_object
except AttributeError:
self.report = {parsed_object['id'].split('--')[1]: parsed_object}
def __load_usual_object(self, parsed_object):
def _load_usual_object(self, parsed_object):
self.event[parsed_object._type][parsed_object['id'].split('--')[1]] = parsed_object
def general_handler(self):
@ -558,34 +555,34 @@ class StixParser():
class StixFromMISPParser(StixParser):
def __init__(self):
super(StixFromMISPParser, self).__init__()
self.objects_mapping = {'asn': {'observable': self.attributes_from_asn_observable, 'pattern': self.pattern_asn},
'credential': {'observable': self.observable_credential, 'pattern': self.pattern_credential},
'domain-ip': {'observable': self.attributes_from_domain_ip_observable, 'pattern': self.pattern_domain_ip},
'email': {'observable': self.observable_email, 'pattern': self.pattern_email},
'file': {'observable': self.observable_file, 'pattern': self.pattern_file},
'ip-port': {'observable': self.observable_ip_port, 'pattern': self.pattern_ip_port},
'network-connection': {'observable': self.observable_connection, 'pattern': self.pattern_connection},
'network-socket': {'observable': self.observable_socket, 'pattern': self.pattern_socket},
'process': {'observable': self.attributes_from_process_observable, 'pattern': self.pattern_process},
'registry-key': {'observable': self.attributes_from_regkey_observable, 'pattern': self.pattern_regkey},
'url': {'observable': self.attributes_from_url_observable, 'pattern': self.pattern_url},
'user-account': {'observable': self.attributes_from_user_account_observable,
'pattern': self.attributes_from_user_account_pattern},
'WindowsPEBinaryFile': {'observable': self.observable_pe, 'pattern': self.pattern_pe},
'x509': {'observable': self.attributes_from_x509_observable, 'pattern': self.pattern_x509}}
self.object_from_refs = {'course-of-action': self.parse_MISP_course_of_action, 'vulnerability': self.parse_vulnerability,
'custom_object': self.parse_custom}
self.object_from_refs.update(dict.fromkeys(['indicator', 'observed-data'], self.parse_usual_object))
self.attributes_fetcher_mapping = {'indicator': self.fetch_attributes_from_indicator,
'observed-data': self.fetch_attributes_from_observable,
'vulnerability': self.fetch_attributes_from_vulnerability}
self.objects_mapping = {'asn': {'observable': 'attributes_from_asn_observable', 'pattern': 'pattern_asn'},
'credential': {'observable': 'observable_credential', 'pattern': 'pattern_credential'},
'domain-ip': {'observable': 'attributes_from_domain_ip_observable', 'pattern': 'pattern_domain_ip'},
'email': {'observable': 'observable_email', 'pattern': 'pattern_email'},
'file': {'observable': 'observable_file', 'pattern': 'pattern_file'},
'ip-port': {'observable': 'observable_ip_port', 'pattern': 'pattern_ip_port'},
'network-connection': {'observable': 'observable_connection', 'pattern': 'pattern_connection'},
'network-socket': {'observable': 'observable_socket', 'pattern': 'pattern_socket'},
'process': {'observable': 'attributes_from_process_observable', 'pattern': 'pattern_process'},
'registry-key': {'observable': 'attributes_from_regkey_observable', 'pattern': 'pattern_regkey'},
'url': {'observable': 'attributes_from_url_observable', 'pattern': 'pattern_url'},
'user-account': {'observable': 'attributes_from_user_account_observable',
'pattern': 'attributes_from_user_account_pattern'},
'WindowsPEBinaryFile': {'observable': 'observable_pe', 'pattern': 'pattern_pe'},
'x509': {'observable': 'attributes_from_x509_observable', 'pattern': 'pattern_x509'}}
self.object_from_refs = {'course-of-action': 'parse_MISP_course_of_action', 'vulnerability': 'parse_vulnerability',
'custom_object': 'parse_custom'}
self.object_from_refs.update(dict.fromkeys(['indicator', 'observed-data'], 'parse_usual_object'))
self.attributes_fetcher_mapping = {'indicator': 'fetch_attributes_from_indicator',
'observed-data': 'fetch_attributes_from_observable',
'vulnerability': 'fetch_attributes_from_vulnerability'}
def handler(self):
self.general_handler()
def parsing_process(self, object2parse, object_type):
labels = object2parse.get('labels')
self.object_from_refs[object_type](object2parse, labels)
getattr(self, self.object_from_refs[object_type])(object2parse, labels)
################################################################################
## PARSING FUNCTIONS. ##
@ -665,7 +662,7 @@ class StixFromMISPParser(StixParser):
misp_object.uuid = uuid
misp_object['meta-category'] = object_category
try:
attributes = self.attributes_fetcher_mapping[stix_type](o, object_type)
attributes = getattr(self, self.attributes_fetcher_mapping[stix_type])(o, object_type)
except KeyError:
print("Unable to map {} object:\n{}".format(stix_type, o), file=sys.stderr)
return
@ -683,11 +680,11 @@ class StixFromMISPParser(StixParser):
pattern = indicator.pattern.replace('\\\\', '\\').split(' AND ')
pattern[0] = pattern[0][1:]
pattern[-1] = pattern[-1][:-1]
return self.objects_mapping[object_type]['pattern'](pattern)
return getattr(self, self.objects_mapping[object_type]['pattern'])(pattern)
def fetch_attributes_from_observable(self, observable, object_type):
observable = observable.objects
return self.objects_mapping[object_type]['observable'](observable)
return getattr(self, self.objects_mapping[object_type]['observable'])(observable)
def fetch_attributes_from_vulnerability(self, vulnerability, _):
attributes = []
@ -1102,52 +1099,52 @@ class StixFromMISPParser(StixParser):
class ExternalStixParser(StixParser):
def __init__(self):
super(ExternalStixParser, self).__init__()
self.object_from_refs = {'course-of-action': self.parse_course_of_action, 'vulnerability': self.parse_external_vulnerability,
'indicator': self.parse_external_indicator, 'observed-data': self.parse_external_observable}
self.observable_mapping = {('artifact', 'file'): self.parse_file_object_observable,
('autonomous-system',): self.parse_asn_observable,
('autonomous-system', 'ipv4-addr'): self.parse_asn_observable,
('autonomous-system', 'ipv6-addr'): self.parse_asn_observable,
('autonomous-system', 'ipv4-addr', 'ipv6-addr'): self.parse_asn_observable,
('domain-name',): self.parse_domain_ip_observable,
('domain-name', 'ipv4-addr'): self.parse_domain_ip_observable,
('domain-name', 'ipv6-addr'): self.parse_domain_ip_observable,
('domain-name', 'ipv4-addr', 'network-traffic'): self.parse_ip_port_or_network_socket_observable,
('domain-name', 'ipv6-addr', 'network-traffic'): self.parse_ip_port_or_network_socket_observable,
('domain-name', 'ipv4-addr', 'ipv6-addr', 'network-traffic'): self.parse_ip_port_or_network_socket_observable,
('domain-name', 'network-traffic'): self.parse_network_socket_observable,
('domain-name', 'network-traffic', 'url'): self.parse_url_object_observable,
('email-addr',): self.parse_email_address_observable,
('email-addr', 'email-message'): self.parse_email_observable,
('email-addr', 'email-message', 'file'): self.parse_email_observable,
('email-message',): self.parse_email_observable,
('file',): self.parse_file_observable,
('ipv4-addr',): self.parse_ip_address_observable,
('ipv6-addr',): self.parse_ip_address_observable,
('ipv4-addr', 'network-traffic'): self.parse_ip_network_traffic_observable,
('ipv6-addr', 'network-traffic'): self.parse_ip_network_traffic_observable,
('mac-addr',): self.parse_mac_address_observable,
('mutex',): self.parse_mutex_observable,
('process',): self.parse_process_observable,
('x509-certificate',): self.parse_x509_observable,
('url',): self.parse_url_observable,
('user-account',): self.parse_user_account_observable,
('windows-registry-key',): self.parse_regkey_observable}
self.pattern_mapping = {('directory',): self.parse_file_pattern,
('directory', 'file'): self.parse_file_pattern,
('domain-name',): self.parse_domain_ip_port_pattern,
('domain-name', 'ipv4-addr', 'url'): self.parse_domain_ip_port_pattern,
('domain-name', 'ipv6-addr', 'url'): self.parse_domain_ip_port_pattern,
('email-addr',): self.parse_email_address_pattern,
('file',): self.parse_file_pattern,
('ipv4-addr',): self.parse_ip_address_pattern,
('ipv6-addr',): self.parse_ip_address_pattern,
('network-traffic',): self.parse_network_traffic_pattern,
('process',): self.parse_process_pattern,
('url',): self.parse_url_pattern,
('user-account',): self.parse_user_account_pattern,
('windows-registry-key',): self.parse_regkey_pattern,
('x509-certificate',): self.parse_x509_pattern}
self.object_from_refs = {'course-of-action': 'parse_course_of_action', 'vulnerability': 'parse_external_vulnerability',
'indicator': 'parse_external_indicator', 'observed-data': 'parse_external_observable'}
self.observable_mapping = {('artifact', 'file'): 'parse_file_object_observable',
('autonomous-system',): 'parse_asn_observable',
('autonomous-system', 'ipv4-addr'): 'parse_asn_observable',
('autonomous-system', 'ipv6-addr'): 'parse_asn_observable',
('autonomous-system', 'ipv4-addr', 'ipv6-addr'): 'parse_asn_observable',
('domain-name',): 'parse_domain_ip_observable',
('domain-name', 'ipv4-addr'): 'parse_domain_ip_observable',
('domain-name', 'ipv6-addr'): 'parse_domain_ip_observable',
('domain-name', 'ipv4-addr', 'network-traffic'): 'parse_ip_port_or_network_socket_observable',
('domain-name', 'ipv6-addr', 'network-traffic'): 'parse_ip_port_or_network_socket_observable',
('domain-name', 'ipv4-addr', 'ipv6-addr', 'network-traffic'): 'parse_ip_port_or_network_socket_observable',
('domain-name', 'network-traffic'): 'parse_network_socket_observable',
('domain-name', 'network-traffic', 'url'): 'parse_url_object_observable',
('email-addr',): 'parse_email_address_observable',
('email-addr', 'email-message'): 'parse_email_observable',
('email-addr', 'email-message', 'file'): 'parse_email_observable',
('email-message',): 'parse_email_observable',
('file',): 'parse_file_observable',
('ipv4-addr',): 'parse_ip_address_observable',
('ipv6-addr',): 'parse_ip_address_observable',
('ipv4-addr', 'network-traffic'): 'parse_ip_network_traffic_observable',
('ipv6-addr', 'network-traffic'): 'parse_ip_network_traffic_observable',
('mac-addr',): 'parse_mac_address_observable',
('mutex',): 'parse_mutex_observable',
('process',): 'parse_process_observable',
('x509-certificate',): 'parse_x509_observable',
('url',): 'parse_url_observable',
('user-account',): 'parse_user_account_observable',
('windows-registry-key',): 'parse_regkey_observable'}
self.pattern_mapping = {('directory',): 'parse_file_pattern',
('directory', 'file'): 'parse_file_pattern',
('domain-name',): 'parse_domain_ip_port_pattern',
('domain-name', 'ipv4-addr', 'url'): 'parse_domain_ip_port_pattern',
('domain-name', 'ipv6-addr', 'url'): 'parse_domain_ip_port_pattern',
('email-addr',): 'parse_email_address_pattern',
('file',): 'parse_file_pattern',
('ipv4-addr',): 'parse_ip_address_pattern',
('ipv6-addr',): 'parse_ip_address_pattern',
('network-traffic',): 'parse_network_traffic_pattern',
('process',): 'parse_process_pattern',
('url',): 'parse_url_pattern',
('user-account',): 'parse_user_account_pattern',
('windows-registry-key',): 'parse_regkey_pattern',
('x509-certificate',): 'parse_x509_pattern'}
self.pattern_forbidden_relations = (' LIKE ', ' FOLLOWEDBY ', ' MATCHES ', ' ISSUBSET ', ' ISSUPERSET ', ' REPEATS ')
self.single_attribute_fields = ('type', 'value', 'to_ids')
@ -1157,7 +1154,7 @@ class ExternalStixParser(StixParser):
def parsing_process(self, object2parse, object_type):
try:
self.object_from_refs[object_type](object2parse)
getattr(self, self.object_from_refs[object_type])(object2parse)
except KeyError:
print("Unknown {} type: {}".format(self.stix_version, object_type), file=sys.stderr)
@ -1198,9 +1195,9 @@ class ExternalStixParser(StixParser):
to_call = self.observable_mapping[types]
observable_id = observable.id.split('--')[1]
if hasattr(observable, 'object_marking_refs'):
to_call(objects, observable_id, marking=observable.object_marking_refs)
getattr(self, to_call)(objects, observable_id, marking=observable.object_marking_refs)
else:
to_call(objects, observable_id)
getattr(self, to_call)(objects, observable_id)
except KeyError:
print('{} not parsed at the moment'.format(types), file=sys.stderr)
# deeper analyse to come, as well as for indicators
@ -1222,7 +1219,7 @@ class ExternalStixParser(StixParser):
for p in pattern:
type_ = tuple([p.split(' = ')[0].split(':')[0]])
try:
self.pattern_mapping[type_]([p.strip()], marking)
getattr(self, self.pattern_mapping[type_])([p.strip()], marking)
except KeyError:
print('{} not parsed at the moment'.format(type_), file=sys.stderr)
raise Exception
@ -1230,7 +1227,7 @@ class ExternalStixParser(StixParser):
pattern = [p.strip() for p in pattern.split(' AND ')]
types = self.parse_external_pattern_types(pattern)
try:
self.pattern_mapping[types](pattern, marking, uuid=uuid)
getattr(self, self.pattern_mapping[types])(pattern, marking, uuid=uuid)
except KeyError:
print('{} not parsed at the moment'.format(types), file=sys.stderr)
raise Exception
@ -1268,7 +1265,7 @@ class ExternalStixParser(StixParser):
## PARSING FUNCTIONS. ##
################################################################################
def add_attributes_from_observable(self, objects, attribute_type, identifier, marking, uuid):
def add_attributes_from_observable(self, objects, attribute_type, identifier, uuid, marking):
attribute = {'to_ids': False}
if len(objects) == 1:
attribute['uuid'] = uuid
@ -1321,11 +1318,11 @@ class ExternalStixParser(StixParser):
file_object.add_reference(pe_uuid, 'includes')
self.misp_event.add_object(**file_object)
def parse_asn_observable(self, objects, marking, uuid):
def parse_asn_observable(self, objects, uuid, marking=None):
attributes = self.attributes_from_asn_observable(objects)
self.handle_import_case(attributes, 'asn', marking, uuid)
def parse_domain_ip_observable(self, objects, marking, uuid):
def parse_domain_ip_observable(self, objects, uuid, marking=None):
attributes = self.attributes_from_domain_ip_observable(objects)
self.handle_import_case(attributes, 'domain-ip', marking, uuid)
@ -1337,7 +1334,7 @@ class ExternalStixParser(StixParser):
attributes = self.attributes_from_dict(values, domain_ip_mapping, True)
self.handle_import_case(attributes, 'domain-ip', marking, uuid)
def parse_email_observable(self, objects, marking, uuid):
def parse_email_observable(self, objects, uuid, marking=None):
to_ids = False
attributes, message = self.parse_complex_fields_observable_email(objects, to_ids)
for m_key, m_value in message.items():
@ -1345,7 +1342,7 @@ class ExternalStixParser(StixParser):
attributes.append(self.append_email_attribute(m_key, m_value, to_ids))
self.handle_import_case(attributes, 'email', marking, uuid)
def parse_email_address_observable(self, objects, marking, uuid):
def parse_email_address_observable(self, objects, uuid, marking=None):
mapping = to_attribute_mapping
attributes = [{'type': 'email-dst', 'object_relation': 'to', 'to_ids': True, 'value': _object.value} for _object in objects.values()]
self.handle_import_case(attributes, 'email', marking, uuid)
@ -1355,7 +1352,7 @@ class ExternalStixParser(StixParser):
attributes = self.fill_pattern_attributes(pattern_types, pattern_values, email_mapping)
self.handle_import_case(attributes, 'email', marking, uuid)
def parse_file_observable(self, objects, marking, uuid):
def parse_file_observable(self, objects, uuid, marking=None):
file, directory = self.split_file_observable(objects)
attributes = self.attributes_from_file_observable(file)
if directory is not None and directory.path:
@ -1375,7 +1372,7 @@ class ExternalStixParser(StixParser):
else:
self.handle_import_case(attributes, 'file', marking, uuid)
def parse_file_object_observable(self, objects, marking, uuid):
def parse_file_object_observable(self, objects, uuid, marking=None):
file, data = self.split_file_observable(objects)
attributes = self.attributes_from_file_observable(file, data)
if hasattr(file, 'extensions') and 'windows-pebinary-ext' in file.extensions:
@ -1383,27 +1380,27 @@ class ExternalStixParser(StixParser):
else:
self.handle_import_case(attributes, file._type, marking, uuid)
def parse_ip_address_observable(self, objects, marking, uuid):
self.add_attributes_from_observable(objects, 'ip-dst', 'value', marking, uuid)
def parse_ip_address_observable(self, objects, uuid, marking=None):
self.add_attributes_from_observable(objects, 'ip-dst', 'value', uuid, marking)
def parse_ip_address_pattern(self, pattern, marking=None, uuid=None):
self.add_attributes_from_pattern('ip-dst', pattern, marking, uuid)
def parse_ip_network_traffic_observable(self, objects, marking, uuid):
def parse_ip_network_traffic_observable(self, objects, uuid, marking=None):
attributes, name = self.attributes_from_network_traffic(objects)
self.handle_import_case(attributes, name, marking, uuid)
def parse_ip_port_or_network_socket_observable(self, objects, marking, uuid):
def parse_ip_port_or_network_socket_observable(self, objects, uuid, marking=None):
attributes, name = self.attributes_from_network_traffic(objects)
self.handle_import_case(attributes, name, marking, uuid)
def parse_mac_address_observable(self, objects, marking, uuid):
self.add_attributes_from_observable(objects, 'mac-address', 'value', marking, uuid)
def parse_mac_address_observable(self, objects, uuid, marking=None):
self.add_attributes_from_observable(objects, 'mac-address', 'value', uuid, marking)
def parse_mutex_observable(self, objects, marking, uuid):
self.add_attributes_from_observable(objects, 'mutex', 'name', marking, uuid)
def parse_mutex_observable(self, objects, uuid, marking=None):
self.add_attributes_from_observable(objects, 'mutex', 'name', uuid, marking)
def parse_network_socket_observable(self, objects, marking, uuid):
def parse_network_socket_observable(self, objects, uuid, marking=None):
attributes, name = self.attributes_from_network_traffic(objects)
self.handle_import_case(attributes, name, marking, uuid)
@ -1412,7 +1409,7 @@ class ExternalStixParser(StixParser):
attributes = self.fill_pattern_attributes(pattern_types, pattern_values, network_traffic_mapping)
self.handle_import_case(attributes, 'ip-port', marking, uuid)
def parse_process_observable(self, objects, marking, uuid):
def parse_process_observable(self, objects, uuid, marking=None):
attributes = self.attributes_from_process_observable(objects)
self.handle_import_case(attributes, 'process', marking, uuid)
@ -1421,7 +1418,7 @@ class ExternalStixParser(StixParser):
attributes = self.fill_pattern_attributes(pattern_types, pattern_values, process_mapping)
self.handle_import_case(attributes, 'process', marking, uuid)
def parse_regkey_observable(self, objects, marking, uuid):
def parse_regkey_observable(self, objects, uuid, marking=None):
_object = objects['0']
attributes = self.attributes_from_regkey_observable(_object)
self.handle_import_case(attributes, 'registry-key', marking, uuid)
@ -1431,17 +1428,17 @@ class ExternalStixParser(StixParser):
attributes = self.fill_pattern_attributes(pattern_types, pattern_values, regkey_mapping)
self.handle_import_case(attributes, 'registry-key', marking, uuid)
def parse_url_observable(self, objects, marking, uuid):
self.add_attributes_from_observable(objects, 'url', 'value', marking, uuid)
def parse_url_observable(self, objects, uuid, marking=None):
self.add_attributes_from_observable(objects, 'url', 'value', uuid, marking)
def parse_url_pattern(self, pattern, marking=None, uuid=None):
self.add_attributes_from_pattern('url', pattern, marking, uuid)
def parse_url_object_observable(self, objects, marking, uuid):
def parse_url_object_observable(self, objects, uuid, marking=None):
attributes = self.attributes_from_url_observable(objects)
self.handle_import_case(attributes, 'url', marking, uuid)
def parse_user_account_observable(self, observable, marking, uuid):
def parse_user_account_observable(self, observable, uuid, marking=None):
attributes = self.attributes_from_user_account_observable(observable)
name = self.__define_user_account_name(attributes)
self.handle_import_case(attributes, name, marking, uuid)
@ -1451,7 +1448,7 @@ class ExternalStixParser(StixParser):
name = self.__define_user_account_name(attributes)
self.handle_import_case(attributes, name, marking, uuid)
def parse_x509_observable(self, objects, marking, uuid):
def parse_x509_observable(self, objects, uuid, marking=None):
attributes = self.attributes_from_x509_observable(objects)
self.handle_import_case(attributes, 'x509', marking, uuid)
@ -1533,7 +1530,7 @@ class ExternalStixParser(StixParser):
values.append(value.strip().strip('\''))
return types, values
def handle_import_case(self, attributes, name, marking=None, uuid=None):
def handle_import_case(self, attributes, name, marking, uuid):
if len(attributes) == 1:
attribute = attributes[0]
attribute = {field: attribute[field] for field in self.single_attribute_fields if attribute.get(field)}

74
app/files/scripts/stix2misp.py
View File

@ -100,41 +100,42 @@ class StixParser():
# Load the mapping dictionary for STIX object types
def load_mapping(self):
self.attribute_types_mapping = {
"AccountObjectType": self.handle_credential,
'AddressObjectType': self.handle_address,
"ArtifactObjectType": self.handle_attachment,
"ASObjectType": self.handle_as,
"CustomObjectType": self.handle_custom,
"DNSRecordObjectType": self.handle_dns,
'DomainNameObjectType': self.handle_domain_or_url,
'EmailMessageObjectType': self.handle_email_attribute,
'FileObjectType': self.handle_file,
'HostnameObjectType': self.handle_hostname,
'HTTPSessionObjectType': self.handle_http,
'MutexObjectType': self.handle_mutex,
'NetworkConnectionObjectType': self.handle_network_connection,
'NetworkSocketObjectType': self.handle_network_socket,
'PDFFileObjectType': self.handle_file,
'PipeObjectType': self.handle_pipe,
'PortObjectType': self.handle_port,
'ProcessObjectType': self.handle_process,
'SocketAddressObjectType': self.handle_socket_address,
'SystemObjectType': self.handle_system,
'UnixUserAccountObjectType': self.handle_unix_user,
'URIObjectType': self.handle_domain_or_url,
'UserAccountObjectType': self.handle_user,
"WhoisObjectType": self.handle_whois,
"WindowsFileObjectType": self.handle_file,
'WindowsRegistryKeyObjectType': self.handle_regkey,
"WindowsExecutableFileObjectType": self.handle_pe,
"WindowsServiceObjectType": self.handle_windows_service,
'WindowsUserAccountObjectType': self.handle_windows_user,
"X509CertificateObjectType": self.handle_x509
"AccountObjectType": 'handle_credential',
'AddressObjectType': 'handle_address',
"ArtifactObjectType": 'handle_attachment',
"ASObjectType": 'handle_as',
"CustomObjectType": 'handle_custom',
"DNSRecordObjectType": 'handle_dns',
'DomainNameObjectType': 'handle_domain_or_url',
'EmailMessageObjectType': 'handle_email_attribute',
'FileObjectType': 'handle_file',
'HostnameObjectType': 'handle_hostname',
'HTTPSessionObjectType': 'handle_http',
'LinkObjectType': 'handle_link',
'MutexObjectType': 'handle_mutex',
'NetworkConnectionObjectType': 'handle_network_connection',
'NetworkSocketObjectType': 'handle_network_socket',
'PDFFileObjectType': 'handle_file',
'PipeObjectType': 'handle_pipe',
'PortObjectType': 'handle_port',
'ProcessObjectType': 'handle_process',
'SocketAddressObjectType': 'handle_socket_address',
'SystemObjectType': 'handle_system',
'UnixUserAccountObjectType': 'handle_unix_user',
'URIObjectType': 'handle_domain_or_url',
'UserAccountObjectType': 'handle_user',
"WhoisObjectType": 'handle_whois',
"WindowsFileObjectType": 'handle_file',
'WindowsRegistryKeyObjectType': 'handle_regkey',
"WindowsExecutableFileObjectType": 'handle_pe',
"WindowsServiceObjectType": 'handle_windows_service',
'WindowsUserAccountObjectType': 'handle_windows_user',
"X509CertificateObjectType": 'handle_x509'
}
self.marking_mapping = {
'AIS:AISMarkingStructure': self.parse_AIS_marking,
'tlpMarking:TLPMarkingStructureType': self.parse_TLP_marking
'AIS:AISMarkingStructure': 'parse_AIS_marking',
'tlpMarking:TLPMarkingStructureType': 'parse_TLP_marking'
}
def parse_marking(self, handling):
@ -142,7 +143,7 @@ class StixParser():
if hasattr(handling, 'marking_structures') and handling.marking_structures:
for marking in handling.marking_structures:
try:
tags.extend(self.marking_mapping[marking._XSI_TYPE](marking))
tags.extend(getattr(self, self.marking_mapping[marking._XSI_TYPE])(marking))
except KeyError:
print(marking._XSI_TYPE, file=sys.stderr)
continue
@ -225,7 +226,7 @@ class StixParser():
args.append(is_object)
elif xsi_type == "ArtifactObjectType":
args.append(title)
return self.attribute_types_mapping[xsi_type](*args)
return getattr(self, self.attribute_types_mapping[xsi_type])(*args)
# except AttributeError:
# # ATM USED TO TEST TYPES
# print("Unparsed type: {}".format(xsi_type))
@ -413,6 +414,11 @@ class StixParser():
value = client_request.http_request_line.http_method.value
return "http-method", value, "method"
# Return type & value of a link attribute
@staticmethod
def handle_link(properties):
return "link", properties.value.value, "link"
# Return type & value of a mutex attribute
@staticmethod
def handle_mutex(properties):

2
app/files/taxonomies

@ -1 +1 @@
Subproject commit 336ceabf31fdacb1a7e1a7b20c1c42ee795fdc0d
Subproject commit d5cc5db3d736e5acede93d514070636834f385d4

2
app/files/warninglists

@ -1 +1 @@
Subproject commit 72732074f1baf79a0a1c34292859b6aed3860f0b
Subproject commit eee124de3401f9f957bb30a5a4989d422324eca8

11
app/webroot/css/main.css
View File

@ -880,6 +880,10 @@ a.proposal_link_red:hover {
border: 0;
}
.modal-body-long {
max-height: 600px;
}
.ajax_popover_form {
display:none;
width: 700px;
@ -942,7 +946,7 @@ a.proposal_link_red:hover {
border-radius: 10px 10px 0px 0px;
padding-left:10px;
margin-bottom:0px;
width:690px;
width:calc(100% - 10px);
background-color:black;
color:white;
font-size: 21px;
@ -976,8 +980,9 @@ a.proposal_link_red:hover {
height:120px !important;
}
.ajax_popover_form .add_attribute_fields {
.ajax_fieldset, .ajax_popover_form .add_attribute_fields {
padding-left:10px;
padding-right:10px;
}
.hover_enrichment_text {
@ -2431,4 +2436,4 @@ table tr:hover .down-expand-button {
background-color: #3b3b3b;
color: white;
border-radius: 3px;
}
}

9
app/webroot/js/decayingModelSimulation.js
View File

@ -504,7 +504,7 @@
this._draw();
},
_create_all_tag_html: function(tag) {
_create_all_tag_html: function(tag, row_i) {
var that = this;
if (tag !== false) {
var html_tag = this._create_tag_html(tag);
@ -517,7 +517,7 @@
}
});
if (overridden_html !== '') {
return '<div style="position:relative;" class="useCursorPointer overridden_tags_container">'
return '<div style="position:relative;" class="useCursorPointer overridden_tags_container" data-row="' + row_i + '">'
+ overridden_html
+ '<div class="attribute_tag_wrapper" style="top:-12px;margin-bottom:-12px; left:4px;margin-right:-4px; float: left; position: relative;">' + html_tag + '</div>'
+ '</div>';
@ -583,7 +583,7 @@
.data(function (tag, row_i) {
var html_computation = that._get_computation_step(tag);
return [
that._create_all_tag_html(tag),
that._create_all_tag_html(tag, row_i),
html_computation[0], html_computation[1], html_computation[2], html_computation[3]
]
});
@ -602,8 +602,9 @@
.style('opacity', 1.0)
.each("end", function(td_content, col_i){
var $div = $(td_content);
var row_i = $div.data('row');
if (col_i == 0 && $div.hasClass('overridden_tags_container')) {
$('.overridden_tags_container').popover({
$('.overridden_tags_container[data-row="' + row_i + '"]').popover({
title: 'Event tag overridden by Attribute tag',
content: that._generateOverridenExplanationPopoverHTML($div),
html: true,

7
app/webroot/js/event-graph.js
View File

@ -1089,6 +1089,11 @@ class EventGraph {
// Do not link already connected nodes
if (that.network.getConnectedEdges(cur_id).length > 0) {
if (nodeData['unreferenced'] !== undefined) {
that.nodes.remove(nodeData.id);
delete nodeData['unreferenced'];
that.nodes.add(nodeData);
}
return;
}
@ -2100,7 +2105,7 @@ function enable_interactive_graph() {
eventGraph = new EventGraph(network_options, nodes, edges);
$(document).on("keydown", function(evt) {
if (evt.target !== undefined && $(evt.target).is('input')) {
if (evt.target !== undefined && ($(evt.target).is('input') || $(evt.target).is('textarea'))) {
return;
}
switch(evt.keyCode) {

198
app/webroot/js/misp.js
View File

@ -1124,110 +1124,150 @@ function clickCreateButton(event, type) {
simplePopup("/" + destination + "/add/" + event);
}
function submitPopoverForm(context_id, referer, update_context_id) {
function openGenericModal(url) {
$.ajax({
type: "get",
url: url,
success: function (data) {
$('#genericModal').remove();
$('body').append(data);
$('#genericModal').modal();
},
error: function (data, textStatus, errorThrown) {
showMessage('fail', textStatus + ": " + errorThrown);
}
});
}
function submitPopoverForm(context_id, referer, update_context_id, modal) {
var url = null;
var context = 'event';
var contextNamingConvention = 'Attribute';
var closePopover = true;
switch (referer) {
case 'add':
url = "/attributes/add/" + context_id;
break;
case 'edit':
url = "/attributes/edit/" + context_id;
break;
case 'propose':
url = "/shadow_attributes/add/" + context_id;
break;
case 'massEdit':
url = "/attributes/editSelected/" + context_id;
break;
case 'addTextElement':
url = "/templateElements/add/text/" + context_id;
context = 'template';
contextNamingConvention = 'TemplateElementText';
break;
case 'editTextElement':
url = "/templateElements/edit/text/" + context_id;
context = 'template';
context_id = update_context_id;
contextNamingConvention = 'TemplateElementText';
break;
case 'addAttributeElement':
url = "/templateElements/add/attribute/" + context_id;
context = 'template';
contextNamingConvention = 'TemplateElementAttribute';
break;
case 'editAttributeElement':
url = "/templateElements/edit/attribute/" + context_id;
context = 'template';
context_id = update_context_id;
contextNamingConvention = 'TemplateElementAttribute';
break;
case 'addFileElement':
url = "/templateElements/add/file/" + context_id;
context = 'template';
contextNamingConvention = 'TemplateElementFile';
break;
case 'editFileElement':
url = "/templateElements/edit/file/" + context_id;
context = 'template';
context_id = update_context_id;
contextNamingConvention = 'TemplateElementFile';
break;
case 'replaceAttributes':
url = "/attributes/attributeReplace/" + context_id;
break;
case 'addSighting':
url = "/sightings/add/" + context_id;
closePopover = false;
break;
case 'addObjectReference':
url = "/objectReferences/add/" + context_id;
break;
}
if (url !== null) {
url = baseurl + url;
$.ajax({
beforeSend: function (XMLHttpRequest) {
$(".loading").show();
if ($("#submitButton").parent().hasClass('modal-footer')) {
var $form = $("#submitButton").parent().parent().find('.modal-body form');
url = baseurl + $form.attr('action');
} else {
var $form = $("#submitButton").closest("form");
url = baseurl + $form.attr('action');
}
$.ajax({
beforeSend: function (XMLHttpRequest) {
if (modal) {
if (closePopover) {
$('#genericModal').modal('hide');
}
} else {
if (closePopover) {
$("#gray_out").fadeOut();
$("#popover_form").fadeOut();
$(".loading").show();
}
},
data: $("#submitButton").closest("form").serialize(),
success:function (data, textStatus) {
var result;
if (closePopover) {
}
},
data: $form.serialize(),
success:function (data, textStatus) {
var result;
if (closePopover) {
if (modal) {
result = handleAjaxModalResponse(data, context_id, url, referer, context, contextNamingConvention);
} else {
result = handleAjaxPopoverResponse(data, context_id, url, referer, context, contextNamingConvention);
}
if (referer == 'addSighting') {
updateIndex(update_context_id, 'event');
$.get( "/sightings/listSightings/" + id + "/attribute", function(data) {
$("#sightingsData").html(data);
});
$('.sightingsToggle').removeClass('btn-primary');
$('.sightingsToggle').addClass('btn-inverse');
$('#sightingsListAllToggle').removeClass('btn-inverse');
$('#sightingsListAllToggle').addClass('btn-primary');
}
if (context == 'event' && (referer == 'add' || referer == 'massEdit' || referer == 'replaceAttributes' || referer == 'addObjectReference')) eventUnpublish();
},
error: function (jqXHR, textStatus, errorThrown) {
showMessage('fail', textStatus + ": " + errorThrown);
},
complete: function () {
$(".loading").hide();
},
type: "post",
url: url,
});
}
}
if (referer == 'addSighting') {
updateIndex(update_context_id, 'event');
$.get( "/sightings/listSightings/" + id + "/attribute", function(data) {
$("#sightingsData").html(data);
});
$('.sightingsToggle').removeClass('btn-primary');
$('.sightingsToggle').addClass('btn-inverse');
$('#sightingsListAllToggle').removeClass('btn-inverse');
$('#sightingsListAllToggle').addClass('btn-primary');
}
if (context == 'event' && (referer == 'add' || referer == 'massEdit' || referer == 'replaceAttributes' || referer == 'addObjectReference')) eventUnpublish();
},
error: function (jqXHR, textStatus, errorThrown) {
showMessage('fail', textStatus + ": " + errorThrown);
},
complete: function () {
$(".loading").hide();
},
type: "post",
url: url,
});
return false;
};
function handleAjaxModalResponse(response, context_id, url, referer, context, contextNamingConvention) {
responseArray = response;
var message = null;
var result = "fail";
if (responseArray.saved) {
updateIndex(context_id, context);
if (responseArray.success) {
showMessage("success", responseArray.success);
result = "success";
}
if (responseArray.errors) {
showMessage("fail", responseArray.errors);
}
} else {
var savedArray = saveValuesForPersistance();
$.ajax({
async:true,
dataType:"html",
success:function (data, textStatus) {
$('#genericModal').remove();
$('body').append(data);
$('#genericModal').modal();
var error_context = context.charAt(0).toUpperCase() + context.slice(1);
handleValidationErrors(responseArray.errors, context, contextNamingConvention);
result = "success";
if (!$.isEmptyObject(responseArray)) {
result = "fail";
}
recoverValuesFromPersistance(savedArray);
$(".loading").hide();
},
url:url
});
}
return result;
}
function handleAjaxPopoverResponse(response, context_id, url, referer, context, contextNamingConvention) {
responseArray = response;
var message = null;
@ -1687,13 +1727,15 @@ function popoverPopup(clicked, id, context, target, admin) {
}
// create a confirm popover on the clicked html node.
function popoverConfirm(clicked) {
function popoverConfirm(clicked, message, placement) {
var $clicked = $(clicked);
var popoverContent = '<div>';
popoverContent += '<button id="popoverConfirmOK" class="btn btn-primary" onclick=submitPopover(this)>Yes</button>';
popoverContent += message === undefined ? '' : '<p>' + message + '</p>';
popoverContent += '<button id="popoverConfirmOK" class="btn btn-primary" style="margin-right: 5px;" onclick=submitPopover(this)>Yes</button>';
popoverContent += '<button class="btn btn-inverse" style="float: right;" onclick=cancelPrompt()>Cancel</button>';
popoverContent += '</div>';
openPopover($clicked, popoverContent);
placement = placement === undefined ? 'auto' : placement;
openPopover($clicked, popoverContent, undefined, placement);
$("#popoverConfirmOK")
.focus()
.bind("keydown", function(e) {
@ -1713,7 +1755,30 @@ function submitPopover(clicked) {
var dismissid = $clicked.closest('div.popover').attr('data-dismissid');
$form = $('[data-dismissid="' + dismissid + '"]').closest('form');
}
$form.submit();
if ($form.data('ajax')) {
$.ajax({
data: $form.serialize(),
beforeSend: function (XMLHttpRequest) {
$(".loading").show();
},
success:function (data, textStatus) {
location.reload();
},
error:function() {
showMessage('fail', 'Could not perform query.');
},
complete:function() {
$(".loading").hide();
$("#popover_form").fadeOut();
$("#gray_out").fadeOut();
$('#temp').remove();
},
type:"post",
url: $form.attr('action')
});
} else {
$form.submit();
}
}
function simplePopup(url) {
@ -2537,12 +2602,16 @@ function moduleResultsSubmit(id) {
var object_uuid = $(this).find('.ObjectUUID').text();
temp = {
uuid: object_uuid,
import_object: $(this).find('.ImportMISPObject')[0].checked,
name: $(this).find('.ObjectName').text(),
meta_category: $(this).find('.ObjectMetaCategory').text(),
distribution: $(this).find('.ObjectDistribution').val(),
sharing_group_id: $(this).find('.ObjectSharingGroup').val(),
comment: $(this).find('.ObjectComment').val()
}
if (!temp['import_object']) {
return true;
}
if (temp['distribution'] != '4') {
temp['sharing_group_id'] = '0';
}
@ -3182,6 +3251,9 @@ function testConnection(id) {
case 7:
$("#connection_test_" + id).html('<span class="red bold" title="The user account on the remote instance is not a sync user.">Remote user not a sync user</span>');
break;
case 8:
$("#connection_test_" + id).html('<span class="orange bold" title="The user account on the remote instance is only a sightings user.">Syncing sightings only</span>');
break;
}
}
})

2
db_schema.json
View File

File diff suppressed because one or more lines are too long

3
docs/INSTALL.rhel7.md
View File

@ -317,8 +317,9 @@ installCake_RHEL ()
sudo chown $WWW_USER:$WWW_USER /usr/share/httpd/.composer
cd $PATH_TO_MISP/app
# Update composer.phar (optional)
#EXPECTED_SIGNATURE="$(wget -q -O - https://composer.github.io/installer.sig)"
#$SUDO_WWW $RUN_PHP -- php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
#$SUDO_WWW $RUN_PHP -- php -r "if (hash_file('SHA384', 'composer-setup.php') === '48e3236262b34d30969dca3c37281b3b4bbe3221bda826ac6a9a62d6444cdb0dcd0615698a5cbe587c3f0fe57a54d8f5') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
#$SUDO_WWW $RUN_PHP -- php -r "if (hash_file('SHA384', 'composer-setup.php') === '$EXPECTED_SIGNATURE') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
#$SUDO_WWW $RUN_PHP "php composer-setup.php"
#$SUDO_WWW $RUN_PHP -- php -r "unlink('composer-setup.php');"
$SUDO_WWW $RUN_PHP "php composer.phar install"

2
docs/INSTALL.rhel8.md
View File

@ -304,7 +304,7 @@ installCake_RHEL ()
cd $PATH_TO_MISP/app
# Update composer.phar (optional)
$SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === 'a5c698ffe4b8e849a443b120cd5ba38043260d5c4023dbf93e1558871f1f07f58274fc6f4c93bcfd858c6bd0775cd8d1') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === 'baf1608c33254d00611ac1705c1d9958c817a1a33bce370c0595974b342601bd80b92a3f46067da89e3b06bff421f182') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
$SUDO_WWW php composer-setup.php
$SUDO_WWW php -r "unlink('composer-setup.php');"
$SUDO_WWW php composer.phar install

4
docs/generic/globalVariables.md
View File

@ -85,9 +85,9 @@ MISPvars () {
# sudo config to run $LUSER commands
if [[ "$(groups ${MISP_USER} |grep -o 'staff')" == "staff" ]]; then
SUDO_USER="sudo -H -u ${MISP_USER} -g staff"
SUDO_CMD="sudo -H -u ${MISP_USER} -g staff"
else
SUDO_USER="sudo -H -u ${MISP_USER}"
SUDO_CMD="sudo -H -u ${MISP_USER}"
fi
SUDO_WWW="sudo -H -u ${WWW_USER} "

26
docs/generic/mail_to_misp-debian.md
View File

@ -8,29 +8,29 @@ mail2misp () {
debug "Installing Mail2${LBLUE}MISP${NC}"
cd /usr/local/src/
sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
$SUDO_USER git clone https://github.com/MISP/mail_to_misp.git
$SUDO_USER git clone git://github.com/stricaud/faup.git faup
$SUDO_USER git clone git://github.com/stricaud/gtcaca.git gtcaca
$SUDO_CMD git clone https://github.com/MISP/mail_to_misp.git
$SUDO_CMD git clone git://github.com/stricaud/faup.git faup
$SUDO_CMD git clone git://github.com/stricaud/gtcaca.git gtcaca
sudo chown -R ${MISP_USER}:${MISP_USER} faup mail_to_misp gtcaca
cd gtcaca
$SUDO_USER mkdir -p build
$SUDO_CMD mkdir -p build
cd build
$SUDO_USER cmake .. && $SUDO_USER make
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
$SUDO_USER mkdir -p build
$SUDO_CMD mkdir -p build
cd build
$SUDO_USER cmake .. && $SUDO_USER make
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
cd ../../mail_to_misp
$SUDO_USER virtualenv -p python3 venv
$SUDO_USER ./venv/bin/pip install https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
$SUDO_USER ./venv/bin/pip install -r requirements.txt
$SUDO_USER cp mail_to_misp_config.py-example mail_to_misp_config.py
$SUDO_CMD virtualenv -p python3 venv
$SUDO_CMD ./venv/bin/pip install https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
$SUDO_CMD ./venv/bin/pip install -r requirements.txt
$SUDO_CMD cp mail_to_misp_config.py-example mail_to_misp_config.py
##$SUDO cp mail_to_misp_config.py-example mail_to_misp_config.py
$SUDO_USER sed -i "s/^misp_url\ =\ 'YOUR_MISP_URL'/misp_url\ =\ 'https:\/\/localhost'/g" /usr/local/src/mail_to_misp/mail_to_misp_config.py
$SUDO_USER sed -i "s/^misp_key\ =\ 'YOUR_KEY_HERE'/misp_key\ =\ '${AUTH_KEY}'/g" /usr/local/src/mail_to_misp/mail_to_misp_config.py
$SUDO_CMD sed -i "s/^misp_url\ =\ 'YOUR_MISP_URL'/misp_url\ =\ 'https:\/\/localhost'/g" /usr/local/src/mail_to_misp/mail_to_misp_config.py
$SUDO_CMD sed -i "s/^misp_key\ =\ 'YOUR_KEY_HERE'/misp_key\ =\ '${AUTH_KEY}'/g" /usr/local/src/mail_to_misp/mail_to_misp_config.py
}
# <snippet-end 5_mail_to_misp.sh>
```

2
docs/generic/misp-modules-debian.md
View File

@ -7,7 +7,7 @@ mispmodules () {
cd /usr/local/src/
## TODO: checkUsrLocalSrc in main doc
debug "Cloning misp-modules"
$SUDO_USER git clone https://github.com/MISP/misp-modules.git
$SUDO_CMD git clone https://github.com/MISP/misp-modules.git
cd misp-modules
# some misp-modules dependencies
sudo apt install libpq5 libjpeg-dev tesseract-ocr libpoppler-cpp-dev imagemagick libopencv-dev zbar-tools libzbar0 libzbar-dev libfuzzy-dev -y

8
docs/generic/ssdeep-debian.md
View File

@ -5,11 +5,11 @@
ssdeep () {
debug "Install ssdeep 2.14.1"
cd /usr/local/src
$SUDO_USER wget https://github.com/ssdeep-project/ssdeep/releases/download/release-2.14.1/ssdeep-2.14.1.tar.gz
$SUDO_USER tar zxvf ssdeep-2.14.1.tar.gz
$SUDO_CMD wget https://github.com/ssdeep-project/ssdeep/releases/download/release-2.14.1/ssdeep-2.14.1.tar.gz
$SUDO_CMD tar zxvf ssdeep-2.14.1.tar.gz
cd ssdeep-2.14.1
$SUDO_USER ./configure --datadir=/usr --prefix=/usr --localstatedir=/var --sysconfdir=/etc
$SUDO_USER make
$SUDO_CMD ./configure --datadir=/usr --prefix=/usr --localstatedir=/var --sysconfdir=/etc
$SUDO_CMD make
sudo make install
#installing ssdeep_php

18
docs/generic/supportFunctions.md
View File

@ -385,12 +385,12 @@ checkID () {
sudo adduser $MISP_USER $WWW_USER
fi
# FIXME: the below SUDO_USER check is a duplicate from global variables, try to have just one check
# FIXME: the below SUDO_CMD check is a duplicate from global variables, try to have just one check
# sudo config to run $LUSER commands
if [[ "$(groups ${MISP_USER} |grep -o 'staff')" == "staff" ]]; then
SUDO_USER="sudo -H -u ${MISP_USER} -g staff"
SUDO_CMD="sudo -H -u ${MISP_USER} -g staff"
else
SUDO_USER="sudo -H -u ${MISP_USER}"
SUDO_CMD="sudo -H -u ${MISP_USER}"
fi
}
@ -798,7 +798,8 @@ composer73 () {
# Update composer.phar
# If hash changes, check here: https://getcomposer.org/download/ and replace with the correct one
# Current Sum for: v1.8.3
SHA384_SUM='48e3236262b34d30969dca3c37281b3b4bbe3221bda826ac6a9a62d6444cdb0dcd0615698a5cbe587c3f0fe57a54d8f5'
SHA384_SUM="$(wget -q -O - https://composer.github.io/installer.sig)"
php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
$SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === '$SHA384_SUM') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); exit(137); } echo PHP_EOL;"
checkFail "composer.phar checksum failed, please investigate manually. " $?
@ -853,11 +854,12 @@ nuke () {
# Final function to let the user know what happened
theEnd () {
space
echo "Admin (root) DB Password: $DBPASSWORD_ADMIN" |$SUDO_USER tee /home/${MISP_USER}/mysql.txt
echo "User (misp) DB Password: $DBPASSWORD_MISP" |$SUDO_USER tee -a /home/${MISP_USER}/mysql.txt
echo "Authkey: $AUTH_KEY" |$SUDO_USER tee -a /home/${MISP_USER}/MISP-authkey.txt
echo "Admin (root) DB Password: $DBPASSWORD_ADMIN" |$SUDO_CMD tee /home/${MISP_USER}/mysql.txt
echo "User (misp) DB Password: $DBPASSWORD_MISP" |$SUDO_CMD tee -a /home/${MISP_USER}/mysql.txt
echo "Authkey: $AUTH_KEY" |$SUDO_CMD tee -a /home/${MISP_USER}/MISP-authkey.txt
clear
# Commenting out, see: https://github.com/MISP/MISP/issues/5368
# clear -x
space
echo -e "${LBLUE}MISP${NC} Installed, access here: ${MISP_BASEURL}"
echo

22
docs/generic/viper-debian.md
View File

@ -18,25 +18,25 @@ viper () {
fi
fi
echo "Cloning Viper"
$SUDO_USER git clone https://github.com/viper-framework/viper.git
$SUDO_USER git clone https://github.com/viper-framework/viper-web.git
$SUDO_CMD git clone https://github.com/viper-framework/viper.git
$SUDO_CMD git clone https://github.com/viper-framework/viper-web.git
sudo chown -R $MISP_USER:$MISP_USER viper
sudo chown -R $MISP_USER:$MISP_USER viper-web
cd viper
echo "Creating virtualenv"
$SUDO_USER virtualenv -p python3 venv
$SUDO_CMD virtualenv -p python3 venv
echo "Submodule update"
# TODO: Check for current user install permissions
$SUDO_USER git submodule update --init --recursive
$SUDO_CMD git submodule update --init --recursive
echo "pip install deps"
$SUDO_USER ./venv/bin/pip install pefile olefile jbxapi Crypto pypdns pypssl r2pipe pdftools virustotal-api SQLAlchemy PrettyTable python-magic scrapy https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
$SUDO_USER ./venv/bin/pip install .
$SUDO_CMD ./venv/bin/pip install pefile olefile jbxapi Crypto pypdns pypssl r2pipe pdftools virustotal-api SQLAlchemy PrettyTable python-magic scrapy https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
$SUDO_CMD ./venv/bin/pip install .
echo 'update-modules' |/usr/local/src/viper/venv/bin/viper
cd /usr/local/src/viper-web
$SUDO_USER sed -i '1 s/^.*$/\#!\/usr\/local\/src\/viper\/venv\/bin\/python/' viper-web
$SUDO_USER /usr/local/src/viper/venv/bin/pip install -r requirements.txt
$SUDO_CMD sed -i '1 s/^.*$/\#!\/usr\/local\/src\/viper\/venv\/bin\/python/' viper-web
$SUDO_CMD /usr/local/src/viper/venv/bin/pip install -r requirements.txt
echo "Launching viper-web"
$SUDO_USER /usr/local/src/viper-web/viper-web -p 8888 -H 0.0.0.0 &
$SUDO_CMD /usr/local/src/viper-web/viper-web -p 8888 -H 0.0.0.0 &
echo 'PATH="/home/misp/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/usr/local/src/viper:/var/www/MISP/app/Console"' |sudo tee /etc/environment
echo ". /etc/environment" >> /home/${MISP_USER}/.profile
@ -48,8 +48,8 @@ viper () {
fi
echo "Setting misp_url/misp_key"
$SUDO_USER sed -i "s/^misp_url\ =/misp_url\ =\ http:\/\/localhost/g" ${VIPER_HOME}/viper.conf
$SUDO_USER sed -i "s/^misp_key\ =/misp_key\ =\ $AUTH_KEY/g" ${VIPER_HOME}/viper.conf
$SUDO_CMD sed -i "s/^misp_url\ =/misp_url\ =\ http:\/\/localhost/g" ${VIPER_HOME}/viper.conf
$SUDO_CMD sed -i "s/^misp_key\ =/misp_key\ =\ $AUTH_KEY/g" ${VIPER_HOME}/viper.conf
# Reset admin password to: admin/Password1234
echo "Fixing admin.db with default password"
VIPER_COUNT=0

2
docs/xINSTALL.NetBSD.md
View File

@ -234,7 +234,7 @@ sudo -u www HOME=/tmp $PATH_TO_MISP/venv/bin/pip install plyara
# Install CakeResque along with its dependencies if you intend to use the built in background jobs:
cd $PATH_TO_MISP/app
sudo -u www php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
sudo -u www php -r "if (hash_file('SHA384', 'composer-setup.php') === 'a5c698ffe4b8e849a443b120cd5ba38043260d5c4023dbf93e1558871f1f07f58274fc6f4c93bcfd858c6bd0775cd8d1') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink
sudo -u www php -r "if (hash_file('SHA384', 'composer-setup.php') === 'baf1608c33254d00611ac1705c1d9958c817a1a33bce370c0595974b342601bd80b92a3f46067da89e3b06bff421f182') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink
('composer-setup.php'); } echo PHP_EOL;"
sudo -u www env HOME=/tmp php composer-setup.php
sudo -u www php -r "unlink('composer-setup.php');"

3
docs/xINSTALL.OpenBSD.md
View File

@ -399,8 +399,9 @@ doas /usr/local/virtualenvs/MISP/bin/pip install git+https://github.com/kbandla/
# Install CakeResque along with its dependencies if you intend to use the built in background jobs:
cd /var/www/htdocs/MISP/app
doas mkdir /var/www/.composer ; doas chown www:www /var/www/.composer
EXPECTED_SIGNATURE="$(wget -q -O - https://composer.github.io/installer.sig)"
doas -u www php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
doas -u www php -r "if (hash_file('SHA384', 'composer-setup.php') === 'a5c698ffe4b8e849a443b120cd5ba38043260d5c4023dbf93e1558871f1f07f58274fc6f4c93bcfd858c6bd0775cd8d1') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
doas -u www php -r "if (hash_file('SHA384', 'composer-setup.php') === '$EXPECTED_SIGNATURE') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
doas -u www env HOME=/var/www php composer-setup.php
doas -u www php -r "unlink('composer-setup.php');"
doas -u www env HOME=/var/www php composer.phar install

3
docs/xINSTALL.centos6.md
View File

@ -240,8 +240,9 @@ sudo mkdir /var/www/.composer/
sudo chown apache:apache /var/www/.composer/
cd $PATH_TO_MISP/app
# Update composer.phar (optional)
#EXPECTED_SIGNATURE="$(wget -q -O - https://composer.github.io/installer.sig)"
#$SUDO_WWW $RUN_PHP -- php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
#$SUDO_WWW $RUN_PHP -- php -r "if (hash_file('SHA384', 'composer-setup.php') === 'a5c698ffe4b8e849a443b120cd5ba38043260d5c4023dbf93e1558871f1f07f58274fc6f4c93bcfd858c6bd0775cd8d1') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
#$SUDO_WWW $RUN_PHP -- php -r "if (hash_file('SHA384', 'composer-setup.php') === '$EXPECTED_SIGNATURE') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
#$SUDO_WWW $RUN_PHP "php composer-setup.php"
#$SUDO_WWW $RUN_PHP -- php -r "unlink('composer-setup.php');"
$SUDO_WWW $RUN_PHP "php composer.phar install"

3
docs/xINSTALL.centos7.md
View File

@ -223,8 +223,9 @@ sudo mkdir /usr/share/httpd/.composer
sudo chown ${WWW_USER}:${WWW_USER} /usr/share/httpd/.composer
cd $PATH_TO_MISP/app
# Update composer.phar (optional)
#EXPECTED_SIGNATURE="$(wget -q -O - https://composer.github.io/installer.sig)"
#$SUDO_WWW $RUN_PHP -- php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
#$SUDO_WWW $RUN_PHP -- php -r "if (hash_file('SHA384', 'composer-setup.php') === 'a5c698ffe4b8e849a443b120cd5ba38043260d5c4023dbf93e1558871f1f07f58274fc6f4c93bcfd858c6bd0775cd8d1') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
#$SUDO_WWW $RUN_PHP -- php -r "if (hash_file('SHA384', 'composer-setup.php') === '$EXPECTED_SIGNATURE') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
#$SUDO_WWW $RUN_PHP "php composer-setup.php"
#$SUDO_WWW $RUN_PHP -- php -r "unlink('composer-setup.php');"
$SUDO_WWW $RUN_PHP "php composer.phar install"

2
docs/xINSTALL.debian10.md
View File

@ -198,7 +198,7 @@ cd $PATH_TO_MISP/app
sudo mkdir /var/www/.composer ; sudo chown $WWW_USER:$WWW_USER /var/www/.composer
# Update composer.phar
$SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === 'a5c698ffe4b8e849a443b120cd5ba38043260d5c4023dbf93e1558871f1f07f58274fc6f4c93bcfd858c6bd0775cd8d1') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
$SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === 'baf1608c33254d00611ac1705c1d9958c817a1a33bce370c0595974b342601bd80b92a3f46067da89e3b06bff421f182') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
$SUDO_WWW php composer-setup.php
$SUDO_WWW php -r "unlink('composer-setup.php');"
$SUDO_WWW php composer.phar install

3
docs/xINSTALL.debian9.md
View File

@ -217,8 +217,9 @@ cd $PATH_TO_MISP/app
# Make composer cache happy
sudo mkdir /var/www/.composer ; sudo chown $WWW_USER:$WWW_USER /var/www/.composer
# Update composer.phar
#EXPECTED_SIGNATURE="$(wget -q -O - https://composer.github.io/installer.sig)"
# $SUDO_WWW php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
# $SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === 'a5c698ffe4b8e849a443b120cd5ba38043260d5c4023dbf93e1558871f1f07f58274fc6f4c93bcfd858c6bd0775cd8d1') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
# $SUDO_WWW php -r "if (hash_file('SHA384', 'composer-setup.php') === '$EXPECTED_SIGNATURE') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
# $SUDO_WWW php composer-setup.php
# $SUDO_WWW php -r "unlink('composer-setup.php');"
$SUDO_WWW php composer.phar install

20
docs/xINSTALL.tsurugi.md
View File

@ -617,24 +617,24 @@ function installMISPonTsurugi() {
git clone git://github.com/stricaud/gtcaca.git gtcaca
chown -R ${MISP_USER}:${MISP_USER} faup mail_to_misp gtcaca
cd gtcaca
$SUDO_USER mkdir -p build
$SUDO_CMD mkdir -p build
cd build
$SUDO_USER cmake .. && $SUDO_USER make
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
cd ../../faup
$SUDO_USER mkdir -p build
$SUDO_CMD mkdir -p build
cd build
$SUDO_USER cmake .. && $SUDO_USER make
$SUDO_CMD cmake .. && $SUDO_CMD make
sudo make install
sudo ldconfig
cd ../../mail_to_misp
$SUDO_USER virtualenv -p python3 venv
$SUDO_USER ./venv/bin/pip install https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
$SUDO_USER ./venv/bin/pip install -r requirements.txt
$SUDO_USER cp mail_to_misp_config.py-example mail_to_misp_config.py
$SUDO_CMD virtualenv -p python3 venv
$SUDO_CMD ./venv/bin/pip install https://github.com/lief-project/packages/raw/lief-master-latest/pylief-0.9.0.dev.zip
$SUDO_CMD ./venv/bin/pip install -r requirements.txt
$SUDO_CMD cp mail_to_misp_config.py-example mail_to_misp_config.py
##$SUDO cp mail_to_misp_config.py-example mail_to_misp_config.py
$SUDO_USER sed -i "s/^misp_url\ =\ 'YOUR_MISP_URL'/misp_url\ =\ 'https:\/\/localhost'/g" /usr/local/src/mail_to_misp/mail_to_misp_config.py
$SUDO_USER sed -i "s/^misp_key\ =\ 'YOUR_KEY_HERE'/misp_key\ =\ '${AUTH_KEY}'/g" /usr/local/src/mail_to_misp/mail_to_misp_config.py
$SUDO_CMD sed -i "s/^misp_url\ =\ 'YOUR_MISP_URL'/misp_url\ =\ 'https:\/\/localhost'/g" /usr/local/src/mail_to_misp/mail_to_misp_config.py
$SUDO_CMD sed -i "s/^misp_key\ =\ 'YOUR_KEY_HERE'/misp_key\ =\ '${AUTH_KEY}'/g" /usr/local/src/mail_to_misp/mail_to_misp_config.py
echo ""
echo "Admin (root) DB Password: $DBPASSWORD_ADMIN" > /home/${MISP_USER}/mysql.txt
echo "User (misp) DB Password: $DBPASSWORD_MISP" >> /home/${MISP_USER}/mysql.txt