MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge remote-tracking branch 'upstream/master'

pull/376/head
Richard van den Berg 2015-02-05 14:09:29 +01:00
parent 253e579307 9cc7a994c3
commit d3c0144b25
25 changed files with 941 additions and 197 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

234
INSTALL/INSTALL.centos6.txt Normal file
View File

@ -0,0 +1,234 @@
INSTALLATION INSTRUCTIONS
------------------------- for CentOS 6.x
1/ Minimal CentOS install
-------------------------
Install a minimal CentOS 6.x system with the software:
- OpenSSH server
- LAMP server (actually, this is done below)
- Mail server
# Make sure your system is up2date:
yum update
2/ Dependencies *
----------------
Once the system is installed you can perform the following steps as root:
# We need some packages from the Extra Packages for Enterprise Linux repository
curl -o epel.rpm http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm
rpm -Uvh epel.rpm
# Because vim is just so practical
yum install vim
# Install the dependencies:
yum install gcc git zip redis mysql-server php-mysql python-devel python-pip libxslt-devel zlib-devel php-devel
yum install php-pear php-pecl-geoip
pear channel-update pear.php.net
pear install Crypt_GPG # we need version >1.3.0
pear install Net_GeoIP
# Enable and start redis
chkconfig redis on
service redis start
3/ MISP code
------------
# Download MISP using git in the /var/www/ directory.
cd /var/www/
git clone https://github.com/MISP/MISP.git
# Make git ignore filesystem permission differences
cd /var/www/MISP
git config core.filemode false
# install Mitre's STIX and its dependencies by running the following commands:
yum install python-importlib python-lxml python-dateutil python-six
cd /var/www/MISP/app/files/scripts
git clone https://github.com/CybOXProject/python-cybox.git
git clone https://github.com/STIXProject/python-stix.git
cd /var/www/MISP/app/files/scripts/python-cybox
git config core.filemode false
# If you umask is has been changed from the default, it is a good idea to reset it to 0022 before installing python modules
UMASK=$(umask)
umask 0022
python setup.py install
cd /var/www/MISP/app/files/scripts/python-stix
git config core.filemode false
python setup.py install
umask $UMASK
4/ CakePHP
-----------
# CakePHP is now included as a submodule of MISP, execute the following commands to let git fetch it
# ignore this message:
# No submodule mapping found in .gitmodules for path 'app/Plugin/CakeResque'
cd /var/www/MISP
git submodule init
git submodule update
# Once done, install CakeResque along with its dependencies if you intend to use the built in background jobs:
cd /var/www/MISP/app
curl -s https://getcomposer.org/installer | php
php composer.phar require --no-update kamisama/cake-resque:4.1.0
php composer.phar config vendor-dir Vendor
php composer.phar install
# CakeResque normally uses phpredis to connect to redis, but it has a (buggy) fallback connector through Redisent. It is highly advised to install phpredis
yum install php-pecl-redis
# If you have not yet set a timezone in php.ini
echo 'date.timezone = "Europe/Amsterdam"' > /etc/php.d/timezone.ini
# To use the scheduler worker for scheduled tasks, do the following:
cp -fa /var/www/MISP/INSTALL/setup/config.php /var/www/MISP/app/Plugin/CakeResque/Config/config.php
5/ Set the permissions
----------------------
# Make sure the permissions are set correctly using the following commands as root:
chown -R root:apache /var/www/MISP
find /var/www/MISP -type d -exec chmod g=rx {} \;
chmod -R g+r,o= /var/www/MISP
chown apache:apache /var/www/MISP/app/files
chown apache:apache /var/www/MISP/app/files/terms
chown apache:apache /var/www/MISP/app/files/scripts/tmp
chown apache:apache /var/www/MISP/app/Plugin/CakeResque/tmp
chown -R apache:apache /var/www/MISP/app/tmp
chown -R apache:apache /var/www/MISP/app/webroot/img/orgs
chown -R apache:apache /var/www/MISP/app/webroot/img/custom
6/ Create a database and user
-----------------------------
# Enable, start and secure your mysql database server
chkconfig mysqld on
service mysqld start
mysql_secure_installation
# Additionally, it is probably a good idea to make the database server listen on localhost only
# Add the following to the [mysqld] of /etc/my.cnf
# bind-address=127.0.0.1
# Enter the mysql shell
mysql -u root -p
mysql> create database misp;
mysql> grant usage on *.* to misp@localhost identified by 'XXXXXXXXX';
mysql> grant all privileges on misp.* to misp@localhost ;
mysql> exit
# Import the empty MySQL database from MYSQL.sql
cd /var/www/MISP
mysql -u misp -p misp < INSTALL/MYSQL.sql
7/ Apache configuration
-----------------------
# Now configure your apache server with the DocumentRoot /var/www/MISP/app/webroot/
# A sample ghost can be found in /var/www/MISP/INSTALL/apache.misp
cp /var/www/MISP/INSTALL/apache.misp /etc/httpd/conf.d/misp.conf
# Edit the misp.conf file and replace /var/log/apache2 with /var/log/httpd
vi /etc/httpd/conf.d/misp.conf
# Allow httpd to connect to the redis server over tcp/ip
setsebool -P httpd_can_network_connect on
# Enable and start the httpd service
chkconfig httpd on
service httpd start
# Open a hole in the iptables firewall
iptables -I INPUT 5 -p tcp --dport 80 -m state --state NEW,ESTABLISHED -j ACCEPT
service iptables save
# We seriously recommend using only SSL !
# Check out the apache.misp.ssl file for an example
8/ MISP configuration
---------------------
# There are 4 sample configuration files in /var/www/MISP/app/Config that need to be copied
cd /var/www/MISP/app/Config
cp -a bootstrap.default.php bootstrap.php
cp -a database.default.php database.php
cp -a core.default.php core.php
cp -a config.default.php config.php
# Configure the fields in the newly created files:
# config.php : baseurl
# database.php : login, port, password, database
# bootstrap.php: uncomment the last 3 lines to enable the background workers (see below)
# CakePlugin::loadAll(array('CakeResque' => array('bootstrap' => true)));
# Setup localhost in database.php:
# 'host' => 'localhost',
# To enable the background workers, if you have installed the package required for it in 4/, uncomment the following lines:
# in core.php (if you have just recently updated MISP, just add this line at the end of the file):
# require_once dirname(__DIR__) . '/Vendor/autoload.php';
# Important! Change the salt key in /var/www/MISP/app/Config/config.php
# The admin user account will be generated on the first login, make sure that the salt is changed before you create that user
# If you forget to do this step, and you are still dealing with a fresh installation, just alter the salt,
# delete the user from mysql and log in again using the default admin credentials (admin@admin.test / admin)
# If you want to be able to change configuration parameters from the webinterface:
chown apache:apache /var/www/MISP/app/Config/config.php
# Generate a GPG encryption key.
mkdir /var/www/MISP/.gnupg
chmod 700 /var/www/MISP/.gnupg
# If the following command gives an error message, try it as root from the console
# can't connect to `/var/www/MISP/.gnupg/S.gpg-agent': No such file or directory
gpg --homedir /var/www/MISP/.gnupg --gen-key
chown -R apache:apache /var/www/MISP/.gnupg
# Recommended key type: RSA
# The email address should match the one set int he config.php configuration file
# And export the public key to the webroot
sudo -u apache gpg --homedir /var/www/MISP/.gnupg --export --armor YOUR-EMAIL > /var/www/MISP/app/webroot/gpg.asc
# Start the workers to enable background jobs
su -s /bin/bash apache -c 'bash /var/www/MISP/app/Console/worker/start.sh'
# To make the background workers start on boot
vi /etc/rc.local
# Add the following line at the end
su -s /bin/bash apache -c 'bash /var/www/MISP/app/Console/worker/start.sh'
# Now log in using the webinterface:
# The default user/pass = admin@admin.test/admin
# Using the server settings tool in the admin interface (Administration -> Server Settings), set MISP up to your preference
# It is especially vital that no critical issues remain!
Don't forget to change the email, password and authentication key after installation.
# Once done, have a look at the diagnostics
# If any of the directories that MISP uses to store files is not writeable to the apache user, change the permissions
# you can do this by running the following commands:
chmod -R 750 /var/www/MISP/<directory path with an indicated issue>
chown -R apache:apache /var/www/MISP/<directory path with an indicated issue>
# Make sure that the STIX libraries and GnuPG work as intended, if not, refer to INSTALL.txt's paragraphs dealing with these two items
Recommended actions
-------------------
- By default CakePHP exposes his name and version in email headers. Apply a patch to remove this behavior.
- You should really harden your OS
- You should really harden the configuration of Apache
- You should really harden the configuration of MySQL
- Keep your software up2date (MISP, CakePHP and everything else)
- Log and audit

248
INSTALL/INSTALL.centos7.txt Normal file
View File

@ -0,0 +1,248 @@
INSTALLATION INSTRUCTIONS
------------------------- for CentOS 7.x
1/ Minimal CentOS install
-------------------------
Install a minimal CentOS 7.x system with the software:
- OpenSSH server
- LAMP server (actually, this is done below)
- Mail server
# Make sure your system is up2date:
yum update
2/ Dependencies *
----------------
Once the system is installed you can perform the following steps as root:
# We need some packages from the Extra Packages for Enterprise Linux repository
curl -o epel.rpm http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-5.noarch.rpm
rpm -Uvh epel.rpm
# Because vim is just so practical
yum install vim
# Install the dependencies:
yum install git httpd zip php redis mysql-server php-mysql python-devel python-pip libxslt-devel zlib-devel php-devel
yum install php-pear php-pecl-geoip
pear channel-update pear.php.net
pear install Crypt_GPG # we need version >1.3.0
pear install Net_GeoIP
# Enable and start redis
systemctl enable redis.service
systemctl start redis.service
3/ MISP code
------------
# Download MISP using git in the /var/www/ directory.
cd /var/www/
git clone https://github.com/MISP/MISP.git
# Make git ignore filesystem permission differences
cd /var/www/MISP
git config core.filemode false
# install Mitre's STIX and its dependencies by running the following commands:
yum install python-importlib python-lxml python-dateutil python-six
cd /var/www/MISP/app/files/scripts
git clone https://github.com/CybOXProject/python-cybox.git
git clone https://github.com/STIXProject/python-stix.git
cd /var/www/MISP/app/files/scripts/python-cybox
git config core.filemode false
# If you umask is has been changed from the default, it is a good idea to reset it to 0022 before installing python modules
UMASK=$(umask)
umask 0022
python setup.py install
cd /var/www/MISP/app/files/scripts/python-stix
git config core.filemode false
python setup.py install
umask $UMASK
4/ CakePHP
-----------
# CakePHP is now included as a submodule of MISP, execute the following commands to let git fetch it
# ignore this message:
# No submodule mapping found in .gitmodules for path 'app/Plugin/CakeResque'
cd /var/www/MISP
git submodule init
git submodule update
# Once done, install CakeResque along with its dependencies if you intend to use the built in background jobs:
cd /var/www/MISP/app
curl -s https://getcomposer.org/installer | php
php composer.phar require --no-update kamisama/cake-resque:4.1.0
php composer.phar config vendor-dir Vendor
php composer.phar install
# CakeResque normally uses phpredis to connect to redis, but it has a (buggy) fallback connector through Redisent. It is highly advised to install phpredis
yum install php-pecl-redis
# If you have not yet set a timezone in php.ini
echo 'date.timezone = "Europe/Amsterdam"' > /etc/php.d/timezone.ini
# To use the scheduler worker for scheduled tasks, do the following:
cp -fa /var/www/MISP/INSTALL/setup/config.php /var/www/MISP/app/Plugin/CakeResque/Config/config.php
5/ Set the permissions
----------------------
# Make sure the permissions are set correctly using the following commands as root:
chown -R root:apache /var/www/MISP
find /var/www/MISP -type d -exec chmod g=rx {} \;
chmod -R g+r,o= /var/www/MISP
chown apache:apache /var/www/MISP/app/files
chown apache:apache /var/www/MISP/app/files/terms
chown apache:apache /var/www/MISP/app/files/scripts/tmp
chown apache:apache /var/www/MISP/app/Plugin/CakeResque/tmp
chown -R apache:apache /var/www/MISP/app/tmp
chown -R apache:apache /var/www/MISP/app/webroot/img/orgs
chown -R apache:apache /var/www/MISP/app/webroot/img/custom
6/ Create a database and user
-----------------------------
yum install mariadb-server
# Enable, start and secure your mysql database server
systemctl enable mariadb.service
systemctl start mariadb.service
mysql_secure_installation
# Additionally, it is probably a good idea to make the database server listen on localhost only
echo [mysqld] > /etc/my.cnf.d/bind-address.cnf
echo bind-address=127.0.0.1 >> /etc/my.cnf.d/bind-address.cnf
systemctl restart mariadb.service
# Enter the mysql shell
mysql -u root -p
MariaDB [(none)]> create database misp;
MariaDB [(none)]> grant usage on *.* to misp@localhost identified by 'XXXXXXXXX';
MariaDB [(none)]> grant all privileges on misp.* to misp@localhost ;
MariaDB [(none)]> exit
# Import the empty MySQL database from MYSQL.sql
cd /var/www/MISP
mysql -u misp -p misp < INSTALL/MYSQL.sql
7/ Apache configuration
-----------------------
# Now configure your apache server with the DocumentRoot /var/www/MISP/app/webroot/
# A sample ghost can be found in /var/www/MISP/INSTALL/apache.misp
cp /var/www/MISP/INSTALL/apache.misp /etc/httpd/conf.d/misp.conf
# Edit the misp.conf file and replace /var/log/apache2 with /var/log/httpd
vi /etc/httpd/conf.d/misp.conf
# Since SELinux is enabled, we need to allow httpd to write to certain directories
chcon -t httpd_sys_content_rw_t /var/www/MISP/app/files
chcon -t httpd_sys_content_rw_t /var/www/MISP/app/files/terms
chcon -t httpd_sys_content_rw_t /var/www/MISP/app/files/scripts/tmp
chcon -t httpd_sys_content_rw_t /var/www/MISP/app/Plugin/CakeResque/tmp
chcon -R -t httpd_sys_content_rw_t /var/www/MISP/app/tmp
chcon -R -t httpd_sys_content_rw_t /var/www/MISP/app/webroot/img/orgs
chcon -R -t httpd_sys_content_rw_t /var/www/MISP/app/webroot/img/custom
# Allow httpd to connect to the redis server over tcp/ip
setsebool -P httpd_can_network_connect on
# Enable and start the httpd service
systemctl enable httpd.service
systemctl start httpd.service
# Open a hole in the iptables firewall
firewall-cmd --zone=public --add-port=80/tcp --permanent
firewall-cmd --reload
# We seriously recommend using only SSL !
# Check out the apache.misp.ssl file for an example
8/ MISP configuration
---------------------
# There are 4 sample configuration files in /var/www/MISP/app/Config that need to be copied
cd /var/www/MISP/app/Config
cp -a bootstrap.default.php bootstrap.php
cp -a database.default.php database.php
cp -a core.default.php core.php
cp -a config.default.php config.php
# Configure the fields in the newly created files:
# config.php : baseurl
# database.php : login, port, password, database
# bootstrap.php: uncomment the last 3 lines to enable the background workers (see below)
# CakePlugin::loadAll(array('CakeResque' => array('bootstrap' => true)));
# Setup localhost in database.php:
# 'host' => 'localhost',
# To enable the background workers, if you have installed the package required for it in 4/, uncomment the following lines:
# in core.php (if you have just recently updated MISP, just add this line at the end of the file):
# require_once dirname(__DIR__) . '/Vendor/autoload.php';
# Important! Change the salt key in /var/www/MISP/app/Config/config.php
# The admin user account will be generated on the first login, make sure that the salt is changed before you create that user
# If you forget to do this step, and you are still dealing with a fresh installation, just alter the salt,
# delete the user from mysql and log in again using the default admin credentials (admin@admin.test / admin)
# If you want to be able to change configuration parameters from the webinterface:
chown apache:apache /var/www/MISP/app/Config/config.php
chcon -t httpd_sys_content_rw_t /var/www/MISP/app/Config/config.php
# Generate a GPG encryption key.
mkdir /var/www/MISP/.gnupg
chmod 700 /var/www/MISP/.gnupg
# If the following command gives an error message, try it as root from the console
# can't connect to `/var/www/MISP/.gnupg/S.gpg-agent': No such file or directory
gpg --homedir /var/www/MISP/.gnupg --gen-key
chown -R apache:apache /var/www/MISP/.gnupg
# Recommended key type: RSA
# The email address should match the one set int he config.php configuration file
# And export the public key to the webroot
sudo -u apache gpg --homedir /var/www/MISP/.gnupg --export --armor YOUR-EMAIL > /var/www/MISP/app/webroot/gpg.asc
# Start the workers to enable background jobs
su -s /bin/bash apache -c 'bash /var/www/MISP/app/Console/worker/start.sh'
# To make the background workers start on boot
vi /etc/rc.local
# Add the following line at the end
su -s /bin/bash apache -c 'bash /var/www/MISP/app/Console/worker/start.sh'
# Now log in using the webinterface:
# The default user/pass = admin@admin.test/admin
# Using the server settings tool in the admin interface (Administration -> Server Settings), set MISP up to your preference
# It is especially vital that no critical issues remain!
Don't forget to change the email, password and authentication key after installation.
# Once done, have a look at the diagnostics
# If any of the directories that MISP uses to store files is not writeable to the apache user, change the permissions
# you can do this by running the following commands:
chmod -R 750 /var/www/MISP/<directory path with an indicated issue>
chown -R apache:apache /var/www/MISP/<directory path with an indicated issue>
# Make sure that the STIX libraries and GnuPG work as intended, if not, refer to INSTALL.txt's paragraphs dealing with these two items
Recommended actions
-------------------
- By default CakePHP exposes his name and version in email headers. Apply a patch to remove this behavior.
- You should really harden your OS
- You should really harden the configuration of Apache
- You should really harden the configuration of MySQL
- Keep your software up2date (MISP, CakePHP and everything else)
- Log and audit

12
INSTALL/INSTALL.txt → INSTALL/INSTALL.ubuntu1404.txt Executable file → Normal file
View File

@ -1,5 +1,5 @@
INSTALLATION INSTRUCTIONS
------------------------- for ubuntu 12.04-server
------------------------- for ubuntu 14.04-server
# steps 2 (besides acquiring git), 4, 5 can be skipped by running INSTALL.SH
# The correct order in this case is 1/, apt get git, 3/ and then executing INSTALL.SH in /var/www/MISP/INSTALL/INSTALL.SH
@ -7,7 +7,7 @@ INSTALLATION INSTRUCTIONS
1/ Minimal ubuntu install
-------------------------
Install a minimal ubuntu 12.04-server system with the software:
Install a minimal ubuntu 14.04-server system with the software:
- OpenSSH server
- LAMP server (don't forget php5-mysql)
@ -33,8 +33,6 @@ apt-get install zip php-pear git redis-server make python-dev python-pip libxml2
pear install Crypt_GPG # we need version >1.3.0
pear install Net_GeoIP
# (for Red Hat users, you will have to install Redis manually, http://redis.io/download)
3/ MISP code
------------
# Download MISP using git in the /var/www/ directory.
@ -91,8 +89,6 @@ chmod -R g+ws /var/www/MISP/app/tmp
chmod -R g+ws /var/www/MISP/app/files
chmod -R g+ws /var/www/MISP/app/files/scripts/tmp
Red Hat users: Keep in mind that your apache user is called apache, not www-data
6/ Create a database and user
-----------------------------
# Enter the mysql shell
@ -118,8 +114,8 @@ cp /var/www/MISP/INSTALL/apache.misp /etc/apache2/sites-available/misp.conf
# The configuration file has to have the .conf extension in the sites-available directory
# For more information, visit http://httpd.apache.org/docs/2.4/upgrading.html
a2dissite default
# default can be called 000-default based on distribution, in which case run a2dissite 000-default
a2dissite 000-default
# 000-default can be called default based on distribution, in which case run a2dissite default
a2ensite misp
# Enable modules

BIN
INSTALL/documentation.pdf Normal file
View File

Binary file not shown.

BIN
INSTALL/documentation1.pdf
View File

Binary file not shown.

BIN
INSTALL/documentation2_user_management.pdf
View File

Binary file not shown.

BIN
INSTALL/documentation3_using_the_system.pdf
View File

Binary file not shown.

BIN
INSTALL/documentation4_administration.pdf
View File

Binary file not shown.

BIN
INSTALL/documentation5_categories_and_types.pdf
View File

Binary file not shown.

2
VERSION.json
View File

@ -1 +1 @@
{"major":2, "minor":3, "hotfix":40}
{"major":2, "minor":3, "hotfix":45}

4
app/Config/database.default.php
View File

@ -62,9 +62,9 @@ class DATABASE_CONFIG {
public $default = array(
'datasource' => 'Database/Mysql',
'persistent' => false,
'host' => '127.0.0.1',
'host' => 'localhost',
'login' => 'db login',
'port' => 8889,
'port' => 3306,
'password' => 'db password',
'database' => 'misp',
'prefix' => '',

4
app/Console/Command/ServerShell.php
View File

@ -90,12 +90,12 @@ class ServerShell extends AppShell
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'push',
'job_type' => 'pull',
'job_input' => 'Server: ' . $server['Server']['id'],
'retries' => 0,
'org' => $user['User']['org'],
'process_id' => 'Part of scheduled pull',
'message' => 'Pushing.',
'message' => 'Pulling.',
);
$this->Job->save($data);
$jobId = $this->Job->id;

22
app/Controller/AttributesController.php
View File

@ -1462,7 +1462,7 @@ class AttributesController extends AppController {
// the last 4 fields accept the following operators:
// && - you can use && between two search values to put a logical OR between them. for value, 1.1.1.1&&2.2.2.2 would find attributes with the value being either of the two.
// ! - you can negate a search term. For example: google.com&&!mail would search for all attributes with value google.com but not ones that include mail. www.google.com would get returned, mail.google.com wouldn't.
public function restSearch($key='download', $value=null, $type=null, $category=null, $org=null, $tags=null) {
public function restSearch($key='download', $value=false, $type=false, $category=false, $org=false, $tags=false, $from=false, $to=false) {
if ($tags) $tags = str_replace(';', ':', $tags);
if ($tags === 'null') $tags = null;
if ($value === 'null') $value = null;
@ -1493,7 +1493,7 @@ class AttributesController extends AppController {
} else {
throw new BadRequestException('Either specify the search terms in the url, or POST a json array / xml (with the root element being "request" and specify the correct accept and content type headers.');
}
$paramArray = array('value', 'type', 'category', 'org', 'tags');
$paramArray = array('value', 'type', 'category', 'org', 'tags', 'from', 'to');
foreach ($paramArray as $p) {
if (isset($data['request'][$p])) ${$p} = $data['request'][$p];
else ${$p} = null;
@ -1577,7 +1577,10 @@ class AttributesController extends AppController {
}
$conditions['AND'][] = $temp;
}
if ($from) $conditions['AND'][] = array('Event.date >=' => $from);
if ($to) $conditions['AND'][] = array('Event.date <=' => $to);
// change the fields here for the attribute export!!!! Don't forget to check for the permissions, since you are not going through fetchevent. Maybe create fetchattribute?
$params = array(
@ -1724,11 +1727,14 @@ class AttributesController extends AppController {
$this->__downloadAttachment($this->Attribute->data['Attribute']);
}
public function text($key='download', $type='all', $tags=false, $eventId=false, $allowNonIDS=false) {
if ($eventId === 'null' || $eventId == '0' || $eventId === 'false') $eventId = false;
if ($allowNonIDS === 'null' || $allowNonIDS === '0' || $allowNonIDS === 'false') $allowNonIDS = false;
public function text($key='download', $type='all', $tags=false, $eventId=false, $allowNonIDS=false, $from=false, $to=false) {
$simpleFalse = array('eventId', 'allowNonIDS', 'tags', 'from', 'to');
foreach ($simpleFalse as $sF) {
if (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false') ${$sF} = false;
}
if ($type === 'null' || $type === '0' || $type === 'false') $type = 'all';
if ($tags === 'null' || $tags === '0' || $tags === 'false') $tags = false;
if ($from && !preg_match('/^[0-9]{4}-[0-9]{2}-[0-9]{2}$/', $from)) $from = false;
if ($to && !preg_match('/^[0-9]{4}-[l0-9]{2}-[0-9]{2}$/', $from)) $from = false;
if ($key != 'download') {
// check if the key is valid -> search for users based on key
$user = $this->checkAuthUser($key);
@ -1743,7 +1749,7 @@ class AttributesController extends AppController {
$this->response->type('txt'); // set the content type
$this->header('Content-Disposition: download; filename="misp.' . $type . '.txt"');
$this->layout = 'text/default';
$attributes = $this->Attribute->text($this->_checkOrg(), $this->_isSiteAdmin(), $type, $tags, $eventId, $allowNonIDS);
$attributes = $this->Attribute->text($this->_checkOrg(), $this->_isSiteAdmin(), $type, $tags, $eventId, $allowNonIDS, $from, $to);
$this->loadModel('Whitelist');
$attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true);
$this->set('attributes', $attributes);

205
app/Controller/EventsController.php
View File

@ -1694,12 +1694,10 @@ class EventsController extends AppController {
return $difference . " " . $periods[$j] . " ago";
}
public function xml($key, $eventid=null, $withAttachment = false, $tags = '') {
$this->helpers[] = 'XmlOutput';
if ($tags != '') $tags = str_replace(';', ':', $tags);
if ($tags === 'null') $tags = null;
if ($eventid === 'null' || $eventid ==='false') $eventid=null;
if ($withAttachment === 'null' || $withAttachment ==='false') $withAttachment = false;
public function xml($key, $eventid=null, $withAttachment = false, $tags = false, $from = false, $to = false) {
App::uses('XMLConverterTool', 'Tools');
$converter = new XMLConverterTool();
$this->loadModel('Whitelist');
// request handler for POSTed queries. If the request is a post, the parameters (apart from the key) will be ignored and replaced by the terms defined in the posted xml object.
// The correct format for a posted xml is a "request" root element, as shown by the examples below:
@ -1710,65 +1708,78 @@ class EventsController extends AppController {
} else {
$data = $this->request->data;
}
$paramArray = array('eventid', 'withAttachment', 'tags');
$paramArray = array('eventid', 'withAttachment', 'tags', 'from', 'to');
foreach ($paramArray as $p) {
if (isset($data['request'][$p])) ${$p} = $data['request'][$p];
else ${$p} = null;
}
}
$simpleFalse = array('tags', 'eventid', 'withAttachment', 'from', 'to');
foreach ($simpleFalse as $sF) {
if (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false') ${$sF} = false;
}
if ($tags) $tags = str_replace(';', ':', $tags);
$eventIdArray = array();
if ($eventid) {
if (!is_numeric($eventid)) throw new MethodNotAllowedException('Invalid Event ID.');
$eventIdArray[] = $eventid;
}
if ($key != 'download') {
// check if the key is valid -> search for users based on key
$user = $this->checkAuthUser($key);
if (!$user) {
throw new UnauthorizedException('This authentication key is not authorized to be used for exports. Contact your administrator.');
}
// display the full xml
$this->response->type('xml'); // set the content type
$this->layout = 'xml/default';
$this->header('Content-Disposition: download; filename="misp.xml"');
$results = $this->__fetchEvent($eventid, null, $user['User']['org'], $user['User']['siteAdmin'], $tags);
$org = $user['User']['org'];
$isSiteAdmin = $user['User']['siteAdmin'];
} else {
if (!$this->Auth->user('id')) {
throw new UnauthorizedException('You have to be logged in to do that.');
}
// display the full xml
$this->response->type('xml'); // set the content type
$this->layout = 'xml/default';
if ($eventid == null) {
$this->header('Content-Disposition: download; filename="misp.export.all.xml"');
} else {
$this->header('Content-Disposition: download; filename="misp.export.event' . $eventid . '.xml"');
}
$results = $this->__fetchEvent($eventid, null, null, false, $tags);
$org = $this->Auth->user('org');
$isSiteAdmin = $this->_isSiteAdmin();
}
if ($withAttachment) {
$this->loadModel('Attribute');
foreach ($results as &$result) {
foreach ($result['Attribute'] as &$attribute) {
if ($this->Attribute->typeIsAttachment($attribute['type'])) {
$encodedFile = $this->Attribute->base64EncodeAttachment($attribute);
if ($eventid) {
$final_filename='misp.event' . $eventid . '.export.xml';
} else {
$final_filename='misp.export.xml';
}
$final = "";
$final .= '<?xml version="1.0" encoding="UTF-8"?>' . PHP_EOL . '<response>' . PHP_EOL;
if (!$eventid) {
$events = $this->Event->fetchEventIds($org, $isSiteAdmin, $from, $to);
foreach ($events as $event) $eventIdArray[] = $event['Event']['id'];
}
foreach ($eventIdArray as $currentEventId) {
$result = $this->__fetchEvent($currentEventId, null, $org, $isSiteAdmin, $tags, $from, $to);
if ($withAttachment) {
foreach ($result[0]['Attribute'] as &$attribute) {
if ($this->Event->Attribute->typeIsAttachment($attribute['type'])) {
$encodedFile = $this->Event->Attribute->base64EncodeAttachment($attribute);
$attribute['data'] = $encodedFile;
}
}
}
$result = $this->Whitelist->removeWhitelistedFromArray($result, false);
$final .= $converter->event2XML($result[0]) . PHP_EOL;
}
// Whitelist check
$this->loadModel('Whitelist');
$results = $this->Whitelist->removeWhitelistedFromArray($results, false);
if ($eventid) {
$this->header('Content-Disposition: download; filename="misp.event' . $eventid . '.export.xml"');
} else {
$this->header('Content-Disposition: download; filename="misp.export.xml"');
}
$final .= '</response>' . PHP_EOL;
$this->response->body($final);
$this->response->type('xml');
$this->set('results', $results);
$this->response->download($final_filename);
return $this->response;
}
// Grab an event or a list of events for the event view or any of the XML exports. The returned object includes an array of events (or an array that only includes a single event if an ID was given)
// Included with the event are the attached attributes, shadow attributes, related events, related attribute information for the event view and the creating user's email address where appropriate
private function __fetchEvent($eventid = null, $idList = null, $orgFromFetch = null, $isSiteAdmin = false, $tags = '') {
private function __fetchEvent($eventid = false, $idList = false, $orgFromFetch = false, $isSiteAdmin = false, $tags = false, $from=false, $to=false) {
// if we come from automation, we may not be logged in - instead we used an auth key in the URL.
if (!empty($orgFromFetch)) {
$org = $orgFromFetch;
@ -1776,18 +1787,16 @@ class EventsController extends AppController {
$org = $this->_checkOrg();
$isSiteAdmin = $this->_isSiteAdmin();
}
if (!empty($orgFromFetch)) $org = $orgFromFetch;
else $org = $this->_checkOrg();
$results = $this->Event->fetchEvent($eventid, $idList, $org, $isSiteAdmin, null, $tags);
$results = $this->Event->fetchEvent($eventid, $idList, $org, $isSiteAdmin, null, $tags, $from, $to);
return $results;
}
public function nids($format = 'suricata', $key = '', $id = null, $continue = false, $tags = '') {
if ($tags != '') $tags = str_replace(';', ':', $tags);
if ($tags === 'null') $tags = null;
if ($id === 'null') $id = null;
if ($continue === 'false') $continue = false;
if ($continue === 'true') $continue = true;
public function nids($format = 'suricata', $key = 'download', $id = false, $continue = false, $tags = false, $from = false, $to = false) {
$simpleFalse = array('id', 'continue', 'tags', 'from', 'to');
foreach ($simpleFalse as $sF) {
if (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false') ${$sF} = false;
}
if ($tags) $tags = str_replace(';', ':', $tags);
// backwards compatibility, swap key and format
if ($format != 'snort' && $format != 'suricata') {
$key = $format;
@ -1808,17 +1817,23 @@ class EventsController extends AppController {
throw new UnauthorizedException('You have to be logged in to do that.');
}
$user = $this->checkAuthUser($this->Auth->user('authkey'));
if (!$user) throw new UnauthorizedException('This authentication key is not authorized to be used for exports. Contact your administrator.');
$user = array('User' => $this->Auth->user());
$user['User']['siteAdmin'] = $this->_isSiteAdmin();
}
// display the full snort rulebase
$this->loadModel('Attribute');
$rules = $this->Attribute->nids($user['User']['siteAdmin'], $user['User']['org'], $format, $user['User']['nids_sid'], $id, $continue, $tags);
$rules = $this->Attribute->nids($user['User']['siteAdmin'], $user['User']['org'], $format, $user['User']['nids_sid'], $id, $continue, $tags, $from, $to);
$this->set('rules', $rules);
}
public function hids($type, $key, $tags = '') {
if ($tags != '') $tags = str_replace(';', ':', $tags);
if ($tags === 'null') $tags = null;
public function hids($type, $key='download', $tags = false, $from = false, $to = false) {
$simpleFalse = array('tags', 'from', 'to');
foreach ($simpleFalse as $sF) {
if (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false') ${$sF} = false;
}
if ($tags) $tags = str_replace(';', ':', $tags);
$this->response->type('txt'); // set the content type
$this->header('Content-Disposition: download; filename="misp.' . $type . '.rules"');
$this->layout = 'text/default';
@ -1833,23 +1848,24 @@ class EventsController extends AppController {
if (!$this->Auth->user('id')) {
throw new UnauthorizedException('You have to be logged in to do that.');
}
$user = $this->checkAuthUser($this->Auth->user('authkey'));
$user = array('User' => $this->Auth->user());
$user['User']['siteAdmin'] = $this->_isSiteAdmin();
}
$this->loadModel('Attribute');
$rules = $this->Attribute->hids($user['User']['siteAdmin'], $user['User']['org'], $type, $tags);
$rules = $this->Attribute->hids($user['User']['siteAdmin'], $user['User']['org'], $type, $tags, $from, $to);
$this->set('rules', $rules);
}
// csv function
// Usage: csv($key, $eventid) - key can be a valid auth key or the string 'download'. Download requires the user to be logged in interactively and will generate a .csv file
// $eventid can be one of 3 options: left empty it will get all the visible to_ids attributes,
// $ignore is a flag that allows the export tool to ignore the ids flag. 0 = only IDS signatures, 1 = everything.
public function csv($key, $eventid=0, $ignore=0, $tags = '', $category=null, $type=null, $includeInfo=null) {
if ($category == 'null') $category = null;
if ($type == 'null') $type = null;
if ($tags == 'null') $tags = '';
if ($includeInfo == 'null') $includeInfo = null;
if ($tags != '') $tags = str_replace(';', ':', $tags);
public function csv($key, $eventid=false, $ignore=false, $tags = false, $category=false, $type=false, $includeInfo=false, $from=false, $to=false) {
$simpleFalse = array('eventid', 'ignore', 'tags', 'category', 'type', 'includeInfo', 'from', 'to');
foreach ($simpleFalse as $sF) {
if (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false') ${$sF} = false;
}
if ($tags) $tags = str_replace(';', ':', $tags);
$list = array();
if ($key != 'download') {
// check if the key is valid -> search for users based on key
@ -1886,18 +1902,18 @@ class EventsController extends AppController {
$list[] = $attribute['Attribute']['id'];
}
}
$attributes = $this->Event->csv($org, $isSiteAdmin, $eventid, $ignore, $list, $tags, $category, $type, $includeInfo);
$attributes = $this->Event->csv($org, $isSiteAdmin, $eventid, $ignore, $list, $tags, $category, $type, $includeInfo, $from, $to);
$this->loadModel('Whitelist');
$final = array();
$attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true);
foreach ($attributes as $attribute) {
$line = $attribute['Attribute']['uuid'] . ',' . $attribute['Attribute']['event_id'] . ',' . $attribute['Attribute']['category'] . ',' . $attribute['Attribute']['type'] . ',' . $attribute['Attribute']['value'] . ',' . intval($attribute['Attribute']['to_ids']) . ',' . $attribute['Attribute']['timestamp'];
if ($includeInfo != null) $line .= ',' . $attribute['Attribute']['event_info'];
if ($includeInfo) $line .= ',' . $attribute['Attribute']['event_info'];
$final[] = $line;
}
$this->response->type('csv'); // set the content type
if ($eventid == 0) {
if (!$eventid) {
$this->header('Content-Disposition: download; filename="misp.all_attributes.csv"');
} else if ($eventid === 'search') {
$this->header('Content-Disposition: download; filename="misp.search_result.csv"');
@ -1906,7 +1922,7 @@ class EventsController extends AppController {
}
$this->layout = 'text/default';
$headers = array('uuid', 'event_id', 'category', 'type', 'value', 'to_ids', 'date');
if ($includeInfo != null) $headers[] = 'event_info';
if ($includeInfo) $headers[] = 'event_info';
$this->set('headers', $headers);
$this->set('final', $final);
}
@ -2340,17 +2356,8 @@ class EventsController extends AppController {
// the last 4 fields accept the following operators:
// && - you can use && between two search values to put a logical OR between them. for value, 1.1.1.1&&2.2.2.2 would find attributes with the value being either of the two.
// ! - you can negate a search term. For example: google.com&&!mail would search for all attributes with value google.com but not ones that include mail. www.google.com would get returned, mail.google.com wouldn't.
public function restSearch($key=null, $value=null, $type=null, $category=null, $org=null, $tags = '', $searchall=null) {
if ($tags != '') $tags = str_replace(';', ':', $tags);
if ($tags === 'null') $tags = '';
if ($value === 'null') $value = null;
if ($type === 'null') $type = null;
if ($tags === 'null') $tags = null;
if ($category === 'null') $category = null;
if ($org === 'null') $org = null;
if ($searchall === 'null') $searchall = '';
if ($searchall === 'true') $searchall = "1";
if ($key!=null && $key!='download') {
public function restSearch($key='download', $value=false, $type=false, $category=false, $org=false, $tags = false, $searchall=false, $from=false, $to=false) {
if ($key!='download') {
$user = $this->checkAuthUser($key);
} else {
if (!$this->Auth->user()) throw new UnauthorizedException('You are not authorized. Please send the Authorization header with your auth key along with an Accept header for application/xml.');
@ -2374,12 +2381,20 @@ class EventsController extends AppController {
} else {
throw new BadRequestException('Either specify the search terms in the url, or POST a json array / xml (with the root element being "request" and specify the correct headers based on content type.');
}
$paramArray = array('value', 'type', 'category', 'org', 'tags', 'searchall');
$paramArray = array('value', 'type', 'category', 'org', 'tags', 'searchall', 'from', 'to');
foreach ($paramArray as $p) {
if (isset($data['request'][$p])) ${$p} = $data['request'][$p];
else ${$p} = null;
}
}
$simpleFalse = array('value' , 'type', 'category', 'org', 'tags', 'searchall', 'from', 'to');
foreach ($simpleFalse as $sF) {
if (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false') ${$sF} = false;
}
if ($tags) $tags = str_replace(';', ':', $tags);
if ($searchall === 'true') $searchall = "1";
if (!isset($this->request->params['ext']) || $this->request->params['ext'] !== 'json') {
$this->response->type('xml'); // set the content type
$this->layout = 'xml/default';
@ -2465,6 +2480,9 @@ class EventsController extends AppController {
'conditions' => $conditions,
'fields' => array('Attribute.event_id'),
);
if ($from) $conditions['AND'][] = array('Event.date >=' => $from);
if ($to) $conditions['AND'][] = array('Event.date <=' => $to);
$attributes = $this->Attribute->find('all', $params);
$eventIds = array();
foreach ($attributes as $attribute) {
@ -2888,7 +2906,7 @@ class EventsController extends AppController {
}
}
public function stix($key, $id = null, $withAttachments = false, $tags = null) {
public function stix($key, $id = false, $withAttachments = false, $tags = false, $from = false, $to = false) {
if ($key != 'download') {
// check if the key is valid -> search for users based on key
$user = $this->checkAuthUser($key);
@ -2904,20 +2922,7 @@ class EventsController extends AppController {
$isSiteAdmin = $this->_isSiteAdmin();
$org = $this->Auth->user('org');
}
// set null if a null string is passed
if ($id == 'false' || $id == 'null') $id = null;
$numeric = false;
if (is_numeric($id)) $numeric = true;
if ($tags == 'false' || $tags == 'null') $tags = null;
if ($withAttachments == 'false' || 'null') $withAttachments = false;
// set the export type based on the request
if ($this->response->type() === 'application/json') $returnType = 'json';
else {
$returnType = 'xml';
$this->response->type('xml'); // set the content type
$this->layout = 'xml/default';
}
// request handler for POSTed queries. If the request is a post, the parameters (apart from the key) will be ignored and replaced by the terms defined in the posted xml object.
// The correct format for a posted xml is a "request" root element, as shown by the examples below:
// For XML: <request><id>!3&amp;!4</id><tags>OSINT</tags></request>
@ -2928,13 +2933,29 @@ class EventsController extends AppController {
} else {
$data = $this->request->data;
}
$paramArray = array('id', 'withAttachment', 'tags');
$paramArray = array('id', 'withAttachment', 'tags', 'from', 'to');
foreach ($paramArray as $p) {
if (isset($data['request'][$p])) ${$p} = $data['request'][$p];
else ${$p} = null;
}
}
$result = $this->Event->stix($id, $tags, $withAttachments, $this->Auth->user('org'), $this->_isSiteAdmin(), $returnType);
$simpleFalse = array('id', 'withAttachments', 'tags', 'from', 'to');
foreach ($simpleFalse as $sF) {
if (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false') ${$sF} = false;
}
// set null if a null string is passed
$numeric = false;
if (is_numeric($id)) $numeric = true;
// set the export type based on the request
if ($this->response->type() === 'application/json') $returnType = 'json';
else {
$returnType = 'xml';
$this->response->type('xml'); // set the content type
$this->layout = 'xml/default';
}
$result = $this->Event->stix($id, $tags, $withAttachments, $org, $isSiteAdmin, $returnType, $from, $to);
if ($result['success'] == 1) {
// read the output file and pass it to the view

98
app/Lib/Tools/XMLConverterTool.php Normal file
View File

@ -0,0 +1,98 @@
<?php
class XMLConverterTool {
public function recursiveEcho($array) {
$text = "";
foreach ($array as $k => $v) {
if (is_array($v)) {
if (empty($v)) $text .= '<' . $k . '/>';
else {
foreach ($v as $element) {
$text .= '<' . $k . '>';
$text .= $this->recursiveEcho($element);
$text .= '</' . $k . '>';
}
}
} else {
if ($v === false) $v = 0;
if ($v === "" || $v === null) $text .= '<' . $k . '/>';
else {
$text .= '<' . $k . '>' . $v . '</' . $k . '>';
}
}
}
return $text;
}
public function event2xmlArray($event) {
$toEscape = array("&", "<", ">", "\"", "'");
$escapeWith = array('&amp;', '&lt;', '&gt;', '&quot;', '&apos;');
$event['Event']['Attribute'] = $event['Attribute'];
$event['Event']['ShadowAttribute'] = $event['ShadowAttribute'];
$event['Event']['RelatedEvent'] = $event['RelatedEvent'];
$event['Event']['info'] = preg_replace ('/[^\x{0009}\x{000a}\x{000d}\x{0020}-\x{D7FF}\x{E000}-\x{FFFD}]+/u', ' ', $event['Event']['info']);
$event['Event']['info'] = str_replace($toEscape, $escapeWith, $event['Event']['info']);
//
// cleanup the array from things we do not want to expose
//
unset($event['Event']['user_id']);
// hide the org field is we are not in showorg mode
if (!Configure::read('MISP.showorg') && !$isSiteAdmin) {
unset($event['Event']['org']);
unset($event['Event']['orgc']);
unset($event['Event']['from']);
}
if (isset($event['Event']['Attribute'])) {
// remove value1 and value2 from the output and remove invalid utf8 characters for the xml parser
foreach ($event['Event']['Attribute'] as $key => $value) {
$event['Event']['Attribute'][$key]['value'] = preg_replace ('/[^\x{0009}\x{000a}\x{000d}\x{0020}-\x{D7FF}\x{E000}-\x{FFFD}]+/u', ' ', $event['Event']['Attribute'][$key]['value']);
$event['Event']['Attribute'][$key]['value'] = str_replace($toEscape, $escapeWith, $event['Event']['Attribute'][$key]['value']);
$event['Event']['Attribute'][$key]['comment'] = preg_replace ('/[^\x{0009}\x{000a}\x{000d}\x{0020}-\x{D7FF}\x{E000}-\x{FFFD}]+/u', ' ', $event['Event']['Attribute'][$key]['comment']);
$event['Event']['Attribute'][$key]['comment'] = str_replace($toEscape, $escapeWith, $event['Event']['Attribute'][$key]['comment']);
unset($event['Event']['Attribute'][$key]['value1']);
unset($event['Event']['Attribute'][$key]['value2']);
unset($event['Event']['Attribute'][$key]['category_order']);
foreach($event['Event']['Attribute'][$key]['ShadowAttribute'] as $skey => $svalue) {
$event['Event']['Attribute'][$key]['ShadowAttribute'][$skey]['value'] = preg_replace ('/[^\x{0009}\x{000a}\x{000d}\x{0020}-\x{D7FF}\x{E000}-\x{FFFD}]+/u', ' ', $event['Event']['Attribute'][$key]['ShadowAttribute'][$skey]['value']);
$event['Event']['Attribute'][$key]['ShadowAttribute'][$skey]['value'] = str_replace($toEscape, $escapeWith, $event['Event']['Attribute'][$key]['ShadowAttribute'][$skey]['value']);
$event['Event']['Attribute'][$key]['ShadowAttribute'][$skey]['comment'] = preg_replace ('/[^\x{0009}\x{000a}\x{000d}\x{0020}-\x{D7FF}\x{E000}-\x{FFFD}]+/u', ' ', $event['Event']['Attribute'][$key]['ShadowAttribute'][$skey]['comment']);
$event['Event']['Attribute'][$key]['ShadowAttribute'][$skey]['comment'] = str_replace($toEscape, $escapeWith, $event['Event']['Attribute'][$key]['ShadowAttribute'][$skey]['comment']);
}
}
}
if (isset($event['Event']['ShadowAttribute'])) {
// remove invalid utf8 characters for the xml parser
foreach($event['Event']['ShadowAttribute'] as $key => $value) {
$event['Event']['ShadowAttribute'][$key]['value'] = preg_replace ('/[^\x{0009}\x{000a}\x{000d}\x{0020}-\x{D7FF}\x{E000}-\x{FFFD}]+/u', ' ', $event['Event']['ShadowAttribute'][$key]['value']);
$event['Event']['ShadowAttribute'][$key]['value'] = str_replace($toEscape, $escapeWith, $event['Event']['ShadowAttribute'][$key]['value']);
$event['Event']['ShadowAttribute'][$key]['comment'] = preg_replace ('/[^\x{0009}\x{000a}\x{000d}\x{0020}-\x{D7FF}\x{E000}-\x{FFFD}]+/u', ' ', $event['Event']['ShadowAttribute'][$key]['comment']);
$event['Event']['ShadowAttribute'][$key]['comment'] = str_replace($toEscape, $escapeWith, $event['Event']['ShadowAttribute'][$key]['comment']);
}
}
if (isset($event['Event']['RelatedEvent'])) {
foreach ($event['Event']['RelatedEvent'] as $key => $value) {
$temp = $value['Event'];
unset($event['Event']['RelatedEvent'][$key]['Event']);
$event['Event']['RelatedEvent'][$key]['Event'][0] = $temp;
unset($event['Event']['RelatedEvent'][$key]['Event'][0]['user_id']);
$event['Event']['RelatedEvent'][$key]['Event'][0]['info'] = preg_replace ('/[^\x{0009}\x{000a}\x{000d}\x{0020}-\x{D7FF}\x{E000}-\x{FFFD}]+/u', ' ', $event['Event']['RelatedEvent'][$key]['Event'][0]['info']);
$event['Event']['RelatedEvent'][$key]['Event'][0]['info'] = str_replace($toEscape, $escapeWith, $event['Event']['RelatedEvent'][$key]['Event'][0]['info']);
if (!Configure::read('MISP.showorg') && !$isAdmin) {
unset($event['Event']['RelatedEvent'][$key]['Event'][0]['org']);
unset($event['Event']['RelatedEvent'][$key]['Event'][0]['orgc']);
}
unset($temp);
}
}
return array('Event' => $event['Event']);
}
public function event2XML($event) {
$xmlArray = $this->event2xmlArray($event);
return $this->recursiveEcho(array('Event' => array(0 => $xmlArray['Event'])));
}
}

25
app/Model/Attribute.php
View File

@ -1159,6 +1159,7 @@ class Attribute extends AppModel {
}
public function hids($isSiteAdmin, $org ,$type, $tags = '') {
if (empty($org)) throw new MethodNotAllowedException('No org supplied.');
// check if it's a valid type
if ($type != 'md5' && $type != 'sha1') {
throw new UnauthorizedException('Invalid hash type.');
@ -1169,6 +1170,7 @@ class Attribute extends AppModel {
$temp = array();
$distribution = array();
array_push($temp, array('Attribute.distribution >' => 0));
array_push($temp, array('AND' => array('Attribute.distribution >' => 0, 'Event.distribution >' => 0)));
array_push($temp, array('(SELECT events.org FROM events WHERE events.id = Attribute.event_id) LIKE' => $org));
$conditions['OR'] = $temp;
}
@ -1202,22 +1204,25 @@ class Attribute extends AppModel {
return $rules;
}
public function nids($isSiteAdmin, $org, $format, $sid, $id = null, $continue = false, $tags = '') {
public function nids($isSiteAdmin, $org, $format, $sid, $id = false, $continue = false, $tags = false, $from = false, $to = false) {
//restricting to non-private or same org if the user is not a site-admin.
$conditions['AND'] = array('Attribute.to_ids' => 1, "Event.published" => 1);
$valid_types = array('ip-dst', 'ip-src', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'domain', 'hostname', 'url', 'user-agent', 'snort');
$conditions['AND']['Attribute.type'] = $valid_types;
if (!$isSiteAdmin) {
$temp = array();
$distribution = array();
array_push($temp, array('Attribute.distribution >' => 0));
array_push($temp, array('AND' => array('Attribute.distribution >' => 0, 'Event.distribution >' => 0)));
array_push($temp, array('(SELECT events.org FROM events WHERE events.id = Attribute.event_id) LIKE' => $org));
$conditions['OR'] = $temp;
}
if ($id) {
array_push($conditions['AND'], array('Event.id' => $id));
}
if ($id) array_push($conditions['AND'], array('Event.id' => $id));
if ($from) array_push($conditions['AND'], array('Event.date >=' => $from));
if ($to) array_push($conditions['AND'], array('Event.date <=' => $to));
// If we sent any tags along, load the associated tag names for each attribute
if ($tags !== '') {
if ($tags) {
$tag = ClassRegistry::init('Tag');
$args = $this->dissectArgs($tags);
$tagArray = $tag->fetchEventTagIds($args[0], $args[1]);
@ -1232,7 +1237,6 @@ class Attribute extends AppModel {
}
$conditions['AND'][] = $temp;
}
$params = array(
'conditions' => $conditions, //array of conditions
'recursive' => 0, //int
@ -1256,11 +1260,13 @@ class Attribute extends AppModel {
return $rules;
}
public function text($org, $isSiteAdmin, $type, $tags = false, $eventId = false, $allowNonIDS = false) {
public function text($org, $isSiteAdmin, $type, $tags = false, $eventId = false, $allowNonIDS = false, $from = false, $to = false) {
//restricting to non-private or same org if the user is not a site-admin.
$conditions['AND'] = array();
if ($allowNonIDS === false) $conditions['AND'] = array('Attribute.to_ids =' => 1, 'Event.published =' => 1);
if ($type !== 'all') $conditions['AND']['Attribute.type'] = $type;
if ($from) $conditions['AND']['Event.date >='] = $from;
if ($to) $conditions['AND']['Event.date <='] = $to;
if (!$isSiteAdmin) {
$temp = array();
$distribution = array();
@ -1268,7 +1274,6 @@ class Attribute extends AppModel {
array_push($temp, array('(SELECT events.org FROM events WHERE events.id = Attribute.event_id) LIKE' => $org));
$conditions['OR'] = $temp;
}
if ($eventId !== false) {
$conditions['AND'][] = array('Event.id' => $eventId);
} elseif ($tags !== false) {
@ -1295,7 +1300,7 @@ class Attribute extends AppModel {
'order' => array('Attribute.value'), //string or array defining order
'group' => array('Attribute.value'), //fields to GROUP BY
'contain' => array('Event' => array(
'fields' => array('Event.id', 'Event.published'),
'fields' => array('Event.id', 'Event.published', 'Event.date'),
)));

75
app/Model/Event.php
View File

@ -795,7 +795,7 @@ class Event extends AppModel {
return null;
}
public function fetchEventIds($org, $isSiteAdmin) {
public function fetchEventIds($org, $isSiteAdmin, $from = false, $to = false) {
$conditions = array();
if (!$isSiteAdmin) {
$conditions['OR'] = array(
@ -806,6 +806,10 @@ class Event extends AppModel {
);
}
$fields = array('Event.id', 'Event.org', 'Event.distribution');
if ($from) $conditions['AND'][] = array('Event.date >=' => $from);
if ($to) $conditions['AND'][] = array('Event.date <=' => $to);
$params = array(
'conditions' => $conditions,
'recursive' => -1,
@ -816,8 +820,8 @@ class Event extends AppModel {
}
//Once the data about the user is gathered from the appropriate sources, fetchEvent is called from the controller.
public function fetchEvent($eventid = null, $idList = null, $org, $isSiteAdmin, $bkgrProcess = null, $tags = '') {
if (isset($eventid)) {
public function fetchEvent($eventid = false, $idList = false, $org, $isSiteAdmin = false, $bkgrProcess = false, $tags = false, $from = false, $to = false) {
if ($eventid) {
$this->id = $eventid;
if (!$this->exists()) {
throw new NotFoundException(__('Invalid event'));
@ -843,12 +847,15 @@ class Event extends AppModel {
'(SELECT events.org FROM events WHERE events.id = Attribute.event_id) LIKE' => $org
);
}
if ($idList && $tags == '') {
if ($from) $conditions['AND'][] = array('Event.date >=' => $from);
if ($to) $conditions['AND'][] = array('Event.date <=' => $to);
if ($idList && !$tags) {
$conditions['AND'][] = array('Event.id' => $idList);
}
// If we sent any tags along, load the associated tag names for each attribute
if ($tags !== '') {
if ($tags) {
$tag = ClassRegistry::init('Tag');
$args = $this->Attribute->dissectArgs($tags);
$tagArray = $tag->fetchEventTagIds($args[0], $args[1]);
@ -917,26 +924,26 @@ class Event extends AppModel {
}
return $results;
}
public function csv($org, $isSiteAdmin, $eventid=0, $ignore=0, $attributeIDList = array(), $tags = '', $category = null, $type = null, $includeInfo = null) {
public function csv($org, $isSiteAdmin, $eventid=false, $ignore=false, $attributeIDList = array(), $tags = false, $category = false, $type = false, $includeInfo = false, $from = false, $to = false) {
$final = array();
$attributeList = array();
$conditions = array();
$econditions = array();
$this->recursive = -1;
// If we are not in the search result csv download function then we need to check what can be downloaded. CSV downloads are already filtered by the search function.
if ($eventid !== 'search') {
if ($from) $econditions['AND'][] = array('Event.date >=' => $from);
if ($to) $econditions['AND'][] = array('Event.date <=' => $to);
// This is for both single event downloads and for full downloads. Org has to be the same as the user's or distribution not org only - if the user is no siteadmin
if(!$isSiteAdmin) {
$econditions['AND']['OR'] = array("AND" => array('Event.distribution >' => 0, 'Event.published =' => 1), 'Event.org =' => $org);
}
if ($eventid == 0 && $ignore == 0) {
$econditions['AND'][] = array('Event.published =' => 1);
}
// If it's a full download (eventid == null) and the user is not a site admin, we need to first find all the events that the user can see and save the IDs
if ($eventid == 0) {
if(!$isSiteAdmin) $econditions['AND']['OR'] = array("AND" => array('Event.distribution >' => 0, 'Event.published =' => 1), 'Event.org =' => $org);
if ($eventid == 0 && $ignore == 0) $econditions['AND'][] = array('Event.published =' => 1);
// If it's a full download (eventid == false) and the user is not a site admin, we need to first find all the events that the user can see and save the IDs
if (!$eventid) {
$this->recursive = -1;
// If we sent any tags along, load the associated tag names for each attribute
if ($tags !== '') {
if (!$tags) {
$tag = ClassRegistry::init('Tag');
$args = $this->Attribute->dissectArgs($tags);
$tagArray = $tag->fetchEventTagIds($args[0], $args[1]);
@ -954,32 +961,21 @@ class Event extends AppModel {
// let's add the conditions if we're dealing with a non-siteadmin user
$params = array(
'conditions' => $econditions,
'fields' => array('id', 'distribution', 'org', 'published'),
'fields' => array('id', 'distribution', 'org', 'published', 'date'),
);
$events = $this->find('all', $params);
}
// if we have items in events, add their IDs to the conditions. If we're a site admin, or we have a single event selected for download, this should be empty
if (isset($events)) {
foreach ($events as $event) {
$conditions['AND']['OR'][] = array('Attribute.event_id' => $event['Event']['id']);
}
foreach ($events as $event) $conditions['AND']['OR'][] = array('Attribute.event_id' => $event['Event']['id']);
}
// if we're downloading a single event, set it as a condition
if ($eventid!=0) {
$conditions['AND'][] = array('Attribute.event_id' => $eventid);
}
if ($eventid) $conditions['AND'][] = array('Attribute.event_id' => $eventid);
//restricting to non-private or same org if the user is not a site-admin.
if ($ignore == 0) {
$conditions['AND'][] = array('Attribute.to_ids' => 1);
}
if ($type!=null) {
$conditions['AND'][] = array('Attribute.type' => $type);
}
if ($category!=null) {
$conditions['AND'][] = array('Attribute.category' => $category);
}
if (!$ignore) $conditions['AND'][] = array('Attribute.to_ids' => 1);
if ($type) $conditions['AND'][] = array('Attribute.type' => $type);
if ($category) $conditions['AND'][] = array('Attribute.category' => $category);
if (!$isSiteAdmin) {
$temp = array();
@ -989,10 +985,9 @@ class Event extends AppModel {
$conditions['OR'] = $temp;
}
}
if ($eventid === 'search') {
foreach ($attributeIDList as $aID) {
$conditions['AND']['OR'][] = array('Attribute.id' => $aID);
}
foreach ($attributeIDList as $aID) $conditions['AND']['OR'][] = array('Attribute.id' => $aID);
}
$params = array(
'conditions' => $conditions, //array of conditions
@ -1005,7 +1000,7 @@ class Event extends AppModel {
$attribute['Attribute']['value'] = '"' . $attribute['Attribute']['value'] . '"';
$attribute['Attribute']['timestamp'] = date('Ymd', $attribute['Attribute']['timestamp']);
}
if ($includeInfo == 'yes') $attributes = $this->attachEventInfoToAttributes($attributes);
if ($includeInfo) $attributes = $this->attachEventInfoToAttributes($attributes);
return $attributes;
}
@ -1817,7 +1812,7 @@ class Event extends AppModel {
}
}
// generate a randomised filename for the temporary file that will be passed to the python script
$randomFileName = $this->__generateRandomFileName();
$randomFileName = $this->generateRandomFileName();
$tempFile = new File (APP . "files" . DS . "scripts" . DS . "tmp" . DS . $randomFileName, true, 0644);
// save the json_encoded event(s) to the temporary file
@ -1868,7 +1863,7 @@ class Event extends AppModel {
return $ids;
}
private function __generateRandomFileName() {
public function generateRandomFileName() {
$length = 12;
$characters = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
$charLen = strlen($characters) - 1;

32
app/Model/Server.php
View File

@ -428,6 +428,22 @@ class Server extends AppModel {
'type' => 'string',
'editable' => false,
),
'password_policy_length' => array(
'level' => 2,
'description' => 'Password length requirement. If it is not set or it is set to 0, then the default value is assumed (6).',
'value' => '',
'errorMessage' => '',
'test' => 'testPasswordLength',
'type' => 'numeric',
),
'password_policy_complexity' => array(
'level' => 2,
'description' => 'Password complexity requirement. Leave it empty for the default setting (3 out of 4, with either a digit or a special char) or enter your own regex. Keep in mind that the length is checked in another key. Example (simple 4 out of 4): /(?=.*[0-9])(?=.*[!@#$%^&*_-])(?=.*[A-Z])(?=.*[a-z]).*$/',
'value' => '',
'errorMessage' => '',
'test' => 'testPasswordRegex',
'type' => 'string',
),
),
'SecureAuth' => array(
'branch' => 1,
@ -467,9 +483,6 @@ class Server extends AppModel {
if ($jobId) {
$job = ClassRegistry::init('Job');
$job->read(null, $jobId);
App::import('Component','Auth');
$this->Auth = new AuthComponent(new ComponentCollection());
$this->Auth->login($user);
$email = "Scheduled job";
} else {
$email = $user['email'];
@ -656,6 +669,7 @@ class Server extends AppModel {
'model_id' => $id,
'email' => $user['email'],
'action' => 'pull',
'user_id' => $user['id'],
'title' => 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $email,
'change' => count($successes) . ' events and ' . count($pulledProposals) . ' proposals pulled or updated. ' . count($fails) . ' events failed or didn\'t need an update.'
));
@ -969,6 +983,18 @@ class Server extends AppModel {
return $this->__testForFile($value, APP . 'webroot' . DS . 'img' . DS . 'custom');
}
public function testPasswordLength($value) {
$numeric = $this->testforNumeric($value);
if ($numeric !== true) return $numeric;
if ($numeric < 0) return 'Length cannot be negative, set a positive integer or 0 (to choose the default option).';
return true;
}
public function testPasswordRegex($value) {
if (!empty($value) && @preg_match($value, 'test') === false) return 'Invalid regex.';
return true;
}
// never come here directly, always go through a secondary check like testForTermsFile in order to also pass along the expected file path
private function __testForFile($value, $path) {

21
app/Model/User.php
View File

@ -38,8 +38,8 @@ class User extends AppModel {
),
'password' => array(
'minlength' => array(
'rule' => array('minlength', 6),
'message' => 'A password of a minimum length of 6 is required.',
'rule' => array('passwordLength'),
'message' => 'Password length requirement not met.',
//'allowEmpty' => false,
'required' => true,
//'last' => false, // Stop validation after this rule
@ -47,7 +47,7 @@ class User extends AppModel {
),
'complexity' => array(
'rule' => array('complexPassword'),
'message' => 'The password must contain at least one upper-case, one lower-case, one (digits or special character).',
'message' => 'Password complexity requirement not met.',
//'allowEmpty' => false,
//'required' => true,
//'last' => false, // Stop validation after this rule
@ -294,17 +294,30 @@ class User extends AppModel {
}
}
public function passwordLength($check) {
$length = Configure::read('Security.password_policy_length');
if (empty($length) || $length < 0) $length = 6;
$value = array_values($check);
$value = $value[0];
if (strlen($value) < $length) return false;
return true;
}
public function complexPassword($check) {
/*
default password:
6 characters minimum
1 or more upper-case letters
1 or more lower-case letters
1 or more digits or special characters
example: "EasyPeasy34"
If Security.password_policy_complexity is set and valid, use the regex provided.
*/
$regex = Configure::read('Security.password_policy_complexity');
if (empty($regex) || @preg_match($regex, 'test') === false) $regex = '/((?=.*\d)|(?=.*\W+))(?![.\n])(?=.*[A-Z])(?=.*[a-z]).*$/';
$value = array_values($check);
$value = $value[0];
return preg_match('/((?=.*\d)|(?=.*\W+))(?![.\n])(?=.*[A-Z])(?=.*[a-z]).*$/', $value);
return preg_match($regex, $value);
}
public function identicalFieldValues($field=array(), $compareField=null) {

4
app/View/Attributes/add.ctp
View File

@ -1,4 +1,4 @@
<div class="attributes <?php if (!$ajax) echo 'form';?>">
<div class="attributes <?php if (!isset($ajax) || !$ajax) echo 'form';?>">
<?php
echo $this->Form->create('Attribute', array('id'));
?>
@ -57,7 +57,7 @@
?>
</div>
</fieldset>
<p style="color:red;font-weight:bold;display:none;<?php if($ajax) echo "text-align:center;"?>" id="warning-message">Warning: You are about to share data that is of a sensitive nature (Attribution / targeting data). Make sure that you are authorised to share this.</p>
<p style="color:red;font-weight:bold;display:none;<?php if(isset($ajax) && $ajax) echo "text-align:center;"?>" id="warning-message">Warning: You are about to share data that is of a sensitive nature (Attribution / targeting data). Make sure that you are authorised to share this.</p>
<?php if ($ajax): ?>
<div class="overlay_spacing">
<table>

2
app/View/Attributes/edit.ctp
View File

@ -43,7 +43,7 @@
$this->Js->get('#AttributeCategory')->event('change', 'formCategoryChanged("#AttributeCategory")');
?>
</fieldset>
<p style="color:red;font-weight:bold;display:none;<?php if($ajax) echo "text-align:center;"?>" id="warning-message">Warning: You are about to share data that is of a sensitive nature (Attribution / targeting data). Make sure that you are authorised to share this.</p>
<p style="color:red;font-weight:bold;display:none;<?php if (isset($ajax) && $ajax) echo "text-align:center;";?> " id="warning-message">Warning: You are about to share data that is of a sensitive nature (Attribution / targeting data). Make sure that you are authorised to share this.</p>
<?php
echo $this->Form->button('Submit', array('class' => 'btn btn-primary'));
echo $this->Form->end();

127
app/View/Events/automation.ctp
View File

@ -16,8 +16,26 @@ You can <?php echo $this->Html->link('reset', array('controller' => 'users', 'ac
<pre><?php echo Configure::read('MISP.baseurl');?>/events/xml/download</pre>
<p>If you only want to fetch a specific event append the eventid number:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/xml/download/1</pre>
<p>The xml download also accepts two additional (optional) parameters: a boolean field that determines whether attachments should be encoded and a second parameter that controls the eligible tags. To include a tag in the results just write its names into this parameter. To exclude a tag prepend it with a '!'. You can also chain several tag commands together with the '&&' operator. Please be aware the colons (:) cannot be used in the tag search. Use semicolons instead (the search will automatically search for colons instead). For example, to include tag1 and tag2 but exclude tag3 you would use:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/xml/download/null/true/tag1&&tag2&&!tag3</pre>
<p>You can post an XML or JSON object containing additional parameters in the following formats:</p>
<p>JSON:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/xml/download.json</pre>
<code>{"request": {"eventid":["!51","!62"],"withAttachment":false,"tags":["APT1","!OSINT"],"from":false,"to":"2015-01-01"}}</code><br /><br />
<p>XML:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/xml/download</pre>
<code>&lt;request&gt;&lt;eventid&gt;!51&lt;/eventid&gt;&lt;eventid&gt;!62&lt;/eventid&gt;&lt;withAttachment&gt;false&lt;/withAttachment&gt;&lt;tags&gt;APT1&lt;/tags&gt;&lt;tags&gt;!OSINT&lt;/tags&gt;&lt;from&gt;false&lt;/from&gt;&lt;to&gt;2015-01-01&lt;/to&gt;&lt;/request&gt;</code><br /><br />
<p>The xml download also accepts two additional the following optional parameters in the url: </p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/xml/download/[eventid]/[withattachments]/[tags]/[from]/[to]</pre>
<p>
<b>eventid</b>: Restrict the download to a single event<br />
<b>withattachments</b>: A boolean field that determines whether attachments should be encoded and a second parameter that controls the eligible tags. <br />
<b>tags</b>: To include a tag in the results just write its names into this parameter. To exclude a tag prepend it with a '!'.
You can also chain several tag commands together with the '&amp;&amp;' operator. Please be aware the colons (:) cannot be used in the tag search.
Use semicolons instead (the search will automatically search for colons instead). For example, to include tag1 and tag2 but exclude tag3 you would use:<br />
<b>from</b>: Events with the date set to a date after the one specified in the from field (format: 2015-02-03)<br />
<b>to</b>: Events with the date set to a date before the one specified in the to field (format: 2015-02-03)<br />
</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/xml/download/false/true/tag1&amp;&amp;tag2&amp;&amp;!tag3</pre>
<p>Also check out the <a href="/pages/display/doc/using_the_system#rest">User Guide</a> to read about the REST API.</p>
<p></p>
<h3>CSV Export</h3>
@ -25,17 +43,39 @@ You can <?php echo $this->Html->link('reset', array('controller' => 'users', 'ac
<p>You can configure your tools to automatically download the following file:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/csv/download/</pre>
<p>You can specify additional flags for CSV exports as follows::</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/csv/download/[event_id]/[ignore_ids_signatures_only_rule]/[tags]/[type]</pre>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/csv/download/[eventid]/[ignore]/[tags]/[category]/[type]/[includeInfo]/[from]/[to]</pre>
<p>
<b>eventid</b>: Restrict the download to a single event<br />
<b>tags</b>: To include a tag in the results just write its names into this parameter. To exclude a tag prepend it with a '!'.
You can also chain several tag commands together with the '&amp;&amp;' operator. Please be aware the colons (:) cannot be used in the tag search.
Use semicolons instead (the search will automatically search for colons instead). For example, to include tag1 and tag2 but exclude tag3 you would use:<br />
<b>ignore</b>: Setting this flag to true will include attributes that are not marked "to_ids".<br />
<b>from</b>: Events with the date set to a date after the one specified in the from field (format: 2015-02-03)<br />
<b>to</b>: Events with the date set to a date before the one specified in the to field (format: 2015-02-03)<br />
</p>
<p>For example, to only download a csv generated of the "domain" type and the "Network Activity" category attributes all events except for the one and further restricting it to events that are tagged "tag1" or "tag2" but not "tag3", only allowing attributes that are IDS flagged use the following syntax:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/csv/download/0/0/tag1&&tag2&&!tag3/Network%20Activity/domain</pre>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/csv/download/false/false/tag1&amp;&amp;tag2&amp;&amp;!tag3/Network%20Activity/domain</pre>
<p>To export the attributes of all events that are of the type "domain", use the following syntax:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/csv/download/0/0/null/null/domain</pre>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/csv/download/false/false/false/false/domain</pre>
<h3>NIDS rules export</h3>
<p>Automatic export of all network related attributes is available under the Snort rule format. Only <em>published</em> events and attributes marked as <em>IDS Signature</em> are exported.</p>
<p>You can configure your tools to automatically download the following file:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/nids/suricata/download
<?php echo Configure::read('MISP.baseurl');?>/events/nids/snort/download</pre>
<p>In addition to the above mentioned, the NIDS exports can accept several additional parameters: an event ID to only create the signature based on a single event (null will still include every event), a boolean flag that determines whether it should be a standalone file with all the descriptions at the start (false) or whether it should just be the signature lines (true). The last parameter is the tagging syntax, as described for the XML export. Please be aware the colons (:) cannot be used in the tag search. Use semicolons instead (the search will automatically search for colons instead). An example for a suricata export for all events excluding those tagged tag1, without all of the commented information at the start of the file would look like this:</p>
<p>The full API syntax is as follows:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/nids/[format]/download/[eventid]/[frame]/[tags]/[from]/[to]</pre>
<p>
<b>format</b>: The export format, can be "suricata" or "snort"<br />
<b>eventid</b>: Restrict the download to a single event<br />
<b>frame</b>: Some commented out explanation framing the data. The reason to disable this would be if you would like to concatenate a list of exports from various select events in order to avoid unnecasary duplication of the comments.<br />
<b>tags</b>: To include a tag in the results just write its names into this parameter. To exclude a tag prepend it with a '!'.
You can also chain several tag commands together with the '&amp;&amp;' operator. Please be aware the colons (:) cannot be used in the tag search.
Use semicolons instead (the search will automatically search for colons instead). For example, to include tag1 and tag2 but exclude tag3 you would use:<br />
<b>from</b>: Events with the date set to a date after the one specified in the from field (format: 2015-02-03)<br />
<b>to</b>: Events with the date set to a date before the one specified in the to field (format: 2015-02-03)<br />
</p>
<p>An example for a suricata export for all events excluding those tagged tag1, without all of the commented information at the start of the file would look like this:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/nids/suricata/download/null/true/!tag1</pre>
<p>Administration is able to maintain a white-list containing host, domain name and IP numbers to exclude from the NIDS export.</p>
@ -46,24 +86,50 @@ You can <?php echo $this->Html->link('reset', array('controller' => 'users', 'ac
<pre><?php echo Configure::read('MISP.baseurl');?>/events/hids/md5/download</pre>
<h4>sha1</h4>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/hids/sha1/download</pre>
<p>You can also use the tag syntax similar to the XML import. Please be aware the colons (:) cannot be used in the tag search. Use semicolons instead (the search will automatically search for colons instead). For example, to only show sha1 values from events tagged tag1, use:</p>
<p>The API's full format is as follows: </p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/hids/[format]/download/[tags]/[from]/[to]</pre>
<b>format</b>: The export format, can be "md5" or "sha1"<br />
<b>tags</b>: To include a tag in the results just write its names into this parameter. To exclude a tag prepend it with a '!'.
You can also chain several tag commands together with the '&amp;&amp;' operator. Please be aware the colons (:) cannot be used in the tag search.
Use semicolons instead (the search will automatically search for colons instead). For example, to include tag1 and tag2 but exclude tag3 you would use:<br />
<b>from</b>: Events with the date set to a date after the one specified in the from field (format: 2015-02-03)<br />
<b>to</b>: Events with the date set to a date before the one specified in the to field (format: 2015-02-03)<br /><br />
<p>For example, to only show sha1 values from events tagged tag1, use:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/hids/sha1/download/tag1</pre>
<h3>STIX export</h3>
<p>You can export MISP events in Mitre's STIX format (to read more about STIX, click <a href="https://stix.mitre.org/">here</a>). The STIX XML export is currently very slow and can lead to timeouts with larger events or collections of events. The JSON return format does not suffer from this issue. Usage:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/stix/download</pre>
<p>Search parameters can be passed to the function via url parameters or by POSTing an xml or json object (depending on the return type). The following parameters can be passed to the STIX export tool: <code>id</code>, <code>withAttachments</code>, <code>tags</code>. Both <code>id</code> and <code>tags</code> can use the <code>&&</code> (and) and <code>!</code> (not) operators to build queries. Using the url parameters, the syntax is as follows:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/stix/download/[id]/[withAttachments]/[tags]</pre>
<p>Search parameters can be passed to the function via url parameters or by POSTing an xml or json object (depending on the return type). The following parameters can be passed to the STIX export tool: <code>id</code>, <code>withAttachments</code>, <code>tags</code>. Both <code>id</code> and <code>tags</code> can use the <code>&amp;&amp;</code> (and) and <code>!</code> (not) operators to build queries. Using the url parameters, the syntax is as follows:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/stix/download/[id]/[withAttachments]/[tags]/[from]/[to]</pre>
<p>
<b>id</b>: The event's ID<br />
<b>withAttachments</b>: Encode attachments where applicable<br />
<b>tags</b>: To include a tag in the results just write its names into this parameter. To exclude a tag prepend it with a '!'.
You can also chain several tag commands together with the '&amp;&amp;' operator. Please be aware the colons (:) cannot be used in the tag search.
Use semicolons instead (the search will automatically search for colons instead). For example, to include tag1 and tag2 but exclude tag3 you would use:<br />
<b>from</b>: Events with the date set to a date after the one specified in the from field (format: 2015-02-03)<br />
<b>to</b>: Events with the date set to a date before the one specified in the to field (format: 2015-02-03)
</p>
<p>You can post an XML or JSON object containing additional parameters in the following formats:</p>
<p>JSON:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/stix/download.json</pre>
<code>{"request": {"id":["!51","!62"],"withAttachment":false,"tags":["APT1","!OSINT"],"from":false,"to":"2015-01-01"}}</code><br /><br />
<p>XML:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/stix/download</pre>
<code>&lt;request&gt;&lt;id&gt;!51&lt;/id&gt;&lt;id&gt;!62&lt;/id&gt;&lt;withAttachment&gt;false&lt;/withAttachment&gt;&lt;tags&gt;APT1&lt;/tags&gt;&lt;tags&gt;!OSINT&lt;/tags&gt;&lt;from&gt;false&lt;/from&gt;&lt;to&gt;2015-01-01&lt;/to&gt;&lt;/request&gt;</code><br /><br />
<h4>Various ways to narrow down the search results of the STIX export</h4>
<p>For example, to retrieve all events tagged "APT1" but excluding events tagged "OSINT" and excluding events #51 and #62 without any attachments:
<pre><?php echo Configure::read('MISP.baseurl');?>/events/stix/download/!51&&!62/false/APT1&&!OSINT</pre>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/stix/download/!51&amp;&amp;!62/false/APT1&amp;&amp;!OSINT/2015-01-01</pre>
<p>To export the same events using a POST request use:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/stix/download.json</pre>
<p>Together with this JSON object in the POST message:</p>
<code>{"request": {"id":["!51","!62"],"tags":["APT1","!OSINT"]}}</code><br /><br />
<code>{"request": {"id":["!51","!62"],"tags":["APT1","!OSINT"],"from":"2015-01-01"}}</code><br /><br />
<p>XML is automatically assumed when using the stix export:</p>
<pre><?php echo Configure::read('MISP.baseurl');?>/events/stix/download</pre>
<p>The same search could be accomplished using the following POSTed XML object (note that ampersands need to be escaped, or alternatively separate id and tag elements can be used): </p>
<code>&lt;request&gt;&lt;id&gt;!51&lt;/id&gt;&lt;id&gt;!62&lt;/id&gt;&lt;tags&gt;APT1&lt;/tags&gt;&lt;tags&gt;!OSINT&lt;/tags&gt;&lt;/request&gt;</code>
<code>&lt;request&gt;&lt;id&gt;!51&lt;/id&gt;&lt;id&gt;!62&lt;/id&gt;&lt;tags&gt;APT1&lt;/tags&gt;&lt;tags&gt;!OSINT&lt;/tags&gt;&lt;from&gt;2015-01-01&lt;/from&gt;&lt;/request&gt;</code>
<h3>Text export</h3>
<p>An automatic export of all attributes of a specific type to a plain text file.</p>
<p>You can configure your tools to automatically download the following files:</p>
@ -84,9 +150,17 @@ foreach ($sigTypes as $sigType) {
<p>As of version 2.3.38, it is possible to restrict the text exports on two additional flags. The first allows the user to restrict based on event ID, whilst the second is a boolean switch allowing non IDS flagged attributes to be exported. Additionally, choosing "all" in the type field will return all eligible attributes. </p>
<pre>
<?php
echo Configure::read('MISP.baseurl').'/attributes/text/download/[type]/[tags]/[event_id]/[ignore_to_ids_restriction]';
echo Configure::read('MISP.baseurl').'/attributes/text/download/[type]/[tags]/[event_id]/[allowNonIDS]/[from]/[to]';
?>
</pre>
<b>type</b>: The attribute type, any valid MISP attribute type is accepted.<br />
<b>tags</b>: To include a tag in the results just write its names into this parameter. To exclude a tag prepend it with a '!'.
You can also chain several tag commands together with the '&amp;&amp;' operator. Please be aware the colons (:) cannot be used in the tag search.
Use semicolons instead (the search will automatically search for colons instead). For example, to include tag1 and tag2 but exclude tag3 you would use:<br />
<b>event_id</b>: Restrict the results to the given event IDs. <br />
<b>allowNonIDS</b>: Allow attributes to be exported that are not marked as "to_ids".<br />
<b>from</b>: Events with the date set to a date after the one specified in the from field (format: 2015-02-03)<br />
<b>to</b>: Events with the date set to a date before the one specified in the to field (format: 2015-02-03)<br />
<p>For example, to retrieve all attributes for event #5, including non IDS marked attributes too, use the following line:</p>
<pre>
<?php
@ -99,10 +173,20 @@ foreach ($sigTypes as $sigType) {
<p>To return an event with all of its attributes, relations, shadowAttributes, use the following syntax:</p>
<pre>
<?php
echo Configure::read('MISP.baseurl').'/events/restSearch/download/[value]/[type]/[category]/[org]/[tag]/[quickfilter]';
echo Configure::read('MISP.baseurl').'/events/restSearch/download/[value]/[type]/[category]/[org]/[tag]/[quickfilter]/[from]/[to]';
?>
</pre>
<p>There is also a new flag called "quickfilter". Enabling this (by passing "1" as the argument) will make the search ignore all of the other arguments, except for the auth key and value. MISP will return an xml / json (depending on the header sent) of all events that have a sub-string match on value in the event info, event orgc, or any of the attribute value1 / value2 fields, or in the attribute comment. For example, to find any event with the term "red october" mentioned, use the following syntax (the example is shown as a POST request instead of a GET, which is highly recommended):</p>
<b>value</b>: Search for the given value in the attributes' value field.<br />
<b>type</b>: The attribute type, any valid MISP attribute type is accepted.<br />
<b>category</b>: The attribute category, any valid MISP attribute category is accepted.<br />
<b>org</b>: Search by the creator organisation by supplying the organisation idenfitier. <br />
<b>tags</b>: To include a tag in the results just write its names into this parameter. To exclude a tag prepend it with a '!'.
You can also chain several tag commands together with the '&amp;&amp;' operator. Please be aware the colons (:) cannot be used in the tag search.
Use semicolons instead (the search will automatically search for colons instead). For example, to include tag1 and tag2 but exclude tag3 you would use:<br />
<b>quickfilter</b>: Enabling this (by passing "1" as the argument) will make the search ignore all of the other arguments, except for the auth key and value. MISP will return an xml / json (depending on the header sent) of all events that have a sub-string match on value in the event info, event orgc, or any of the attribute value1 / value2 fields, or in the attribute comment. <br />
<b>from</b>: Events with the date set to a date after the one specified in the from field (format: 2015-02-03)<br />
<b>to</b>: Events with the date set to a date before the one specified in the to field (format: 2015-02-03)<br /><br />
<p>For example, to find any event with the term "red october" mentioned, use the following syntax (the example is shown as a POST request instead of a GET, which is highly recommended):</p>
<p>POST to:</p>
<pre>
<?php
@ -118,16 +202,25 @@ foreach ($sigTypes as $sigType) {
{"request": {"value":"red october","searchall":1}}
</code></p>
<p>To just return a list of attributes, use the following syntax:</p>
<b>value</b>: Search for the given value in the attributes' value field.<br />
<b>type</b>: The attribute type, any valid MISP attribute type is accepted.<br />
<b>category</b>: The attribute category, any valid MISP attribute category is accepted.<br />
<b>org</b>: Search by the creator organisation by supplying the organisation idenfitier. <br />
<b>tags</b>: To include a tag in the results just write its names into this parameter. To exclude a tag prepend it with a '!'.
You can also chain several tag commands together with the '&amp;&amp;' operator. Please be aware the colons (:) cannot be used in the tag search.
Use semicolons instead (the search will automatically search for colons instead). For example, to include tag1 and tag2 but exclude tag3 you would use:<br />
<b>from</b>: Events with the date set to a date after the one specified in the from field (format: 2015-02-03)<br />
<b>to</b>: Events with the date set to a date before the one specified in the to field (format: 2015-02-03)<br /><br />
<pre>
<?php
echo Configure::read('MISP.baseurl').'/attributes/restSearch/download/[value]/[type]/[category]/[org]/[tag]';
echo Configure::read('MISP.baseurl').'/attributes/restSearch/download/[value]/[type]/[category]/[org]/[tag]/[from]/[to]';
?>
</pre>
<p>value, type, category and org are optional. It is possible to search for several terms in each category by joining them with the '&amp;&amp;' operator. It is also possible to negate a term with the '!' operator. Please be aware the colons (:) cannot be used in the tag search. Use semicolons instead (the search will automatically search for colons instead).
For example, in order to search for all attributes created by your organisation that contain 192.168 or 127.0 but not 0.1 and are of the type ip-src, excluding the events that were tagged tag1 use the following syntax:</p>
<pre>
<?php
echo Configure::read('MISP.baseurl').'/attributes/restSearch/download/192.168&&127.0&&!0.1/ip-src/null/' . $me['org'] . '/!tag1';
echo Configure::read('MISP.baseurl').'/attributes/restSearch/download/192.168&&127.0&&!0.1/ip-src/false/' . $me['org'] . '/!tag1';
?>
</pre>
<p>You can also use search for IP addresses using CIDR. Make sure that you use '|' (pipe) instead of '/' (slashes). Please be aware the colons (:) cannot be used in the tag search. Use semicolons instead (the search will automatically search for colons instead). See below for an example: </p>

3
app/View/Helper/XmlOutputHelper.php
View File

@ -1,8 +1,5 @@
<?php
App::uses('AppHelper', 'View/Helper');
// This helper helps determining the brightness of a colour (initially only used for the tagging) in order to decide
// what text colour to use against the background (black or white)
class XmlOutputHelper extends AppHelper {
public function recursiveEcho($array) {
foreach ($array as $k => $v) {

6
app/files/scripts/misp2cybox.py
View File

@ -66,8 +66,8 @@ def generateFileObservable(filenameValue, hashValue):
file_object.file_name = ntpath.basename(filenameValue)
else:
file_object.file_name = filenameValue
if (hashValue != ""):
file_object.add_hash(Hash(hashValue))
if (hashValue != ""):
file_object.add_hash(Hash(hashValue))
return file_object
def generateIPObservable(attribute):
@ -190,6 +190,8 @@ def returnAttachmentComposition(attribute):
observable.observable_composition = composition
else:
observable = Observable(file_object)
if attribute["comment"] != "":
observable.description = attribute["comment"]
return observable
# email-attachment are mapped to an email message observable that contains the attachment as a file object

14
app/files/scripts/misp2stix.py
View File

@ -214,7 +214,10 @@ def handleNonIndicatorAttribute(incident, ttps, attribute):
addJournalEntry(incident, entry_line)
elif attribute["type"] == "target-machine":
aa = AffectedAsset()
aa.description = attribute["value"]
if attribute["comment"] != "":
aa.description = attribute["value"] + " (" + attribute["comment"] + ")"
else:
aa.description = attribute["value"]
incident.affected_assets.append(aa)
elif attribute["type"] == "vulnerability":
generateTTP(incident, attribute)
@ -248,6 +251,8 @@ def generateTTP(incident, attribute):
malware.add_name(attribute["value"])
ttp.behavior = Behavior()
ttp.behavior.add_malware_instance(malware)
if attribute["comment"] != "":
ttp.description = attribute["comment"]
relatedTTP = RelatedTTP(ttp, relationship=attribute["category"])
incident.leveraged_ttps.append(relatedTTP)
@ -256,13 +261,18 @@ def generateThreatActor(attribute):
ta = ThreatActor()
ta.id_= namespace[1] + ":threatactor-" + attribute["uuid"]
ta.title = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"]
ta.description = attribute["value"]
if attribute["comment"] != "":
ta.description = attribute["value"] + " (" + attribute["comment"] + ")"
else:
ta.description = attribute["value"]
return ta
# generate the indicator and add the relevant information
def generateIndicator(attribute):
indicator = Indicator()
indicator.id_= namespace[1] + ":indicator-" + attribute["uuid"]
if attribute["comment"] != "":
indicator.description = attribute["comment"]
setTLP(indicator, attribute["distribution"])
indicator.title = "MISP Attribute #" + attribute["id"] + " uuid: " + attribute["uuid"]
confidence_description = "Derived from MISP's IDS flag. If an attribute is marked for IDS exports, the confidence will be high, otherwise none"