MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'feature/api_rework' into 2.4

pull/3608/head
iglocska 2018-08-31 12:58:44 +02:00
parent 7f46c2d82d d8cc162cfe
commit f3558fb18a
19 changed files with 1255 additions and 865 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
.travis.yml
View File

@ -1,7 +1,6 @@
language: php
php:
- 5.6
- 7.0
- 7.1
- 7.2
@ -11,37 +10,30 @@ services:
- redis
sudo: required
dist: trusty
dist: bionic
group: edge
addons:
mariadb:
- '10.0'
mariadb: '10.2'
hosts:
- misp.local
- localhost
apt:
packages:
- python3-dev
- python3-pip
- python3-nose
- libxml2-dev
- libzmq3-dev
- zlib1g-dev
- apache2
- curl
- php5-mysql
- php5-dev
- php5-cli
- libapache2-mod-php5
before_install:
- git config --global user.name "TravisCI"
- export PATH="$HOME/.local/bin:$PATH"
install:
- sudo pip3 install --upgrade pip setuptools requests
- pip3 install --user --upgrade -r requirements.txt
- sudo add-apt-repository -y ppa:deadsnakes/ppa
- sudo apt-get -y update
- sudo apt-get -y install python3.6 python3-pip python3.6-dev python3-nose libxml2-dev libzmq3-dev zlib1g-dev apache2 curl php-mysql php-dev php-cli libapache2-mod-php libfuzzy-dev
- sudo apt-get -y dist-upgrade
- wget https://bootstrap.pypa.io/get-pip.py
- sudo python3.6 get-pip.py
- hash -r
- sudo pip3.6 install --upgrade pip setuptools requests
- hash -r
- sudo pip3.6 install --upgrade -r requirements.txt
- phpenv rehash
- pushd app
- composer install
@ -69,6 +61,7 @@ install:
- mysql -u misp -pblah misp < INSTALL/MYSQL.sql
# configure apache virtual hosts
- sudo chmod -R 777 `pwd`/build
- sudo mkdir -p /etc/apache2/sites-available
- sudo cp -f build/travis-ci-apache /etc/apache2/sites-available/misp.local.conf
- sudo sed -e "s?%TRAVIS_BUILD_DIR%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.local.conf
- sudo a2dissite 000-default
@ -94,6 +87,7 @@ install:
# Get authkey
- sudo usermod -a -G www-data $USER
- sudo -E su $USER -c 'app/Console/cake userInit -q | sudo tee ./key.txt'
- sudo app/Console/cake Live 1
- sudo chmod 777 ./key.txt
- sudo chmod -R 777 ./tests
- sudo chown -R www-data:www-data `pwd`
@ -116,13 +110,20 @@ script:
- ./curl_tests.sh $AUTH
- popd
- pushd PyMISP
- pip3 install --user .
- nosetests --with-coverage --cover-package=pymisp tests/test_*.py
- python3 tests/test.py
- sudo pip3.6 install nose python-dateutil
- sudo pip3.6 install git+https://github.com/kbandla/pydeep.git
- sudo pip3.6 install -e .[fileobjects,neo,openioc,virustotal]
- pushd tests
- git clone https://github.com/viper-framework/viper-test-files.git
- popd
- python3.6 tests/test.py
- python3.6 tests/test_mispevent.py
- python3.6 tests/test_offline.py
- python3.6 tests/testlive_comprehensive.py
- popd
- cp PyMISP/tests/keys.py PyMISP/examples/events/
- pushd PyMISP/examples/events/
- python3 ./create_massive_dummy_events.py -l 5 -a 30
- python3.6 ./create_massive_dummy_events.py -l 5 -a 30
- popd
- pushd app/files/feed-metadata
- jsonschema -i defaults.json schema.json

64
app/Controller/AppController.php
View File

@ -46,7 +46,7 @@ class AppController extends Controller
public $helpers = array('Utility', 'OrgImg');
private $__queryVersion = '43';
private $__queryVersion = '44';
public $pyMispVersion = '2.4.93';
public $phpmin = '5.6.5';
public $phprec = '7.0.16';
@ -535,6 +535,68 @@ class AppController extends Controller
return $this->Auth->user('org_id');
}
protected function _getApiAuthUser($key, &$exception)
{
if (strlen($key) == 40) {
// check if the key is valid -> search for users based on key
$user = $this->checkAuthUser($key);
if (!$user) {
$exception = $this->RestResponse->throwException(
401,
__('This authentication key is not authorized to be used for exports. Contact your administrator.')
);
return false;
}
} else {
if (!$this->Auth->user('id')) {
$exception = $this->RestResponse->throwException(
401,
__('You have to be logged in to do that.')
);
return false;
}
$user = $this->Auth->user();
}
return $user;
}
// generic function to standardise on the collection of parameters. Accepts posted request objects, url params, named url params
protected function _harvestParameters($options, &$exception)
{
$data = array();
if (!empty($options['request']->is('post'))) {
if (empty($options['request']->data)) {
$exception = $this->RestResponse->throwException(
400,
__('Either specify the search terms in the url, or POST a json with the filter parameters.'),
'/' . $this->request->params['controller'] . '/' . $this->action
);
return false;
} else {
if (isset($options['request']->data['request'])) {
$data = $options['request']->data['request'];
} else {
$data = $options['request']->data;
}
}
}
if (!empty($options['paramArray'])) {
foreach ($options['paramArray'] as $p) {
if (
isset($options['ordered_url_params'][$p]) &&
(!in_array(strtolower($options['ordered_url_params'][$p]), array('null', '0', false, 'false', null)))
) {
$data[$p] = $options['ordered_url_params'][$p];
$data[$p] = str_replace(';', ':', $data[$p]);
}
if (isset($options['named_params'][$p])) {
$data[$p] = $options['named_params'][$p];
}
}
}
return $data;
}
// pass an action to this method for it to check the active user's access to the action
public function checkAction($action = 'perm_sync')
{

268
app/Controller/AttributesController.php
View File

@ -2082,183 +2082,97 @@ class AttributesController extends AppController
$this->set('fails', $this->Attribute->checkComposites());
}
// Use the rest interface to search for attributes. Usage:
// MISP-base-url/attributes/restSearch/[api-key]/[value]/[type]/[category]/[orgc]
// value, type, category, orgc are optional
// the last 4 fields accept the following operators:
// && - you can use && between two search values to put a logical OR between them. for value, 1.1.1.1&&2.2.2.2 would find attributes with the value being either of the two.
// ! - you can negate a search term. For example: google.com&&!mail would search for all attributes with value google.com but not ones that include mail. www.google.com would get returned, mail.google.com wouldn't.
public function restSearch($key = 'download', $value = false, $type = false, $category = false, $org = false, $tags = false, $from = false, $to = false, $last = false, $eventid = false, $withAttachments = false, $uuid = false, $publish_timestamp = false, $published = false, $timestamp = false, $enforceWarninglist = false, $to_ids = false, $deleted = false, $includeEventUuid = false, $event_timestamp = false, $threat_level_id = false)
{
if ($tags) {
$tags = str_replace(';', ':', $tags);
}
$simpleFalse = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id');
foreach ($simpleFalse as $sF) {
if (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false') {
${$sF} = false;
}
}
if ($key != null && strlen($key) == 40) {
$user = $this->checkAuthUser($key);
if (!$user) {
throw new UnauthorizedException(__('This authentication key is not authorized to be used for exports. Contact your administrator.'));
}
} else {
$key = strtolower($key);
if (!$this->Auth->user()) {
throw new UnauthorizedException(__('You are not authorized. Please send the Authorization header with your auth key along with an Accept header for application/xml.'));
}
}
// request handler for POSTed queries. If the request is a post, the parameters (apart from the key) will be ignored and replaced by the terms defined in the posted json or xml object.
// The correct format for both is a "request" root element, as shown by the examples below:
// For Json: {"request":{"value": "7.7.7.7&&1.1.1.1","type":"ip-src"}}
// For XML: <request><value>7.7.7.7&amp;&amp;1.1.1.1</value><type>ip-src</type></request>
// the response type is used to determine the parsing method (xml/json)
if ($this->request->is('post')) {
if ($this->response->type() === 'application/json') {
if ($key == 'xml') {
throw new MethodNotAllowedException(__('Content type and parameter mismatch. Expecting JSON.'));
}
$data = $this->request->input('json_decode', true);
} elseif ($this->response->type() === 'application/xml' && !empty($this->request->data)) {
if ($key == 'json') {
throw new MethodNotAllowedException(__('Content type and parameter mismatch. Expecting XML.'));
}
$data = $this->request->data;
} else {
throw new BadRequestException(__('Either specify the search terms in the url, or POST a json array / xml (with the root element being "request" and specify the correct accept and content type headers).'));
}
if (!isset($data['request'])) {
$data['request'] = $data;
}
$paramArray = array('value', 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'uuid', 'published', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id');
foreach ($paramArray as $p) {
if (isset($data['request'][$p])) {
${$p} = $data['request'][$p];
} else {
${$p} = null;
}
}
}
$simpleFalse = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'event_timestamp', 'threat_level_id');
foreach ($simpleFalse as $sF) {
if (!is_array(${$sF}) && (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false')) {
${$sF} = false;
}
}
if ($from) {
$from = $this->Attribute->Event->dateFieldCheck($from);
}
if ($to) {
$to = $this->Attribute->Event->dateFieldCheck($to);
}
if ($last) {
$last = $this->Attribute->Event->resolveTimeDelta($last);
}
$conditions['AND'] = array();
$subcondition = array();
$this->loadModel('Attribute');
// add the values as specified in the 2nd parameter to the conditions
$parameters = array('value', 'type', 'category', 'org', 'eventid', 'uuid');
foreach ($parameters as $k => $param) {
if (isset(${$parameters[$k]}) && ${$parameters[$k]} !== false) {
$conditions = $this->Attribute->Event->setSimpleConditions($parameters[$k], ${$parameters[$k]}, $conditions);
}
}
// If we sent any tags along, load the associated tag names for each attribute
if ($tags) {
$conditions = $this->Attribute->setTagConditions($tags, $conditions, 'attribute');
}
if ($from) {
$conditions['AND'][] = array('Event.date >=' => $from);
}
if ($to) {
$conditions['AND'][] = array('Event.date <=' => $to);
}
if ($publish_timestamp) {
$conditions = $this->Attribute->setTimestampConditions($publish_timestamp, $conditions, 'Event.publish_timestamp');
}
if ($last) {
$conditions['AND'][] = array('Event.publish_timestamp >=' => $last);
}
if ($published) {
$conditions['AND'][] = array('Event.published' => $published);
}
if ($timestamp) {
$conditions = $this->Attribute->setTimestampConditions($timestamp, $conditions, 'Attribute.timestamp');
}
if ($event_timestamp) {
$conditions = $this->Attribute->setTimestampConditions($event_timestamp, $conditions, 'Event.timestamp');
}
if ($threat_level_id) {
if (!is_array($threat_level_id)) {
$threat_level_id = array($threat_level_id);
}
$threat_level_lookup = array('high' => 1, 'medium' => 2, 'low' => 3, 'undefined' => 4);
foreach ($threat_level_id as $tldk => $tld) {
if (!is_numeric($tld)) {
if (isset($threat_level_lookup[strtolower($tld)])) {
$threat_level_id[$tldk] = $threat_level_lookup[strtolower($tld)];
}
}
}
$conditions['AND'][] = array('Event.threat_level_id' => $threat_level_id);
}
if ($to_ids) {
$conditions = $this->Attribute->setToIDSConditions($to_ids, $conditions);
}
// change the fields here for the attribute export!!!! Don't forget to check for the permissions, since you are not going through fetchevent. Maybe create fetchattribute?
$params = array(
'conditions' => $conditions,
'fields' => array('Attribute.*', 'Event.org_id', 'Event.distribution'),
'withAttachments' => $withAttachments,
'enforceWarninglist' => $enforceWarninglist,
'includeAllTags' => true,
'flatten' => 1,
'includeEventUuid' => $includeEventUuid
);
if ($deleted) {
$params['deleted'] = 1;
if ($deleted === 'only') {
$params['conditions']['AND'][] = array('Attribute.deleted' => 1);
}
}
$results = $this->Attribute->fetchAttributes($this->Auth->user(), $params);
$this->loadModel('Whitelist');
$results = $this->Whitelist->removeWhitelistedFromArray($results, true);
if ($key == 'openioc') {
App::uses('IOCExportTool', 'Tools');
$this->IOCExport = new IOCExportTool();
$results = $this->IOCExport->buildAll($this->Auth->user(), $results, 'attribute');
} else {
if (!empty($results)) {
$results = array('response' => array('Attribute' => $results));
foreach ($results['response']['Attribute'] as $k => $v) {
if (isset($results['response']['Attribute'][$k]['AttributeTag'])) {
foreach ($results['response']['Attribute'][$k]['AttributeTag'] as $tk => $tag) {
$results['response']['Attribute'][$k]['Attribute']['Tag'][$tk] = $tag['Tag'];
}
}
$results['response']['Attribute'][$k] = $results['response']['Attribute'][$k]['Attribute'];
unset(
$results['response']['Attribute'][$k]['value1'],
$results['response']['Attribute'][$k]['value2']
);
}
} else {
$results = array('response' => array());
}
}
$responseType = $this->response->type();
if ($key == 'openioc') {
$responseType = 'openioc';
}
return $this->RestResponse->viewData($results, $responseType);
public function restSearch($returnFormat = 'json', $value = false, $type = false, $category = false, $org = false, $tags = false, $from = false, $to = false, $last = false, $eventid = false, $withAttachments = false, $uuid = false, $publish_timestamp = false, $published = false, $timestamp = false, $enforceWarninglist = false, $to_ids = false, $deleted = false, $includeEventUuid = false, $event_timestamp = false, $threat_level_id = false) {
$paramArray = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id');
$filterData = array(
'request' => $this->request,
'named_params' => $this->params['named'],
'paramArray' => $paramArray,
'ordered_url_params' => compact($paramArray)
);
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
unset($filterData);
if ($filters === false) {
return $exception;
}
$list = array();
$user = $this->_getApiAuthUser($returnFormat, $exception);
if ($user === false) {
return $exception;
}
if (isset($filters['returnFormat'])) {
$returnFormat = $filters['returnFormat'];
}
$conditions = $this->Attribute->buildFilterConditions($this->Auth->user(), $filters);
$params = array(
'conditions' => $conditions,
'fields' => array('Attribute.*', 'Event.org_id', 'Event.distribution'),
'withAttachments' => !empty($filters['withAttachments']) ? $filters['withAttachments'] : 0,
'enforceWarninglist' => !empty($filters['enforceWarninglist']) ? $filters['enforceWarninglist'] : 0,
'includeAllTags' => true,
'flatten' => 1,
'includeEventUuid' => !empty($filters['includeEventUuid']) ? $filters['includeEventUuid'] : 0,
);
if (!empty($filtes['deleted'])) {
$params['deleted'] = 1;
if ($params['deleted'] === 'only') {
$params['conditions']['AND'][] = array('Attribute.deleted' => 1);
$params['conditions']['AND'][] = array('Object.deleted' => 1);
}
}
$results = $this->Attribute->fetchAttributes($this->Auth->user(), $params);
$this->loadModel('Whitelist');
$results = $this->Whitelist->removeWhitelistedFromArray($results, true);
$validFormats = array(
'openioc' => 'OpeniocExport',
'json' => 'AttributeExport',
'xml' => 'AttributeExport'
);
$final = '';
App::uses($validFormats[$returnFormat], 'Export');
$exportTool = new $validFormats[$returnFormat]();
$exportToolParams = array(
'user' => $this->Auth->user(),
'params' => $params,
'returnFormat' => $returnFormat
);
$final .= $exportTool->header($exportToolParams);
foreach ($results as $attribute) {
$final .= $exportTool->handler($attribute, $exportToolParams);
$final .= $exportTool->separator($exportToolParams);
}
$final .= $exportTool->footer($exportToolParams);
/*
if ($returnFormat == 'openioc') {
App::uses('IOCExportTool', 'Tools');
$this->IOCExport = new IOCExportTool();
$results = $this->IOCExport->buildAll($this->Auth->user(), $results, 'attribute');
} else {
if (!empty($results)) {
$results = array('response' => array('Attribute' => $results));
foreach ($results['response']['Attribute'] as $k => $v) {
if (isset($results['response']['Attribute'][$k]['AttributeTag'])) {
foreach ($results['response']['Attribute'][$k]['AttributeTag'] as $tk => $tag) {
$results['response']['Attribute'][$k]['Attribute']['Tag'][$tk] = $tag['Tag'];
}
}
$results['response']['Attribute'][$k] = $results['response']['Attribute'][$k]['Attribute'];
unset(
$results['response']['Attribute'][$k]['value1'],
$results['response']['Attribute'][$k]['value2']
);
}
} else {
$results = array('response' => array('Attribute' => array()));
}
}
*/
$responseType = $this->response->type();
if ($returnFormat == 'openioc') {
$responseType = 'openioc';
}
return $this->RestResponse->viewData($final, $responseType);
}
// returns an XML with attributes that belong to an event. The type of attributes to be returned can be restricted by type using the 3rd parameter.

7
app/Controller/Component/RestResponseComponent.php
View File

@ -278,8 +278,13 @@ class RestResponseComponent extends Component
return $this->__sendResponse($data, 200, $format, $raw, $download);
}
public function throwException($code, $message, $format, $raw)
public function throwException($code, $message, $url = '', $format = false, $raw = false)
{
$message = array(
'name' => $message,
'message' => $message,
'url' => $url
);
return $this->__sendResponse($message, $code, $format, $raw);
}

368
app/Controller/EventsController.php
View File

@ -2679,56 +2679,26 @@ class EventsController extends AppController
public function csv($key, $eventid = false, $ignore = false, $tags = false, $category = false, $type = false, $includeContext = false, $from = false, $to = false, $last = false, $headerless = false, $enforceWarninglist = false, $value = false, $timestamp = false)
{
$paramArray = array('eventid', 'ignore', 'tags', 'category', 'type', 'includeContext', 'from', 'to', 'last', 'headerless', 'enforceWarninglist', 'value', 'timestamp');
if ($this->request->is('post')) {
if (empty($this->request->data)) {
return $this->RestResponse->throwException(400, __('Either specify the search terms in the url, or POST a json or xml with the filter parameters.'), 'csv', true);
} else {
$data = $this->request->data;
}
if (!isset($data['request'])) {
$data = array('request' => $data);
}
foreach ($paramArray as $p) {
if (isset($data['request'][$p])) {
${$p} = $data['request'][$p];
}
}
}
foreach ($paramArray as $p) {
if (!is_array(${$p}) && (${$p} === 'null' || ${$p} == '0' || ${$p} === false || strtolower(${$p}) === 'false')) {
${$p} = false;
}
}
$exportType = $eventid;
if ($from) {
$from = $this->Event->dateFieldCheck($from);
}
if ($to) {
$to = $this->Event->dateFieldCheck($to);
}
if ($tags) {
$tags = str_replace(';', ':', $tags);
}
if ($last) {
$last = $this->Event->resolveTimeDelta($last);
$filterData = array(
'request' => $this->request,
'named_params' => $this->params['named'],
'paramArray' => $paramArray,
'ordered_url_params' => compact($paramArray)
);
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
if ($filters === false) {
return $exception;
}
$list = array();
if ($key != 'download') {
// check if the key is valid -> search for users based on key
$user = $this->checkAuthUser($key);
if (!$user) {
return $this->RestResponse->throwException(401, __('This authentication key is not authorized to be used for exports. Contact your administrator.'), 'csv', true);
}
} else {
if (!$this->Auth->user('id')) {
return $this->RestResponse->throwException(401, __('You have to be logged in to do that.'), 'csv', true);
}
$user = $this->Auth->user();
$user = $this->_getApiAuthUser($key, $exception);
if ($user === false) {
return $exception;
}
// if it's a search, grab the attributeIDList from the session and get the IDs from it. Use those as the condition
// We don't need to look out for permissions since that's filtered by the search itself
// We just want all the attributes found by the search
if ($eventid === 'search') {
if (!empty($filters['eventid']) && $filters['eventid'] === 'search') {
$ioc = $this->Session->read('paginate_conditions_ioc');
$paginateConditions = $this->Session->read('paginate_conditions');
unset($paginateConditions['contain']['Event']['Orgc']);
@ -2752,31 +2722,10 @@ class EventsController extends AppController
);
$list[] = $attribute['Attribute']['id'];
}
$events = array($eventid);
} elseif ($eventid === false) {
$events = $this->Event->fetchEventIds($this->Auth->user(), $from, $to, $last, true);
if (empty($events)) {
$events = array(0 => -1);
}
} else {
$events = array($eventid);
} elseif (!empty($filters['eventid']) && $filters['eventid'] !== 'all') {
$events = $filters['eventid'];
}
$final = array();
$this->loadModel('Whitelist');
if ($tags) {
$args = $this->Event->Attribute->dissectArgs($tags);
$tagArray = $this->Event->EventTag->Tag->fetchEventTagIds($args[0], $args[1]);
if (!empty($tagArray[0])) {
$events = array_intersect($events, $tagArray[0]);
}
if (!empty($tagArray[1])) {
foreach ($events as $k => $eventid) {
if (in_array($eventid, $tagArray[1])) {
unset($events[$k]);
}
}
}
}
$requested_attributes = array('uuid', 'event_id', 'category', 'type',
'value', 'comment', 'to_ids', 'timestamp', 'object_relation');
$requested_obj_attributes = array('uuid', 'name', 'meta-category');
@ -2801,65 +2750,50 @@ class EventsController extends AppController
if (isset($data['request']['obj_attributes'])) {
$requested_obj_attributes = $data['request']['obj_attributes'];
}
if (isset($events)) {
$events = array_chunk($events, 100);
foreach ($events as $k => $eventid) {
$attributes = $this->Event->csv($user, $eventid, $ignore, $list, false, $category, $type, $includeContext, false, false, false, $enforceWarninglist, $value, $timestamp);
$attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true);
foreach ($attributes as $attribute) {
$line1 = '';
$line2 = '';
foreach ($requested_attributes as $requested_attribute) {
$line1 .= $attribute['Attribute'][$requested_attribute] . ',';
}
$line1 = rtrim($line1, ",");
foreach ($requested_obj_attributes as $requested_obj_attribute) {
$line2 .= $attribute['Object'][$requested_obj_attribute] . ',';
}
$line2 = rtrim($line2, ",");
$line = $line1 . ',' . $line2;
$line = rtrim($line, ",");
if ($includeContext) {
foreach ($this->Event->csv_event_context_fields_to_fetch as $header => $field) {
if ($field['object']) {
$line .= ',' . $attribute['Event'][$field['object']][$field['var']];
} else {
$line .= ',' . str_replace(array("\n","\t","\r"), " ", $attribute['Event'][$field['var']]);
}
}
}
$final[] = $line;
}
$possibleParams = array(
'ignore', 'list', 'category', 'type', 'includeContext',
'enforceWarninglist', 'value', 'timestamp', 'tags',
'last', 'from', 'to'
);
$params = array();
if (!empty($events)) {
$filters['eventid'] = $events;
}
foreach ($possibleParams as $possibleParam) {
if (isset($filters[$possibleParam])) {
$params[$possibleParam] = $filters[$possibleParam];
}
}
$params['limit'] = 1000;
$params['page'] = 1;
$i = 0;
$continue = true;
$options = array(
'requested_obj_attributes' => $requested_obj_attributes,
'requested_attributes' => $requested_attributes,
'includeContext' => $includeContext
);
App::uses('CsvExport', 'Export');
$export = new CsvExport();
$final = $export->header($options);
while ($continue) {
$attributes = $this->Event->csv($user, $params, false, $continue);
$params['page'] += 1;
$final .= $export->handler($attributes, $final, $options);
$final .= $export->separator($attributes, $final);
}
$export->footer();
$this->response->type('csv'); // set the content type
if (!$exportType) {
$filename = "misp.all_attributes.csv";
} elseif ($exportType === 'search') {
if (empty($filters['eventid'])) {
$filename = "misp.filtered_attributes.csv";
} elseif ($filters['eventid'] === 'search') {
$filename = "misp.search_result.csv";
} else {
$filename = "misp.event_" . $exportType . ".csv";
}
$this->layout = 'text/default';
if (!empty($requested_obj_attributes)) {
array_walk($requested_obj_attributes, function (&$value, $key) {
$value = 'object-'.$value;
});
}
$headers = array_merge($requested_attributes, $requested_obj_attributes);
if ($includeContext) {
$headers = array_merge($headers, array_keys($this->Event->csv_event_context_fields_to_fetch));
}
foreach ($headers as $k => $v) {
$headers[$k] = str_replace('-', '_', $v);
if ($v == 'timestamp') {
$headers[$k] = 'date';
if (is_array($filters['eventid'])) {
$filters['eventid'] = 'list';
}
$filename = "misp.event_" . $filters['eventid'] . ".csv";
}
$headers = implode(',', $headers);
$final = array_merge(array($headers), $final);
$final = implode(PHP_EOL, $final);
$final .= PHP_EOL;
return $this->RestResponse->viewData($final, 'csv', false, true, $filename);
}
@ -3072,168 +3006,38 @@ class EventsController extends AppController
// the last 4 fields accept the following operators:
// && - you can use && between two search values to put a logical OR between them. for value, 1.1.1.1&&2.2.2.2 would find attributes with the value being either of the two.
// ! - you can negate a search term. For example: google.com&&!mail would search for all attributes with value google.com but not ones that include mail. www.google.com would get returned, mail.google.com wouldn't.
public function restSearch($key = 'download', $value = false, $type = false, $category = false, $org = false, $tags = false, $searchall = false, $from = false, $to = false, $last = false, $eventid = false, $withAttachments = false, $metadata = false, $uuid = false, $publish_timestamp = false, $timestamp = false, $published = false, $enforceWarninglist = false, $sgReferenceOnly = false)
public function restSearch($returnFormat = 'json', $value = false, $type = false, $category = false, $org = false, $tags = false, $searchall = false, $from = false, $to = false, $last = false, $eventid = false, $withAttachments = false, $metadata = false, $uuid = false, $publish_timestamp = false, $timestamp = false, $published = false, $enforceWarninglist = false, $sgReferenceOnly = false)
{
if ($key != null && strlen($key) == 40) {
if (!$this->checkAuthUser($key)) {
throw new UnauthorizedException(__('This authentication key is not authorized to be used for exports. Contact your administrator.'));
}
} else {
$key = strtolower($key);
if (!$this->Auth->user()) {
throw new UnauthorizedException(__('You are not authorized. Please send the Authorization header with your auth key along with an Accept header for application/xml.'));
}
$paramArray = array('value', 'type', 'category', 'org', 'tag', 'tags', 'searchall', 'from', 'to', 'last', 'eventid', 'withAttachments', 'metadata', 'uuid', 'published', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'sgReferenceOnly');
$filterData = array(
'request' => $this->request,
'named_params' => $this->params['named'],
'paramArray' => $paramArray,
'ordered_url_params' => compact($paramArray)
);
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
unset($filterData);
if ($filters === false) {
return $exception;
}
if (!is_array($value) && $value !== false) {
$value = str_replace('|', '/', $value);
$list = array();
$user = $this->_getApiAuthUser($returnFormat, $exception);
if ($user === false) {
return $exception;
}
// request handler for POSTed queries. If the request is a post, the parameters (apart from the key) will be ignored and replaced by the terms defined in the posted json or xml object.
// The correct format for both is a "request" root element, as shown by the examples below:
// For Json: {"request":{"value": "7.7.7.7&&1.1.1.1","type":"ip-src"}}
// For XML: <request><value>7.7.7.7&amp;&amp;1.1.1.1</value><type>ip-src</type></request>
// the response type is used to determine the parsing method (xml/json)
if ($this->request->is('post')) {
if ($this->response->type() === 'application/json') {
$data = $this->request->input('json_decode', true);
} elseif ($this->response->type() === 'application/xml') {
$data = $this->request->data;
} else {
throw new BadRequestException(__('Either specify the search terms in the url, or POST a json array / xml (with the root element being "request" and specify the correct headers based on content type).'));
}
if (!isset($data['request'])) {
$data['request'] = $data;
}
$paramArray = array('value', 'type', 'category', 'org', 'tag', 'tags', 'searchall', 'from', 'to', 'last', 'eventid', 'withAttachments', 'metadata', 'uuid', 'published', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'sgReferenceOnly');
foreach ($paramArray as $p) {
if (isset($data['request'][$p])) {
${$p} = $data['request'][$p];
} else {
${$p} = null;
}
}
if (isset($filters['returnFormat'])) {
$returnFormat = $filters['returnFormat'];
}
$simpleFalse = array('value' , 'type', 'category', 'org', 'tags', 'searchall', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'sgReferenceOnly');
foreach ($simpleFalse as $sF) {
if (!is_array(${$sF}) && (${$sF} === 'null' || ${$sF} == '0' || ${$sF} === false || strtolower(${$sF}) === 'false')) {
${$sF} = false;
}
}
if ($from) {
$from = $this->Event->dateFieldCheck($from);
}
if ($to) {
$to = $this->Event->dateFieldCheck($to);
}
if (!empty($tag) && !$tags) {
$tags = $tag;
}
if ($tags) {
$tags = str_replace(';', ':', $tags);
}
if ($last) {
$last = $this->Event->resolveTimeDelta($last);
}
if ($searchall === 'true') {
$searchall = "1";
}
$conditions['AND'] = array();
$subcondition = array();
$this->loadModel('Attribute');
// add the values as specified in the 2nd parameter to the conditions
if (isset($searchall) && ($searchall == 1 || $searchall === true || $searchall == 'true')) {
$eventIds = $this->__quickFilter($value);
} else {
$parameters = array('value', 'type', 'category', 'org', 'uuid', 'eventid');
$attributeLevelFilters = array('value', 'type', 'category', 'uuid');
$preFilterLevel = 'event';
foreach ($parameters as $k => $param) {
if (${$parameters[$k]} !== null && ${$parameters[$k]} !== false) {
if (in_array($param, $attributeLevelFilters)) {
$preFilterLevel = 'attribute';
}
if ($param == 'eventid') {
$restrictScopeToEvents = true;
}
$conditions = $this->Event->setSimpleConditions($parameters[$k], ${$parameters[$k]}, $conditions, !empty($restrictScopeToEvents));
}
}
// If we sent any tags along, load the associated tag names for each attribute
if ($tags) {
$conditions = $this->Event->Attribute->setTagConditions($tags, $conditions);
}
$blockedAttributeTags = array();
if (!empty($tags)) {
if (!is_array($tags)) {
$tags = explode('&&', $tags);
}
foreach ($tags as $tag) {
if ($tag[0] == '!') {
$blockedAttributeTags[] = ltrim($tag, '!');
}
}
$preFilterLevel = 'attribute';
}
if ($from) {
$conditions['AND'][] = array('Event.date >=' => $from);
}
if ($to) {
$conditions['AND'][] = array('Event.date <=' => $to);
}
if ($publish_timestamp) {
$conditions = $this->Event->Attribute->setTimestampConditions($publish_timestamp, $conditions, 'Event.publish_timestamp');
}
if ($timestamp) {
$conditions = $this->Event->Attribute->setTimestampConditions($timestamp, $conditions, 'Event.timestamp');
}
if ($last) {
$conditions['AND'][] = array('Event.publish_timestamp >=' => $last);
}
if ($published !== null && $published !== false) {
$conditions['AND'][] = array('Event.published' => $published);
}
if ($preFilterLevel == 'event') {
$params = array(
'conditions' => $conditions
);
$eventIds = $this->Event->fetchSimpleEventIds($this->Auth->user(), $params);
} else {
$params = array(
'conditions' => $conditions,
'fields' => array('DISTINCT(Attribute.event_id)'),
'contain' => array(),
'recursive' => -1,
'list' => true,
'event_ids' => true,
'flatten' => 1
);
$attributes = $this->Event->Attribute->fetchAttributes($this->Auth->user(), $params);
$eventIds = array();
if (!empty($attributes)) {
$eventIds = array_values($attributes);
}
if (is_array($eventid)) {
foreach ($eventid as $temp_id) {
if (!in_array($temp_id, $eventIds)) {
$eventIds[] = $temp_id;
}
}
} else {
if ($eventid && !in_array($eventid, $eventIds)) {
$eventIds[] = $eventid;
}
}
unset($attributes);
}
}
$this->loadModel('Whitelist');
$responseType = 'xml';
$eventid = $this->Event->filterEventIds($user, $filters);
$responseType = 'json';
$converters = array(
'xml' => 'XMLConverterTool',
'json' => 'JSONConverterTool',
'openioc' => 'IOCExportTool'
);
if (in_array($key, array('json', 'xml', 'openioc'))) {
$responseType = $key;
if (in_array($returnFormat, array('json', 'xml', 'openioc'))) {
$responseType = $returnFormat;
} elseif (((isset($this->request->params['ext']) && $this->request->params['ext'] == 'xml')) || $this->response->type() == 'application/xml') {
$responseType = 'xml';
} else {
@ -3242,22 +3046,20 @@ class EventsController extends AppController
App::uses($converters[$responseType], 'Tools');
$converter = new $converters[$responseType]();
$final = $converter->generateTop($this->Auth->user());
$eventCount = count($eventIds);
$eventCount = count($eventid);
$i = 0;
foreach ($eventIds as $k => $currentEventId) {
foreach ($eventid as $k => $currentEventId) {
$filters['eventid'] = $currentEventId;
if (!empty($filters['tags']['NOT'])) {
$filters['blockedAttributeTags'] = $filters['tags']['NOT'];
}
$result = $this->Event->fetchEvent(
$this->Auth->user(),
array(
'blockedAttributeTags' => $blockedAttributeTags,
'eventid' => $currentEventId,
'includeAttachments' => $withAttachments,
'metadata' => $metadata,
'enforceWarninglist' => $enforceWarninglist,
'sgReferenceOnly' => $sgReferenceOnly
),
$filters,
true
);
if (!empty($result)) {
$this->loadModel('Whitelist');
$result = $this->Whitelist->removeWhitelistedFromArray($result, false);
if ($i != 0) {
$final .= ',' . PHP_EOL;
@ -3271,7 +3073,7 @@ class EventsController extends AppController
}
$final .= $converter->generateBottom($responseType, $final);
$extension = $responseType;
if ($key == 'openioc') {
if ($returnFormat == 'openioc') {
$extension = '.ioc';
}
if (isset($eventid) && $eventid) {

8
app/Controller/ServersController.php
View File

@ -1611,7 +1611,8 @@ class ServersController extends AppController
return $this->RestResponse->viewData(array('uuid' => Configure::read('MISP.uuid')), $this->response->type());
}
public function rest() {
public function rest()
{
if ($this->request->is('post')) {
$request = $this->request->data;
if (!empty($request['Server'])) {
@ -1631,7 +1632,8 @@ class ServersController extends AppController
$this->set('header', $header);
}
private function __doRestQuery($request) {
private function __doRestQuery($request)
{
App::uses('SyncTool', 'Tools');
$params = array(
@ -1667,7 +1669,7 @@ class ServersController extends AppController
$request['method'] === 'GET'
) {
$response = $HttpSocket->get($url, false, array('header' => $request['header']));
} else if (
} elseif (
!empty($request['method']) &&
$request['method'] === 'POST' &&
!empty($request['body'])

16
app/Controller/UsersController.php
View File

@ -750,7 +750,6 @@ class UsersController extends AppController
array_push($fields, $field);
}
}
// TODO Audit, __extralog, fields get orig
$fieldsOldValues = array();
foreach ($fields as $field) {
if ($field == 'enable_password') {
@ -762,7 +761,6 @@ class UsersController extends AppController
array_push($fieldsOldValues, $this->User->field('password'));
}
}
// TODO Audit, __extralog, fields get orig END
if (
isset($this->request->data['User']['enable_password']) && $this->request->data['User']['enable_password'] != '0' &&
isset($this->request->data['User']['password']) && "" != $this->request->data['User']['password']
@ -785,7 +783,6 @@ class UsersController extends AppController
}
}
if ($this->User->save($this->request->data, true, $fields)) {
// TODO Audit, __extralog, fields compare
// newValues to array
$fieldsNewValues = array();
foreach ($fields as $field) {
@ -822,8 +819,7 @@ class UsersController extends AppController
$c++;
}
$fieldsResultStr = substr($fieldsResultStr, 2);
$this->__extralog("edit", "user", $fieldsResultStr); // TODO Audit, check: modify User
// TODO Audit, __extralog, fields compare END
$this->__extralog("edit", "user", $fieldsResultStr);
if ($this->_isRest()) {
$user = $this->User->find('first', array(
'conditions' => array('User.id' => $this->User->id),
@ -853,7 +849,7 @@ class UsersController extends AppController
$this->redirect(array('controller' => 'users', 'action' => 'index', 'admin' => true));
}
$this->User->set('password', '');
$this->request->data = $this->User->data; // TODO CHECK
$this->request->data = $this->User->data;
}
if ($this->_isSiteAdmin()) {
$orgs = $this->User->Organisation->find('list', array(
@ -964,7 +960,7 @@ class UsersController extends AppController
}
}
if ($this->Auth->login()) {
$this->__extralog("login"); // TODO Audit, __extralog, check: customLog i.s.o. __extralog, no auth user?: $this->User->customLog('login', $this->Auth->user('id'), array('title' => '','user_id' => $this->Auth->user('id'),'email' => $this->Auth->user('email'),'org' => 'IN2'));
$this->__extralog("login");
$this->User->Behaviors->disable('SysLogLogable.SysLogLogable');
$this->User->id = $this->Auth->user('id');
$user = $this->User->find('first', array(
@ -1074,8 +1070,8 @@ class UsersController extends AppController
public function logout()
{
if ($this->Session->check('Auth.User')) { // TODO session, user is logged in, so ..
$this->__extralog("logout"); // TODO Audit, __extralog, check: customLog i.s.o. __extralog, $this->User->customLog('logout', $this->Auth->user('id'), array());
if ($this->Session->check('Auth.User')) {
$this->__extralog("logout");
}
$this->Flash->info(__('Good-Bye'));
$user = $this->User->find('first', array(
@ -1245,7 +1241,7 @@ class UsersController extends AppController
}
private function __extralog($action = null, $description = null, $fieldsResult = null)
{ // TODO move audit to AuditsController?
{
// new data
$model = 'User';
$modelId = $this->Auth->user('id');

79
app/Lib/Export/CsvExport.php Normal file
View File

@ -0,0 +1,79 @@
<?php
class CsvExport
{
public $csv_event_context_fields_to_fetch = array(
'event_info' => array('object' => false, 'var' => 'info'),
'event_member_org' => array('object' => 'Org', 'var' => 'name'),
'event_source_org' => array('object' => 'Orgc', 'var' => 'name'),
'event_distribution' => array('object' => false, 'var' => 'distribution'),
'event_threat_level_id' => array('object' => 'ThreatLevel', 'var' => 'name'),
'event_analysis' => array('object' => false, 'var' => 'analysis'),
'event_date' => array('object' => false, 'var' => 'date'),
'event_tag' => array('object' => 'Tag', 'var' => 'name')
);
public function handler($attributes, $final, $options = array())
{
$result = array();
foreach ($attributes as $attribute) {
$line1 = '';
$line2 = '';
foreach ($options['requested_attributes'] as $requested_attribute) {
$line1 .= $attribute['Attribute'][$requested_attribute] . ',';
}
$line1 = rtrim($line1, ",");
foreach ($options['requested_obj_attributes'] as $requested_obj_attribute) {
$line2 .= $attribute['Object'][$requested_obj_attribute] . ',';
}
$line2 = rtrim($line2, ",");
$line = $line1 . ',' . $line2;
$line = rtrim($line, ",");
if (!empty($options['includeContext'])) {
foreach ($this->Event->csv_event_context_fields_to_fetch as $header => $field) {
if ($field['object']) {
$line .= ',' . $attribute['Event'][$field['object']][$field['var']];
} else {
$line .= ',' . str_replace(array("\n","\t","\r"), " ", $attribute['Event'][$field['var']]);
}
}
}
$result[] = $line;
}
$result = implode(PHP_EOL, $result);
return $result;
}
public function header($options = array())
{
if (!empty($options['requested_obj_attributes'])) {
array_walk($options['requested_obj_attributes'], function (&$value, $key) {
$value = 'object-'.$value;
});
}
$headers = array_merge($options['requested_attributes'], $options['requested_obj_attributes']);
if (!empty($options['includeContext'])) {
$headers = array_merge($headers, array_keys($this->csv_event_context_fields_to_fetch));
}
foreach ($headers as $k => $v) {
$headers[$k] = str_replace('-', '_', $v);
if ($v == 'timestamp') {
$headers[$k] = 'date';
}
}
$headers = implode(',', $headers) . PHP_EOL;
return $headers;
}
public function footer()
{
return PHP_EOL;
}
public function separator()
{
return PHP_EOL;
}
}

3
app/Lib/Tools/FinancialTool.php
View File

@ -232,7 +232,8 @@ class FinancialTool
$out = array_fill(0, 25, 0);
for ($i=0;$i<strlen($input);$i++) {
if (($p=strpos($alphabet, $input[$i]))===false) {
$p = strpos($alphabet, $input[$i]);
if ($p === false) {
return false;
}
$c = $p;

92
app/Model/AppModel.php
View File

@ -1194,7 +1194,8 @@ class AppModel extends Model
return $version_array;
}
public function validateAuthkey($value) {
public function validateAuthkey($value)
{
if (empty($value['authkey'])) {
return 'Empty authkey found. Make sure you set the 40 character long authkey.';
}
@ -1458,7 +1459,8 @@ class AppModel extends Model
$this->elasticSearchClient = $client;
}
public function getS3Client() {
public function getS3Client()
{
if (!$this->s3Client) {
$this->s3Client = $this->loadS3Client();
}
@ -1466,14 +1468,16 @@ class AppModel extends Model
return $this->s3Client;
}
public function loadS3Client() {
public function loadS3Client()
{
App::uses('AWSS3Client', 'Tools');
$client = new AWSS3Client();
$client->initTool();
return $client;
}
public function attachmentDirIsS3() {
public function attachmentDirIsS3()
{
// Naive way to detect if we're working in S3
return substr(Configure::read('MISP.attachments_dir'), 0, 2) === "s3";
}
@ -1655,4 +1659,84 @@ class AppModel extends Model
}
return $request;
}
// take filters in the {"OR" => [foo], "NOT" => [bar]} format along with conditions and set the conditions
public function generic_add_filter($conditions, &$filter, $keys)
{
$operator_composition = array(
'NOT' => 'AND',
'OR' => 'OR',
'AND' => 'AND'
);
if (!is_array($keys)) {
$keys = array($keys);
}
if (!isset($filter['OR']) && !isset($filter['AND']) && !isset($filter['OR'])) {
return $conditions;
}
foreach ($filter as $operator => $filters) {
$temp = array();
foreach ($filters as $f) {
// split the filter params into two lists, one for substring searches one for exact ones
if ($f[strlen($f) - 1] === '%' || $f[0] === '%') {
foreach ($keys as $key) {
if ($operator === 'NOT') {
$temp[] = array($key . ' NOT LIKE' => $f);
} else {
$temp[] = array($key . ' LIKE' => $f);
}
}
} else {
foreach ($keys as $key) {
if ($operator === 'NOT') {
$temp[$key . ' !='][] = $f;
} else {
$temp['OR'][$key][] = $f;
}
}
}
}
$conditions['AND'][] = array($operator_composition[$operator] => $temp);
if ($operator !== 'NOT') {
unset($filter[$operator]);
}
}
return $conditions;
}
/*
* Get filters in one of the following formats:
* [foo, bar]
* ["OR" => [foo, bar], "NOT" => [baz]]
* "foo"
* "foo&&bar&&!baz"
* and convert it into the same format ["OR" => [foo, bar], "NOT" => [baz]]
*/
public function convert_filters($filter)
{
if (!is_array($filter)) {
$temp = explode('&&', $filter);
$filter = array();
foreach ($temp as $f) {
if ($f[0] === '!') {
$filter['NOT'][] = $f;
} else {
$filter['OR'][] = $f;
}
}
return $filter;
}
if (!isset($filter['OR']) && !isset($filter['NOT']) && !isset($filter['AND'])) {
$temp = array();
foreach ($filter as $param) {
if ($param[0] === '!') {
$temp['NOT'][] = substr($param, 1);
} else {
$temp['OR'][] = $param;
}
}
$filter = $temp;
}
return $filter;
}
}

241
app/Model/Attribute.php
View File

@ -668,8 +668,7 @@ class Attribute extends AppModel
// We're working in S3
$s3 = $this->getS3Client();
$s3->delete($this->data['Attribute']['event_id'] . DS . $this->data['Attribute']['id']);
}
else {
} else {
// Standard delete
$filepath = $attachments_dir . DS . $this->data['Attribute']['event_id'] . DS . $this->data['Attribute']['id'];
$file = new File($filepath);
@ -2068,6 +2067,88 @@ class Attribute extends AppModel
return $rules;
}
public function set_filter_tags(&$params, $conditions, $options)
{
if (empty($params['tags'])) {
return $conditions;
}
$tag = ClassRegistry::init('Tag');
$params['tags'] = $this->dissectArgs($params['tags']);
$tagArray = $tag->fetchTagIds($params['tags'][0], $params['tags'][1]);
if (!empty($params['tags'][0]) && empty($tagArray[0])) {
$tagArray[0] = array(-1);
}
$temp = array();
if (!empty($tagArray[0])) {
$subquery_options = array(
'conditions' => array(
'tag_id' => $tagArray[0]
),
'fields' => array(
'event_id'
)
);
$lookup_field = ($options['scope'] === 'Event') ? 'Event.id' : 'Attribute.event_id';
$temp = array_merge(
$temp,
$this->subQueryGenerator($tag->EventTag, $subquery_options, $lookup_field)
);
$subquery_options = array(
'conditions' => array(
'tag_id' => $tagArray[0]
),
'fields' => array(
$options['scope'] === 'Event' ? 'Event.id' : 'attribute_id'
)
);
$lookup_field = $options['scope'] === 'Event' ? 'Event.id' : 'Attribute.id';
$temp = array_merge(
$temp,
$this->subQueryGenerator($tag->AttributeTag, $subquery_options, $lookup_field)
);
$conditions['AND'][] = array('OR' => $temp);
}
$temp = array();
if (!empty($tagArray[1])) {
if ($options['scope'] == 'all' || $options['scope'] == 'Event') {
$subquery_options = array(
'conditions' => array(
'tag_id' => $tagArray[1]
),
'fields' => array(
'event_id'
)
);
$lookup_field = ($options['scope'] === 'Event') ? 'Event.id' : 'Attribute.event_id';
$conditions['AND'][] = array_merge($temp, $this->subQueryGenerator($tag->EventTag, $subquery_options, $lookup_field, 1));
}
if ($options['scope'] == 'all' || $options['scope'] == 'Attribute') {
$subquery_options = array(
'conditions' => array(
'tag_id' => $tagArray[1]
),
'fields' => array(
$options['scope'] === 'Event' ? 'event.id' : 'attribute_id'
)
);
$lookup_field = $options['scope'] === 'Event' ? 'Event.id' : 'Attribute.id';
$conditions['AND'][] = array_merge($temp, $this->subQueryGenerator($tag->AttributeTag, $subquery_options, $lookup_field, 1));
}
}
$params['tags'] = array();
if (!empty($tagArray[0]) && empty($options['pop'])) {
$params['tags']['OR'] = $tagArray[0];
}
if (!empty($tagArray[1])) {
$params['tags']['NOT'] = $tagArray[1];
}
if (empty($params['tags'])) {
unset($params['tags']);
}
return $conditions;
}
public function text($user, $type, $tags = false, $eventId = false, $allowNonIDS = false, $from = false, $to = false, $last = false, $enforceWarninglist = false, $allowNotPublished = false)
{
//permissions are taken care of in fetchAttributes()
@ -2094,56 +2175,7 @@ class Attribute extends AppModel
if ($eventId !== false) {
$conditions['AND'][] = array('Event.id' => $eventId);
} elseif ($tags !== false) {
// If we sent any tags along, load the associated tag names for each attribute
$tag = ClassRegistry::init('Tag');
$args = $this->dissectArgs($tags);
$tagArray = $tag->fetchTagIds($args[0], $args[1]);
$temp = array();
if (!empty($tagArray[0])) {
$options = array(
'conditions' => array(
'tag_id' => $tagArray[0]
),
'fields' => array(
'event_id'
)
);
$temp = array_merge($temp, $this->subQueryGenerator($tag->EventTag, $options, 'Attribute.event_id'));
$options = array(
'conditions' => array(
'tag_id' => $tagArray[0]
),
'fields' => array(
'attribute_id'
)
);
$temp = array_merge($temp, $this->subQueryGenerator($tag->AttributeTag, $options, 'Attribute.id'));
$temp2 = array('OR' => $temp);
$conditions['AND'][] = $temp2;
}
$temp = array();
if (!empty($tagArray[1])) {
$options = array(
'conditions' => array(
'tag_id' => $tagArray[1]
),
'fields' => array(
'event_id'
)
);
$temp = array_merge($temp, $this->subQueryGenerator($tag->EventTag, $options, 'Attribute.event_id', 1));
$options = array(
'conditions' => array(
'tag_id' => $tagArray[1]
),
'fields' => array(
'attribute_id'
)
);
$temp = array_merge($temp, $this->subQueryGenerator($tag->AttributeTag, $options, 'Attribute.id', 1));
$temp2 = array('AND' => $temp);
$conditions['AND'][] = $temp2;
}
$conditions = $this->set_filter_tags(array('tags' => $tags), $conditions);
}
$attributes = $this->fetchAttributes($user, array(
'conditions' => $conditions,
@ -2419,25 +2451,26 @@ class Attribute extends AppModel
// array 1 will have all of the non negated terms and array 2 all the negated terms
public function dissectArgs($args)
{
if (!$args) {
return array(null, null);
if (!is_array($args)) {
$args = explode('&&', $args);
}
if (is_array($args)) {
$argArray = $args;
$result = array(0 => array(), 1 => array());
if (isset($args['OR']) || isset($args['NOT']) || isset($args['AND'])) {
if (!empty($args['OR'])) {
$result[0] = $args['OR'];
}
if (!empty($args['NOT'])) {
$result[1] = $args['NOT'];
}
} else {
$argArray = explode('&&', $args);
}
$accept = $reject = $result = array();
$reject = array();
foreach ($argArray as $arg) {
if (substr($arg, 0, 1) == '!') {
$reject[] = substr($arg, 1);
} else {
$accept[] = $arg;
foreach ($args as $arg) {
if (substr($arg, 0, 1) == '!') {
$result[1][] = substr($arg, 1);
} else {
$result[0][] = $arg;
}
}
}
$result[0] = $accept;
$result[1] = $reject;
return $result;
}
@ -2725,7 +2758,7 @@ class Attribute extends AppModel
// conditions
// order
// group
public function fetchAttributes($user, $options = array())
public function fetchAttributes($user, $options = array(), &$continue = true)
{
$params = array(
'conditions' => $this->buildConditions($user),
@ -2822,9 +2855,8 @@ class Attribute extends AppModel
$this->Warninglist = ClassRegistry::init('Warninglist');
$warninglists = $this->Warninglist->fetchForEventView();
}
if (empty($params['limit'])) {
$loopLimit = 100000;
$loopLimit = 50000;
$loop = true;
$params['limit'] = $loopLimit;
$params['page'] = 0;
@ -2833,7 +2865,6 @@ class Attribute extends AppModel
$pagesToFetch = 1;
}
$attributes = array();
$continue = true;
while ($continue) {
if ($loop) {
$params['page'] = $params['page'] + 1;
@ -2841,13 +2872,17 @@ class Attribute extends AppModel
$continue = false;
continue;
}
} else {
$continue = false;
}
$results = $this->find('all', $params);
if (!$loop) {
if (!empty($params['limit']) && count($results) < $params['limit']) {
$continue = false;
}
$break = true;
}
// return false if we're paginating
if (isset($options['limit']) && empty($results)) {
return false;
return array();
}
$results = array_values($results);
$proposals_block_attributes = Configure::read('MISP.proposals_block_attributes');
@ -2873,6 +2908,9 @@ class Attribute extends AppModel
}
$attributes[] = $results[$key];
}
if (!empty($break)) {
break;
}
}
return $attributes;
}
@ -3128,11 +3166,11 @@ class Attribute extends AppModel
if (is_array($timestamp)) {
$timestamp[0] = $this->Event->resolveTimeDelta($timestamp[0]);
$timestamp[1] = $this->Event->resolveTimeDelta($timestamp[1]);
$conditions['AND'][] = array($scope . ' >=' => $timestamp[0]);
$conditions['AND'][] = array($scope . ' <=' => $timestamp[1]);
$conditions['AND'][] = array($scope . ' >=' => intval($timestamp[0]));
$conditions['AND'][] = array($scope . ' <=' => intval($timestamp[1]));
} else {
$timestamp = $this->Event->resolveTimeDelta($timestamp);
$conditions['AND'][] = array($scope . ' >=' => $timestamp);
$conditions['AND'][] = array($scope . ' >=' => intval($timestamp));
}
return $conditions;
}
@ -3545,6 +3583,7 @@ class Attribute extends AppModel
}
return true;
}
public function attachValidationWarnings($adata)
{
if (!$this->__fTool) {
@ -3555,4 +3594,52 @@ class Attribute extends AppModel
}
return $adata;
}
public function buildFilterConditions($user, &$params)
{
$conditions = $this->buildConditions($user);
$attribute_conditions = array();
$object_conditions = array();
$simple_params = array(
'Attribute' => array(
'value' => array('function' => 'set_filter_value'),
'category' => array('function' => 'set_filter_simple_attribute'),
'type' => array('function' => 'set_filter_simple_attribute'),
'tags' => array('function' => 'set_filter_tags'),
'uuid' => array('function' => 'set_filter_uuid'),
'deleted' => array('function' => 'set_filter_deleted')
),
'Event' => array(
'eventid' => array('function' => 'set_filter_eventid'),
'ignore' => array('function' => 'set_filter_ignore'),
'tags' => array('function' => 'set_filter_tags'),
'tag' => array('function' => 'set_filter_tags'),
'from' => array('function' => 'set_filter_timestamp'),
'to' => array('function' => 'set_filter_timestamp'),
'last' => array('function' => 'set_filter_timestamp'),
'timestamp' => array('function' => 'set_filter_timestamp'),
'publish_timestamp' => array('function' => 'set_filter_timestamp'),
'org' => array('function' => 'set_filter_org'),
'uuid' => array('function' => 'set_filter_uuid'),
'published' => array('function' => 'set_filter_published')
),
'Object' => array(
'object_name' => array('function' => 'set_filter_object_name'),
'deleted' => array('function' => 'set_filter_deleted')
)
);
foreach ($params as $param => $paramData) {
foreach ($simple_params as $scope => $simple_param_scoped) {
if (isset($simple_param_scoped[$param]) && $params[$param] !== false) {
$options = array(
'filter' => $param,
'scope' => $scope,
'pop' => !empty($simple_param_scoped[$param]['pop'])
);
$conditions = $this->Event->{$simple_param_scoped[$param]['function']}($params, $conditions, $options);
}
}
}
return $conditions;
}
}

375
app/Model/Event.php
View File

@ -913,7 +913,8 @@ class Event extends AppModel
return 'Success';
}
private function __prepareForPushToServer($event, $server) {
private function __prepareForPushToServer($event, $server)
{
if ($event['Event']['distribution'] == 4) {
if (!empty($event['SharingGroup']['SharingGroupServer'])) {
$found = false;
@ -965,6 +966,7 @@ class Event extends AppModel
}
return $jsonArray['name'];
}
// no break
case '302': // Found
$newLocation = $response->headers['Location'];
$newTextBody = $response->body();
@ -984,7 +986,9 @@ class Event extends AppModel
public function restfulEventToServer($event, $server, $urlPath, &$newLocation, &$newTextBody, $HttpSocket = null)
{
$event = $this->__prepareForPushToServer($event, $server);
if (is_numeric($event)) return $event;
if (is_numeric($event)) {
return $event;
}
$url = $server['Server']['url'];
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
@ -1028,7 +1032,8 @@ class Event extends AppModel
return $data;
}
private function __prepareAttributesForSync($data, $server) {
private function __prepareAttributesForSync($data, $server)
{
// prepare attribute for sync
if (!empty($data['Attribute'])) {
foreach ($data['Attribute'] as $key => $attribute) {
@ -1044,7 +1049,8 @@ class Event extends AppModel
return $data;
}
private function __prepareObjectsForSync($data, $server) {
private function __prepareObjectsForSync($data, $server)
{
// prepare Object for sync
if (!empty($data['Object'])) {
foreach ($data['Object'] as $key => $object) {
@ -1288,6 +1294,69 @@ class Event extends AppModel
return $conditions;
}
public function filterEventIds($user, &$params = array())
{
$conditions = $this->createEventConditions($user);
$simple_params = array(
'Event' => array(
'eventid' => array('function' => 'set_filter_eventid', 'pop' => true),
'ignore' => array('function' => 'set_filter_ignore'),
'tags' => array('function' => 'set_filter_tags'),
'tag' => array('function' => 'set_filter_tags'),
'from' => array('function' => 'set_filter_timestamp', 'pop' => true),
'to' => array('function' => 'set_filter_timestamp', 'pop' => true),
'last' => array('function' => 'set_filter_timestamp', 'pop' => true),
'timestamp' => array('function' => 'set_filter_timestamp', 'pop' => true),
'publish_timestamp' => array('function' => 'set_filter_timestamp', 'pop' => true),
'org' => array('function' => 'set_filter_org', 'pop' => true),
'uuid' => array('function' => 'set_filter_uuid', 'pop' => true),
'published' => array('function' => 'set_filter_published', 'pop' => true)
),
'Object' => array(
'object_name' => array('function' => 'set_filter_object_name')
),
'Attribute' => array(
'value' => array('function' => 'set_filter_value', 'pop' => true),
'category' => array('function' => 'set_filter_simple_attribute'),
'type' => array('function' => 'set_filter_simple_attribute'),
'tags' => array('function' => 'set_filter_tags', 'pop' => true),
'uuid' => array('function' => 'set_filter_uuid')
)
);
foreach ($params as $param => $paramData) {
foreach ($simple_params as $scope => $simple_param_scoped) {
if (isset($simple_param_scoped[$param]) && $params[$param] !== false) {
$options = array(
'filter' => $param,
'scope' => $scope,
'pop' => !empty($simple_param_scoped[$param]['pop'])
);
if ($scope === 'Event') {
$conditions = $this->{$simple_param_scoped[$param]['function']}($params, $conditions, $options);
} else {
$temp = array();
$temp = $this->{$simple_param_scoped[$param]['function']}($params, $temp, $options);
if (!empty($temp)) {
$subQueryOptions = array(
'conditions' => $temp,
'fields' => array(
'event_id'
)
);
$conditions['AND'][] = $this->subQueryGenerator($this->{$scope}, $subQueryOptions, 'Event.id');
}
}
}
}
}
$results = array_values($this->find('list', array(
'conditions' => $conditions,
'recursive' => -1,
'fields' => array('Event.id')
)));
return $results;
}
public function fetchSimpleEventIds($user, $params = array())
{
$conditions = $this->createEventConditions($user);
@ -1891,90 +1960,189 @@ class Event extends AppModel
$field = '"' . $field . '"';
}
public function csv($user, $eventid=false, $ignore=false, $attributeIDList = array(), $tags = false, $category = false, $type = false, $includeContext = false, $from = false, $to = false, $last = false, $enforceWarninglist = false, $value = false, $timestamp = false)
public function set_filter_org(&$params, $conditions, $options)
{
$this->recursive = -1;
$conditions = array();
// If we are not in the search result csv download function then we need to check what can be downloaded. CSV downloads are already filtered by the search function.
if ($eventid !== 'search') {
if ($from) {
$conditions['AND']['Event.date >='] = $from;
}
if ($to) {
$conditions['AND']['Event.date <='] = $to;
}
if ($last) {
$conditions['AND']['Event.publish_timestamp >='] = $last;
}
if ($timestamp) {
$conditions['AND']['Attribute.timestamp >='] = $timestamp;
$conditions['AND']['Event.timestamp >='] = $timestamp;
}
// This is for both single event downloads and for full downloads. Org has to be the same as the user's or distribution not org only - if the user is no siteadmin
if ($ignore == false) {
$conditions['AND']['Event.published'] = 1;
}
if (!empty($params['org'])) {
$params['org'] = $this->convert_filters($params['org']);
$conditions = $this->generic_add_filter($conditions, $params['org'], 'Event.orgc_id');
}
return $conditions;
}
// If we sent any tags along, load the associated tag names for each attribute
if ($tags) {
$tag = ClassRegistry::init('Tag');
$args = $this->Attribute->dissectArgs($tags);
$tagArray = $tag->fetchEventTagIds($args[0], $args[1]);
$temp = array();
foreach ($tagArray[0] as $accepted) {
$temp['OR'][] = array('Event.id' => $accepted);
}
if (!empty($temp)) {
$conditions['AND'][] = $temp;
}
$temp = array();
foreach ($tagArray[1] as $rejected) {
$temp['AND'][] = array('Event.id !=' => $rejected);
}
if (!empty($temp)) {
$conditions['AND'][] = $temp;
}
}
// if we're downloading a single event, set it as a condition
if ($eventid) {
$conditions['AND'][] = array('Event.id' => $eventid);
}
public function set_filter_eventid(&$params, $conditions, $options)
{
if (!empty($params['eventid']) && $params['eventid'] !== 'all') {
$params['eventid'] = $this->convert_filters($params['eventid']);
$conditions = $this->generic_add_filter($conditions, $params['eventid'], 'Event.id');
}
return $conditions;
}
//restricting to non-private or same org if the user is not a site-admin.
if (!$ignore) {
$conditions['AND']['Attribute.to_ids'] = 1;
public function set_filter_uuid(&$params, $conditions, $options)
{
if (!empty($params['uuid'])) {
$params['uuid'] = $this->convert_filters($params['uuid']);
if (!empty($options['scope']) || $options['scope'] === 'Event') {
$conditions = $this->generic_add_filter($conditions, $params['uuid'], 'Event.uuid');
}
if ($type) {
$conditions['AND']['Attribute.type'] = $type;
}
if ($category) {
$conditions['AND']['Attribute.category'] = $category;
}
if ($value) {
$temp = array(
'OR' => array(
'Attribute.value1' => $value,
'Attribute.value2' => $value
)
);
$conditions['AND'][] = $temp;
if (!empty($options['scope']) || $options['scope'] === 'Attribute') {
$conditions = $this->generic_add_filter($conditions, $params['uuid'], 'Attribute.uuid');
}
}
return $conditions;
}
if ($eventid === 'search') {
foreach ($attributeIDList as $aID) {
public function set_filter_deleted(&$params, $conditions, $options)
{
if (!empty($params['deleted'])) {
if (empty($options['scope'])) {
$scope = 'Attribute';
} else {
$scope = $options['scope'];
}
if ($params['deleted'])
$conditions = $this->
$conditions = $this->generic_add_filter($conditions, $params['deleted'], $scope . '.deleted');
}
return $conditions;
}
public function set_filter_ignore(&$params, $conditions, $options)
{
if (empty($params['ignore'])) {
$conditions['AND']['Event.published'] = 1;
$conditions['AND']['Attribute.to_ids'] = 1;
}
return $conditions;
}
public function set_filter_published(&$params, $conditions, $options)
{
if (isset($params['published'])) {
$conditions['AND']['Event.published'] = $params['published'];
}
return $conditions;
}
public function set_filter_tags(&$params, $conditions, $options)
{
if (!empty($params['tags'])) {
$conditions = $this->Attribute->set_filter_tags($params, $conditions, $options);
}
return $conditions;
}
public function set_filter_simple_attribute(&$params, $conditions, $options)
{
if (!empty($params[$options['filter']])) {
$params[$options['filter']] = $this->convert_filters($params[$options['filter']]);
$conditions = $this->generic_add_filter($conditions, $params[$options['filter']], 'Attribute.' . $options['filter']);
}
return $conditions;
}
public function set_filter_attribute_id(&$params, $conditions, $options)
{
if (!empty($params[$options['filter']])) {
$params[$options['filter']] = $this->convert_filters($params[$options['filter']]);
$conditions = $this->generic_add_filter($conditions, $params[$options['filter']], 'Attribute.' . $options['filter']);
}
return $conditions;
}
public function set_filter_value(&$params, $conditions, $options)
{
if (!empty($params['value'])) {
$params[$options['filter']] = $this->convert_filters($params[$options['filter']]);
$conditions = $this->generic_add_filter($conditions, $params[$options['filter']], array('Attribute.value1', 'Attribute.value2'));
}
return $conditions;
}
public function set_filter_timestamp(&$params, $conditions, $options)
{
if ($options['filter'] == 'from') {
$conditions['AND']['Event.date >='] = $params['from'];
} elseif ($options['filter'] == 'to') {
$conditions['AND']['Event.date <='] = $params['to'];
} else {
$filters = array(
'timestamp' => array(
'Event.timestamp'
),
'publish_timestamp' => array(
'Event.publish_timestamp'
),
'last' => array(
'Event.publish_timestamp'
)
);
foreach ($filters[$options['filter']] as $f) {
$conditions = $this->Attribute->setTimestampConditions($params[$options['filter']], $conditions, $f);
}
}
return $conditions;
}
public function csv($user, $params, $search = false, &$continue = true)
{
$conditions = array();
$simple_params = array(
'eventid' => array('function' => 'set_filter_eventid'),
'ignore' => array('function' => 'set_filter_ignore'),
'tags' => array('function' => 'set_filter_tags'),
'category' => array('function' => 'set_filter_simple_attribute'),
'type' => array('function' => 'set_filter_simple_attribute'),
'from' => array('function' => 'set_filter_timestamp'),
'to' => array('function' => 'set_filter_timestamp'),
'last' => array('function' => 'set_filter_timestamp'),
'value' => array('function' => 'set_filter_value'),
'timestamp' => array('function' => 'set_filter_timestamp'),
'attributeIDList' => array('functon' => 'set_filter_attribute_id')
);
foreach ($params as $param => $paramData) {
if (isset($simple_params[$param]) && $params[$param] !== false) {
$options = array(
'filter' => $param,
'scope' => 'Event',
'pop' => !empty($simple_param_scoped[$param]['pop'])
);
$conditions = $this->{$simple_params[$param]['function']}($params, $conditions, $options);
}
}
//$attributeIDList = array(), $includeContext = false, $enforceWarninglist = false
$this->recursive = -1;
if (!empty($params['eventid']) && $params['eventid'] === 'search') {
foreach ($params['attributeIDList'] as $aID) {
$conditions['AND']['OR'][] = array('Attribute.id' => $aID);
}
}
$params = array(
$csv_params = array(
'conditions' => $conditions, //array of conditions
'fields' => array('Attribute.event_id', 'Attribute.distribution', 'Attribute.category', 'Attribute.type', 'Attribute.value', 'Attribute.comment', 'Attribute.uuid', 'Attribute.to_ids', 'Attribute.timestamp', 'Attribute.id', 'Attribute.object_relation'),
'order' => array('Attribute.uuid ASC'),
'enforceWarninglist' => $enforceWarninglist,
'flatten' => true
);
// copy over the parameters that have to deal with pagination or additional functionality to be executed
$control_params = array(
'limit', 'page', 'enforceWarninglist'
);
foreach ($control_params as $control_param) {
if (!empty($params[$control_param])) {
$csv_params[$control_param] = $params[$control_param];
}
}
$csv_params = $this->__appendIncludesCSV($csv_params, !empty($params['includeContext']));
$attributes = $this->Attribute->fetchAttributes($user, $csv_params, $continue);
$attributes = $this->__sanitiseCSVAttributes($attributes, !empty($params['includeContext']), !empty($params['ignore']));
return $attributes;
}
private function __appendIncludesCSV($params, $includeContext)
{
if ($includeContext) {
$params['contain'] = array(
'Event' => array(
@ -1994,9 +2162,14 @@ class Event extends AppModel
);
}
$params['contain']['Object'] = array('fields' => array('id', 'uuid', 'name', 'meta-category'));
$attributes = $this->Attribute->fetchAttributes($user, $params);
if (empty($attributes)) {
return array();
return $params;
}
private function __sanitiseCSVAttributes($attributes, $includeContext, $ignore)
{
if (!empty($ignore)) {
$this->Log = ClassRegistry::init('Log');
$attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true);
}
foreach ($attributes as &$attribute) {
$this->__escapeCSVField($attribute['Attribute']['value']);
@ -2354,6 +2527,20 @@ class Event extends AppModel
if (empty($orgMembers)) {
return false;
}
$temp = $this->__buildContactEventEmailBody($user, $message, $event, $targetUser, $id);
$bodyevent = $temp[0];
$body = $temp[1];
$result = true;
$tplColorString = !empty(Configure::read('MISP.email_subject_TLP_string')) ? Configure::read('MISP.email_subject_TLP_string') : "TLP Amber";
foreach ($orgMembers as &$reporter) {
$subject = "[" . Configure::read('MISP.org') . " MISP] Need info about event " . $id . " - ".$tplColorString;
$result = $this->User->sendEmail($reporter, $bodyevent, $body, $subject, $user) && $result;
}
return $result;
}
private function __buildContactEventEmailBody($user, $message, $event, $targetUser, $id)
{
// The mail body, h() is NOT needed as we are sending plain-text mails.
$body = "";
$body .= "Hello, \n";
@ -2415,13 +2602,7 @@ class Event extends AppModel
}
$bodyevent .= "\n";
$bodyevent .= $bodyTempOther; // append the 'other' attribute types to the bottom.
$result = true;
$tplColorString = !empty(Configure::read('MISP.email_subject_TLP_string')) ? Configure::read('MISP.email_subject_TLP_string') : "TLP Amber";
foreach ($orgMembers as &$reporter) {
$subject = "[" . Configure::read('MISP.org') . " MISP] Need info about event " . $id . " - ".$tplColorString;
$result = $this->User->sendEmail($reporter, $bodyevent, $body, $subject, $user) && $result;
}
return $result;
return array($bodyevent, $body);
}
private function __captureSGForElement($element, $user)
@ -3184,7 +3365,8 @@ class Event extends AppModel
}
}
private function __getPrioWorkerIfPossible() {
private function __getPrioWorkerIfPossible()
{
$this->ResqueStatus = new ResqueStatus\ResqueStatus(Resque::redis());
$workers = $this->ResqueStatus->getWorkers();
$workerType = 'default';
@ -3705,18 +3887,22 @@ class Event extends AppModel
public function resolveTimeDelta($delta)
{
if (is_int($delta)) {
if (is_numeric($delta)) {
return $delta;
}
$multiplierArray = array('d' => 86400, 'h' => 3600, 'm' => 60);
$multiplierArray = array('d' => 86400, 'h' => 3600, 'm' => 60, 's' => 1);
$multiplier = $multiplierArray['d'];
$lastChar = strtolower(substr($delta, -1));
if (!is_numeric($lastChar) && array_key_exists($lastChar, $multiplierArray)) {
$multiplier = $multiplierArray[$lastChar];
$delta = substr($delta, 0, -1);
} else {
// invalid filter, make sure we don't return anything
return time() + 1;
}
if (!is_numeric($delta)) {
return false;
// Same here. (returning false dumps the whole database)
return time() + 1;
}
return time() - ($delta * $multiplier);
}
@ -4291,7 +4477,7 @@ class Event extends AppModel
if ($parameterKey === 'org') {
$found_orgs = $this->Org->find('all', array(
'recursive' => -1,
'conditions' => array('LOWER(name) LIKE' => '%' . strtolower(substr($v, 1)) . '%'),
'conditions' => array('name' => substr($v, 1)),
));
foreach ($found_orgs as $o) {
$subcondition['AND'][] = array('Event.orgc_id !=' => $o['Org']['id']);
@ -4306,7 +4492,12 @@ class Event extends AppModel
$subcondition['AND'][] = array('Event.uuid !=' => substr($v, 1));
$subcondition['AND'][] = array('Attribute.uuid !=' => substr($v, 1));
} else {
$subcondition['AND'][] = array('Attribute.' . $parameterKey . ' NOT LIKE' => '%'.substr($v, 1).'%');
$lookup = substr($v, 1);
if (strlen($lookup) != strlen(trim($lookup, '%'))) {
$subcondition['AND'][] = array('Attribute.' . $parameterKey . ' NOT LIKE' => $lookup);
} else {
$subcondition['AND'][] = array('NOT' => array('Attribute.' . $parameterKey => $lookup));
}
}
}
} else {
@ -4322,7 +4513,7 @@ class Event extends AppModel
if ($parameterKey === 'org') {
$found_orgs = $this->Org->find('all', array(
'recursive' => -1,
'conditions' => array('LOWER(name) LIKE' => '%' . strtolower($v) . '%'),
'conditions' => array('name' => $v),
));
foreach ($found_orgs as $o) {
$subcondition['OR'][] = array('Event.orgc_id' => $o['Org']['id']);
@ -4338,7 +4529,11 @@ class Event extends AppModel
$subcondition['OR'][] = array('Event.uuid' => $v);
} else {
if (!empty($v)) {
$subcondition['OR'][] = array('Attribute.' . $parameterKey . ' LIKE' => '%'.$v.'%');
if (strlen($v) != strlen(trim($v, '%'))) {
$subcondition['AND'][] = array('Attribute.' . $parameterKey . ' LIKE' => $v);
} else {
$subcondition['AND'][] = array('Attribute.' . $parameterKey => $v);
}
}
}
}

316
app/Model/Server.php
View File

@ -1783,7 +1783,8 @@ class Server extends AppModel
return true;
}
private function __getEventIdListBasedOnPullTechnique($technique, $server) {
private function __getEventIdListBasedOnPullTechnique($technique, $server)
{
if ("full" === $technique) {
// get a list of the event_ids on the server
$eventIds = $this->getEventIdsFromServer($server);
@ -1819,6 +1820,165 @@ class Server extends AppModel
return $eventIds;
}
private function __checkIfEventIsBlockedBeforePull($event)
{
if (Configure::read('MISP.enableEventBlacklisting') !== false) {
$this->EventBlacklist = ClassRegistry::init('EventBlacklist');
$r = $this->EventBlacklist->find('first', array('conditions' => array('event_uuid' => $event['Event']['uuid'])));
if (!empty($r)) {
return true;
}
}
return false;
}
private function __updatePulledEventBeforeInsert($event, $server, $user)
{
// we have an Event array
// The event came from a pull, so it should be locked.
$event['Event']['locked'] = true;
if (!isset($event['Event']['distribution'])) { // version 1
$event['Event']['distribution'] = '1';
}
// Distribution
if (empty(Configure::read('MISP.host_org_id')) || !$server['Server']['internal'] || Configure::read('MISP.host_org_id') != $server['Server']['org_id']) {
switch ($event['Event']['distribution']) {
case 1:
// if community only, downgrade to org only after pull
$event['Event']['distribution'] = '0';
break;
case 2:
// if connected communities downgrade to community only
$event['Event']['distribution'] = '1';
break;
}
if (isset($event['Event']['Attribute']) && !empty($event['Event']['Attribute'])) {
foreach ($event['Event']['Attribute'] as $key => $a) {
switch ($a['distribution']) {
case '1':
$event['Event']['Attribute'][$key]['distribution'] = '0';
break;
case '2':
$event['Event']['Attribute'][$key]['distribution'] = '1';
break;
}
}
}
}
// Distribution, set reporter of the event, being the admin that initiated the pull
$event['Event']['user_id'] = $user['id'];
return $event;
}
private function __checkIfPulledEventExistsAndAddOrUpdate($event, &$successes, &$fails, $eventModel, $server, $user, $passAlong, $job, $jobId)
{
// check if the event already exist (using the uuid)
$existingEvent = null;
$existingEvent = $eventModel->find('first', array('conditions' => array('Event.uuid' => $event['Event']['uuid'])));
if (!$existingEvent) {
// add data for newly imported events
$passAlong = $server['Server']['id'];
$result = $eventModel->_add($event, true, $user, $server['Server']['org_id'], $passAlong, true, $jobId);
if ($result) {
$successes[] = $eventId;
} else {
$fails[$eventId] = 'Failed (partially?) because of validation errors: '. json_encode($eventModel->validationErrors, true);
}
} else {
$tempUser = $user;
$tempUser['Role']['perm_site_admin'] = 0;
$result = $eventModel->_edit($event, $tempUser, $existingEvent['Event']['id'], $jobId);
if ($result === true) {
$successes[] = $eventId;
} elseif (isset($result['error'])) {
$fails[$eventId] = $result['error'];
} else {
$fails[$eventId] = json_encode($result);
}
}
}
private function __pullEvents($eventId, $successes, $fails, $eventModel, $server, $user, $passAlong, $job, $jobId)
{
$event = $eventModel->downloadEventFromServer(
$eventId,
$server
);
if (!empty($event)) {
if ($this->__checkIfEventIsBlockedBeforePull($event)) {
return false;
}
$this->__updatePulledEventBeforeInsert($event, $server, $user);
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $successes, $fails, $eventModel, $server, $user, $passAlong, $job, $jobId);
} else {
// error
$fails[$eventId] = 'failed downloading the event';
}
if ($jobId) {
if ($k % 10 == 0) {
$job->id = $jobId;
$job->saveField('progress', 50 * (($k + 1) / count($eventIds)));
}
}
return true;
}
private function __handlePulledProposals($proposals, $events, $job, $jobId)
{
$pulledProposals = array();
if (!empty($proposals)) {
$shadowAttribute = ClassRegistry::init('ShadowAttribute');
$shadowAttribute->recursive = -1;
$uuidEvents = array_flip($events);
foreach ($proposals as $k => &$proposal) {
$proposal = $proposal['ShadowAttribute'];
$oldsa = $shadowAttribute->findOldProposal($proposal);
$proposal['event_id'] = $uuidEvents[$proposal['event_uuid']];
if (!$oldsa || $oldsa['timestamp'] < $proposal['timestamp']) {
if ($oldsa) {
$shadowAttribute->delete($oldsa['id']);
}
if (!isset($pulledProposals[$proposal['event_id']])) {
$pulledProposals[$proposal['event_id']] = 0;
}
$pulledProposals[$proposal['event_id']]++;
if (isset($proposal['old_id'])) {
$oldAttribute = $eventModel->Attribute->find('first', array('recursive' => -1, 'conditions' => array('uuid' => $proposal['uuid'])));
if ($oldAttribute) {
$proposal['old_id'] = $oldAttribute['Attribute']['id'];
} else {
$proposal['old_id'] = 0;
}
}
// check if this is a proposal from an old MISP instance
if (!isset($proposal['Org']) && isset($proposal['org']) && !empty($proposal['org'])) {
$proposal['Org'] = $proposal['org'];
$proposal['EventOrg'] = $proposal['event_org'];
} elseif (!isset($proposal['Org']) && !isset($proposal['EventOrg'])) {
continue;
}
$proposal['org_id'] = $this->Organisation->captureOrg($proposal['Org'], $user);
$proposal['event_org_id'] = $this->Organisation->captureOrg($proposal['EventOrg'], $user);
unset($proposal['Org']);
unset($proposal['EventOrg']);
$shadowAttribute->create();
if (!isset($proposal['deleted']) || !$proposal['deleted']) {
if ($shadowAttribute->save($proposal)) {
$shadowAttribute->sendProposalAlertEmail($proposal['event_id']);
}
}
}
if ($jobId) {
if ($k % 50 == 0) {
$job->id = $jobId;
$job->saveField('progress', 50 * (($k + 1) / count($proposals)));
}
}
}
}
return $pulledProposals;
}
public function pull($user, $id = null, $technique=false, $server, $jobId = false, $percent = 100, $current = 0)
{
if ($jobId) {
@ -1826,6 +1986,7 @@ class Server extends AppModel
$job->read(null, $jobId);
$email = "Scheduled job";
} else {
$job = false;
$email = $user['email'];
}
$eventModel = ClassRegistry::init('Event');
@ -1856,101 +2017,12 @@ class Server extends AppModel
}
$successes = array();
$fails = array();
$pulledProposals = array();
// now process the $eventIds to pull each of the events sequentially
if (!empty($eventIds)) {
// download each event
if (null != $eventIds) {
$HttpSocket = $this->setupHttpSocket($server);
foreach ($eventIds as $k => $eventId) {
$event = $eventModel->downloadEventFromServer(
$eventId,
$server
);
if (null != $event) {
$blocked = false;
if (Configure::read('MISP.enableEventBlacklisting') !== false) {
$this->EventBlacklist = ClassRegistry::init('EventBlacklist');
$r = $this->EventBlacklist->find('first', array('conditions' => array('event_uuid' => $event['Event']['uuid'])));
if (!empty($r)) {
continue;
}
}
// we have an Event array
// The event came from a pull, so it should be locked.
$event['Event']['locked'] = true;
if (!isset($event['Event']['distribution'])) { // version 1
$event['Event']['distribution'] = '1';
}
// Distribution
if (empty(Configure::read('MISP.host_org_id')) || !$server['Server']['internal'] || Configure::read('MISP.host_org_id') != $server['Server']['org_id']) {
switch ($event['Event']['distribution']) {
case 1:
// if community only, downgrade to org only after pull
$event['Event']['distribution'] = '0';
break;
case 2:
// if connected communities downgrade to community only
$event['Event']['distribution'] = '1';
break;
}
if (isset($event['Event']['Attribute']) && !empty($event['Event']['Attribute'])) {
foreach ($event['Event']['Attribute'] as $key => $a) {
switch ($a['distribution']) {
case '1':
$event['Event']['Attribute'][$key]['distribution'] = '0';
break;
case '2':
$event['Event']['Attribute'][$key]['distribution'] = '1';
break;
}
}
}
}
// Distribution, set reporter of the event, being the admin that initiated the pull
$event['Event']['user_id'] = $user['id'];
// check if the event already exist (using the uuid)
$existingEvent = null;
$existingEvent = $eventModel->find('first', array('conditions' => array('Event.uuid' => $event['Event']['uuid'])));
if (!$existingEvent) {
// add data for newly imported events
$passAlong = $server['Server']['id'];
$result = $eventModel->_add($event, true, $user, $server['Server']['org_id'], $passAlong, true, $jobId);
if ($result) {
$successes[] = $eventId;
} else {
$fails[$eventId] = 'Failed (partially?) because of validation errors: '. json_encode($eventModel->validationErrors, true);
}
} else {
$tempUser = $user;
$tempUser['Role']['perm_site_admin'] = 0;
$result = $eventModel->_edit($event, $tempUser, $existingEvent['Event']['id'], $jobId);
if ($result === true) {
$successes[] = $eventId;
} elseif (isset($result['error'])) {
$fails[$eventId] = $result['error'];
} else {
$fails[$eventId] = json_encode($result);
}
}
} else {
// error
$fails[$eventId] = 'failed downloading the event';
}
if ($jobId) {
if ($k % 10 == 0) {
$job->id = $jobId;
$job->saveField('progress', 50 * (($k + 1) / count($eventIds)));
}
}
}
if (count($fails) > 0) {
// there are fails, take the lowest fail
$lastpulledid = min(array_keys($fails));
} else {
// no fails, take the highest success
$lastpulledid = count($successes) > 0 ? max($successes) : 0;
}
$HttpSocket = $this->setupHttpSocket($server);
foreach ($eventIds as $k => $eventId) {
$this->__pullEvents($eventId, $successes, $fails, $eventModel, $server, $user, $passAlong, $job, $jobId);
}
}
if ($jobId) {
@ -1961,58 +2033,9 @@ class Server extends AppModel
'recursive' => -1,
'conditions' => $conditions
));
$shadowAttribute = ClassRegistry::init('ShadowAttribute');
$shadowAttribute->recursive = -1;
if (!empty($events)) {
$proposals = $eventModel->downloadProposalsFromServer($events, $server);
if (!empty($proposals)) {
$uuidEvents = array_flip($events);
foreach ($proposals as $k => &$proposal) {
$proposal = $proposal['ShadowAttribute'];
$oldsa = $shadowAttribute->findOldProposal($proposal);
$proposal['event_id'] = $uuidEvents[$proposal['event_uuid']];
if (!$oldsa || $oldsa['timestamp'] < $proposal['timestamp']) {
if ($oldsa) {
$shadowAttribute->delete($oldsa['id']);
}
if (!isset($pulledProposals[$proposal['event_id']])) {
$pulledProposals[$proposal['event_id']] = 0;
}
$pulledProposals[$proposal['event_id']]++;
if (isset($proposal['old_id'])) {
$oldAttribute = $eventModel->Attribute->find('first', array('recursive' => -1, 'conditions' => array('uuid' => $proposal['uuid'])));
if ($oldAttribute) {
$proposal['old_id'] = $oldAttribute['Attribute']['id'];
} else {
$proposal['old_id'] = 0;
}
}
// check if this is a proposal from an old MISP instance
if (!isset($proposal['Org']) && isset($proposal['org']) && !empty($proposal['org'])) {
$proposal['Org'] = $proposal['org'];
$proposal['EventOrg'] = $proposal['event_org'];
} elseif (!isset($proposal['Org']) && !isset($proposal['EventOrg'])) {
continue;
}
$proposal['org_id'] = $this->Organisation->captureOrg($proposal['Org'], $user);
$proposal['event_org_id'] = $this->Organisation->captureOrg($proposal['EventOrg'], $user);
unset($proposal['Org']);
unset($proposal['EventOrg']);
$shadowAttribute->create();
if (!isset($proposal['deleted']) || !$proposal['deleted']) {
if ($shadowAttribute->save($proposal)) {
$shadowAttribute->sendProposalAlertEmail($proposal['event_id']);
}
}
}
if ($jobId) {
if ($k % 50 == 0) {
$job->id = $jobId;
$job->saveField('progress', 50 * (($k + 1) / count($proposals)));
}
}
}
}
$pulledProposals = $this->__handlePulledProposals($proposals, $events, $job, $jobId);
}
if ($jobId) {
$job->saveField('progress', 100);
@ -2029,12 +2052,9 @@ class Server extends AppModel
'action' => 'pull',
'user_id' => $user['id'],
'title' => 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $email,
'change' => count($successes) . ' events and ' . count($pulledProposals) . ' proposals pulled or updated. ' . count($fails) . ' events failed or didn\'t need an update.'
'change' => count($successes) . ' events and ' . array_sum($pulledProposals) . ' proposals pulled or updated. ' . count($fails) . ' events failed or didn\'t need an update.'
));
if (!isset($lastpulledid)) {
$lastpulledid = 0;
}
return array($successes, $fails, $pulledProposals, $lastpulledid);
return array($successes, $fails, $pulledProposals);
}
public function filterRuleToParameter($filter_rules)

38
app/Model/Tag.php
View File

@ -195,23 +195,41 @@ class Tag extends AppModel
public function findTagIdsByTagNames($array)
{
$ids = array();
foreach ($array as $a) {
$conditions['OR'][] = array('LOWER(Tag.name) like' => strtolower($a));
$tag_ids = array();
if (!is_array($array)) {
$array = array($array);
}
$params = array(
'recursive' => 1,
'conditions' => $conditions,
'fields' => array('Tag.id', 'Tag.id')
);
$result = $this->find('list', $params);
return array_values($result);
foreach ($array as $k => $tag) {
if (is_numeric($tag)) {
$tag_ids[] = $tag;
unset($array[$k]);
}
}
$array = array_values($array);
if (!empty($array)) {
foreach ($array as $a) {
$conditions['OR'][] = array('LOWER(Tag.name) like' => strtolower($a));
}
$params = array(
'recursive' => 1,
'conditions' => $conditions,
'fields' => array('Tag.id', 'Tag.id')
);
$result = $this->find('list', $params);
$tag_ids = array_merge($result, $tag_ids);
}
return array_values($tag_ids);
}
public function findEventIdsByTagNames($array)
{
$ids = array();
foreach ($array as $a) {
$conditions['OR'][] = array('LOWER(name) like' => strtolower($a));
if (is_numeric($a)) {
$conditions['OR'][] = array('id' => $a);
} else {
$conditions['OR'][] = array('LOWER(name) like' => strtolower($a));
}
}
$params = array(
'recursive' => 1,

3
app/Model/User.php
View File

@ -529,7 +529,8 @@ class User extends AppModel
return $results;
}
private function testSmimeCertificate($certif_public) {
private function testSmimeCertificate($certif_public)
{
$result = array();
try {
App::uses('Folder', 'Utility');

41
app/View/Servers/rest.ctp
View File

@ -49,18 +49,51 @@
<hr />
</fieldset>
<?php
if (!empty($data['data'])) {
$formats = array('Raw', 'JSON', 'HTML');
if (!empty($data['data'])):
echo sprintf('<h3>%s</h3>', __('Response'));
echo sprintf('<div><span class="bold">%s</span>: %d</div>', __('Response code'), h($data['code']));
echo sprintf('<div><span class="bold">%s</span>: %s</div>', __('Request duration'), h($data['duration']));
echo sprintf('<div class="bold">%s</div>', __('Headers'));
foreach ($data['headers'] as $header => $value) {
echo sprintf('&nbsp;&nbsp;<span class="bold">%s</span>: %s<br />', h($header), h($value));
if (is_array($value)) {
$value = implode(',', $value);
}
echo sprintf('&nbsp;&nbsp;<span class="bold">%s</span>: %s<br />', h($header), h($value));
}
echo sprintf('<pre>%s</pre>', h($data['data']));
}
$format_toggles = '';
foreach ($formats as $k => $format) {
$position = '';
if ($k == 0) {
$position = '-left';
}
if ($k == (count($formats) -1)) {
$position = '-right';
}
$format_toggles .= sprintf('<span class="btn btn-inverse qet toggle%s format-toggle-button" data-toggle-type="%s">%s</span>', $position, $format, $format);
}
echo sprintf('<div style="padding-bottom:24px;">%s</div>', $format_toggles);
echo sprintf('<div class="hidden" id="rest-response-hidden-container">%s</div>', h($data['data']));
echo '<div id="rest-response-container"></div>';
endif;
?>
</div>
<?php
echo $this->element('side_menu', array('menuList' => 'event-collection', 'menuItem' => 'rest'));
?>
<script type="text/javascript">
// tooltips
$(document).ready(function () {
insertRawRestResponse();
$('.format-toggle-button').bind('click', function() {
$('#rest-response-container').empty();
if ($(this).data('toggle-type') == 'Raw') {
insertRawRestResponse();
} else if ($(this).data('toggle-type') == 'HTML') {
insertHTMLRestResponse();
} else if ($(this).data('toggle-type') == 'JSON') {
insertJSONRestResponse();
}
});
});
</script>

13
app/webroot/css/main.css
View File

@ -1559,6 +1559,19 @@ a.proposal_link_red:hover {
color: white;
}
.json_key {
color: #690;
font-weight: bold;
}
.json_string, .json_boolean, .json_number {
color: #905;
}
.json_null {
color: #a8a8a8;
}
.background-lightblue {
background-color:#cfefff;
}

40
app/webroot/js/misp.js
View File

@ -3431,6 +3431,46 @@ function checkIfLoggedIn() {
setTimeout(function() { checkIfLoggedIn(); }, 5000);
}
function insertRawRestResponse() {
$('#rest-response-container').append('<pre id="raw-response-container" />');
$('#raw-response-container').text($('#rest-response-hidden-container').text());
}
function insertHTMLRestResponse() {
$('#rest-response-container').append('<div id="html-response-container" style="border: 1px solid blue; padding:5px;" />');
$('#html-response-container').html($('#rest-response-hidden-container').text());
}
function insertJSONRestResponse() {
$('#rest-response-container').append('<p id="json-response-container" style="border: 1px solid blue; padding:5px;" />');
var parsedJson = syntaxHighlightJson($('#rest-response-hidden-container').text());
console.log(parsedJson);
$('#json-response-container').html(parsedJson);
}
function syntaxHighlightJson(json) {
if (typeof json == 'string') {
json = JSON.parse(json);
}
json = JSON.stringify(json, undefined, 2);
json = json.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/(?:\r\n|\r|\n)/g, '<br>').replace(/ /g, '&nbsp;');
return json.replace(/("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?)/g, function (match) {
var cls = 'json_number';
if (/^"/.test(match)) {
if (/:$/.test(match)) {
cls = 'json_key';
} else {
cls = 'json_string';
}
} else if (/true|false/.test(match)) {
cls = 'json_boolean';
} else if (/null/.test(match)) {
cls = 'json_null';
}
return '<span class="' + cls + '">' + match + '</span>';
});
}
(function(){
"use strict";
$(".datepicker").datepicker({

97
tests/event.csv
View File

@ -1,31 +1,68 @@
uuid,event_id,category,type,value,comment,to_ids,date,object_relation,object_uuid,object_name,object_meta_category
"548847d8-01e0-4231-a739-15bb950d210b",750,"Payload installation","md5","744c07e886497f7b68f6f7fe57b7ab54","Regin samples collected.",1,20141210,"","","",""
"548847d8-05f8-49e7-af79-15bb950d210b",750,"Payload installation","md5","47d0e8f9d7a6429920329207a32ecc2e","Regin samples collected.",1,20141210,"","","",""
"548847d8-3fbc-4a06-ba82-15bb950d210b",750,"Payload installation","md5","2c8b9d2885543d7ade3cae98225e263b","Regin samples collected.",1,20141210,"","","",""
"548847d8-9db0-4df6-8206-15bb950d210b",750,"Payload installation","md5","26297dc3cd0b688de3b846983c5385e5","Regin samples collected.",1,20141210,"","","",""
"548847d8-a33c-41f3-9f7a-15bb950d210b",750,"Payload installation","md5","01c2f321b6bfdb9473c079b0797567ba","Regin samples collected.",1,20141210,"","","",""
"548847d8-c950-48eb-b960-15bb950d210b",750,"Payload installation","md5","4b6b86c7fec1c574706cecedf44abded","Regin samples collected.",1,20141210,"","","",""
"548847d9-1404-4331-ae3c-15bb950d210b",750,"Payload installation","md5","90fecc6a89b2e22d82d58878d93477d4","Regin samples collected.",1,20141210,"","","",""
"548847d9-39dc-4247-b23d-15bb950d210b",750,"Payload installation","md5","06665b96e293b23acc80451abb413e50","Regin samples collected.",1,20141210,"","","",""
"548847d9-3b28-449e-b527-15bb950d210b",750,"Payload installation","md5","e94393561901895cb0783edc34740fd4","Regin samples collected.",1,20141210,"","","",""
"548847d9-4020-41da-b5f3-15bb950d210b",750,"Payload installation","md5","db405ad775ac887a337b02ea8b07fddc","Regin samples collected.",1,20141210,"","","",""
"548847d9-6340-44a0-8f33-15bb950d210b",750,"Payload installation","md5","ffb0b9b5b610191051a7bdf0806e1e47","Regin samples collected.",1,20141210,"","","",""
"548847d9-8b18-4654-9766-15bb950d210b",750,"Payload installation","md5","f3ffc2aaaa1e2ab55ec26ff098653347","Regin samples collected.",1,20141210,"","","",""
"548847d9-a564-4178-b8e6-15bb950d210b",750,"Payload installation","md5","6662c390b2bbbd291ec7987388fc75d7","Regin samples collected.",1,20141210,"","","",""
"548847d9-afe0-4531-a4b0-15bb950d210b",750,"Payload installation","md5","187044596bc1328efa0ed636d8aa4a5c","Regin samples collected.",1,20141210,"","","",""
"548847d9-b63c-4c95-a2bd-15bb950d210b",750,"Payload installation","md5","1800def71006ca6790767e202fae9b9a","Regin samples collected.",1,20141210,"","","",""
"548847d9-e6fc-4b93-a773-15bb950d210b",750,"Payload installation","md5","bfbe8c3ee78750c3a520480700e440f8","Regin samples collected.",1,20141210,"","","",""
"548847d9-fd54-4e49-909b-15bb950d210b",750,"Payload installation","md5","89003e9a1ae635c97ebad07aebc67f00","Regin samples collected.",1,20141210,"","","",""
"548847da-1660-4562-a1f8-15bb950d210b",750,"Payload installation","md5","b505d65721bb2453d5039a389113b566","Regin samples collected.",1,20141210,"","","",""
"548847da-2134-43d7-ba22-15bb950d210b",750,"Payload installation","md5","8fcf4e53ece6111758a1dd3139dc7cad","Regin samples collected.",1,20141210,"","","",""
"548847da-3e40-4ab2-a5eb-15bb950d210b",750,"Payload installation","md5","1c024e599ac055312a4ab75b3950040a","Regin samples collected.",1,20141210,"","","",""
"548847da-49c0-404d-ae42-15bb950d210b",750,"Payload installation","md5","d240f06e98c8d3e647cbf4d442d79475","Regin samples collected.",1,20141210,"","","",""
"548847da-71ec-4b2b-bae5-15bb950d210b",750,"Payload installation","md5","148c1bb9d405d717252c77593aff4bd8","Regin samples collected.",1,20141210,"","","",""
"548847da-9798-4b6d-b422-15bb950d210b",750,"Payload installation","md5","ba7bb65634ce1e30c1e5415be3d1db1d","Regin samples collected.",1,20141210,"","","",""
"548847da-ac78-474c-86fe-15bb950d210b",750,"Payload installation","md5","b29ca4f22ae7b7b25f79c1d4a421139d","Regin samples collected.",1,20141210,"","","",""
"548847da-c2d0-4d24-821e-15bb950d210b",750,"Payload installation","md5","b269894f434657db2b15949641a67532","Regin samples collected.",1,20141210,"","","",""
"548847da-ffe4-4a90-9f2a-15bb950d210b",750,"Payload installation","md5","22bfc970f707fd775d49e875b63c2f0c","Regin samples collected.",1,20141210,"","","",""
"548847db-060c-4275-a0c7-15bb950d210b",750,"Payload installation","md5","049436bb90f71cf38549817d9b90e2da","Regin samples collected.",1,20141210,"","","",""
"5488486c-1418-4624-b87c-15ba950d210b",750,"Artifacts dropped","regkey","Class\{4F20E605-9452-4787-B793-D0204917CA58}","",1,20141210,"","","",""
"5488486c-47ec-4952-8e60-15ba950d210b",750,"Artifacts dropped","regkey","Class\{9B9A8ADB-8864-4BC4-8AD5-B17DFDBB9F58}","",1,20141210,"","","",""
"5488486c-a044-4c31-830c-15ba950d210b",750,"Artifacts dropped","regkey","HKLM\System\CurrentControlSet\Control\","",1,20141210,"","","",""
"5488466a-f0d0-4b58-89a5-15bc950d210b",1635,"External analysis","link","https://www.f-secure.com/documents/996508/1030745/w32_regin_stage_1.pdf","",,20141210,"","","",""
"548847d8-01e0-4231-a739-15bb950d210b",1635,"Payload installation","md5","744c07e886497f7b68f6f7fe57b7ab54","Regin samples collected.",1,20141210,"","","",""
"548847d8-05f8-49e7-af79-15bb950d210b",1635,"Payload installation","md5","47d0e8f9d7a6429920329207a32ecc2e","Regin samples collected.",1,20141210,"","","",""
"548847d8-3fbc-4a06-ba82-15bb950d210b",1635,"Payload installation","md5","2c8b9d2885543d7ade3cae98225e263b","Regin samples collected.",1,20141210,"","","",""
"548847d8-9db0-4df6-8206-15bb950d210b",1635,"Payload installation","md5","26297dc3cd0b688de3b846983c5385e5","Regin samples collected.",1,20141210,"","","",""
"548847d8-a33c-41f3-9f7a-15bb950d210b",1635,"Payload installation","md5","01c2f321b6bfdb9473c079b0797567ba","Regin samples collected.",1,20141210,"","","",""
"548847d8-c950-48eb-b960-15bb950d210b",1635,"Payload installation","md5","4b6b86c7fec1c574706cecedf44abded","Regin samples collected.",1,20141210,"","","",""
"548847d9-1404-4331-ae3c-15bb950d210b",1635,"Payload installation","md5","90fecc6a89b2e22d82d58878d93477d4","Regin samples collected.",1,20141210,"","","",""
"548847d9-39dc-4247-b23d-15bb950d210b",1635,"Payload installation","md5","06665b96e293b23acc80451abb413e50","Regin samples collected.",1,20141210,"","","",""
"548847d9-3b28-449e-b527-15bb950d210b",1635,"Payload installation","md5","e94393561901895cb0783edc34740fd4","Regin samples collected.",1,20141210,"","","",""
"548847d9-4020-41da-b5f3-15bb950d210b",1635,"Payload installation","md5","db405ad775ac887a337b02ea8b07fddc","Regin samples collected.",1,20141210,"","","",""
"548847d9-6340-44a0-8f33-15bb950d210b",1635,"Payload installation","md5","ffb0b9b5b610191051a7bdf0806e1e47","Regin samples collected.",1,20141210,"","","",""
"548847d9-8b18-4654-9766-15bb950d210b",1635,"Payload installation","md5","f3ffc2aaaa1e2ab55ec26ff098653347","Regin samples collected.",1,20141210,"","","",""
"548847d9-a564-4178-b8e6-15bb950d210b",1635,"Payload installation","md5","6662c390b2bbbd291ec7987388fc75d7","Regin samples collected.",1,20141210,"","","",""
"548847d9-afe0-4531-a4b0-15bb950d210b",1635,"Payload installation","md5","187044596bc1328efa0ed636d8aa4a5c","Regin samples collected.",1,20141210,"","","",""
"548847d9-b63c-4c95-a2bd-15bb950d210b",1635,"Payload installation","md5","1800def71006ca6790767e202fae9b9a","Regin samples collected.",1,20141210,"","","",""
"548847d9-e6fc-4b93-a773-15bb950d210b",1635,"Payload installation","md5","bfbe8c3ee78750c3a520480700e440f8","Regin samples collected.",1,20141210,"","","",""
"548847d9-fd54-4e49-909b-15bb950d210b",1635,"Payload installation","md5","89003e9a1ae635c97ebad07aebc67f00","Regin samples collected.",1,20141210,"","","",""
"548847da-1660-4562-a1f8-15bb950d210b",1635,"Payload installation","md5","b505d65721bb2453d5039a389113b566","Regin samples collected.",1,20141210,"","","",""
"548847da-2134-43d7-ba22-15bb950d210b",1635,"Payload installation","md5","8fcf4e53ece6111758a1dd3139dc7cad","Regin samples collected.",1,20141210,"","","",""
"548847da-3e40-4ab2-a5eb-15bb950d210b",1635,"Payload installation","md5","1c024e599ac055312a4ab75b3950040a","Regin samples collected.",1,20141210,"","","",""
"548847da-49c0-404d-ae42-15bb950d210b",1635,"Payload installation","md5","d240f06e98c8d3e647cbf4d442d79475","Regin samples collected.",1,20141210,"","","",""
"548847da-71ec-4b2b-bae5-15bb950d210b",1635,"Payload installation","md5","148c1bb9d405d717252c77593aff4bd8","Regin samples collected.",1,20141210,"","","",""
"548847da-9798-4b6d-b422-15bb950d210b",1635,"Payload installation","md5","ba7bb65634ce1e30c1e5415be3d1db1d","Regin samples collected.",1,20141210,"","","",""
"548847da-ac78-474c-86fe-15bb950d210b",1635,"Payload installation","md5","b29ca4f22ae7b7b25f79c1d4a421139d","Regin samples collected.",1,20141210,"","","",""
"548847da-c2d0-4d24-821e-15bb950d210b",1635,"Payload installation","md5","b269894f434657db2b15949641a67532","Regin samples collected.",1,20141210,"","","",""
"548847da-ffe4-4a90-9f2a-15bb950d210b",1635,"Payload installation","md5","22bfc970f707fd775d49e875b63c2f0c","Regin samples collected.",1,20141210,"","","",""
"548847db-060c-4275-a0c7-15bb950d210b",1635,"Payload installation","md5","049436bb90f71cf38549817d9b90e2da","Regin samples collected.",1,20141210,"","","",""
"54884832-2608-4fe6-959e-1ac6950d210b",1635,"Artifacts dropped","filename","ser8uart.sys","",,20141210,"","","",""
"54884832-5134-460e-bea2-1ac6950d210b",1635,"Artifacts dropped","filename","atdisk.sys","",,20141210,"","","",""
"54884832-6fb4-4c63-937c-1ac6950d210b",1635,"Artifacts dropped","filename","rdpmdd.sys","",,20141210,"","","",""
"54884832-93a4-4fb0-aeba-1ac6950d210b",1635,"Artifacts dropped","filename","usbclass.sys","",,20141210,"","","",""
"54884832-983c-4e4c-a692-1ac6950d210b",1635,"Artifacts dropped","filename","pcidump.sys","",,20141210,"","","",""
"54884832-f2a8-46ff-be58-1ac6950d210b",1635,"Artifacts dropped","filename","abiosdsk.sys","",,20141210,"","","",""
"5488486c-1418-4624-b87c-15ba950d210b",1635,"Artifacts dropped","regkey","Class\{4F20E605-9452-4787-B793-D0204917CA58}","",1,20141210,"","","",""
"5488486c-47ec-4952-8e60-15ba950d210b",1635,"Artifacts dropped","regkey","Class\{9B9A8ADB-8864-4BC4-8AD5-B17DFDBB9F58}","",1,20141210,"","","",""
"5488486c-a044-4c31-830c-15ba950d210b",1635,"Artifacts dropped","regkey","HKLM\System\CurrentControlSet\Control\","",1,20141210,"","","",""
"5488488d-a4ec-4b40-bd7d-15c7950d210b",1635,"External analysis","text","In this document we analyze a set of 32-bit samples
which represents stage #1 of the complex threat that is
known as Regin. Based on our analysis of the malware’s
functionalities, this part of the Regin threat can be
considered just a support module — its sole purpose
is to facilitate and enable the operations of stage #2
by loading it and making it more difficult to detect by
security products.
Regin’s stage #1 targets the Windows platform and
support various versions of the operating system,
beginning with Windows NT 4.0. Based on our analysis,
the samples may be classified into two categories: “pure”
samples that do not feature any extra, non-malicious
code; and “augmented” ones which feature malware
code as part of another device driver. The existence of
“augmented” samples indicates the intention of the
attacker to remain undiscovered for as long as possible.
When activated, samples of Regin stage #1 will
retrieve encrypted content from specific locations of
an already compromised system, map it into kernel
memory and transfer control to it. In terms of technical
sophistication, stage #1’s import resolution process is
of particular interest, as the malware uses the unusual
“trampoline” technique to mask the payload’s access to
API functions.
It is clear that this support component, that represents
the initial stage of a very complex threat, has been
instrumental in securing long-term persistence in the
attacks that made use of this threat.","",,20141210,"","","",""
"54884899-35b8-48a3-9da2-15c6950d210b",1635,"Other","text","Regin","",,20141210,"","","",""

1 uuid event_id category type value comment to_ids date object_relation object_uuid object_name object_meta_category
2 548847d8-01e0-4231-a739-15bb950d210b 5488466a-f0d0-4b58-89a5-15bc950d210b 750 1635 Payload installation External analysis md5 link 744c07e886497f7b68f6f7fe57b7ab54 https://www.f-secure.com/documents/996508/1030745/w32_regin_stage_1.pdf Regin samples collected. 1 20141210
3 548847d8-05f8-49e7-af79-15bb950d210b 548847d8-01e0-4231-a739-15bb950d210b 750 1635 Payload installation md5 47d0e8f9d7a6429920329207a32ecc2e 744c07e886497f7b68f6f7fe57b7ab54 Regin samples collected. 1 20141210
4 548847d8-3fbc-4a06-ba82-15bb950d210b 548847d8-05f8-49e7-af79-15bb950d210b 750 1635 Payload installation md5 2c8b9d2885543d7ade3cae98225e263b 47d0e8f9d7a6429920329207a32ecc2e Regin samples collected. 1 20141210
5 548847d8-9db0-4df6-8206-15bb950d210b 548847d8-3fbc-4a06-ba82-15bb950d210b 750 1635 Payload installation md5 26297dc3cd0b688de3b846983c5385e5 2c8b9d2885543d7ade3cae98225e263b Regin samples collected. 1 20141210
6 548847d8-a33c-41f3-9f7a-15bb950d210b 548847d8-9db0-4df6-8206-15bb950d210b 750 1635 Payload installation md5 01c2f321b6bfdb9473c079b0797567ba 26297dc3cd0b688de3b846983c5385e5 Regin samples collected. 1 20141210
7 548847d8-c950-48eb-b960-15bb950d210b 548847d8-a33c-41f3-9f7a-15bb950d210b 750 1635 Payload installation md5 4b6b86c7fec1c574706cecedf44abded 01c2f321b6bfdb9473c079b0797567ba Regin samples collected. 1 20141210
8 548847d9-1404-4331-ae3c-15bb950d210b 548847d8-c950-48eb-b960-15bb950d210b 750 1635 Payload installation md5 90fecc6a89b2e22d82d58878d93477d4 4b6b86c7fec1c574706cecedf44abded Regin samples collected. 1 20141210
9 548847d9-39dc-4247-b23d-15bb950d210b 548847d9-1404-4331-ae3c-15bb950d210b 750 1635 Payload installation md5 06665b96e293b23acc80451abb413e50 90fecc6a89b2e22d82d58878d93477d4 Regin samples collected. 1 20141210
10 548847d9-3b28-449e-b527-15bb950d210b 548847d9-39dc-4247-b23d-15bb950d210b 750 1635 Payload installation md5 e94393561901895cb0783edc34740fd4 06665b96e293b23acc80451abb413e50 Regin samples collected. 1 20141210
11 548847d9-4020-41da-b5f3-15bb950d210b 548847d9-3b28-449e-b527-15bb950d210b 750 1635 Payload installation md5 db405ad775ac887a337b02ea8b07fddc e94393561901895cb0783edc34740fd4 Regin samples collected. 1 20141210
12 548847d9-6340-44a0-8f33-15bb950d210b 548847d9-4020-41da-b5f3-15bb950d210b 750 1635 Payload installation md5 ffb0b9b5b610191051a7bdf0806e1e47 db405ad775ac887a337b02ea8b07fddc Regin samples collected. 1 20141210
13 548847d9-8b18-4654-9766-15bb950d210b 548847d9-6340-44a0-8f33-15bb950d210b 750 1635 Payload installation md5 f3ffc2aaaa1e2ab55ec26ff098653347 ffb0b9b5b610191051a7bdf0806e1e47 Regin samples collected. 1 20141210
14 548847d9-a564-4178-b8e6-15bb950d210b 548847d9-8b18-4654-9766-15bb950d210b 750 1635 Payload installation md5 6662c390b2bbbd291ec7987388fc75d7 f3ffc2aaaa1e2ab55ec26ff098653347 Regin samples collected. 1 20141210
15 548847d9-afe0-4531-a4b0-15bb950d210b 548847d9-a564-4178-b8e6-15bb950d210b 750 1635 Payload installation md5 187044596bc1328efa0ed636d8aa4a5c 6662c390b2bbbd291ec7987388fc75d7 Regin samples collected. 1 20141210
16 548847d9-b63c-4c95-a2bd-15bb950d210b 548847d9-afe0-4531-a4b0-15bb950d210b 750 1635 Payload installation md5 1800def71006ca6790767e202fae9b9a 187044596bc1328efa0ed636d8aa4a5c Regin samples collected. 1 20141210
17 548847d9-e6fc-4b93-a773-15bb950d210b 548847d9-b63c-4c95-a2bd-15bb950d210b 750 1635 Payload installation md5 bfbe8c3ee78750c3a520480700e440f8 1800def71006ca6790767e202fae9b9a Regin samples collected. 1 20141210
18 548847d9-fd54-4e49-909b-15bb950d210b 548847d9-e6fc-4b93-a773-15bb950d210b 750 1635 Payload installation md5 89003e9a1ae635c97ebad07aebc67f00 bfbe8c3ee78750c3a520480700e440f8 Regin samples collected. 1 20141210
19 548847da-1660-4562-a1f8-15bb950d210b 548847d9-fd54-4e49-909b-15bb950d210b 750 1635 Payload installation md5 b505d65721bb2453d5039a389113b566 89003e9a1ae635c97ebad07aebc67f00 Regin samples collected. 1 20141210
20 548847da-2134-43d7-ba22-15bb950d210b 548847da-1660-4562-a1f8-15bb950d210b 750 1635 Payload installation md5 8fcf4e53ece6111758a1dd3139dc7cad b505d65721bb2453d5039a389113b566 Regin samples collected. 1 20141210
21 548847da-3e40-4ab2-a5eb-15bb950d210b 548847da-2134-43d7-ba22-15bb950d210b 750 1635 Payload installation md5 1c024e599ac055312a4ab75b3950040a 8fcf4e53ece6111758a1dd3139dc7cad Regin samples collected. 1 20141210
22 548847da-49c0-404d-ae42-15bb950d210b 548847da-3e40-4ab2-a5eb-15bb950d210b 750 1635 Payload installation md5 d240f06e98c8d3e647cbf4d442d79475 1c024e599ac055312a4ab75b3950040a Regin samples collected. 1 20141210
23 548847da-71ec-4b2b-bae5-15bb950d210b 548847da-49c0-404d-ae42-15bb950d210b 750 1635 Payload installation md5 148c1bb9d405d717252c77593aff4bd8 d240f06e98c8d3e647cbf4d442d79475 Regin samples collected. 1 20141210
24 548847da-9798-4b6d-b422-15bb950d210b 548847da-71ec-4b2b-bae5-15bb950d210b 750 1635 Payload installation md5 ba7bb65634ce1e30c1e5415be3d1db1d 148c1bb9d405d717252c77593aff4bd8 Regin samples collected. 1 20141210
25 548847da-ac78-474c-86fe-15bb950d210b 548847da-9798-4b6d-b422-15bb950d210b 750 1635 Payload installation md5 b29ca4f22ae7b7b25f79c1d4a421139d ba7bb65634ce1e30c1e5415be3d1db1d Regin samples collected. 1 20141210
26 548847da-c2d0-4d24-821e-15bb950d210b 548847da-ac78-474c-86fe-15bb950d210b 750 1635 Payload installation md5 b269894f434657db2b15949641a67532 b29ca4f22ae7b7b25f79c1d4a421139d Regin samples collected. 1 20141210
27 548847da-ffe4-4a90-9f2a-15bb950d210b 548847da-c2d0-4d24-821e-15bb950d210b 750 1635 Payload installation md5 22bfc970f707fd775d49e875b63c2f0c b269894f434657db2b15949641a67532 Regin samples collected. 1 20141210
28 548847db-060c-4275-a0c7-15bb950d210b 548847da-ffe4-4a90-9f2a-15bb950d210b 750 1635 Payload installation md5 049436bb90f71cf38549817d9b90e2da 22bfc970f707fd775d49e875b63c2f0c Regin samples collected. 1 20141210
29 5488486c-1418-4624-b87c-15ba950d210b 548847db-060c-4275-a0c7-15bb950d210b 750 1635 Artifacts dropped Payload installation regkey md5 Class\{4F20E605-9452-4787-B793-D0204917CA58} 049436bb90f71cf38549817d9b90e2da Regin samples collected. 1 20141210
30 5488486c-47ec-4952-8e60-15ba950d210b 54884832-2608-4fe6-959e-1ac6950d210b 750 1635 Artifacts dropped regkey filename Class\{9B9A8ADB-8864-4BC4-8AD5-B17DFDBB9F58} ser8uart.sys 1 20141210
31 5488486c-a044-4c31-830c-15ba950d210b 54884832-5134-460e-bea2-1ac6950d210b 750 1635 Artifacts dropped regkey filename HKLM\System\CurrentControlSet\Control\ atdisk.sys 1 20141210
32 54884832-6fb4-4c63-937c-1ac6950d210b 1635 Artifacts dropped filename rdpmdd.sys 20141210
33 54884832-93a4-4fb0-aeba-1ac6950d210b 1635 Artifacts dropped filename usbclass.sys 20141210
34 54884832-983c-4e4c-a692-1ac6950d210b 1635 Artifacts dropped filename pcidump.sys 20141210
35 54884832-f2a8-46ff-be58-1ac6950d210b 1635 Artifacts dropped filename abiosdsk.sys 20141210
36 5488486c-1418-4624-b87c-15ba950d210b 1635 Artifacts dropped regkey Class\{4F20E605-9452-4787-B793-D0204917CA58} 1 20141210
37 5488486c-47ec-4952-8e60-15ba950d210b 1635 Artifacts dropped regkey Class\{9B9A8ADB-8864-4BC4-8AD5-B17DFDBB9F58} 1 20141210
38 5488486c-a044-4c31-830c-15ba950d210b 1635 Artifacts dropped regkey HKLM\System\CurrentControlSet\Control\ 1 20141210
39 5488488d-a4ec-4b40-bd7d-15c7950d210b 1635 External analysis text In this document we analyze a set of 32-bit samples which represents stage #1 of the complex threat that is known as Regin. Based on our analysis of the malware’s functionalities, this part of the Regin threat can be considered just a support module — its sole purpose is to facilitate and enable the operations of stage #2 by loading it and making it more difficult to detect by security products. Regin’s stage #1 targets the Windows platform and support various versions of the operating system, beginning with Windows NT 4.0. Based on our analysis, the samples may be classified into two categories: “pure” samples that do not feature any extra, non-malicious code; and “augmented” ones which feature malware code as part of another device driver. The existence of “augmented” samples indicates the intention of the attacker to remain undiscovered for as long as possible. When activated, samples of Regin stage #1 will retrieve encrypted content from specific locations of an already compromised system, map it into kernel memory and transfer control to it. In terms of technical sophistication, stage #1’s import resolution process is of particular interest, as the malware uses the unusual “trampoline” technique to mask the payload’s access to API functions. It is clear that this support component, that represents the initial stage of a very complex threat, has been instrumental in securing long-term persistence in the attacks that made use of this threat. 20141210
40 54884899-35b8-48a3-9da2-15c6950d210b 1635 Other text Regin 20141210
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68