MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge remote-tracking branch 'upstream/2.4' into 2.4

pull/3435/head
Steve Clement 2018-07-04 17:33:16 +02:00
parent 6af1d02bfe 73c18f8833
commit 43e6dfb699
38 changed files with 452 additions and 544 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
INSTALL/UPDATE.txt
View File

@ -3,13 +3,13 @@
# Option 1: To update to the latest commit from the 2.4 branch simply pull the latest commit
cd /var/www/MISP
git pull origin 2.4
git submodule update --init --force
git submodule update --init --recursive
# Option 2: If you want to stick to a point release instead of pulling the latest commit directly:
cd /var/www/MISP
git fetch
git checkout tags/$(git describe --tags `git rev-list --tags --max-count=1`)
git submodule update --init --force
git submodule update --init --recursive
# If you would like to upgrade from a minor version to another, look at the UPGRADE.txt file instead (such as 2.3.142 -> 2.4.13)
@ -30,7 +30,7 @@ git checkout tags/$(git describe --tags `git rev-list --tags --max-count=1`)
# 2. Update CakePHP to the latest supported version (if for some reason it doesn't get updated automatically with git submodule)
cd /var/www/MISP
git submodule update --init --force
git submodule update --init --recursive
# 3. Update Mitre's STIX and its dependencies

2
INSTALL/UPGRADE.txt
View File

@ -14,7 +14,7 @@ git checkout tags/$(git describe --tags `git rev-list --tags --max-count=1`)
# 3. Update CakePHP to the latest supported version
cd /var/www/MISP
rm -rf app/Lib/cakephp/
git submodule update --init --force
git submodule update --init --recursive
# 4. delete everything from MISP's cache directory to get rid of the cached models
find /var/www/MISP/app/tmp/cache/ -type f -not -name 'empty' -delete

2
INSTALL/ansible/roles/nginx/templates/nginx/misp
View File

@ -22,6 +22,6 @@ server {
fastcgi_split_path_info ^(.+\.php)(/.+)$;
fastcgi_pass unix:/var/run/php5-fpm.sock;
fastcgi_index index.php;
include fastcgi_params;
include fastcgi.conf;
}
}

22
INSTALL/get-misp-automation.py
View File

@ -1,22 +0,0 @@
#!/usr/bin/env python
'''
Example file on how to get the exported IDS data from MISP
Add your API key, set the MISP host and define the output file.
'''
import urllib2
MISP_HOST="http:/"
API_KEY=""
EXPORT_DATA="events/nids/suricata/download"
OUTPUT_FILE="misp-suricata"
URL="%s/%s" % (MISP_HOST, EXPORT_DATA)
request = urllib2.Request(URL)
f = open(OUTPUT_FILE,'w')
request.add_header('Authorization', API_KEY)
data = urllib2.urlopen(request).read()
f.write(data)
f.close()

1
app/Config/routes.php
View File

@ -33,6 +33,7 @@
Router::connect('/roles/admin_index/*', array('controller' => 'roles', 'action' => 'index', 'admin' => true));
Router::connect('/logs/admin_search/*', array('controller' => 'logs', 'action' => 'search', 'admin' => true));
Router::connect('/logs/admin_index/*', array('controller' => 'logs', 'action' => 'index', 'admin' => true));
Router::connect('/regexp/admin_index/*', array('controller' => 'regexp', 'action' => 'index', 'admin' => true));
// Activate REST
Router::mapResources(array('events', 'attributes'));

2
app/Console/Command/AdminShell.php
View File

@ -78,6 +78,8 @@ class AdminShell extends AppShell
public function setSetting() {
$setting = !isset($this->args[0]) ? null : $this->args[0];
$value = !isset($this->args[1]) ? null : $this->args[1];
if ($value === 'false') $value = 0;
if ($value === 'true') $value = 1;
if (empty($setting) || $value === null) {
echo 'Invalid parameters. Usage: ' . APP . 'Console/cake Admin setSetting [setting_name] [setting_value]';
} else {

6
app/Controller/AttributesController.php
View File

@ -1983,11 +1983,11 @@ class AttributesController extends AppController {
if ($tags) $conditions = $this->Attribute->setTagConditions($tags, $conditions, 'attribute');
if ($from) $conditions['AND'][] = array('Event.date >=' => $from);
if ($to) $conditions['AND'][] = array('Event.date <=' => $to);
if ($publish_timestamp) $conditions = $this->Attribute->setPublishTimestampConditions($publish_timestamp, $conditions);
if ($publish_timestamp) $conditions = $this->Attribute->setTimestampConditions($publish_timestamp, $conditions, 'Event.publish_timestamp');
if ($last) $conditions['AND'][] = array('Event.publish_timestamp >=' => $last);
if ($published) $conditions['AND'][] = array('Event.published' => $published);
if ($timestamp) $conditions['AND'][] = array('Attribute.timestamp >=' => $timestamp);
if ($event_timestamp) $conditions['AND'][] = array('Event.timestamp >=' => $event_timestamp);
if ($timestamp) $conditions = $this->Attribute->setTimestampConditions($timestamp, $conditions, 'Attribute.timestamp');
if ($event_timestamp) $conditions = $this->Attribute->setTimestampConditions($event_timestamp, $conditions, 'Event.timestamp');
if ($threat_level_id) {
if (!is_array($threat_level_id)) {
$threat_level_id = array($threat_level_id);

1
app/Controller/Component/ACLComponent.php
View File

@ -106,6 +106,7 @@ class ACLComponent extends Component {
'filterEventIdsForPush' => array('perm_sync'),
'filterEventIndex' => array('*'),
'freeTextImport' => array('perm_add'),
'getEditStrategy' => array('perm_add'),
'getEventInfoById' => array('*'),
'getEventGraphReferences' => array('*'),
'getEventGraphTags' => array('*'),

45
app/Controller/EventsController.php
View File

@ -3081,8 +3081,8 @@ class EventsController extends AppController {
}
if ($from) $conditions['AND'][] = array('Event.date >=' => $from);
if ($to) $conditions['AND'][] = array('Event.date <=' => $to);
if ($publish_timestamp) $conditions = $this->Event->Attribute->setPublishTimestampConditions($publish_timestamp, $conditions);
if ($timestamp) $conditions = $this->Event->Attribute->setTimestampConditions($timestamp, $conditions);
if ($publish_timestamp) $conditions = $this->Event->Attribute->setTimestampConditions($publish_timestamp, $conditions, 'Event.publish_timestamp');
if ($timestamp) $conditions = $this->Event->Attribute->setTimestampConditions($timestamp, $conditions, 'Event.timestamp');
if ($last) $conditions['AND'][] = array('Event.publish_timestamp >=' => $last);
if ($published !== null && $published !== false) $conditions['AND'][] = array('Event.published' => $published);
if ($preFilterLevel == 'event') {
@ -5098,4 +5098,45 @@ class EventsController extends AppController {
return $this->RestResponse->viewData(array(), $this->response->type());
}
}
public function getEditStrategy($id) {
// find the id of the event, change $id to it and proceed to read the event as if the ID was entered.
if (Validation::uuid($id)) {
$this->Event->recursive = -1;
$event = $this->Event->find('first', array(
'recursive' => -1,
'conditions' => array('Event.uuid' => $id),
'fields' => array('Event.id', 'Event.uuid', 'Event.orgc_id')
));
if ($event == null) throw new NotFoundException('Invalid event');
$id = $event['Event']['id'];
} else if (!is_numeric($id)) {
throw new NotFoundException(__('Invalid event'));
} else {
$event = $this->Event->find('first', array(
'recursive' => -1,
'conditions' => array('Event.id' => $id),
'fields' => array('Event.id', 'Event.uuid', 'Event.orgc_id')
));
}
if (empty($event)) throw new NotFoundException(__('Invalid event'));
$response = array('extensions' => array());
if ($event['Event']['orgc_id'] === $this->Auth->user('org_id')) {
$response['strategy'] = 'edit';
} else {
$response['strategy'] = 'extend';
}
$extendedEvents = $this->Event->find('all', array(
'recursive' => -1,
'fields' => array('Event.id', 'Event.info', 'Event.uuid'),
'conditions' => array(
'Event.extends_uuid' => $event['Event']['uuid'],
'Event.orgc_id' => $this->Auth->user('org_id')
)
));
foreach ($extendedEvents as $extendedEvent) {
$response['extensions'][] = $extendedEvent['Event'];
}
return $this->RestResponse->viewData($response, $this->response->type());
}
}

5
app/Controller/GalaxiesController.php
View File

@ -27,7 +27,10 @@ class GalaxiesController extends AppController {
public function update() {
if (!$this->request->is('post')) throw new MethodNotAllowedException('This action is only accessible via POST requests.');
$result = $this->Galaxy->update();
if (!empty($this->params['named']['force'])) {
$force = 1;
}
$result = $this->Galaxy->update($force);
$message = 'Galaxies updated.';
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('Galaxy', 'update', false, $this->response->type(), $message);

2
app/Controller/ServersController.php
View File

@ -377,7 +377,7 @@ class ServersController extends AppController {
}
if (!$fail) {
// say what fields are to be updated
$fieldList = array('id', 'url', 'push', 'pull', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal');
$fieldList = array('id', 'url', 'push', 'pull', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
$this->request->data['Server']['id'] = $id;
if (isset($this->request->data['Server']['authkey']) && "" != $this->request->data['Server']['authkey']) $fieldList[] = 'authkey';
if(isset($this->request->data['Server']['organisation_type']) && isset($json)) {

8
app/Lib/Tools/FinancialTool.php
View File

@ -95,6 +95,7 @@ class FinancialTool {
'bic' => 'BIC',
'iban' => 'IBAN',
'btc' => 'BTC',
'xmr' => 'XMR'
);
if (in_array($type, array_keys($validationRoutes))) return $this->{'validate' . strtoupper($validationRoutes[$type])}($value);
return true;
@ -191,6 +192,13 @@ private function my_bcmod( $x, $y )
return true;
}
public function validateXMR($address) {
if (!preg_match('/^4[0-9AB][1-9A-HJ-NP-Za-km-z]{93}$/', $address)) {
return false;
}
return true;
}
private function __decodeBase58($input) {
$alphabet = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";

7
app/Lib/Tools/SyncTool.php
View File

@ -16,9 +16,10 @@ class SyncTool {
}
}
$HttpSocket = new HttpSocket($params);
$proxy = Configure::read('Proxy');
if (isset($proxy['host']) && !empty($proxy['host'])) $HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']);
if (empty($server['Server']['skip_proxy'])) {
$proxy = Configure::read('Proxy');
if (isset($proxy['host']) && !empty($proxy['host'])) $HttpSocket->configProxy($proxy['host'], $proxy['port'], $proxy['method'], $proxy['user'], $proxy['password']);
}
return $HttpSocket;
}

6
app/Model/AppModel.php
View File

@ -63,7 +63,8 @@ class AppModel extends Model {
public $db_changes = array(
1 => false, 2 => false, 3 => false, 4 => true, 5 => false, 6 => false,
7 => false, 8 => false, 9 => false, 10 => false, 11 => false, 12 => false
7 => false, 8 => false, 9 => false, 10 => false, 11 => false, 12 => false,
13 => false
);
function afterSave($created, $options = array()) {
@ -957,6 +958,9 @@ class AppModel extends Model {
INDEX `timestamp` (`timestamp`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;";
break;
case 13:
$sqlArray[] = "ALTER TABLE `servers` ADD `skip_proxy` tinyint(1) NOT NULL DEFAULT 0;";
break;
case 'fixNonEmptySharingGroupID':
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';

93
app/Model/Attribute.php
View File

@ -152,6 +152,7 @@ class Attribute extends AppModel {
'target-location' => array('desc' => 'Attack Targets Physical Location(s)', 'default_category' => 'Targeting data', 'to_ids' => 0),
'target-external' => array('desc' => 'External Target Organizations Affected by this Attack', 'default_category' => 'Targeting data', 'to_ids' => 0),
'btc' => array('desc' => 'Bitcoin Address', 'default_category' => 'Financial fraud', 'to_ids' => 1),
'xmr' => array('desc' => 'Monero Address', 'default_category' => 'Financial fraud', 'to_ids' => 1),
'iban' => array('desc' => 'International Bank Account Number', 'default_category' => 'Financial fraud', 'to_ids' => 1),
'bic' => array('desc' => 'Bank Identifier Code Number also known as SWIFT-BIC, SWIFT code or ISO 9362 code', 'default_category' => 'Financial fraud', 'to_ids' => 1),
'bank-account-nr' => array('desc' => 'Bank account number without any routing number', 'default_category' => 'Financial fraud', 'to_ids' => 1),
@ -316,7 +317,7 @@ class Attribute extends AppModel {
'Financial fraud' => array(
'desc' => 'Financial Fraud indicators',
'formdesc' => 'Financial Fraud indicators, for example: IBAN Numbers, BIC codes, Credit card numbers, etc.',
'types' => array('btc', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'phone-number', 'comment', 'text', 'other', 'hex'),
'types' => array('btc', 'xmr', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'phone-number', 'comment', 'text', 'other', 'hex'),
),
'Support Tool' => array(
'desc' => 'Tools supporting analysis or detection of the event',
@ -383,7 +384,7 @@ class Attribute extends AppModel {
public $typeGroupings = array(
'file' => array('attachment', 'pattern-in-file', 'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'impfuzzy','authentihash', 'pehash', 'tlsh', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|authentihash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|pehash', 'malware-sample', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'x509-fingerprint-md5'),
'network' => array('ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port', 'mac-address', 'mac-eui-64', 'hostname', 'hostname|port', 'domain', 'domain|ip', 'email-dst', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'pattern-in-traffic', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256'),
'financial' => array('btc', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'phone-number')
'financial' => array('btc', 'xmr', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'phone-number')
);
private $__fTool = false;
@ -1170,6 +1171,7 @@ class Attribute extends AppModel {
case 'iban':
case 'bic':
case 'btc':
case 'xmr':
if (preg_match('/^[a-zA-Z0-9]+$/', $value)) {
$returnValue = true;
}
@ -2605,39 +2607,57 @@ class Attribute extends AppModel {
));
return $results;
}
$results = $this->find('all', $params);
// return false if we're paginating
if (isset($options['limit']) && empty($results)) return false;
if ($options['enforceWarninglist']) {
$this->Warninglist = ClassRegistry::init('Warninglist');
$warninglists = $this->Warninglist->fetchForEventView();
}
$results = array_values($results);
$proposals_block_attributes = Configure::read('MISP.proposals_block_attributes');
foreach ($results as $key => $attribute) {
if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttributes($warninglists, $attribute['Attribute'])) {
unset($results[$key]);
continue;
if (empty($params['limit'])) {
$pagesToFetch = $this->find('count', array('conditions' => $params['conditions']));
$loopLimit = 100000;
$pagesToFetch = ceil($pagesToFetch / $loopLimit);
$loop = true;
} else {
$loop = false;
$pagesToFetch = 1;
}
$attributes = array();
for ($i = 0; $i < $pagesToFetch; $i++) {
if ($loop) {
$params['limit'] = $loopLimit;
$params['page'] = $i+1;
}
if (!empty($options['includeAttributeUuid']) || !empty($options['includeEventUuid'])) {
$results[$key]['Attribute']['event_uuid'] = $results[$key]['Event']['uuid'];
}
if ($proposals_block_attributes) {
if (!empty($attribute['ShadowAttribute'])) {
unset($results[$key]);
} else {
unset($results[$key]['ShadowAttribute']);
$results = $this->find('all', $params);
// return false if we're paginating
if (isset($options['limit']) && empty($results)) return false;
$results = array_values($results);
$proposals_block_attributes = Configure::read('MISP.proposals_block_attributes');
foreach ($results as $key => $attribute) {
if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttributes($warninglists, $attribute['Attribute'])) {
continue;
}
}
if ($options['withAttachments']) {
if ($this->typeIsAttachment($attribute['Attribute']['type'])) {
$encodedFile = $this->base64EncodeAttachment($attribute['Attribute']);
$results[$key]['Attribute']['data'] = $encodedFile;
if (!empty($options['includeAttributeUuid']) || !empty($options['includeEventUuid'])) {
$results[$key]['Attribute']['event_uuid'] = $results[$key]['Event']['uuid'];
}
if ($proposals_block_attributes) {
if (!empty($attribute['ShadowAttribute'])) {
continue;
} else {
unset($results[$key]['ShadowAttribute']);
}
}
if ($options['withAttachments']) {
if ($this->typeIsAttachment($attribute['Attribute']['type'])) {
$encodedFile = $this->base64EncodeAttachment($attribute['Attribute']);
$results[$key]['Attribute']['data'] = $encodedFile;
}
}
$attributes[] = $results[$key];
}
}
$results = array_values($results);
return $results;
return $attributes;
}
// Method gets and converts the contents of a file passed along as a base64 encoded string with the original filename into a zip archive
@ -2846,22 +2866,15 @@ class Attribute extends AppModel {
return $conditions;
}
public function setPublishTimestampConditions($publish_timestamp, $conditions) {
if (is_array($publish_timestamp)) {
$conditions['AND'][] = array('Event.publish_timestamp >=' => $publish_timestamp[0]);
$conditions['AND'][] = array('Event.publish_timestamp <=' => $publish_timestamp[1]);
} else {
$conditions['AND'][] = array('Event.publish_timestamp >=' => $publish_timestamp);
}
return $conditions;
}
public function setTimestampConditions($timestamp, $conditions, $scope = 'Event') {
public function setTimestampConditions($timestamp, $conditions, $scope = 'Event.timestamp') {
if (is_array($timestamp)) {
$conditions['AND'][] = array($scope . '.timestamp >=' => $timestamp[0]);
$conditions['AND'][] = array($scope . '.timestamp <=' => $timestamp[1]);
$timestamp[0] = $this->Event->resolveTimeDelta($timestamp[0]);
$timestamp[1] = $this->Event->resolveTimeDelta($timestamp[1]);
$conditions['AND'][] = array($scope . ' >=' => $timestamp[0]);
$conditions['AND'][] = array($scope . ' <=' => $timestamp[1]);
} else {
$conditions['AND'][] = array($scope . '.timestamp >=' => $timestamp);
$timestamp = $this->Event->resolveTimeDelta($timestamp);
$conditions['AND'][] = array($scope . ' >=' => $timestamp);
}
return $conditions;
}

1
app/Model/Event.php
View File

@ -3450,6 +3450,7 @@ class Event extends AppModel {
}
public function resolveTimeDelta($delta) {
if (is_int($delta)) return $delta;
$multiplierArray = array('d' => 86400, 'h' => 3600, 'm' => 60);
$multiplier = $multiplierArray['d'];
$lastChar = strtolower(substr($delta, -1));

60
app/Model/Galaxy.php
View File

@ -26,7 +26,7 @@ class Galaxy extends AppModel{
$this->GalaxyCluster->deleteAll(array('GalaxyCluster.galaxy_id' => $this->id));
}
private function __load_galaxies() {
private function __load_galaxies($force = false) {
$dir = new Folder(APP . 'files' . DS . 'misp-galaxy' . DS . 'galaxies');
$files = $dir->find('.*\.json');
$galaxies = array();
@ -50,6 +50,7 @@ class Galaxy extends AppModel{
foreach ($galaxies as $k => $galaxy) {
if (isset($existingGalaxies[$galaxy['uuid']])) {
if (
$force ||
$existingGalaxies[$galaxy['uuid']]['version'] < $galaxy['version'] ||
(!empty($galaxy['icon']) && ($existingGalaxies[$galaxy['uuid']]['icon'] != $galaxy['icon']))
) {
@ -64,8 +65,8 @@ class Galaxy extends AppModel{
return $this->find('list', array('recursive' => -1, 'fields' => array('type', 'id')));
}
public function update() {
$galaxies = $this->__load_galaxies();
public function update($force = false) {
$galaxies = $this->__load_galaxies($force);
$dir = new Folder(APP . 'files' . DS . 'misp-galaxy' . DS . 'clusters');
$files = $dir->find('.*\.json');
$cluster_packages = array();
@ -90,27 +91,43 @@ class Galaxy extends AppModel{
'GalaxyCluster.galaxy_id' => $galaxies[$cluster_package['type']]
),
'recursive' => -1,
'fields' => array('version', 'id', 'value')
'fields' => array('version', 'id', 'value', 'uuid')
));
$existingClusters = array();
foreach ($temp as $k => $v) {
$existingClusters[$v['GalaxyCluster']['value']] = $v;
}
foreach ($cluster_package['values'] as $cluster) {
$clusters_to_delete = array();
// Delete all existing outdated clusters
foreach ($cluster_package['values'] as $k => $cluster) {
if (empty($cluster['value'])) {
debug($cluster);
throw new Exception();
}
if (isset($cluster['version'])) {
$template['version'] = $cluster['version'];
} else if (!empty($cluster_package['version'])) {
$template['version'] = $cluster_package['version'];
$cluster_package['values'][$k]['version'] = $cluster_package['version'];
} else {
$template['version'] = 0;
$cluster_package['values'][$k]['version'] = 0;
}
if (!empty($existingClusters[$cluster['value']])){
if ($existingClusters[$cluster['value']]['GalaxyCluster']['version'] < $template['version']) {
$this->GalaxyCluster->delete($existingClusters[$cluster['value']]['GalaxyCluster']['id']);
if ($force || $existingClusters[$cluster['value']]['GalaxyCluster']['version'] < $cluster_package['values'][$k]['version']) {
$clusters_to_delete[] = $existingClusters[$cluster['value']]['GalaxyCluster']['id'];
} else {
continue;
unset($cluster_package['values'][$k]);
}
}
}
if (!empty($clusters_to_delete)) {
$this->GalaxyCluster->GalaxyElement->deleteAll(array('GalaxyElement.galaxy_cluster_id' => $clusters_to_delete), false, false);
$this->GalaxyCluster->delete($clusters_to_delete, false, false);
}
// create all clusters
foreach ($cluster_package['values'] as $cluster) {
$template['version'] = $cluster['version'];
$this->GalaxyCluster->create();
$cluster_to_save = $template;
if (isset($cluster['description'])) {
@ -120,29 +137,34 @@ class Galaxy extends AppModel{
$cluster_to_save['value'] = $cluster['value'];
$cluster_to_save['tag_name'] = $cluster_to_save['tag_name'] . $cluster['value'] . '"';
unset($cluster['value']);
$result = $this->GalaxyCluster->save($cluster_to_save);
if (empty($cluster_to_save['description'])) $cluster_to_save['description'] = '';
$result = $this->GalaxyCluster->save($cluster_to_save, false);
$galaxyClusterId = $this->GalaxyCluster->id;
if (isset($cluster['meta'])) {
foreach ($cluster['meta'] as $key => $value) {
if (is_array($value)) {
foreach ($value as $v) {
$elements[] = array(
'galaxy_cluster_id' => $galaxyClusterId,
'key' => $key,
'value' => $v
$galaxyClusterId,
$key,
$v
);
}
} else {
$elements[] = array(
'galaxy_cluster_id' => $this->GalaxyCluster->id,
'key' => $key,
'value' => $value
$this->GalaxyCluster->id,
$key,
$value
);
}
}
}
}
$this->GalaxyCluster->GalaxyElement->saveMany($elements);
$db = $this->getDataSource();
$fields = array('galaxy_cluster_id', 'key', 'value');
if (!empty($elements)) {
$db->insertMulti('galaxy_elements', $fields, $elements);
}
}
return true;
}

38
app/Model/MispObject.php
View File

@ -216,6 +216,25 @@ class MispObject extends AppModel {
// group
public function fetchObjects($user, $options = array()) {
$sgsids = $this->SharingGroup->fetchAllAuthorised($user);
$attributeConditions = array();
if (!$user['Role']['perm_site_admin']) {
$attributeConditions = array(
'OR' => array(
array(
'(SELECT events.org_id FROM events WHERE events.id = Attribute.event_id)' => $user['org_id']
),
array(
'OR' => array(
'Attribute.distribution' => array(1, 2, 3, 5),
array(
'Attribute.distribution' => 4,
'Attribute.sharing_group_id' => $sgsids
)
)
)
)
);
}
$params = array(
'conditions' => $this->buildConditions($user),
'recursive' => -1,
@ -224,23 +243,8 @@ class MispObject extends AppModel {
'fields' => array('id', 'info', 'org_id', 'orgc_id'),
),
'Attribute' => array(
'conditions' => array(
'OR' => array(
array(
'Event.org_id' => $user['org_id'],
),
array(
'OR' => array(
'Attribute.distribution' => array(1, 2, 3, 5),
array(
'Attribute.distribution' => 4,
'Attribute.sharing_group_id' => $sgids
)
)
)
)
),
'ShadowAttribute',
'conditions' => $attributeConditions,
//'ShadowAttribute',
'AttributeTag' => array(
'Tag'
)

46
app/View/Elements/Events/View/attribute_correlations.ctp Normal file
View File

@ -0,0 +1,46 @@
<?php
$i = 0;
$linkColour = ($scope == 'Attribute') ? 'red' : 'white';
$count = count($event['Related' . $scope][$object['id']]);
foreach ($event['Related' . $scope][$object['id']] as $relatedAttribute) {
if ($i == 4) {
$expandButton = __('Show ') . (count($event['Related' . $scope][$object['id']]) - 4) . __(' more...');
echo sprintf(
'<li class="no-side-padding correlation-expand-button useCursorPointer linkButton %s">%s</li>',
$linkColour,
$expandButton
);
}
$relatedData = array(
'Orgc' => !empty($orgTable[$relatedAttribute['org_id']]) ? $orgTable[$relatedAttribute['org_id']] : 'N/A',
'Date' => isset($relatedAttribute['date']) ? $relatedAttribute['date'] : 'N/A',
'Info' => $relatedAttribute['info'],
'Correlating Value' => $relatedAttribute['value']
);
$popover = '';
foreach ($relatedData as $k => $v) {
$popover .= '<span class=\'bold black\'>' . h($k) . '</span>: <span class="blue">' . h($v) . '</span><br />';
}
$link = $this->Html->link(
$relatedAttribute['id'],
array('controller' => 'events', 'action' => 'view', $relatedAttribute['id'], true, $event['Event']['id']),
array('class' => ($relatedAttribute['org_id'] == $me['org_id']) ? $linkColour : 'blue')
);
echo sprintf(
'<li class="no-side-padding %s" %s data-toggle="popover" data-content="%s" data-trigger="hover">%s&nbsp;</li>',
($i > 3) ? 'correlation-expanded-area' : '',
($i > 3) ? 'style="display:none;"' : '',
h($popover),
$link
);
$i++;
}
if ($i > 4) {
echo sprintf(
'<li class="no-side-padding correlation-collapse-button useCursorPointer linkButton %s" style="display:none;">%s</li>',
$linkColour,
__('Collapse…')
);
}
?>

49
app/View/Elements/Events/View/row_attribute.ctp
View File

@ -1,7 +1,6 @@
<?php
$tr_class = '';
$linkClass = 'blue';
$otherColour = 'blue';
if ($event['Event']['id'] != $object['event_id']) {
if (!$isSiteAdmin && $event['extensionEvents'][$object['event_id']]['Orgc']['id'] != $me['org_id']) {
$mayModify = false;
@ -161,47 +160,13 @@
<td class="shortish">
<ul class="inline" style="margin:0px;">
<?php
$relatedObject = 'Attribute';
if (!empty($event['Related' . $relatedObject][$object['id']])):
$i = 0;
$count = count($event['Related' . $relatedObject][$object['id']]);
foreach ($event['Related' . $relatedObject][$object['id']] as $relatedAttribute):
if ($i == 4):
?>
<li class="no-side-padding correlation-expand-button useCursorPointer linkButton blue">
<?php echo __('Show ') . (count($event['Related' . $relatedObject][$object['id']]) - 4); echo __(' more...');?>
</li>
<?php
endif;
$relatedData = array(
'Orgc' => !empty($orgTable[$relatedAttribute['org_id']]) ? $orgTable[$relatedAttribute['org_id']] : 'N/A',
'Date' => isset($relatedAttribute['date']) ? $relatedAttribute['date'] : 'N/A',
'Info' => $relatedAttribute['info'],
'Correlating Value' => $relatedAttribute['value']
);
$popover = '';
foreach ($relatedData as $k => $v):
$popover .= '<span class=\'bold black\'>' . h($k) . '</span>: <span class="blue">' . h($v) . '</span><br />';
endforeach;
?>
<li class="no-side-padding <?php if ($i > 3) echo 'correlation-expanded-area'; ?>" <?php if ($i > 3) echo 'style="display:none;"'; ?> data-toggle="popover" data-content="<?php echo h($popover); ?>" data-trigger="hover">
<?php
if ($relatedAttribute['org_id'] == $me['org_id']):
echo $this->Html->link($relatedAttribute['id'], array('controller' => 'events', 'action' => 'view', $relatedAttribute['id'], true, $event['Event']['id']), array('class' => 'red'));
else:
echo $this->Html->link($relatedAttribute['id'], array('controller' => 'events', 'action' => 'view', $relatedAttribute['id'], true, $event['Event']['id']), array('class' => $otherColour));
endif;
?>
</li>
<?php
$i++;
endforeach;
if ($i > 4):
?>
<li class="no-side-padding correlation-collapse-button useCursorPointer linkButton blue" style="display:none;"><?php echo __('Collapse…');?></li>
<?php
endif;
endif;
if (!empty($event['RelatedAttribute'][$object['id']])) {
echo $this->element('Events/View/attribute_correlations', array(
'scope' => 'Attribute',
'object' => $object,
'event' => $event,
));
}
?>
</ul>
</td>

17
app/View/Elements/Events/View/row_proposal.ctp
View File

@ -125,18 +125,11 @@
<ul class="inline" style="margin:0px;">
<?php
if (!empty($event['RelatedShadowAttribute'][$object['id']])) {
foreach ($event['RelatedShadowAttribute'][$object['id']] as $relatedAttribute) {
$relatedData = array('Event info' => $relatedAttribute['info'], 'Correlating Value' => $relatedAttribute['value'], 'date' => isset($relatedAttribute['date']) ? $relatedAttribute['date'] : 'N/A');
$popover = '';
foreach ($relatedData as $k => $v) {
$popover .= '<span class=\'bold black\'>' . h($k) . '</span>: <span class="blue">' . h($v) . '</span><br />';
}
echo '<li style="padding-right: 0px; padding-left:0px;" data-toggle="popover" data-content="' . h($popover) . '" data-trigger="hover"><span>';
$correlationClass = 'white' . ($relatedAttribute['org_id'] == $me['org_id'] ? ' bold' : '');
echo $this->Html->link($relatedAttribute['id'], array('controller' => 'events', 'action' => 'view', $relatedAttribute['id'], true, $event['Event']['id']), array('class' => $correlationClass));
echo "</span></li>";
echo ' ';
}
echo $this->element('Events/View/attribute_correlations', array(
'scope' => 'ShadowAttribute',
'object' => $object,
'event' => $event,
));
}
?>
</ul>

1
app/View/Elements/side_menu.ctp
View File

@ -526,6 +526,7 @@
if ($isSiteAdmin):
?>
<li><?php echo $this->Form->postLink(__('Update Galaxies'), array('controller' => 'galaxies', 'action' => 'update'), null, __('Are you sure you want to reimport all galaxies from the submodule?')); ?></li>
<li><?php echo $this->Form->postLink(__('Force Update Galaxies'), array('controller' => 'galaxies', 'action' => 'update', 'force' => 1), null, __('Are you sure you want to drop and reimport all galaxies from the submodule?')); ?></li>
<?php
endif;
if ($menuItem === 'viewGraph' || $menuItem === 'view_cluster'): ?>

12
app/View/Elements/view_mitre_attack_matrix.ctp
View File

@ -1,9 +1,9 @@
<div class="attack-matrix-options" style="right: initial; background: transparent;">
<ul id="attack-matrix-tabscontroller" class="nav nav-tabs" style="margin-bottom: 2px;">
<?php
<?php
$enterpriseTag = "mitre-enterprise-attack-attack-pattern";
foreach($attackTactic as $tactic):
$galaxy = $tactic['galaxy'];
foreach($attackTactic as $tactic):
$galaxy = $tactic['galaxy'];
?>
<li class="tactic <?php echo $galaxy['type']==$enterpriseTag ? "active" : ""; ?>"><span href="#tabMatrix-<?php echo h($galaxy['type']); ?>" data-toggle="tab" style="padding-top: 3px; padding-bottom: 3px;"><?php echo h($galaxy['name']); ?></span></li>
<?php endforeach; ?>
@ -52,7 +52,7 @@ foreach($attackTactic as $tactic):
<?php echo h(ucfirst($name)); ?>
<div class="th-inner"><?php echo h(ucfirst($name)); ?></div>
</th>
<?php endforeach; ?>
</tr>
</thead>
@ -108,8 +108,8 @@ foreach($attackTactic as $tactic):
</div>
<?php if($pickingMode): ?>
<div role="button" tabindex="0" aria-label="Submit" title="Submit" class="templateChoiceButton btn-matrix-submit" onClick="cancelPopoverForm('#popover_form_large');"><?php echo _('Submit'); ?></div>
<div role="button" tabindex="0" aria-label="Cancel" title="Cancel" class="templateChoiceButton templateChoiceButtonLast" onClick="cancelPopoverForm('#popover_form_large');"><?php echo _('Cancel'); ?></div>
<div role="button" tabindex="0" aria-label="Submit" title="Submit" class="templateChoiceButton btn-matrix-submit" onClick="cancelPopoverForm('#popover_form_large');"><?php echo __('Submit'); ?></div>
<div role="button" tabindex="0" aria-label="Cancel" title="Cancel" class="templateChoiceButton templateChoiceButtonLast" onClick="cancelPopoverForm('#popover_form_large');"><?php echo __('Cancel'); ?></div>
<?php endif; ?>
<?php

4
app/View/Servers/add.ctp
View File

@ -88,6 +88,10 @@
echo $this->Form->input('self_signed', array(
'type' => 'checkbox',
));
?>
<div class = "input clear"></div>
<?php
echo $this->Form->input('skip_proxy', array('type' => 'checkbox', 'label' => 'Skip proxy (if applicable)'));
echo $this->Form->input('Server.submitted_cert', array(
'label' => '<b>' . __('Server certificate file') . '</b>',

4
app/View/Servers/edit.ctp
View File

@ -94,6 +94,10 @@
echo $this->Form->input('self_signed', array(
'type' => 'checkbox',
));
?>
<div class = "input clear"></div>
<?php
echo $this->Form->input('skip_proxy', array('type' => 'checkbox', 'label' => 'Skip proxy (if applicable)'));
?>
<div class="clear">
<p>

2
app/View/Servers/index.ctp
View File

@ -31,6 +31,7 @@
<th><?php echo $this->Paginator->sort('cert_file');?></th>
<th><?php echo $this->Paginator->sort('client_cert_file');?></th>
<th><?php echo $this->Paginator->sort('self_signed');?></th>
<th><?php echo $this->Paginator->sort('skip_proxy');?></th>
<th><?php echo $this->Paginator->sort('org');?></th>
<th class="actions"><?php echo __('Actions');?></th>
</tr>
@ -78,6 +79,7 @@ foreach ($servers as $server):
<td class="short"><?php echo h($server['Server']['cert_file']); ?>&nbsp;</td>
<td class="short"><?php echo h($server['Server']['client_cert_file']); ?>&nbsp;</td>
<td class="short"><span class="<?php echo ($server['Server']['self_signed'] ? 'icon-ok' : 'icon-remove'); ?>"></span></td>
<td class="short"><span class="<?php echo ($server['Server']['skip_proxy'] ? 'icon-ok' : 'icon-remove'); ?>"></span></td>
<td class="short"><a href="/organisations/view/<?php echo h($server['Organisation']['id']); ?>"><?php echo h($server['Organisation']['name']); ?></a></td>
<td class="short action-links">
<?php

2
app/files/misp-galaxy

@ -1 +1 @@
Subproject commit 1bd0fb34d76ad82692ccbcf92bdfa7b650cab5d0
Subproject commit b7d52a8bac7be7da93cd3c17b24114f918a6ff1f

1
app/files/scripts/generate_file_objects.py
View File

@ -53,6 +53,7 @@ def make_objects(path):
to_return['references'] += fo.ObjectReference
return json.dumps(to_return, cls=MISPEncode)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Extract indicators out of binaries and returns MISP objects.')
group = parser.add_mutually_exclusive_group()

1
app/files/scripts/misp2stix.py
View File

@ -1,3 +1,4 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import sys, json, uuid, os, time, datetime, re, ntpath, socket
from pymisp import MISPEvent

2
app/files/scripts/misp2stix_framing.py
View File

@ -1,3 +1,5 @@
#!/usr/bin/env python3
import sys, json, uuid, os, time, datetime, re
from dateutil.tz import tzutc
from stix.indicator import Indicator

190
app/files/scripts/mispzmq/mispzmq.py
View File

@ -1,3 +1,5 @@
#!/usr/bin/env python3
import zmq
import sys
import redis
@ -5,99 +7,115 @@ import json
import os
import time
socket = None
r = None
namespace = None
settings = None
current_location = os.path.dirname(os.path.realpath(__file__))
pidfile = current_location + "/mispzmq.pid"
timestamp = time.time()
timestampSettings = timestamp
publishCount = 0
from pathlib import Path
def setup():
global namespace
global socket
global r
global settings
global timestampSettings
with open(current_location + '/settings.json') as settings_file:
settings = json.load(settings_file)
namespace = settings["redis_namespace"]
r = redis.StrictRedis(host=settings["redis_host"], db=settings["redis_database"], password=settings["redis_password"], port=settings["redis_port"])
timestampSettings = time.time()
def handleCommand(command):
if command == "kill":
print("Kill command received, shutting down.\n")
removePidFile()
sys.exit()
if command == "reload":
print("Reload command received, reloading settings from file.\n")
setup()
if command == "status":
print("Status command received, responding with latest stats.\n")
r.delete(namespace + ":status")
r.lpush(namespace + ":status", json.dumps({"timestamp": timestamp, "timestampSettings": timestampSettings, "publishCount": publishCount}))
return
def check_pid(pid):
""" Check For the existence of a unix pid. """
try:
os.kill(pid, 0)
except OSError:
return False
else:
return True
def removePidFile():
os.unlink(pidfile)
def createPidFile():
pid = str(os.getpid())
open(pidfile, 'w').write(pid)
class MISPZMQ():
def pubMessage(topic, data, socket):
socket.send_string("%s %s" % (topic, data))
if topic is 'misp_json':
global publishCount
publishCount = publishCount + 1
def __init__(self):
self.current_location = Path(__file__).cwd()
self.pidfile = self.current_location / "mispzmq.pid"
self.publishCount = 0
if self.pidfile.exists():
with open(self.pidfile) as f:
pid = f.read()
if check_pid(pid):
raise Exception('mispzmq already running on PID {}'.format(pid))
else:
# Cleanup
self.pidfile.unlink()
if (self.current_location / 'settings.json').exists():
self.setup()
else:
raise Exception("The settings file is missing.")
def main(args):
start_time = int(time.time())
setup()
createPidFile()
status_array = [
'And when you\'re dead I will be still alive.',
'And believe me I am still alive.',
'I\'m doing science and I\'m still alive.',
'I feel FANTASTIC and I\'m still alive.',
'While you\'re dying I\'ll be still alive.'
def setup(self):
with open(self.current_location / 'settings.json') as settings_file:
self.settings = json.load(settings_file)
self.namespace = self.settings["redis_namespace"]
self.r = redis.StrictRedis(host=self.settings["redis_host"], db=self.settings["redis_database"],
password=self.settings["redis_password"], port=self.settings["redis_port"])
self.timestampSettings = time.time()
]
context = zmq.Context()
socket = context.socket(zmq.PUB)
socket.bind("tcp://*:%s" % settings["port"])
time.sleep(1)
def handleCommand(self, command):
if command == "kill":
print("Kill command received, shutting down.")
self.pidfile.unlink()
sys.exit()
if command == "reload":
print("Reload command received, reloading settings from file.")
self.setup()
if command == "status":
print("Status command received, responding with latest stats.")
self.r.delete("{self.namespace}:status".format(self.namespace))
self.r.lpush("{self.namespace}:status".format(self.namespace),
json.dumps({"timestamp": time.time(),
"timestampSettings": self.timestampSettings,
"publishCount": self.publishCount}))
def createPidFile(self):
with open(self.pidfile, 'w') as f:
f.write(str(os.getpid()))
def pubMessage(self, topic, data, socket):
socket.send_string("{} {}".format(topic, data))
if topic is 'misp_json':
self.publishCount += 1
def main(self):
start_time = int(time.time())
self.createPidFile()
status_array = [
"And when you're dead I will be still alive.",
"And believe me I am still alive.",
"I'm doing science and I'm still alive.",
"I feel FANTASTIC and I'm still alive.",
"While you're dying I'll be still alive."
]
context = zmq.Context()
socket = context.socket(zmq.PUB)
socket.bind("tcp://*:{}".format(self.settings["port"]))
time.sleep(1)
while True:
command = self.r.lpop("{}:command".format(self.namespace))
if command is not None:
self.handleCommand(command)
topics = ["misp_json", "misp_json_event", "misp_json_attribute", "misp_json_sighting",
"misp_json_organisation", "misp_json_user", "misp_json_conversation",
"misp_json_object", "misp_json_object_reference", "misp_json_audit",
"misp_json_tag"
]
message_received = False
for topic in topics:
data = self.r.lpop("{}:data:{}".format(self.namespace, topic))
if data is not None:
self.pubMessage(topic, data, socket)
message_received = True
if not message_received:
time.sleep(0.1)
current_time = 10 * time.time()
temp_start_time = 10 * start_time
time_delta = int(current_time - temp_start_time)
if (time_delta % 100 == 0):
status_entry = int(time_delta / 100 % 5)
status_message = {
'status': status_array[status_entry],
'uptime': int(time.time()) - start_time
}
self.pubMessage('misp_json_self', json.dumps(status_message), socket)
while True:
command = r.lpop(namespace + ":command")
if command is not None:
handleCommand(command)
topics = ["misp_json", "misp_json_event", "misp_json_attribute", "misp_json_sighting",
"misp_json_organisation", "misp_json_user", "misp_json_conversation",
"misp_json_object", "misp_json_object_reference", "misp_json_audit",
"misp_json_tag"
]
message_received = False
for topic in topics:
data = r.lpop(namespace + ":data:" + topic)
if data is not None:
pubMessage(topic, data, socket)
message_received = True
if (message_received == False):
time.sleep(0.1)
current_time = 10*time.time()
temp_start_time = 10*start_time
time_delta = int(current_time - temp_start_time)
if (time_delta % 100 == 0):
status_entry = time_delta/100 % 5
status_message = {
'status': status_array[status_entry],
'uptime': int(time.time()) - start_time
}
pubMessage('misp_json_self', json.dumps(status_message), socket)
if __name__ == "__main__":
main(sys.argv)
mzq = MISPZMQ()
mzq.main()

2
app/files/scripts/mispzmq/mispzmqtest.py
View File

@ -1,3 +1,5 @@
#!/usr/bin/env python3
try:
import zmq
except ImportError:

50
app/files/scripts/stix2/misp2stix2.py
View File

@ -1,3 +1,4 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright (C) 2017-2018 CIRCL Computer Incident Response Center Luxembourg (smile gie)
# Copyright (C) 2017-2018 Christian Studer
@ -507,7 +508,7 @@ class StixBuilder():
pattern = pattern_arg
else:
name = misp_object.name
pattern = self.objects_mapping[name]['pattern'](misp_object.attribute)
pattern = self.objects_mapping[name]['pattern'](misp_object.attributes)
indicator_id = 'indicator--{}'.format(misp_object.uuid)
category = misp_object.get('meta-category')
killchain = self.create_killchain(category)
@ -526,7 +527,7 @@ class StixBuilder():
observable_objects = observable_arg
else:
name = misp_object.name
observable_objects = self.objects_mapping[name]['observable'](misp_object.attribute)
observable_objects = self.objects_mapping[name]['observable'](misp_object.attributes)
observed_data_id = 'observed-data--{}'.format(misp_object.uuid)
category = misp_object.get('meta-category')
labels = self.create_object_labels(name, category, False)
@ -536,7 +537,7 @@ class StixBuilder():
'first_observed': timestamp, 'last_observed': timestamp,
'created_by_ref': self.identity_id}
try:
observed_data = ObservedData(**observed_data_args)
observed_data = ObservedData(**observed_data_args, allow_custom=True)
except exceptions.InvalidValueError:
observed_data = self.fix_enumeration_issues(name, observed_data_args)
self.append_object(observed_data, observed_data_id)
@ -653,7 +654,7 @@ class StixBuilder():
try:
stix_type = asnObjectMapping[relation]
except KeyError:
continue
stix_type = "x_misp_{}_{}".format(attribute.type, relation)
attribute_value = attribute.value
if relation == "subnet-announced":
observable[str(object_num)] = {'type': define_address_type(attribute_value), 'value': attribute_value}
@ -662,7 +663,7 @@ class StixBuilder():
asn[stix_type] = int(attribute_value[2:]) if (stix_type == 'number' and attribute_value.startswith("AS")) else attribute_value
observable[str(object_num)] = asn
for n in range(object_num):
observable[n]['belongs_to_refs'] = [str(object_num)]
observable[str(n)]['belongs_to_refs'] = [str(object_num)]
return observable
@staticmethod
@ -674,7 +675,7 @@ class StixBuilder():
try:
stix_type = asnObjectMapping[relation]
except KeyError:
continue
stix_type = "'x_misp_{}_{}'".format(attribute.type, relation)
attribute_value = attribute.value
if relation == "subnet-announced":
pattern += "{0}:{1} = '{2}' AND ".format(define_address_type(attribute_value), stix_type, attribute_value)
@ -711,36 +712,36 @@ class StixBuilder():
reply_to = []
object_num = 0
for attribute in attributes:
attribute_type = attribute.type
relation = attribute.object_relation
attribute_value = attribute.value
try:
mapping = emailObjectMapping[attribute_type]
mapping = emailObjectMapping[relation]['stix_type']
except:
continue
if attribute_type in ('email-src', 'email-dst'):
mapping = "x_misp_{}_{}".format(attribute.type, relation)
if relation in ('from', 'to', 'cc'):
object_str = str(object_num)
observable[object_str] = {'type': 'email-addr', 'value': attribute_value}
try:
message[mapping['stix_type'][attribute.object_relation]].append(object_str)
except:
message[mapping['stix_type']] = object_str
if relation == 'from':
message[mapping] = object_str
else:
message[mapping].append(object_str)
object_num += 1
elif attribute_type == 'email-reply-to':
elif relation == 'reply-to':
reply_to.append(attribute_value)
elif attribute_type == 'email-attachment':
elif relation == 'attachment':
object_str = str(object_num)
body = {"content_disposition": "attachment; filename='{}'".format(attribute_value),
"body_raw_ref": object_str}
message['body_multipart'].append(body)
observable[object_str] = {'type': 'file', 'name': attribute_value}
object_num += 1
elif attribute_type == 'email-x-mailer':
elif relation == 'x-mailer':
if 'additional_header_fields' in message:
message['additional_header_fields']['X-Mailer'] = attribute_value
else:
message['additional_header_fields'] = {'X-Mailer': attribute_value}
else:
message[mapping['stix_type']] = attribute_value
message[mapping] = attribute_value
if reply_to and 'additional_header_fields' in message:
message['additional_header_fields']['Reply-To'] = reply_to
message['type'] = 'email-message'
@ -757,14 +758,13 @@ class StixBuilder():
pattern = ""
for attribute in attributes:
try:
mapping = emailObjectMapping[attribute.type]
except:
continue
try:
stix_type = mapping['stix_type'][attribute.object_relation]
except:
mapping = emailObjectMapping[attribute.object_relation]
stix_type = mapping['stix_type']
pattern += pattern_mapping.format(mapping['email_type'], stix_type, attribute.value)
email_type = mapping['email_type']
except:
stix_type = "'x_misp_{}_{}'".format(attribute.type, attribute.object_relation)
email_type = 'message'
pattern += pattern_mapping.format(email_type, stix_type, attribute.value)
return pattern[:-5]
@staticmethod

18
app/files/scripts/stix2/misp2stix2_mapping.py
View File

@ -257,15 +257,15 @@ asnObjectMapping = {'asn': 'number', 'description': 'name', 'subnet-announced':
domainIpObjectMapping = {'ip-dst': 'resolves_to_refs[*].value', 'domain': 'value'}
emailObjectMapping = {'email-body': {'email_type': 'message', 'stix_type': 'body'},
'email-subject': {'email_type': 'message', 'stix_type': 'subject'},
'email-dst': {'email_type': 'message', 'stix_type': {'to': 'to_refs', 'cc': 'cc_refs'}},
'email-dst-display-name': {'email_type': 'addr', 'stix_type': 'display_name'},
'email-src': {'email_type': 'message', 'stix_type': 'from_ref'},
'email-src-display-name': {'email_type': 'addr', 'stix_type': 'display_name'},
'email-reply-to': {'email_type': 'message', 'stix_type': 'additional_header_fields.reply_to'},
'email-attachment': {'email_type': 'message', 'stix_type': 'body_multipart[*].body_raw_ref.name'},
'datetime': {'email_type': 'message', 'stix_type': 'date'},
'email-x-mailer': {'email_type': 'message', 'stix_type': 'additional_header_fields.x_mailer'}}
'subject': {'email_type': 'message', 'stix_type': 'subject'},
'to': {'email_type': 'message', 'stix_type': 'to_refs'}, 'cc': {'email_type': 'message', 'stix_type': 'cc_refs'},
'to-display-name': {'email_type': 'addr', 'stix_type': 'display_name'},
'from': {'email_type': 'message', 'stix_type': 'from_ref'},
'from-display-name': {'email_type': 'addr', 'stix_type': 'display_name'},
'reply-to': {'email_type': 'message', 'stix_type': 'additional_header_fields.reply_to'},
'attachment': {'email_type': 'message', 'stix_type': 'body_multipart[*].body_raw_ref.name'},
'send-date': {'email_type': 'message', 'stix_type': 'date'},
'x-mailer': {'email_type': 'message', 'stix_type': 'additional_header_fields.x_mailer'}}
fileMapping = {'hashes': "hashes.'{0}'", 'size-in-bytes': 'size', 'filename': 'name', 'mime-type': 'mime_type'}

7
app/files/scripts/stix2/stix2misp.py
View File

@ -1,3 +1,4 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright (C) 2017-2018 CIRCL Computer Incident Response Center Luxembourg (smile gie)
# Copyright (C) 2017-2018 Christian Studer
@ -42,7 +43,7 @@ class StixParser():
for o in event.get('objects'):
try:
try:
self.event.append(stix2.parse(o))
self.event.append(stix2.parse(o, allow_custom=True))
except:
self.parse_custom(o)
except:
@ -215,10 +216,10 @@ class StixParser():
def parse_object(self, o, labels):
object_type = self.get_misp_type(labels)
misp_object_type = 'file' is object_type == 'WindowsPEBinaryFile' else object_type
name = 'file' if object_type == 'WindowsPEBinaryFile' else object_type
object_category = self.get_misp_category(labels)
stix_type = o._type
misp_object = MISPObject(misp_object_type)
misp_object = MISPObject(name)
misp_object['meta-category'] = object_category
if stix_type == 'indicator':
pattern = o.get('pattern').replace('\\\\', '\\').split(' AND ')

38
app/files/scripts/stix2/stix2misp_mapping.py
View File

@ -164,8 +164,8 @@ asn_mapping = {'number': as_number_attribute_mapping,
'autonomous-system:number': as_number_attribute_mapping,
'name': asn_description_attribute_mapping,
'autonomous-system:name': asn_description_attribute_mapping,
'ipv4-address:value': asn_subnet_attribute_mapping,
'ipv6-address:value': asn_subnet_attribute_mapping}
'ipv4-addr:value': asn_subnet_attribute_mapping,
'ipv6-addr:value': asn_subnet_attribute_mapping}
domain_ip_mapping = {'domain-name': domain_attribute_mapping,
'domain-name:value': domain_attribute_mapping,
@ -277,13 +277,19 @@ x509_mapping = {'issuer': issuer_attribute_mapping,
}
def fill_observable_attributes(attributes, stix_object, object_mapping):
for o in stix_object:
for o_key, o_value in stix_object.items():
print(o_key, o_value)
try:
mapping = object_mapping[o]
mapping = object_mapping[o_key]
attributes.append({'type': mapping.get('type'), 'object_relation': mapping.get('relation'),
'value': o_value, 'to_ids': False})
except:
continue
attributes.append({'type': mapping.get('type'), 'object_relation': mapping.get('relation'),
'value': stix_object.get(o)})
if "x_misp_" in o_key:
attribute_type, relation = o_key.split("x_misp_")[1].split("_")
attributes.append({'type': attribute_type, 'object_relation': relation,
'value': o_value, 'to_ids': False})
else:
continue
def fill_pattern_attributes(pattern, object_mapping):
attributes = []
@ -291,15 +297,23 @@ def fill_pattern_attributes(pattern, object_mapping):
p_type, p_value = p.split(' = ')
try:
mapping = object_mapping[p_type]
attributes.append({'type': mapping['type'], 'object_relation': mapping['relation'],
'value': p_value[1:-1], 'to_ids': True})
except KeyError:
continue
attributes.append({'type': mapping['type'], 'object_relation': mapping['relation'],
'value': p_value[1:-1]})
if "x_misp_" in p_type:
attribute_type, relation = p_type.split("x_misp_")[1][:-1].split("_")
attributes.append({'type': attribute_type, 'object_relation': relation,
'value': p_value[1:-1], 'to_ids': True})
else:
continue
return attributes
def observable_asn(observable):
attributes = []
fill_observable_attributes(attributes, observable, asn_mapping)
fill_observable_attributes(attributes, observable.pop(str(len(observable) - 1)), asn_mapping)
for o_dict in observable.values():
attributes.append({'type': 'ip-src', 'object_relation': 'subnet-announced',
'value': o_dict['value'], 'to_ids': False})
return attributes
def pattern_asn(pattern):
@ -333,7 +347,7 @@ def observable_email(observable):
message = dict(observable_part)
attributes.append({'type': 'email-src', 'object_relation': 'from',
'value': addresses[message.pop('from_ref')]})
for ref in ('to_refs', 'cc_refs', 'ouioui'):
for ref in ('to_refs', 'cc_refs'):
if ref in message:
for item in message.pop(ref):
mapping = email_mapping[ref]

15
app/files/scripts/stixtest.py
View File

@ -1,4 +1,7 @@
import json, sys
#!/usr/bin/env python3
import json
import sys
results = {
'success': 1,
'stix': 0,
@ -38,11 +41,11 @@ except Exception:
results['success'] = 0
print(json.dumps({
'success' : results['success'],
'stix' : results['stix'],
'cybox' : results['cybox'],
'mixbox' : results['mixbox'],
'success': results['success'],
'stix': results['stix'],
'cybox': results['cybox'],
'mixbox': results['mixbox'],
'maec': results['maec'],
'pymisp' : results['pymisp']
'pymisp': results['pymisp']
}))
sys.exit(1)

228
tools/example-rest.py
View File

File diff suppressed because one or more lines are too long