MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch '2.4' into guides

pull/4840/head
Steve Clement 2019-07-07 16:44:27 +09:00
parent 81d035f569 ab76eaff05
commit c26ea4f3ef
No known key found for this signature in database
GPG Key ID: 69A20F509BE4AEE9
23 changed files with 18501 additions and 1225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.travis.yml
View File

@ -53,6 +53,7 @@ install:
- sudo chmod -R g+ws `pwd`/app/tmp/logs
- sudo chmod -R g+ws `pwd`/app/files
- sudo chmod -R g+ws `pwd`/app/files/scripts/tmp
- sudo chown -R www-data:www-data `pwd`
# Resque perms
- sudo chown -R www-data:www-data `pwd`/app/Plugin/CakeResque/tmp
- sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp
@ -84,7 +85,7 @@ install:
- sudo mkdir `pwd`/.gnupg
# /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS
- sudo cp -a /dev/urandom /dev/random
- sudo gpg --homedir `pwd`/.gnupg --gen-key --batch `pwd`/travis/gpg
- sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/travis/gpg
- sudo chown www-data:www-data `pwd`/.gnupg
- sudo chmod 700 `pwd`/.gnupg
# Get authkey

18
INSTALL/INSTALL.sh
View File

@ -260,7 +260,7 @@ checkCoreOS () {
# Extract debian flavour
checkFlavour () {
lsb_dist=""
FLAVOUR=""
# Every system that we officially support has /etc/os-release
if [ -r /etc/os-release ]; then
FLAVOUR="$(. /etc/os-release && echo "$ID"| tr '[:upper:]' '[:lower:]')"
@ -290,14 +290,14 @@ checkFlavour () {
if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then
dist_version="$(. /etc/os-release && echo "$VERSION_ID")"
fi
echo "$lsb_dist not supported at the moment"
echo "$FLAVOUR not supported at the moment"
exit 1
;;
rhel|ol|sles)
if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then
dist_version="$(. /etc/os-release && echo "$VERSION_ID")"
fi
echo "$lsb_dist not supported at the moment"
echo "$FLAVOUR not supported at the moment"
exit 1
;;
*)
@ -334,20 +334,20 @@ check_forked () {
if [ "$lsb_release_exit_code" = "0" ]; then
# Print info about current distro
cat <<-EOF
You're using '$lsb_dist' version '$dist_version'.
You're using '$FLAVOUR' version '$dist_version'.
EOF
# Get the upstream release info
lsb_dist=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'id' | cut -d ':' -f 2 | tr -d '[:space:]')
FLAVOUR=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'id' | cut -d ':' -f 2 | tr -d '[:space:]')
dist_version=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'codename' | cut -d ':' -f 2 | tr -d '[:space:]')
# Print info about upstream distro
cat <<-EOF
Upstream release is '$lsb_dist' version '$dist_version'.
Upstream release is '$FLAVOUR' version '$dist_version'.
EOF
else
if [ -r /etc/debian_version ] && [ "$lsb_dist" != "ubuntu" ] && [ "$lsb_dist" != "raspbian" ]; then
if [ -r /etc/debian_version ] && [ "$FLAVOUR" != "ubuntu" ] && [ "$FLAVOUR" != "raspbian" ]; then
# We're Debian and don't even know it!
lsb_dist=debian
FLAVOUR=debian
dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')"
case "$dist_version" in
10)
@ -2480,7 +2480,7 @@ armv7l-ubuntu-bionic
"
# Check if we actually support this configuration
if ! echo "$SUPPORT_MAP" | grep "$(uname -m)-$lsb_dist-$dist_version" >/dev/null; then
if ! echo "$SUPPORT_MAP" | grep "$(uname -m)-$FLAVOUR-$dist_version" >/dev/null; then
cat >&2 <<-'EOF'
Either your platform is not easily detectable or is not supported by this
installer script.

6
INSTALL/INSTALL.sh.sfv
View File

@ -1,5 +1,5 @@
; Generated by RHash v1.3.8 on 2019-07-02 at 11:32.43
; Generated by RHash v1.3.8 on 2019-07-02 at 12:28.32
; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
;
; 99486 11:32.43 2019-07-02 INSTALL.sh
INSTALL.sh 9CF5E9CD8F7AE5D3A558ECD3C8B21718D4733FC6 5A47A2545E36D2933F284B7CE0A90B88109A11DD50A4CD994B5EEBDA6C895484 69565AA17784FBE0D148D391B8412751B5CBF68763C0F430BB3776E7D0E591E74585FC6D48701EE38946B6C798D1C21F 760D264E464A9EB3C51A9BE32965567F1277E64DF7A7AFE0BC9A3D239741EAAF819A282765F5F8918348DE588EABCCB6F87A4966D60906F624A767ACA003645B
; 99476 12:28.32 2019-07-02 INSTALL.sh
INSTALL.sh 064C49340280B33DA2078761BB649794D8E31F6C 86BA12B0F85366B89106C901341C43D39ABB4F1EF546143D40EE44CD7DA578F7 AAE62AA039BA822CD39A5937DCBD22F0AC991802DED6F94A0F9754BE87CA091E9EAF457C5AB1831E6C583F4F0B286BE3 D77A1290A34453EFCCA50970FED5BC7035C49652E35EDC4108D82A7C4BA3CF1F020B41D36DD2E80AA9A557CF3F176F370CFC797130E9956C859E48AC272832E8

2
INSTALL/INSTALL.sh.sha1
View File

@ -1 +1 @@
9cf5e9cd8f7ae5d3a558ecd3c8b21718d4733fc6 INSTALL.sh
064c49340280b33da2078761bb649794d8e31f6c INSTALL.sh

2
INSTALL/INSTALL.sh.sha256
View File

@ -1 +1 @@
5a47a2545e36d2933f284b7ce0a90b88109a11dd50a4cd994b5eebda6c895484 INSTALL.sh
86ba12b0f85366b89106c901341c43d39abb4f1ef546143d40ee44cd7da578f7 INSTALL.sh

2
INSTALL/INSTALL.sh.sha384
View File

@ -1 +1 @@
69565aa17784fbe0d148d391b8412751b5cbf68763c0f430bb3776e7d0e591e74585fc6d48701ee38946b6c798d1c21f INSTALL.sh
aae62aa039ba822cd39a5937dcbd22f0ac991802ded6f94a0f9754be87ca091e9eaf457c5ab1831e6c583f4f0b286be3 INSTALL.sh

2
INSTALL/INSTALL.sh.sha512
View File

@ -1 +1 @@
760d264e464a9eb3c51a9be32965567f1277e64df7a7afe0bc9a3d239741eaaf819a282765f5f8918348de588eabccb6f87a4966d60906f624a767aca003645b INSTALL.sh
d77a1290a34453efcca50970fed5bc7035c49652e35edc4108d82a7c4ba3cf1f020b41d36dd2e80aa9a557cf3f176f370cfc797130e9956c859e48ac272832e8 INSTALL.sh

2
INSTALL/INSTALL.tpl.sh
View File

@ -686,7 +686,7 @@ armv7l-ubuntu-bionic
"
# Check if we actually support this configuration
if ! echo "$SUPPORT_MAP" | grep "$(uname -m)-$lsb_dist-$dist_version" >/dev/null; then
if ! echo "$SUPPORT_MAP" | grep "$(uname -m)-$FLAVOUR-$dist_version" >/dev/null; then
cat >&2 <<-'EOF'
Either your platform is not easily detectable or is not supported by this
installer script.

2
PyMISP

@ -1 +1 @@
Subproject commit 583fb6592495ea358aad47a8a1ec92d43c13348a
Subproject commit c850ec25485a5bcc60baf60f041f30f6cb15232c

2
app/Controller/AppController.php
View File

@ -46,7 +46,7 @@ class AppController extends Controller
public $helpers = array('Utility', 'OrgImg', 'FontAwesome');
private $__queryVersion = '78';
private $__queryVersion = '79';
public $pyMispVersion = '2.4.106';
public $phpmin = '7.0';
public $phprec = '7.2';

7
app/Controller/Component/IOCImportComponent.php
View File

@ -262,7 +262,12 @@ class IOCImportComponent extends Component
private function __analyseIndicator($attribute)
{
$attribute['distribution'] = $this->distribution;
$temp = $this->__checkType($attribute['search'], $attribute['type']);
if (isset($attribute['type'])) {
$type = $attribute['type'];
} else {
$type = '';
}
$temp = $this->__checkType($attribute['search'], $type);
if ($attribute['condition'] !== 'containsnot') {
if (!$temp) {
return false;

37
app/Controller/EventsController.php
View File

@ -3507,7 +3507,6 @@ class EventsController extends AppController
'org_id' => $this->Auth->user('org_id'),
'orgc_id' => $this->Auth->user('org_id'),
'timestamp' => $ts,
'uuid' => CakeText::uuid(),
'user_id' => $this->Auth->user('id'),
));
$default['Event']['info'] = 'A junk event for load testing';
@ -3517,6 +3516,7 @@ class EventsController extends AppController
$default['Event']['distribution'] = '0';
for ($i = 0; $i < 50; $i++) {
$data = $default;
$data['Event']['uuid'] = CakeText::uuid();
for ($j = 0; $j < 3000; $j++) {
$value = mt_rand();
$data['Attribute'][] = array(
@ -3530,6 +3530,7 @@ class EventsController extends AppController
'comment' => '',
'uuid' => CakeText::uuid(),
'timestamp' => $ts,
'disable_correlation' => 1
);
}
$this->Event->create();
@ -5176,40 +5177,8 @@ class EventsController extends AppController
$event['Event']['orgc_name'] = $org_name['Orgc']['name'];
if ($attribute[0]['Object']['id']) {
$object_id = $attribute[0]['Object']['id'];
$initial_object = $this->Event->Object->find('first', array(
'conditions' => array('Object.id' => $object_id,
'Object.event_id' => $event_id,
'Object.deleted' => 0),
'recursive' => -1,
'fields' => array('Object.id', 'Object.uuid', 'Object.name')
));
$initial_object = $this->Event->fetchInitialObject($event_id, $object_id);
if (!empty($initial_object)) {
$initial_attributes = $this->Event->Attribute->find('all', array(
'conditions' => array('Attribute.object_id' => $object_id,
'Attribute.deleted' => 0),
'recursive' => -1,
'fields' => array('Attribute.id', 'Attribute.uuid', 'Attribute.type',
'Attribute.object_relation', 'Attribute.value')
));
if (!empty($initial_attributes)) {
$initial_object['Attribute'] = array();
foreach ($initial_attributes as $initial_attribute) {
array_push($initial_object['Attribute'], $initial_attribute['Attribute']);
}
}
$initial_references = $this->Event->Object->ObjectReference->find('all', array(
'conditions' => array('ObjectReference.object_id' => $object_id,
'ObjectReference.event_id' => $event_id,
'ObjectReference.deleted' => 0),
'recursive' => -1,
'fields' => array('ObjectReference.referenced_uuid', 'ObjectReference.relationship_type')
));
if (!empty($initial_references)) {
$initial_object['ObjectReference'] = array();
foreach ($initial_references as $initial_reference) {
array_push($initial_object['ObjectReference'], $initial_reference['ObjectReference']);
}
}
$event['initialObject'] = $initial_object;
}
}

2596
app/Locale/default.pot
View File

File diff suppressed because it is too large Load Diff

16659
app/Locale/no/LC_MESSAGES/default.po Normal file
View File

File diff suppressed because it is too large Load Diff

113
app/Model/Event.php
View File

@ -5179,6 +5179,46 @@ class Event extends AppModel
return $conditions;
}
public function fetchInitialObject($event_id, $object_id)
{
$initial_object = $this->Object->find('first', array(
'conditions' => array('Object.id' => $object_id,
'Object.event_id' => $event_id,
'Object.deleted' => 0),
'recursive' => -1,
'fields' => array('Object.id', 'Object.uuid', 'Object.name')
));
if (!empty($initial_object)) {
$initial_attributes = $this->Attribute->find('all', array(
'conditions' => array('Attribute.object_id' => $object_id,
'Attribute.deleted' => 0),
'recursive' => -1,
'fields' => array('Attribute.id', 'Attribute.uuid', 'Attribute.type',
'Attribute.object_relation', 'Attribute.value')
));
if (!empty($initial_attributes)) {
$initial_object['Attribute'] = array();
foreach ($initial_attributes as $initial_attribute) {
array_push($initial_object['Attribute'], $initial_attribute['Attribute']);
}
}
$initial_references = $this->Object->ObjectReference->find('all', array(
'conditions' => array('ObjectReference.object_id' => $object_id,
'ObjectReference.event_id' => $event_id,
'ObjectReference.deleted' => 0),
'recursive' => -1,
'fields' => array('ObjectReference.referenced_uuid', 'ObjectReference.relationship_type')
));
if (!empty($initial_references)) {
$initial_object['ObjectReference'] = array();
foreach ($initial_references as $initial_reference) {
array_push($initial_object['ObjectReference'], $initial_reference['ObjectReference']);
}
}
}
return $initial_object;
}
public function handleModuleResult($result, $event_id)
{
$resultArray = array();
@ -5806,38 +5846,57 @@ class Event extends AppModel
throw new MethodNotAllowedException('Invalid event.');
}
$attributes_added = 0;
$initial_objects = array();
$event_id = $event[0]['Event']['id'];
foreach ($event[0]['Attribute'] as $attribute) {
$object_id = $attribute['object_id'];
foreach ($enabledModules['modules'] as $module) {
if (in_array($module['name'], $params['modules'])) {
if (in_array($attribute['type'], $module['mispattributes']['input'])) {
$data = array('module' => $module['name'], $attribute['type'] => $attribute['value'], 'event_id' => $attribute['event_id'], 'attribute_uuid' => $attribute['uuid']);
$data = array('module' => $module['name'], 'event_id' => $event_id, 'attribute_uuid' => $attribute['uuid']);
if (!empty($module['config'])) {
$data['config'] = $module['config'];
}
if (!empty($module['mispattributes']['format']) && $module['mispattributes']['format'] == 'misp_standard') {
$data['attribute'] = $attribute;
if ($object_id != '0' && empty($initial_objects[$object_id])) {
$initial_objects[$object_id] = $this->fetchInitialObject($event_id, $object_id);
}
} else {
$data[$attribute['type']] = $attribute['value'];
}
$data = json_encode($data);
$result = $this->Module->queryModuleServer('/query', $data, false, 'Enrichment');
if (!$result) {
throw new MethodNotAllowedException($type . ' service not reachable.');
throw new MethodNotAllowedException(h($module['name']) . ' service not reachable.');
}
//if (isset($result['error'])) $this->Session->setFlash($result['error']);
if (!is_array($result)) {
throw new Exception($result);
}
$attributes = $this->handleModuleResult($result, $attribute['event_id']);
foreach ($attributes as $a) {
$this->Attribute->create();
$a['distribution'] = $attribute['distribution'];
$a['sharing_group_id'] = $attribute['sharing_group_id'];
$comment = 'Attribute #' . $attribute['id'] . ' enriched by ' . $module['name'] . '.';
if (!empty($a['comment'])) {
$a['comment'] .= PHP_EOL . $comment;
} else {
$a['comment'] = $comment;
if (!empty($module['mispattributes']['format']) && $module['mispattributes']['format'] == 'misp_standard') {
if ($object_id != '0' && !empty($initial_objects[$object_id])) {
$result['initialObject'] = $initial_objects[$object_id];
}
$a['type'] = empty($a['default_type']) ? $a['types'][0] : $a['default_type'];
$result = $this->Attribute->save($a);
if ($result) {
$attributes_added++;
$default_comment = $attribute['value'] . ': enriched via the ' . $module['name'] . ' module.';
$attributes_added += $this->processModuleResultsData($params['user'], $result['results'], $event_id, $default_comment, false, false, true);
} else {
$attributes = $this->handleModuleResult($result, $event_id);
foreach ($attributes as $a) {
$this->Attribute->create();
$a['distribution'] = $attribute['distribution'];
$a['sharing_group_id'] = $attribute['sharing_group_id'];
$comment = 'Attribute #' . $attribute['id'] . ' enriched by ' . $module['name'] . '.';
if (!empty($a['comment'])) {
$a['comment'] .= PHP_EOL . $comment;
} else {
$a['comment'] = $comment;
}
$a['type'] = empty($a['default_type']) ? $a['types'][0] : $a['default_type'];
$result = $this->Attribute->save($a);
if ($result) {
$attributes_added++;
}
}
}
}
@ -6064,11 +6123,12 @@ class Event extends AppModel
return $message;
}
public function processModuleResultsData($user, $resolved_data, $id, $default_comment = '', $jobId = false, $adhereToWarninglists = false)
public function processModuleResultsData($user, $resolved_data, $id, $default_comment = '', $jobId = false, $adhereToWarninglists = false, $event_level = false)
{
if ($jobId) {
$this->Job = ClassRegistry::init('Job');
$this->Job->id = $jobId;
}
$failed_attributes = $failed_objects = $failed_object_attributes = 0;
$saved_attributes = $saved_objects = $saved_object_attributes = 0;
@ -6135,8 +6195,10 @@ class Event extends AppModel
$total_objects = count($resolved_data['Object']);
$references = array();
foreach ($resolved_data['Object'] as $o => $object) {
$object['meta-category'] = $object['meta_category'];
unset($object['meta_category']);
if (isset($object['meta_category']) && !isset($object['meta-category'])) {
$object['meta-category'] = $object['meta_category'];
unset($object['meta_category']);
}
$object['event_id'] = $id;
if (isset($object['id']) && $object['id'] == $initial_object_id) {
$initial_object = $resolved_data['initialObject'];
@ -6165,8 +6227,8 @@ class Event extends AppModel
if ($this->__saveObjectAttribute($object_attribute, $default_comment, $id, $initial_object_id, $user)) {
$saved_object_attributes++;
} else {
$failed_object_attributes++;
$lastObjectAttributeError = $this->Attribute->validationErrors;
$failed_object_attributes++;
$lastObjectAttributeError = $this->Attribute->validationErrors;
}
}
}
@ -6262,8 +6324,12 @@ class Event extends AppModel
}
list($referenced_id, $referenced_uuid, $referenced_type) = $this->Object->ObjectReference->getReferencedInfo(
$reference['referenced_uuid'],
array('Event' => array('id' => $id))
array('Event' => array('id' => $id)),
false
);
if (!$referenced_id && !$referenced_uuid && !$referenced_type) {
continue;
}
$reference = array(
'event_id' => $id,
'referenced_id' => $referenced_id,
@ -6291,6 +6357,9 @@ class Event extends AppModel
$event['Event']['timestamp'] = $date->getTimestamp();
$this->save($event);
}
if ($event_level) {
return $saved_attributes + $saved_object_attributes;
}
$message = '';
if ($saved_attributes > 0) {
$message .= $saved_attributes . ' ' . $this->__apply_inflector($saved_attributes, 'attribute') . ' created. ';

8
app/Model/ObjectReference.php
View File

@ -263,7 +263,7 @@ class ObjectReference extends AppModel
return true;
}
public function getReferencedInfo($referencedUuid, $object)
public function getReferencedInfo($referencedUuid, $object, $strict = true)
{
$referenced_type = 1;
$target_object = $this->Object->find('first', array(
@ -284,7 +284,11 @@ class ObjectReference extends AppModel
'fields' => array('Attribute.id', 'Attribute.uuid', 'Attribute.event_id')
));
if (empty($target_attribute)) {
throw new NotFoundException('Invalid target.');
if ($strict) {
throw new NotFoundException('Invalid target.');
} else {
return array(0, 0, 0);
}
}
if ($target_attribute['Attribute']['event_id'] != $object['Event']['id']) {
throw new NotFoundException('Invalid target. Target has to be within the same event.');

2
app/files/misp-galaxy

@ -1 +1 @@
Subproject commit 9517c8b8782a9385874940eec6741a78597cad12
Subproject commit 483f048f7fac1045e54a8d7bfc85825a2ff9d58d

2
app/files/misp-objects

@ -1 +1 @@
Subproject commit e7bb12af7daa0d771dd558bf7707610d7e9b46ed
Subproject commit d502c254cc8d763b20290c2fdbf7b8fd32ae8f2b

226
app/files/scripts/stix2/stix2misp.py
View File

@ -229,20 +229,30 @@ class StixParser():
continue
return attributes
def attributes_from_ip_port_observable(self, objects):
ordered_objects = defaultdict(dict)
for key, value in objects.items():
if isinstance(value, (stix2.IPv4Address, stix2.IPv6Address, stix2.NetworkTraffic)):
ordered_objects[value._type.split('-')[1]][key] = value
else:
attributes = self.fill_observable_attributes(value, network_traffic_mapping)
for traffic in ordered_objects['traffic'].values():
if hasattr(traffic, 'dst_ref'):
mapping = ip_attribute_mapping
attributes.append({'type': mapping['type'], 'object_relation': mapping['relation'],
'to_ids': False, 'value': ordered_objects['addr'][traffic.dst_ref].value})
attributes.extend(self.fill_observable_attributes(traffic, network_traffic_mapping))
return attributes
def attributes_from_network_traffic(self, objects, name=None):
network_traffic, references = self.fetch_network_traffic_objects_and_references(objects)
attributes = self.fill_observable_attributes(network_traffic, network_traffic_mapping)
if name is not None:
mapping = network_socket_types
for protocol in network_traffic.protocols:
try:
layer = connection_protocols[protocol]
attributes.append({'type': 'text', 'value': protocol, 'to_ids': False,
'object_relation': 'layer{}-protocol'.format(layer)})
except KeyError:
continue
elif hasattr(network_traffic, 'extensions') and network_traffic.extensions:
extension_type, extension_value = list(network_traffic.extensions.items())[0]
name = network_traffic_extensions[extension_type]
attributes.extend(self.parse_socket_extension(extension_value))
mapping = network_traffic_references_mapping['with_extensions']
else:
name = 'ip-port'
mapping = network_traffic_references_mapping['without_extensions']
attributes.extend(self.parse_network_traffic_references(references, network_traffic, mapping))
if references:
attributes.extend(self.parse_remaining_references(references, mapping))
return attributes, name
def attributes_from_process_observable(self, objects):
main_process = None
@ -306,16 +316,9 @@ class StixParser():
file = value
return file, data
@staticmethod
def fetch_network_traffic_objects(objects):
network_traffics = []
for value in objects.values():
if isinstance(value, stix2.NetworkTraffic):
return value
@staticmethod
def fetch_network_traffic_objects_and_references(objects):
references = defaultdict(dict)
references = {}
for key, value in objects.items():
if isinstance(value, (stix2.DomainName, stix2.IPv4Address, stix2.IPv6Address)):
references[key] = value
@ -376,15 +379,23 @@ class StixParser():
self.misp_event.add_object(**misp_object)
@staticmethod
def parse_network_traffic_references(objects, network_traffic, mapping, attr='get'):
def __parse_network_traffic_reference(ref_object, ref, mapping):
origin = ref.split('_')[0]
misp_type, relation = mapping[ref_object._type]
return {'type': misp_type.format(origin), 'object_relation': relation.format(origin),
'to_ids': False, 'value': ref_object.value}
def parse_network_traffic_references(self, objects, network_traffic, mapping):
attributes= []
for ref in ('src_ref', 'dst_ref'):
if hasattr(network_traffic, ref):
ref_object = getattr(objects, attr)(getattr(network_traffic, ref))
origin = ref.split('_')[0]
misp_type, relation = mapping[ref_object._type]
attributes.append({'type': misp_type.format(origin), 'object_relation': relation.format(origin),
'to_ids': False, 'value': ref_object.value})
ref_object = objects.pop(getattr(network_traffic, ref))
attributes.append(self.__parse_network_traffic_reference(ref_object, ref, mapping))
for refs in ('src_refs', 'dst_refs'):
if hasattr(network_traffic, refs):
for ref in getattr(network_traffic, refs):
ref_object = objects.pop(ref)
attributes.append(self.__parse_network_traffic_reference(ref_object, refs, mapping))
return attributes
def parse_pe(self, extension):
@ -416,6 +427,22 @@ class StixParser():
'to_ids': False, 'value': reference.value})
return attributes
@staticmethod
def parse_socket_extension(extension):
attributes = []
for element in extension:
try:
mapping = network_traffic_mapping[element]
except KeyError:
continue
if element in ('is_listening', 'is_blocking'):
attribute_value = element.split('_')[1]
else:
attribute_value = extension[element]
attributes.append({'type': mapping['type'], 'object_relation': mapping['relation'],
'value': attribute_value})
return attributes
@staticmethod
def pyMISPify(attribute_dict):
attribute = MISPAttribute()
@ -441,10 +468,12 @@ class StixFromMISPParser(StixParser):
def __init__(self):
super(StixFromMISPParser, self).__init__()
self.objects_mapping = {'asn': {'observable': self.attributes_from_asn_observable, 'pattern': self.pattern_asn},
'credential': {'observable': self.observable_credential, 'pattern': self.pattern_credential},
'domain-ip': {'observable': self.attributes_from_domain_ip_observable, 'pattern': self.pattern_domain_ip},
'email': {'observable': self.observable_email, 'pattern': self.pattern_email},
'file': {'observable': self.observable_file, 'pattern': self.pattern_file},
'ip-port': {'observable': self.attributes_from_ip_port_observable, 'pattern': self.pattern_ip_port},
'ip-port': {'observable': self.observable_ip_port, 'pattern': self.pattern_ip_port},
'network-connection': {'observable': self.observable_connection, 'pattern': self.pattern_connection},
'network-socket': {'observable': self.observable_socket, 'pattern': self.pattern_socket},
'process': {'observable': self.attributes_from_process_observable, 'pattern': self.pattern_process},
'registry-key': {'observable': self.attributes_from_regkey_observable, 'pattern': self.pattern_regkey},
@ -592,6 +621,47 @@ class StixFromMISPParser(StixParser):
else:
self.misp_event['Galaxy'].append(self.parse_galaxy(o, labels))
def observable_connection(self, observable):
attributes, _ = self.attributes_from_network_traffic(observable, 'network-connection')
return attributes
def pattern_connection(self, pattern):
attributes = []
for p in pattern:
p_type, p_value = p.split(' = ')
p_value = p_value[1:-1]
try:
mapping = network_traffic_mapping[p_type]
attributes.append({'type': mapping['type'], 'object_relation': mapping['relation'],
'value': p_value})
except KeyError:
if not p_type.startswith('network-traffic:protocols['):
continue
attributes.append({'type': 'text', 'value': p_value,
'object_relation': 'layer{}-protocol'.format(connection_protocols[p_value])})
return attributes
def observable_credential(self, observable):
return self.fill_observable_attributes(observable['0'], credential_mapping)
def pattern_credential(self, pattern):
attributes = []
for p in pattern:
p_type, p_value = p.split(' = ')
p_type = p_type.split(':')[1]
p_value = p_value[1:-1]
try:
mapping = credential_mapping[p_type]
attributes.append({'type': mapping['type'], 'object_relation': mapping['relation'],
'value': p_value})
except KeyError:
if not p_type.startswith('x_misp_'):
continue
attribute_type, relation = p_type.strip('x_misp_').split('_')
attributes.append({'type': attribute_type, 'object_relation': relation,
'value': p_value})
return attributes
def observable_email(self, observable):
to_ids = False
attributes, message = self.parse_complex_fields_observable_email(observable, to_ids)
@ -751,6 +821,10 @@ class StixFromMISPParser(StixParser):
def pattern_domain_ip(self, pattern):
return self.fill_pattern_attributes(pattern, domain_ip_mapping)
def observable_ip_port(self, observable):
attributes, _ = self.attributes_from_network_traffic(observable)
return attributes
def pattern_ip_port(self, pattern):
return self.fill_pattern_attributes(pattern, network_traffic_mapping)
@ -786,52 +860,7 @@ class StixFromMISPParser(StixParser):
return attributes
def observable_socket(self, observable):
observable_object = dict(observable['0']) if len(observable) == 1 else self.parse_socket_observable(observable)
try:
extension = observable_object.pop('extensions')
attributes = self.parse_socket_extension(extension['socket-ext'])
except KeyError:
attributes = []
for o_key, o_value in observable_object.items():
if o_key in ('src_ref', 'dst_ref'):
element_object = observable[o_value]
if 'domain-name' in element_object['type']:
attribute_type = 'hostname'
relation = 'hostname-{}'.format(o_key.split('_')[0])
else:
attribute_type = relation = "ip-{}".format(o_key.split('_')[0])
attributes.append({'type': attribute_type, 'object_relation': relation,
'value': element_object['value']})
continue
try:
mapping = network_traffic_mapping[o_key]
except KeyError:
continue
attributes.append({'type': mapping['type'], 'object_relation': mapping['relation'],
'value': o_value})
return attributes
@staticmethod
def parse_socket_observable(observable):
for key in observable:
observable_object = observable[key]
if observable_object['type'] == 'network-traffic':
return dict(observable_object)
@staticmethod
def parse_socket_extension(extension):
attributes = []
for element in extension:
try:
mapping = network_traffic_mapping[element]
except KeyError:
continue
if element in ('is_listening', 'is_blocking'):
attribute_value = element.split('_')[1]
else:
attribute_value = extension[element]
attributes.append({'type': mapping['type'], 'object_relation': mapping['relation'],
'value': attribute_value})
attributes, _ = self.attributes_from_network_traffic(observable)
return attributes
@staticmethod
@ -839,6 +868,7 @@ class StixFromMISPParser(StixParser):
attributes = []
for p in pattern:
p_type, p_value = p.split(' = ')
p_value = p_value[1:-1]
try:
mapping = network_traffic_mapping[p_type]
except KeyError:
@ -964,7 +994,7 @@ class ExternalStixParser(StixParser):
('domain-name', 'ipv6-addr'): self.parse_domain_ip_observable,
('domain-name', 'ipv4-addr', 'network-traffic'): self.parse_ip_port_or_network_socket_observable,
('domain-name', 'ipv6-addr', 'network-traffic'): self.parse_ip_port_or_network_socket_observable,
('domain-name', 'ipv4-addr', 'ipv6-addr', 'network-traffic'): self.parse_ip_port_observable,
('domain-name', 'ipv4-addr', 'ipv6-addr', 'network-traffic'): self.parse_ip_port_or_network_socket_observable,
('domain-name', 'network-traffic'): self.parse_network_socket_observable,
('domain-name', 'network-traffic', 'url'): self.parse_url_object_observable,
('email-addr', 'email-message'): self.parse_email_observable,
@ -1213,37 +1243,11 @@ class ExternalStixParser(StixParser):
self.add_attributes_from_pattern('ip-dst', pattern, marking, uuid)
def parse_ip_network_traffic_observable(self, objects, marking, uuid):
network_traffic = self.fetch_network_traffic_objects(objects)
attributes = self.fill_observable_attributes(network_traffic, network_traffic_mapping)
if hasattr(network_traffic, 'extensions') and network_traffic.extensions:
extension_type, extension_value = list(network_traffic.extensions.items())[0]
name = network_traffic_extensions[extension_type]
attributes.extend(self.fill_observable_attributes(extension_value, network_traffic_mapping))
mapping = network_traffic_references_mapping['with_extensions']
else:
name = 'ip-port'
mapping = network_traffic_references_mapping['without_extensions']
attributes.extend(self.parse_network_traffic_references(objects, network_traffic, mapping))
attributes, name = self.attributes_from_network_traffic(objects)
self.handle_import_case(attributes, name, marking, uuid)
def parse_ip_port_observable(self, objects, marking, uuid):
attributes = self.attributes_from_ip_port_observable(objects)
self.handle_import_case(attributes, 'ip-port', marking, uuid)
def parse_ip_port_or_network_socket_observable(self, objects, marking, uuid):
network_traffic, references = self.fetch_network_traffic_objects_and_references(objects)
attributes = self.fill_observable_attributes(network_traffic, network_traffic_mapping)
if hasattr(network_traffic, 'extensions') and network_traffic.extensions:
extension_type, extension_value = list(network_traffic.extensions.items())[0]
name = network_traffic_extensions[extension_type]
attributes.extend(self.fill_observable_attributes(extension_value, network_traffic_mapping))
mapping = network_traffic_references_mapping['with_extensions']
else:
name = 'ip-port'
mapping = network_traffic_references_mapping['without_extensions']
attributes.extend(self.parse_network_traffic_references(references, network_traffic, mapping, attr='pop'))
if references:
attributes.extend(self.parse_remaining_references(references, mapping))
attributes, name = self.attributes_from_network_traffic(objects)
self.handle_import_case(attributes, name, marking, uuid)
def parse_mac_address_observable(self, objects, marking, uuid):
@ -1253,19 +1257,7 @@ class ExternalStixParser(StixParser):
self.add_attributes_from_observable(objects, 'mutex', 'name', marking, uuid)
def parse_network_socket_observable(self, objects, marking, uuid):
network_traffic, references = self.fetch_network_traffic_objects_and_references(objects)
attributes = self.fill_observable_attributes(network_traffic, network_traffic_mapping)
if hasattr(network_traffic, 'extensions') and network_traffic.extensions:
extension_type, extension_value = list(network_traffic.extensions.items())[0]
name = network_traffic_extensions[extension_type]
attributes.extend(self.fill_observable_attributes(extension_value, network_traffic_mapping))
mapping = network_traffic_references_mapping['with_extensions']
else:
name = 'ip-port'
mapping = network_traffic_references_mapping['without_extensions']
attributes.extend(self.parse_network_traffic_references(objects, network_traffic, mapping, attr='pop'))
if references:
attributes.extend(self.parse_remaining_references(references, mapping))
attributes, name = self.attributes_from_network_traffic(objects)
self.handle_import_case(attributes, name, marking, uuid)
def parse_network_traffic_pattern(self, pattern, marking=None, uuid=None):

13
app/files/scripts/stix2/stix2misp_mapping.py
View File

@ -180,6 +180,9 @@ asn_mapping = {'number': as_number_attribute_mapping,
'ipv4-addr:value': asn_subnet_attribute_mapping,
'ipv6-addr:value': asn_subnet_attribute_mapping}
credential_mapping = {'credential': {'type': 'text', 'relation': 'password'},
'user_id': {'type': 'text', 'relation': 'username'}}
domain_ip_mapping = {'domain-name': domain_attribute_mapping,
'domain-name:value': domain_attribute_mapping,
'ipv4-addr': ip_attribute_mapping,
@ -220,7 +223,8 @@ network_traffic_mapping = {'src_port': src_port_attribute_mapping,
'network-traffic:end': end_datetime_attribute_mapping,
'value': domain_attribute_mapping,
'domain-name:value': domain_attribute_mapping,
'network-traffic:dst_ref.value': ip_attribute_mapping,
'network-traffic:dst_ref.value': {'type': 'ip-dst', 'relation': 'ip-dst'},
'network-traffic:src_red.value': {'type': 'ip-src', 'relation': 'ip-src'},
'address_family': address_family_attribute_mapping,
"network-traffic:extensions.'socket-ext'.address_family": address_family_attribute_mapping,
'protocol_family': domain_family_attribute_mapping,
@ -233,8 +237,7 @@ network_traffic_mapping = {'src_port': src_port_attribute_mapping,
network_traffic_extensions = {'socket-ext': 'network-socket'}
network_traffic_ip = ('ip-{}', 'ip-{}')
ip_port_ip = ('ip-dst', 'ip')
ip_port_types = {'domain-name': ('domain', 'domain'), 'ipv4-addr': ip_port_ip, 'ipv6-addr': ip_port_ip}
ip_port_types = {'domain-name': ('domain', 'domain'), 'ipv4-addr': network_traffic_ip, 'ipv6-addr': network_traffic_ip}
network_socket_types = {'domain-name': ('hostname', 'hostname-{}'), 'ipv4-addr': network_traffic_ip, 'ipv6-addr': network_traffic_ip}
network_traffic_references_mapping = {'with_extensions': network_socket_types, 'without_extensions': ip_port_types}
@ -311,3 +314,7 @@ external_pattern_mapping = {'domain-name': domain_pattern_mapping,
'url': {'value':{'type': 'url'}},
'x509-certificate': x509_mapping
}
connection_protocols = {"IP": "3", "ICMP": "3", "ARP": "3",
"TCP": "4", "UDP": "4",
"HTTP": "7", "HTTPS": "7", "FTP": "7"}

2
app/files/scripts/stix2misp.py
View File

@ -29,7 +29,7 @@ from stix.core import STIXPackage
from collections import defaultdict
try:
import stix_edh
except (ModuleNotFoundError, ImportError):
except ImportError:
pass
_MISP_dir = "/".join([p for p in os.path.dirname(os.path.realpath(__file__)).split('/')[:-3]])

4
app/webroot/js/event-graph.js
View File

@ -1383,8 +1383,8 @@ class DataHandler {
}
// search if this field exists in the object
for (var attr of obj.Attribute) { // for each field
var attr_type = attr.type;
if (template_req.indexOf(attr_type) != -1) {
var attr_rel = attr.object_relation;
if (template_req.indexOf(attr_rel) != -1) {
label += ": " + attr.value;
return label;
}

16
docs/generic/supportFunctions.md
View File

@ -106,7 +106,7 @@ checkCoreOS () {
# Extract debian flavour
checkFlavour () {
lsb_dist=""
FLAVOUR=""
# Every system that we officially support has /etc/os-release
if [ -r /etc/os-release ]; then
FLAVOUR="$(. /etc/os-release && echo "$ID"| tr '[:upper:]' '[:lower:]')"
@ -136,14 +136,14 @@ checkFlavour () {
if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then
dist_version="$(. /etc/os-release && echo "$VERSION_ID")"
fi
echo "$lsb_dist not supported at the moment"
echo "$FLAVOUR not supported at the moment"
exit 1
;;
rhel|ol|sles)
if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then
dist_version="$(. /etc/os-release && echo "$VERSION_ID")"
fi
echo "$lsb_dist not supported at the moment"
echo "$FLAVOUR not supported at the moment"
exit 1
;;
*)
@ -180,20 +180,20 @@ check_forked () {
if [ "$lsb_release_exit_code" = "0" ]; then
# Print info about current distro
cat <<-EOF
You're using '$lsb_dist' version '$dist_version'.
You're using '$FLAVOUR' version '$dist_version'.
EOF
# Get the upstream release info
lsb_dist=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'id' | cut -d ':' -f 2 | tr -d '[:space:]')
FLAVOUR=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'id' | cut -d ':' -f 2 | tr -d '[:space:]')
dist_version=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'codename' | cut -d ':' -f 2 | tr -d '[:space:]')
# Print info about upstream distro
cat <<-EOF
Upstream release is '$lsb_dist' version '$dist_version'.
Upstream release is '$FLAVOUR' version '$dist_version'.
EOF
else
if [ -r /etc/debian_version ] && [ "$lsb_dist" != "ubuntu" ] && [ "$lsb_dist" != "raspbian" ]; then
if [ -r /etc/debian_version ] && [ "$FLAVOUR" != "ubuntu" ] && [ "$FLAVOUR" != "raspbian" ]; then
# We're Debian and don't even know it!
lsb_dist=debian
FLAVOUR=debian
dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')"
case "$dist_version" in
10)