MISP
/
PyMISP
mirror of https://github.com/MISP/PyMISP
2
0
Fork 0
Code Issues Releases Wiki Activity

correction for https://travis-ci.org/MISP/misp-modules/jobs/231065469 

code refactoring

add new composite
pull/84/head
Tristan METAYER 2017-05-15 20:25:16 +02:00
parent 143de76bc3
commit d72d3dceb7
1 changed files with 73 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

179
pymisp/tools/openioc.py
View File

@ -129,8 +129,10 @@ iocMispMapping = {
'UrlHistoryItem/URL' : {'type': 'url','comment': 'UrlHistory. '}, 'UrlHistoryItem/URL' : {'type': 'url','comment': 'UrlHistory. '},
'UrlHistoryItem/HostName': {'type': 'hostname','comment': 'UrlHistory. '}, 'UrlHistoryItem/HostName': {'type': 'hostname','comment': 'UrlHistory. '},
'Yara/Yara' : {'type': 'yara'}, 'Yara/Yara' : {'type': 'yara'}
}
iocMispCompositeMapping = {
# mapping for composite object # mapping for composite object
# maybe later filename|sizeinbyte # maybe later filename|sizeinbyte
'FileItem/FileName|FileItem/Md5sum' : {'type': 'filename|md5'}, 'FileItem/FileName|FileItem/Md5sum' : {'type': 'filename|md5'},
@ -140,13 +142,14 @@ iocMispMapping = {
'PortItem/remoteIP|PortItem/remotePort' : {'comment': 'ip-dst|port'}, 'PortItem/remoteIP|PortItem/remotePort' : {'comment': 'ip-dst|port'},
'RegistryItem/Path|RegistryItem/Value' : {'type': 'regkey|value'}, 'RegistryItem/Path|RegistryItem/Value' : {'type': 'regkey|value'},
'RegistryItem/KeyPath|RegistryItem/Value' : {'type': 'regkey|value'}, 'RegistryItem/KeyPath|RegistryItem/Value' : {'type': 'regkey|value'},
'RegistryItem/Path|RegistryItem/Text' : {'type': 'regkey|value'}
} }
def extract_field(report, field_name): def extract_field(report, field_name):
data = report.find(field_name.lower()) if report:
if data and hasattr(data, 'text'): data = report.find(field_name.lower())
return data.text if data and hasattr(data, 'text'):
return data.text
return None return None
@ -186,44 +189,61 @@ def load_openioc(openioc):
return misp_event return misp_event
def get_mapping(openioc_type): def get_mapping(openioc_type, mappingDict=iocMispMapping):
t = openioc_type.lower() t = openioc_type.lower()
for k, v in iocMispMapping.items(): for k, v in mappingDict.items():
if k.lower() == t: if k.lower() == t:
return v return v
return False return False
def set_composite_values(value1, value2):
attribute_values = {'comment': ''} def set_values(value1, value2=None):
attribute_values = {}
# construct attribut composite type
compositeMapping = value1.find('context')['search']+'|'+value2.find('context')['search'] if value2 is not None:
mapping = get_mapping(compositeMapping) # construct attribut composite value
value = "{}|{}".format(
extract_field(value1, 'Content'),
extract_field(value2, 'Content')
)
else:
value = extract_field(value1, 'Content')
if value:
attribute_values['value'] = value
else:
return None
if value2 is not None:
# construct attribut composite type
compositeMapping = value1.find('context')['search']+'|'+value2.find('context')['search']
mapping = get_mapping(compositeMapping, mappingDict=iocMispCompositeMapping)
else:
mapping = get_mapping(value1.find('context')['search'])
if mapping: if mapping:
attribute_values.update(mapping) attribute_values.update(mapping)
else: else:
# prevent some mistake error # Unknown mapping, assign to default
attribute_values.update({'category': 'External analysis', 'type': 'other'}) attribute_values['category'] = 'External analysis'
attribute_values['type'] = 'other'
# construct attribut composite value # change value to composite
compositeValue = value1.find('content').text + "|" +value2.find('content').text # 127.0.0.1:80 ip-* to 127.0.0.1|80 ip-*|port
if compositeValue: if attribute_values['type'] in ['ip-src', 'ip-dst'] and attribute_values['value'].count(':') == 1:
attribute_values['value'] = compositeValue attribute_values['type'] = attribute_values['type'] + '|port'
attribute_values['value'] = attribute_values['value'].replace(':', '|')
attribute_values["comment"] = '{}{}'.format(
extract_field(value1, 'Comment'),
extract_field(value2, 'Comment')
)
# construct composite comment
compositeComment = ""
if value1.find('comment'):
compositeComment += value1.find('comment').text
if value2.find('comment'):
compositeComment += value2.find('comment').text
attribute_values["comment"] = compositeComment
return attribute_values return attribute_values
def set_all_attributes(openioc, misp_event): def set_all_attributes(openioc, misp_event):
processed = set() processed = set()
hashName = ["FileItem/Md5sum","FileItem/Sha1sum","FileItem/Sha256sum"]
# check for composite item # check for composite item
for composite in openioc.find_all("indicator", operator="AND"): for composite in openioc.find_all("indicator", operator="AND"):
@ -233,95 +253,42 @@ def set_all_attributes(openioc, misp_event):
if len(childs) == 2: if len(childs) == 2:
childList = [child.find('context')['search'] for child in childs] childList = [child.find('context')['search'] for child in childs]
if ('FileItem/FileName' in childList) and\ def check_and_add(value1, value2):
(set(hashName) - set(childList) != set(hashName)): if (value1 and value2) in childList:
if childs[0].find('context')['search'] == 'FileItem/FileName': if childs[0].find('context')['search'] == value1:
value1, value2 = childs[0], childs[1] attribute_values = set_values(childs[0], childs[1])
else: else:
value1, value2 = childs[1], childs[0] attribute_values = set_values(childs[1], childs[0])
attribute_values = set_composite_values(value1, value2) misp_event.add_attribute(**attribute_values)
misp_event.add_attribute(**attribute_values) processed.add(childs[0]['id'])
processed.add(childs[0]['id']) processed.add(childs[1]['id'])
processed.add(childs[1]['id'])
for k in iocMispCompositeMapping:
elif ("Network/DNS" and "PortItem/RemoteIP") in childList: check_and_add(k.split('|')[0], k.split('|')[1])
if childs[0].find('context')['search'] == 'Network/DNS':
value1, value2 = childs[0], childs[1]
else:
value1, value2 = childs[1], childs[0]
attribute_values = set_composite_values(value1, value2)
misp_event.add_attribute(**attribute_values)
processed.add(childs[0]['id'])
processed.add(childs[1]['id'])
elif ("PortItem/RemoteIP" and "PortItem/RemotePort") in childList:
if childs[0].find('context')['search'] == 'PortItem/RemoteIP':
value1, value2 = childs[0], childs[1]
else:
value1, value2 = childs[1], childs[0]
attribute_values = set_composite_values(value1, value2)
misp_event.add_attribute(**attribute_values)
processed.add(childs[0]['id'])
processed.add(childs[1]['id'])
elif ("RegistryItem/Path" and "RegistryItem/Value") in childList:
if childs[0].find('context')['search'] == 'RegistryItem/PathP':
value1, value2 = childs[0], childs[1]
else:
value1, value2 = childs[1], childs[0]
attribute_values = set_composite_values(value1, value2)
misp_event.add_attribute(**attribute_values)
processed.add(childs[0]['id'])
processed.add(childs[1]['id'])
for item in openioc.find_all("indicatoritem"): for item in openioc.find_all("indicatoritem"):
# check if id in processed list # check if id in processed list
if item['id'] in processed: if item['id'] in processed:
continue continue
attribute_values = {'comment': ''} attribute_values = set_values(item)
if item.find('context'):
mapping = get_mapping(item.find('context')['search']) if attribute_values is None:
if mapping:
attribute_values.update(mapping)
else:
# Unknown mapping, assign to default
attribute_values.update({'category': 'External analysis', 'type': 'other'})
#continue
else:
continue continue
value = extract_field(item, 'Content')
if value:
attribute_values['value'] = value
else:
# No value, ignoring
continue
comment = extract_field(item, 'Comment')
if comment:
attribute_values["comment"] = '{} {}'.format(attribute_values["comment"], comment)
# change value to composite
# 127.0.0.1:80 ip-* to 127.0.0.1|80 ip-*|port
if mapping['type'] in ['ip-src', 'ip-dst'] and value.count(':') == 1:
attribute_values['type'] = mapping['type'] + '|port'
attribute_values['value'] = attribute_values['value'].replace(':', '|')
misp_event.add_attribute(**attribute_values) misp_event.add_attribute(**attribute_values)
return misp_event return misp_event
if __name__ == '__main__': if __name__ == '__main__':
import requests
# test file for composite # test file for composite
# https://github.com/fireeye/iocs/blob/master/BlogPosts/9cee306d-5441-4cd3-932d-f3119752634c.ioc url = 'https://raw.githubusercontent.com/fireeye/iocs/master/BlogPosts/9cee306d-5441-4cd3-932d-f3119752634c.ioc'
x = open('test.ioc', 'r') #~ url = 'https://raw.githubusercontent.com/MISP/misp-modules/master/tests/openioc.xml'
mispEvent = load_openioc(x.read()) x = requests.get(url)
#~ print(mispEvent._json_full()) mispEvent = load_openioc(x.text)
from pymisp import PyMISP print(mispEvent)
misp = PyMISP('http://misp.local', 'xxxxx') #~ from pymisp import PyMISP
r = misp.add_event(mispEvent) #~ misp = PyMISP('http://misp.local', 'xxxxx')
print(r) #~ r = misp.add_event(mispEvent)
#~ print(r)