MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #367 from joesecurity/master

Browse Source 
joe: (1) allow users to disable PE object import (2) set 'to_ids' to False
pull/368/head
Christian Studer 3 years ago committed by GitHub
parent 840ada484b 04685ea63e
commit 1de6890152
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 91 additions and 46 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 87
      misp_modules/lib/joe_parser.py
  2. 28
      misp_modules/modules/expansion/joesandbox_query.py
  3. 22
      misp_modules/modules/import_mod/joe_import.py

87
misp_modules/lib/joe_parser.py
Unescape View File

@ -51,12 +51,15 @@ signerinfo_object_mapping = {'sigissuer': ('text', 'issuer'),
class JoeParser():
def __init__(self):
def __init__(self, config):
self.misp_event = MISPEvent()
self.references = defaultdict(list)
self.attributes = defaultdict(lambda: defaultdict(set))
self.process_references = {}
self.import_pe = config["import_pe"]
self.create_mitre_attack = config["mitre_attack"]
def parse_data(self, data):
self.data = data
if self.analysis_type() == "file":
@ -72,7 +75,9 @@ class JoeParser():
if self.attributes:
self.handle_attributes()
self.parse_mitre_attack()
if self.create_mitre_attack:
self.parse_mitre_attack()
def build_references(self):
for misp_object in self.misp_event.objects:
@ -97,12 +102,12 @@ class JoeParser():
file_object = MISPObject('file')
for key, mapping in dropped_file_mapping.items():
attribute_type, object_relation = mapping
file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': droppedfile[key]})
file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': droppedfile[key], 'to_ids': False})
if droppedfile['@malicious'] == 'true':
file_object.add_attribute('state', **{'type': 'text', 'value': 'Malicious'})
file_object.add_attribute('state', **{'type': 'text', 'value': 'Malicious', 'to_ids': False})
for h in droppedfile['value']:
hash_type = dropped_hash_mapping[h['@algo']]
file_object.add_attribute(hash_type, **{'type': hash_type, 'value': h['$']})
file_object.add_attribute(hash_type, **{'type': hash_type, 'value': h['$'], 'to_ids': False})
self.misp_event.add_object(**file_object)
self.references[self.process_references[(int(droppedfile['@targetid']), droppedfile['@process'])]].append({
'referenced_uuid': file_object.uuid,
@ -132,9 +137,12 @@ class JoeParser():
for object_relation, attribute in attributes.items():
network_connection_object.add_attribute(object_relation, **attribute)
network_connection_object.add_attribute('first-packet-seen',
**{'type': 'datetime', 'value': min(tuple(min(timestamp) for timestamp in data.values()))})
**{'type': 'datetime',
'value': min(tuple(min(timestamp) for timestamp in data.values())),
'to_ids': False})
for protocol in data.keys():
network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol})
network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]),
**{'type': 'text', 'value': protocol, 'to_ids': False})
self.misp_event.add_object(**network_connection_object)
self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=network_connection_object.uuid,
relationship_type='initiates'))
@ -143,8 +151,8 @@ class JoeParser():
network_connection_object = MISPObject('network-connection')
for object_relation, attribute in attributes.items():
network_connection_object.add_attribute(object_relation, **attribute)
network_connection_object.add_attribute('first-packet-seen', **{'type': 'datetime', 'value': min(timestamps)})
network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol})
network_connection_object.add_attribute('first-packet-seen', **{'type': 'datetime', 'value': min(timestamps), 'to_ids': False})
network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol, 'to_ids': False})
self.misp_event.add_object(**network_connection_object)
self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=network_connection_object.uuid,
relationship_type='initiates'))
@ -154,7 +162,8 @@ class JoeParser():
if screenshotdata:
screenshotdata = screenshotdata['interesting']['$']
attribute = {'type': 'attachment', 'value': 'screenshot.jpg',
'data': screenshotdata, 'disable_correlation': True}
'data': screenshotdata, 'disable_correlation': True,
'to_ids': False}
self.misp_event.add_attribute(**attribute)
def parse_system_behavior(self):
@ -166,9 +175,9 @@ class JoeParser():
general = process['general']
process_object = MISPObject('process')
for feature, relation in process_object_fields.items():
process_object.add_attribute(relation, **{'type': 'text', 'value': general[feature]})
process_object.add_attribute(relation, **{'type': 'text', 'value': general[feature], 'to_ids': False})
start_time = datetime.strptime('{} {}'.format(general['date'], general['time']), '%d/%m/%Y %H:%M:%S')
process_object.add_attribute('start-time', **{'type': 'datetime', 'value': start_time})
process_object.add_attribute('start-time', **{'type': 'datetime', 'value': start_time, 'to_ids': False})
self.misp_event.add_object(**process_object)
for field, to_call in process_activities.items():
if process.get(field):
@ -203,7 +212,7 @@ class JoeParser():
url_object = MISPObject("url")
self.analysisinfo_uuid = url_object.uuid
url_object.add_attribute("url", generalinfo["target"]["url"])
url_object.add_attribute("url", generalinfo["target"]["url"], to_ids=False)
self.misp_event.add_object(**url_object)
def parse_fileinfo(self):
@ -213,10 +222,10 @@ class JoeParser():
self.analysisinfo_uuid = file_object.uuid
for field in file_object_fields:
file_object.add_attribute(field, **{'type': field, 'value': fileinfo[field]})
file_object.add_attribute(field, **{'type': field, 'value': fileinfo[field], 'to_ids': False})
for field, mapping in file_object_mapping.items():
attribute_type, object_relation = mapping
file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': fileinfo[field]})
file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': fileinfo[field], 'to_ids': False})
arch = self.data['generalinfo']['arch']
if arch in arch_type_mapping:
to_call = arch_type_mapping[arch]
@ -234,9 +243,9 @@ class JoeParser():
attribute_type = 'text'
for comment, permissions in permission_lists.items():
permission_object = MISPObject('android-permission')
permission_object.add_attribute('comment', **dict(type=attribute_type, value=comment))
permission_object.add_attribute('comment', **dict(type=attribute_type, value=comment, to_ids=False))
for permission in permissions:
permission_object.add_attribute('permission', **dict(type=attribute_type, value=permission))
permission_object.add_attribute('permission', **dict(type=attribute_type, value=permission, to_ids=False))
self.misp_event.add_object(**permission_object)
self.references[file_object.uuid].append(dict(referenced_uuid=permission_object.uuid,
relationship_type='grants'))
@ -255,24 +264,24 @@ class JoeParser():
if elf.get('type'):
# Haven't seen anything but EXEC yet in the files I tested
attribute_value = "EXECUTABLE" if elf['type'] == "EXEC (Executable file)" else elf['type']
elf_object.add_attribute('type', **dict(type=attribute_type, value=attribute_value))
elf_object.add_attribute('type', **dict(type=attribute_type, value=attribute_value, to_ids=False))
for feature, relation in elf_object_mapping.items():
if elf.get(feature):
elf_object.add_attribute(relation, **dict(type=attribute_type, value=elf[feature]))
elf_object.add_attribute(relation, **dict(type=attribute_type, value=elf[feature], to_ids=False))
sections_number = len(fileinfo['sections']['section'])
elf_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number})
elf_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number, 'to_ids': False})
self.misp_event.add_object(**elf_object)
for section in fileinfo['sections']['section']:
section_object = MISPObject('elf-section')
for feature in ('name', 'type'):
if section.get(feature):
section_object.add_attribute(feature, **dict(type=attribute_type, value=section[feature]))
section_object.add_attribute(feature, **dict(type=attribute_type, value=section[feature], to_ids=False))
if section.get('size'):
section_object.add_attribute(size, **dict(type=size, value=int(section['size'], 16)))
section_object.add_attribute(size, **dict(type=size, value=int(section['size'], 16), to_ids=False))
for flag in section['flagsdesc']:
try:
attribute_value = elf_section_flags_mapping[flag]
section_object.add_attribute('flag', **dict(type=attribute_type, value=attribute_value))
section_object.add_attribute('flag', **dict(type=attribute_type, value=attribute_value, to_ids=False))
except KeyError:
print(f'Unknown elf section flag: {flag}')
continue
@ -281,6 +290,8 @@ class JoeParser():
relationship_type=relationship))
def parse_pe(self, fileinfo, file_object):
if not self.import_pe:
return
try:
peinfo = fileinfo['pe']
except KeyError:
@ -292,8 +303,8 @@ class JoeParser():
self.misp_event.add_object(**file_object)
for field, mapping in pe_object_fields.items():
attribute_type, object_relation = mapping
pe_object.add_attribute(object_relation, **{'type': attribute_type, 'value': peinfo[field]})
pe_object.add_attribute('compilation-timestamp', **{'type': 'datetime', 'value': int(peinfo['timestamp'].split()[0], 16)})
pe_object.add_attribute(object_relation, **{'type': attribute_type, 'value': peinfo[field], 'to_ids': False})
pe_object.add_attribute('compilation-timestamp', **{'type': 'datetime', 'value': int(peinfo['timestamp'].split()[0], 16), 'to_ids': False})
program_name = fileinfo['filename']
if peinfo['versions']:
for feature in peinfo['versions']['version']:
@ -301,18 +312,18 @@ class JoeParser():
if name == 'InternalName':
program_name = feature['value']
if name in pe_object_mapping:
pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value']})
pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value'], 'to_ids': False})
sections_number = len(peinfo['sections']['section'])
pe_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number})
pe_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number, 'to_ids': False})
signatureinfo = peinfo['signature']
if signatureinfo['signed']:
signerinfo_object = MISPObject('authenticode-signerinfo')
pe_object.add_reference(signerinfo_object.uuid, 'signed-by')
self.misp_event.add_object(**pe_object)
signerinfo_object.add_attribute('program-name', **{'type': 'text', 'value': program_name})
signerinfo_object.add_attribute('program-name', **{'type': 'text', 'value': program_name, 'to_ids': False})
for feature, mapping in signerinfo_object_mapping.items():
attribute_type, object_relation = mapping
signerinfo_object.add_attribute(object_relation, **{'type': attribute_type, 'value': signatureinfo[feature]})
signerinfo_object.add_attribute(object_relation, **{'type': attribute_type, 'value': signatureinfo[feature], 'to_ids': False})
self.misp_event.add_object(**signerinfo_object)
else:
self.misp_event.add_object(**pe_object)
@ -327,7 +338,7 @@ class JoeParser():
for feature, mapping in pe_section_object_mapping.items():
if section.get(feature):
attribute_type, object_relation = mapping
section_object.add_attribute(object_relation, **{'type': attribute_type, 'value': section[feature]})
section_object.add_attribute(object_relation, **{'type': attribute_type, 'value': section[feature], 'to_ids': False})
return section_object
def parse_network_interactions(self):
@ -339,13 +350,13 @@ class JoeParser():
for key, mapping in domain_object_mapping.items():
attribute_type, object_relation = mapping
domain_object.add_attribute(object_relation,
**{'type': attribute_type, 'value': domain[key]})
**{'type': attribute_type, 'value': domain[key], 'to_ids': False})
self.misp_event.add_object(**domain_object)
reference = dict(referenced_uuid=domain_object.uuid, relationship_type='contacts')
self.add_process_reference(domain['@targetid'], domain['@currentpath'], reference)
else:
attribute = MISPAttribute()
attribute.from_dict(**{'type': 'domain', 'value': domain['@name']})
attribute.from_dict(**{'type': 'domain', 'value': domain['@name'], 'to_ids': False})
self.misp_event.add_attribute(**attribute)
reference = dict(referenced_uuid=attribute.uuid, relationship_type='contacts')
self.add_process_reference(domain['@targetid'], domain['@currentpath'], reference)
@ -353,7 +364,7 @@ class JoeParser():
if ipinfo:
for ip in ipinfo['ip']:
attribute = MISPAttribute()
attribute.from_dict(**{'type': 'ip-dst', 'value': ip['@ip']})
attribute.from_dict(**{'type': 'ip-dst', 'value': ip['@ip'], 'to_ids': False})
self.misp_event.add_attribute(**attribute)
reference = dict(referenced_uuid=attribute.uuid, relationship_type='contacts')
self.add_process_reference(ip['@targetid'], ip['@currentpath'], reference)
@ -363,7 +374,7 @@ class JoeParser():
target_id = int(url['@targetid'])
current_path = url['@currentpath']
attribute = MISPAttribute()
attribute_dict = {'type': 'url', 'value': url['@name']}
attribute_dict = {'type': 'url', 'value': url['@name'], 'to_ids': False}
if target_id != -1 and current_path != 'unknown':
self.references[self.process_references[(target_id, current_path)]].append({
'referenced_uuid': attribute.uuid,
@ -384,8 +395,8 @@ class JoeParser():
registry_key = MISPObject('registry-key')
for field, mapping in regkey_object_mapping.items():
attribute_type, object_relation = mapping
registry_key.add_attribute(object_relation, **{'type': attribute_type, 'value': call[field]})
registry_key.add_attribute('data-type', **{'type': 'text', 'value': 'REG_{}'.format(call['type'].upper())})
registry_key.add_attribute(object_relation, **{'type': attribute_type, 'value': call[field], 'to_ids': False})
registry_key.add_attribute('data-type', **{'type': 'text', 'value': 'REG_{}'.format(call['type'].upper()), 'to_ids': False})
self.misp_event.add_object(**registry_key)
self.references[process_uuid].append(dict(referenced_uuid=registry_key.uuid,
relationship_type=relationship))
@ -398,7 +409,7 @@ class JoeParser():
def create_attribute(self, attribute_type, attribute_value):
attribute = MISPAttribute()
attribute.from_dict(**{'type': attribute_type, 'value': attribute_value})
attribute.from_dict(**{'type': attribute_type, 'value': attribute_value, 'to_ids': False})
self.misp_event.add_attribute(**attribute)
return attribute.uuid
@ -419,5 +430,5 @@ class JoeParser():
attributes = {}
for field, value in zip(network_behavior_fields, connection):
attribute_type, object_relation = network_connection_object_mapping[field]
attributes[object_relation] = {'type': attribute_type, 'value': value}
attributes[object_relation] = {'type': attribute_type, 'value': value, 'to_ids': False}
return attributes

28
misp_modules/modules/expansion/joesandbox_query.py
Unescape View File

@ -4,12 +4,13 @@ import json
from joe_parser import JoeParser
misperrors = {'error': 'Error'}
mispattributes = {'input': ['link'], 'format': 'misp_standard'}
moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
inputSource = ['link']
moduleinfo = {'version': '0.2', 'author': 'Christian Studer',
'description': 'Query Joe Sandbox API with a report URL to get the parsed data.',
'module-type': ['expansion']}
moduleconfig = ['apiurl', 'apikey']
moduleconfig = ['apiurl', 'apikey', 'import_pe', 'import_mitre_attack']
def handler(q=False):
@ -18,6 +19,11 @@ def handler(q=False):
request = json.loads(q)
apiurl = request['config'].get('apiurl') or 'https://jbxcloud.joesecurity.org/api'
apikey = request['config'].get('apikey')
parser_config = {
"import_pe": request["config"].get('import_pe', "false") == "true",
"mitre_attack": request["config"].get('import_mitre_attack', "false") == "true",
}
if not apikey:
return {'error': 'No API key provided'}
@ -41,7 +47,7 @@ def handler(q=False):
analysis_webid = joe_info['most_relevant_analysis']['webid']
joe_parser = JoeParser()
joe_parser = JoeParser(parser_config)
joe_data = json.loads(joe.analysis_download(analysis_webid, 'jsonfixed')[1])
joe_parser.parse_data(joe_data['analysis'])
joe_parser.finalize_results()
@ -50,7 +56,19 @@ def handler(q=False):
def introspection():
return mispattributes
modulesetup = {}
try:
userConfig
modulesetup['userConfig'] = userConfig
except NameError:
pass
try:
inputSource
modulesetup['input'] = inputSource
except NameError:
pass
modulesetup['format'] = 'misp_standard'
return modulesetup
def version():

22
misp_modules/modules/import_mod/joe_import.py
Unescape View File

@ -4,10 +4,20 @@ import json
from joe_parser import JoeParser
misperrors = {'error': 'Error'}
userConfig = {}
userConfig = {
"Import PE": {
"type": "Boolean",
"message": "Import PE Information",
},
"Mitre Att&ck" : {
"type": "Boolean",
"message": "Import Mitre Att&ck techniques",
},
}
inputSource = ['file']
moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
moduleinfo = {'version': '0.2', 'author': 'Christian Studer',
'description': 'Import for Joe Sandbox JSON reports',
'module-type': ['import']}
@ -18,10 +28,16 @@ def handler(q=False):
if q is False:
return False
q = json.loads(q)
config = {
"import_pe": bool(int(q["config"]["Import PE"])),
"mitre_attack": bool(int(q["config"]["Mitre Att&ck"])),
}
data = base64.b64decode(q.get('data')).decode('utf-8')
if not data:
return json.dumps({'success': 0})
joe_parser = JoeParser()
joe_parser = JoeParser(config)
joe_parser.parse_data(json.loads(data)['analysis'])
joe_parser.finalize_results()
return {'results': joe_parser.results}

Write Preview
Loading…
Cancel
Save