mirror of https://github.com/MISP/misp-modules
Merge pull request #367 from joesecurity/master
joe: (1) allow users to disable PE object import (2) set 'to_ids' to Falsepull/368/head
Christian Studer
GitHub
commit
1de6890152
|
|
@ -51,12 +51,15 @@ signerinfo_object_mapping = {'sigissuer': ('text', 'issuer'),
|
|||
|
||||
|
||||
class JoeParser():
|
||||
def __init__(self):
|
||||
def __init__(self, config):
|
||||
self.misp_event = MISPEvent()
|
||||
self.references = defaultdict(list)
|
||||
self.attributes = defaultdict(lambda: defaultdict(set))
|
||||
self.process_references = {}
|
||||
|
||||
self.import_pe = config["import_pe"]
|
||||
self.create_mitre_attack = config["mitre_attack"]
|
||||
|
||||
def parse_data(self, data):
|
||||
self.data = data
|
||||
if self.analysis_type() == "file":
|
||||
|
|
@ -72,7 +75,9 @@ class JoeParser():
|
|||
|
||||
if self.attributes:
|
||||
self.handle_attributes()
|
||||
self.parse_mitre_attack()
|
||||
|
||||
if self.create_mitre_attack:
|
||||
self.parse_mitre_attack()
|
||||
|
||||
def build_references(self):
|
||||
for misp_object in self.misp_event.objects:
|
||||
|
|
@ -97,12 +102,12 @@ class JoeParser():
|
|||
file_object = MISPObject('file')
|
||||
for key, mapping in dropped_file_mapping.items():
|
||||
attribute_type, object_relation = mapping
|
||||
file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': droppedfile[key]})
|
||||
file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': droppedfile[key], 'to_ids': False})
|
||||
if droppedfile['@malicious'] == 'true':
|
||||
file_object.add_attribute('state', **{'type': 'text', 'value': 'Malicious'})
|
||||
file_object.add_attribute('state', **{'type': 'text', 'value': 'Malicious', 'to_ids': False})
|
||||
for h in droppedfile['value']:
|
||||
hash_type = dropped_hash_mapping[h['@algo']]
|
||||
file_object.add_attribute(hash_type, **{'type': hash_type, 'value': h['$']})
|
||||
file_object.add_attribute(hash_type, **{'type': hash_type, 'value': h['$'], 'to_ids': False})
|
||||
self.misp_event.add_object(**file_object)
|
||||
self.references[self.process_references[(int(droppedfile['@targetid']), droppedfile['@process'])]].append({
|
||||
'referenced_uuid': file_object.uuid,
|
||||
|
|
@ -132,9 +137,12 @@ class JoeParser():
|
|||
for object_relation, attribute in attributes.items():
|
||||
network_connection_object.add_attribute(object_relation, **attribute)
|
||||
network_connection_object.add_attribute('first-packet-seen',
|
||||
**{'type': 'datetime', 'value': min(tuple(min(timestamp) for timestamp in data.values()))})
|
||||
**{'type': 'datetime',
|
||||
'value': min(tuple(min(timestamp) for timestamp in data.values())),
|
||||
'to_ids': False})
|
||||
for protocol in data.keys():
|
||||
network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol})
|
||||
network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]),
|
||||
**{'type': 'text', 'value': protocol, 'to_ids': False})
|
||||
self.misp_event.add_object(**network_connection_object)
|
||||
self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=network_connection_object.uuid,
|
||||
relationship_type='initiates'))
|
||||
|
|
@ -143,8 +151,8 @@ class JoeParser():
|
|||
network_connection_object = MISPObject('network-connection')
|
||||
for object_relation, attribute in attributes.items():
|
||||
network_connection_object.add_attribute(object_relation, **attribute)
|
||||
network_connection_object.add_attribute('first-packet-seen', **{'type': 'datetime', 'value': min(timestamps)})
|
||||
network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol})
|
||||
network_connection_object.add_attribute('first-packet-seen', **{'type': 'datetime', 'value': min(timestamps), 'to_ids': False})
|
||||
network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol, 'to_ids': False})
|
||||
self.misp_event.add_object(**network_connection_object)
|
||||
self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=network_connection_object.uuid,
|
||||
relationship_type='initiates'))
|
||||
|
|
@ -154,7 +162,8 @@ class JoeParser():
|
|||
if screenshotdata:
|
||||
screenshotdata = screenshotdata['interesting']['$']
|
||||
attribute = {'type': 'attachment', 'value': 'screenshot.jpg',
|
||||
'data': screenshotdata, 'disable_correlation': True}
|
||||
'data': screenshotdata, 'disable_correlation': True,
|
||||
'to_ids': False}
|
||||
self.misp_event.add_attribute(**attribute)
|
||||
|
||||
def parse_system_behavior(self):
|
||||
|
|
@ -166,9 +175,9 @@ class JoeParser():
|
|||
general = process['general']
|
||||
process_object = MISPObject('process')
|
||||
for feature, relation in process_object_fields.items():
|
||||
process_object.add_attribute(relation, **{'type': 'text', 'value': general[feature]})
|
||||
process_object.add_attribute(relation, **{'type': 'text', 'value': general[feature], 'to_ids': False})
|
||||
start_time = datetime.strptime('{} {}'.format(general['date'], general['time']), '%d/%m/%Y %H:%M:%S')
|
||||
process_object.add_attribute('start-time', **{'type': 'datetime', 'value': start_time})
|
||||
process_object.add_attribute('start-time', **{'type': 'datetime', 'value': start_time, 'to_ids': False})
|
||||
self.misp_event.add_object(**process_object)
|
||||
for field, to_call in process_activities.items():
|
||||
if process.get(field):
|
||||
|
|
@ -203,7 +212,7 @@ class JoeParser():
|
|||
url_object = MISPObject("url")
|
||||
self.analysisinfo_uuid = url_object.uuid
|
||||
|
||||
url_object.add_attribute("url", generalinfo["target"]["url"])
|
||||
url_object.add_attribute("url", generalinfo["target"]["url"], to_ids=False)
|
||||
self.misp_event.add_object(**url_object)
|
||||
|
||||
def parse_fileinfo(self):
|
||||
|
|
@ -213,10 +222,10 @@ class JoeParser():
|
|||
self.analysisinfo_uuid = file_object.uuid
|
||||
|
||||
for field in file_object_fields:
|
||||
file_object.add_attribute(field, **{'type': field, 'value': fileinfo[field]})
|
||||
file_object.add_attribute(field, **{'type': field, 'value': fileinfo[field], 'to_ids': False})
|
||||
for field, mapping in file_object_mapping.items():
|
||||
attribute_type, object_relation = mapping
|
||||
file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': fileinfo[field]})
|
||||
file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': fileinfo[field], 'to_ids': False})
|
||||
arch = self.data['generalinfo']['arch']
|
||||
if arch in arch_type_mapping:
|
||||
to_call = arch_type_mapping[arch]
|
||||
|
|
@ -234,9 +243,9 @@ class JoeParser():
|
|||
attribute_type = 'text'
|
||||
for comment, permissions in permission_lists.items():
|
||||
permission_object = MISPObject('android-permission')
|
||||
permission_object.add_attribute('comment', **dict(type=attribute_type, value=comment))
|
||||
permission_object.add_attribute('comment', **dict(type=attribute_type, value=comment, to_ids=False))
|
||||
for permission in permissions:
|
||||
permission_object.add_attribute('permission', **dict(type=attribute_type, value=permission))
|
||||
permission_object.add_attribute('permission', **dict(type=attribute_type, value=permission, to_ids=False))
|
||||
self.misp_event.add_object(**permission_object)
|
||||
self.references[file_object.uuid].append(dict(referenced_uuid=permission_object.uuid,
|
||||
relationship_type='grants'))
|
||||
|
|
@ -255,24 +264,24 @@ class JoeParser():
|
|||
if elf.get('type'):
|
||||
# Haven't seen anything but EXEC yet in the files I tested
|
||||
attribute_value = "EXECUTABLE" if elf['type'] == "EXEC (Executable file)" else elf['type']
|
||||
elf_object.add_attribute('type', **dict(type=attribute_type, value=attribute_value))
|
||||
elf_object.add_attribute('type', **dict(type=attribute_type, value=attribute_value, to_ids=False))
|
||||
for feature, relation in elf_object_mapping.items():
|
||||
if elf.get(feature):
|
||||
elf_object.add_attribute(relation, **dict(type=attribute_type, value=elf[feature]))
|
||||
elf_object.add_attribute(relation, **dict(type=attribute_type, value=elf[feature], to_ids=False))
|
||||
sections_number = len(fileinfo['sections']['section'])
|
||||
elf_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number})
|
||||
elf_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number, 'to_ids': False})
|
||||
self.misp_event.add_object(**elf_object)
|
||||
for section in fileinfo['sections']['section']:
|
||||
section_object = MISPObject('elf-section')
|
||||
for feature in ('name', 'type'):
|
||||
if section.get(feature):
|
||||
section_object.add_attribute(feature, **dict(type=attribute_type, value=section[feature]))
|
||||
section_object.add_attribute(feature, **dict(type=attribute_type, value=section[feature], to_ids=False))
|
||||
if section.get('size'):
|
||||
section_object.add_attribute(size, **dict(type=size, value=int(section['size'], 16)))
|
||||
section_object.add_attribute(size, **dict(type=size, value=int(section['size'], 16), to_ids=False))
|
||||
for flag in section['flagsdesc']:
|
||||
try:
|
||||
attribute_value = elf_section_flags_mapping[flag]
|
||||
section_object.add_attribute('flag', **dict(type=attribute_type, value=attribute_value))
|
||||
section_object.add_attribute('flag', **dict(type=attribute_type, value=attribute_value, to_ids=False))
|
||||
except KeyError:
|
||||
print(f'Unknown elf section flag: {flag}')
|
||||
continue
|
||||
|
|
@ -281,6 +290,8 @@ class JoeParser():
|
|||
relationship_type=relationship))
|
||||
|
||||
def parse_pe(self, fileinfo, file_object):
|
||||
if not self.import_pe:
|
||||
return
|
||||
try:
|
||||
peinfo = fileinfo['pe']
|
||||
except KeyError:
|
||||
|
|
@ -292,8 +303,8 @@ class JoeParser():
|
|||
self.misp_event.add_object(**file_object)
|
||||
for field, mapping in pe_object_fields.items():
|
||||
attribute_type, object_relation = mapping
|
||||
pe_object.add_attribute(object_relation, **{'type': attribute_type, 'value': peinfo[field]})
|
||||
pe_object.add_attribute('compilation-timestamp', **{'type': 'datetime', 'value': int(peinfo['timestamp'].split()[0], 16)})
|
||||
pe_object.add_attribute(object_relation, **{'type': attribute_type, 'value': peinfo[field], 'to_ids': False})
|
||||
pe_object.add_attribute('compilation-timestamp', **{'type': 'datetime', 'value': int(peinfo['timestamp'].split()[0], 16), 'to_ids': False})
|
||||
program_name = fileinfo['filename']
|
||||
if peinfo['versions']:
|
||||
for feature in peinfo['versions']['version']:
|
||||
|
|
@ -301,18 +312,18 @@ class JoeParser():
|
|||
if name == 'InternalName':
|
||||
program_name = feature['value']
|
||||
if name in pe_object_mapping:
|
||||
pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value']})
|
||||
pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value'], 'to_ids': False})
|
||||
sections_number = len(peinfo['sections']['section'])
|
||||
pe_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number})
|
||||
pe_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number, 'to_ids': False})
|
||||
signatureinfo = peinfo['signature']
|
||||
if signatureinfo['signed']:
|
||||
signerinfo_object = MISPObject('authenticode-signerinfo')
|
||||
pe_object.add_reference(signerinfo_object.uuid, 'signed-by')
|
||||
self.misp_event.add_object(**pe_object)
|
||||
signerinfo_object.add_attribute('program-name', **{'type': 'text', 'value': program_name})
|
||||
signerinfo_object.add_attribute('program-name', **{'type': 'text', 'value': program_name, 'to_ids': False})
|
||||
for feature, mapping in signerinfo_object_mapping.items():
|
||||
attribute_type, object_relation = mapping
|
||||
signerinfo_object.add_attribute(object_relation, **{'type': attribute_type, 'value': signatureinfo[feature]})
|
||||
signerinfo_object.add_attribute(object_relation, **{'type': attribute_type, 'value': signatureinfo[feature], 'to_ids': False})
|
||||
self.misp_event.add_object(**signerinfo_object)
|
||||
else:
|
||||
self.misp_event.add_object(**pe_object)
|
||||
|
|
@ -327,7 +338,7 @@ class JoeParser():
|
|||
for feature, mapping in pe_section_object_mapping.items():
|
||||
if section.get(feature):
|
||||
attribute_type, object_relation = mapping
|
||||
section_object.add_attribute(object_relation, **{'type': attribute_type, 'value': section[feature]})
|
||||
section_object.add_attribute(object_relation, **{'type': attribute_type, 'value': section[feature], 'to_ids': False})
|
||||
return section_object
|
||||
|
||||
def parse_network_interactions(self):
|
||||
|
|
@ -339,13 +350,13 @@ class JoeParser():
|
|||
for key, mapping in domain_object_mapping.items():
|
||||
attribute_type, object_relation = mapping
|
||||
domain_object.add_attribute(object_relation,
|
||||
**{'type': attribute_type, 'value': domain[key]})
|
||||
**{'type': attribute_type, 'value': domain[key], 'to_ids': False})
|
||||
self.misp_event.add_object(**domain_object)
|
||||
reference = dict(referenced_uuid=domain_object.uuid, relationship_type='contacts')
|
||||
self.add_process_reference(domain['@targetid'], domain['@currentpath'], reference)
|
||||
else:
|
||||
attribute = MISPAttribute()
|
||||
attribute.from_dict(**{'type': 'domain', 'value': domain['@name']})
|
||||
attribute.from_dict(**{'type': 'domain', 'value': domain['@name'], 'to_ids': False})
|
||||
self.misp_event.add_attribute(**attribute)
|
||||
reference = dict(referenced_uuid=attribute.uuid, relationship_type='contacts')
|
||||
self.add_process_reference(domain['@targetid'], domain['@currentpath'], reference)
|
||||
|
|
@ -353,7 +364,7 @@ class JoeParser():
|
|||
if ipinfo:
|
||||
for ip in ipinfo['ip']:
|
||||
attribute = MISPAttribute()
|
||||
attribute.from_dict(**{'type': 'ip-dst', 'value': ip['@ip']})
|
||||
attribute.from_dict(**{'type': 'ip-dst', 'value': ip['@ip'], 'to_ids': False})
|
||||
self.misp_event.add_attribute(**attribute)
|
||||
reference = dict(referenced_uuid=attribute.uuid, relationship_type='contacts')
|
||||
self.add_process_reference(ip['@targetid'], ip['@currentpath'], reference)
|
||||
|
|
@ -363,7 +374,7 @@ class JoeParser():
|
|||
target_id = int(url['@targetid'])
|
||||
current_path = url['@currentpath']
|
||||
attribute = MISPAttribute()
|
||||
attribute_dict = {'type': 'url', 'value': url['@name']}
|
||||
attribute_dict = {'type': 'url', 'value': url['@name'], 'to_ids': False}
|
||||
if target_id != -1 and current_path != 'unknown':
|
||||
self.references[self.process_references[(target_id, current_path)]].append({
|
||||
'referenced_uuid': attribute.uuid,
|
||||
|
|
@ -384,8 +395,8 @@ class JoeParser():
|
|||
registry_key = MISPObject('registry-key')
|
||||
for field, mapping in regkey_object_mapping.items():
|
||||
attribute_type, object_relation = mapping
|
||||
registry_key.add_attribute(object_relation, **{'type': attribute_type, 'value': call[field]})
|
||||
registry_key.add_attribute('data-type', **{'type': 'text', 'value': 'REG_{}'.format(call['type'].upper())})
|
||||
registry_key.add_attribute(object_relation, **{'type': attribute_type, 'value': call[field], 'to_ids': False})
|
||||
registry_key.add_attribute('data-type', **{'type': 'text', 'value': 'REG_{}'.format(call['type'].upper()), 'to_ids': False})
|
||||
self.misp_event.add_object(**registry_key)
|
||||
self.references[process_uuid].append(dict(referenced_uuid=registry_key.uuid,
|
||||
relationship_type=relationship))
|
||||
|
|
@ -398,7 +409,7 @@ class JoeParser():
|
|||
|
||||
def create_attribute(self, attribute_type, attribute_value):
|
||||
attribute = MISPAttribute()
|
||||
attribute.from_dict(**{'type': attribute_type, 'value': attribute_value})
|
||||
attribute.from_dict(**{'type': attribute_type, 'value': attribute_value, 'to_ids': False})
|
||||
self.misp_event.add_attribute(**attribute)
|
||||
return attribute.uuid
|
||||
|
||||
|
|
@ -419,5 +430,5 @@ class JoeParser():
|
|||
attributes = {}
|
||||
for field, value in zip(network_behavior_fields, connection):
|
||||
attribute_type, object_relation = network_connection_object_mapping[field]
|
||||
attributes[object_relation] = {'type': attribute_type, 'value': value}
|
||||
attributes[object_relation] = {'type': attribute_type, 'value': value, 'to_ids': False}
|
||||
return attributes
|
||||
|
|
|
|||
|
|
@ -4,12 +4,13 @@ import json
|
|||
from joe_parser import JoeParser
|
||||
|
||||
misperrors = {'error': 'Error'}
|
||||
mispattributes = {'input': ['link'], 'format': 'misp_standard'}
|
||||
|
||||
moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
|
||||
inputSource = ['link']
|
||||
|
||||
moduleinfo = {'version': '0.2', 'author': 'Christian Studer',
|
||||
'description': 'Query Joe Sandbox API with a report URL to get the parsed data.',
|
||||
'module-type': ['expansion']}
|
||||
moduleconfig = ['apiurl', 'apikey']
|
||||
moduleconfig = ['apiurl', 'apikey', 'import_pe', 'import_mitre_attack']
|
||||
|
||||
|
||||
def handler(q=False):
|
||||
|
|
@ -18,6 +19,11 @@ def handler(q=False):
|
|||
request = json.loads(q)
|
||||
apiurl = request['config'].get('apiurl') or 'https://jbxcloud.joesecurity.org/api'
|
||||
apikey = request['config'].get('apikey')
|
||||
parser_config = {
|
||||
"import_pe": request["config"].get('import_pe', "false") == "true",
|
||||
"mitre_attack": request["config"].get('import_mitre_attack', "false") == "true",
|
||||
}
|
||||
|
||||
if not apikey:
|
||||
return {'error': 'No API key provided'}
|
||||
|
||||
|
|
@ -41,7 +47,7 @@ def handler(q=False):
|
|||
|
||||
analysis_webid = joe_info['most_relevant_analysis']['webid']
|
||||
|
||||
joe_parser = JoeParser()
|
||||
joe_parser = JoeParser(parser_config)
|
||||
joe_data = json.loads(joe.analysis_download(analysis_webid, 'jsonfixed')[1])
|
||||
joe_parser.parse_data(joe_data['analysis'])
|
||||
joe_parser.finalize_results()
|
||||
|
|
@ -50,7 +56,19 @@ def handler(q=False):
|
|||
|
||||
|
||||
def introspection():
|
||||
return mispattributes
|
||||
modulesetup = {}
|
||||
try:
|
||||
userConfig
|
||||
modulesetup['userConfig'] = userConfig
|
||||
except NameError:
|
||||
pass
|
||||
try:
|
||||
inputSource
|
||||
modulesetup['input'] = inputSource
|
||||
except NameError:
|
||||
pass
|
||||
modulesetup['format'] = 'misp_standard'
|
||||
return modulesetup
|
||||
|
||||
|
||||
def version():
|
||||
|
|
|
|||
|
|
@ -4,10 +4,20 @@ import json
|
|||
from joe_parser import JoeParser
|
||||
|
||||
misperrors = {'error': 'Error'}
|
||||
userConfig = {}
|
||||
userConfig = {
|
||||
"Import PE": {
|
||||
"type": "Boolean",
|
||||
"message": "Import PE Information",
|
||||
},
|
||||
"Mitre Att&ck" : {
|
||||
"type": "Boolean",
|
||||
"message": "Import Mitre Att&ck techniques",
|
||||
},
|
||||
}
|
||||
|
||||
inputSource = ['file']
|
||||
|
||||
moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
|
||||
moduleinfo = {'version': '0.2', 'author': 'Christian Studer',
|
||||
'description': 'Import for Joe Sandbox JSON reports',
|
||||
'module-type': ['import']}
|
||||
|
||||
|
|
@ -18,10 +28,16 @@ def handler(q=False):
|
|||
if q is False:
|
||||
return False
|
||||
q = json.loads(q)
|
||||
config = {
|
||||
"import_pe": bool(int(q["config"]["Import PE"])),
|
||||
"mitre_attack": bool(int(q["config"]["Mitre Att&ck"])),
|
||||
}
|
||||
|
||||
data = base64.b64decode(q.get('data')).decode('utf-8')
|
||||
if not data:
|
||||
return json.dumps({'success': 0})
|
||||
joe_parser = JoeParser()
|
||||
|
||||
joe_parser = JoeParser(config)
|
||||
joe_parser.parse_data(json.loads(data)['analysis'])
|
||||
joe_parser.finalize_results()
|
||||
return {'results': joe_parser.results}
|
||||
|
|
|
|||
Loading…
Reference in New Issue