MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

fix: [joe parser] Some clean-up on the Joe parser

pull/560/head
chrisr3d 2022-03-07 17:53:43 +01:00
parent db902275b3
commit cba06ab372
No known key found for this signature in database
GPG Key ID: 6BBED1B63A6D639F
3 changed files with 386 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
misp_modules/lib/__init__.py
View File

@ -1,3 +1,4 @@
import joe_mapping
from .vt_graph_parser import * # noqa from .vt_graph_parser import * # noqa
all = ['joe_parser', 'lastline_api', 'cof2misp', 'qintel_helper'] all = ['joe_parser', 'lastline_api', 'cof2misp', 'qintel_helper']

114
misp_modules/lib/joe_mapping.py Normal file
View File

@ -0,0 +1,114 @@
arch_type_mapping = {
'ANDROID': 'parse_apk',
'LINUX': 'parse_elf',
'WINDOWS': 'parse_pe'
}
domain_object_mapping = {
'@ip': {'type': 'ip-dst', 'object_relation': 'ip'},
'@name': {'type': 'domain', 'object_relation': 'domain'}
}
dropped_file_mapping = {
'@entropy': {'type': 'float', 'object_relation': 'entropy'},
'@file': {'type': 'filename', 'object_relation': 'filename'},
'@size': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'},
'@type': {'type': 'mime-type', 'object_relation': 'mimetype'}
}
dropped_hash_mapping = {
'MD5': 'md5',
'SHA': 'sha1',
'SHA-256': 'sha256',
'SHA-512': 'sha512'
}
elf_object_mapping = {
'epaddr': 'entrypoint-address',
'machine': 'arch',
'osabi': 'os_abi'
}
elf_section_flags_mapping = {
'A': 'ALLOC',
'I': 'INFO_LINK',
'M': 'MERGE',
'S': 'STRINGS',
'T': 'TLS',
'W': 'WRITE',
'X': 'EXECINSTR'
}
file_object_fields = (
'filename',
'md5',
'sha1',
'sha256',
'sha512',
'ssdeep'
)
file_object_mapping = {
'entropy': {'type': 'float', 'object_relation': 'entropy'},
'filesize': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'},
'filetype': {'type': 'mime-type', 'object_relation': 'mimetype'}
}
file_references_mapping = {
'fileCreated': 'creates',
'fileDeleted': 'deletes',
'fileMoved': 'moves',
'fileRead': 'reads',
'fileWritten': 'writes'
}
network_behavior_fields = ('srcip', 'dstip', 'srcport', 'dstport')
network_connection_object_mapping = {
'srcip': {'type': 'ip-src', 'object_relation': 'ip-src'},
'dstip': {'type': 'ip-dst', 'object_relation': 'ip-dst'},
'srcport': {'type': 'port', 'object_relation': 'src-port'},
'dstport': {'type': 'port', 'object_relation': 'dst-port'}
}
pe_object_fields = {
'entrypoint': {'type': 'text', 'object_relation': 'entrypoint-address'},
'imphash': {'type': 'imphash', 'object_relation': 'imphash'}
}
pe_object_mapping = {
'CompanyName': 'company-name',
'FileDescription': 'file-description',
'FileVersion': 'file-version',
'InternalName': 'internal-filename',
'LegalCopyright': 'legal-copyright',
'OriginalFilename': 'original-filename',
'ProductName': 'product-filename',
'ProductVersion': 'product-version',
'Translation': 'lang-id'
}
pe_section_object_mapping = {
'characteristics': {'type': 'text', 'object_relation': 'characteristic'},
'entropy': {'type': 'float', 'object_relation': 'entropy'},
'name': {'type': 'text', 'object_relation': 'name'},
'rawaddr': {'type': 'hex', 'object_relation': 'offset'},
'rawsize': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'},
'virtaddr': {'type': 'hex', 'object_relation': 'virtual_address'},
'virtsize': {'type': 'size-in-bytes', 'object_relation': 'virtual_size'}
}
process_object_fields = {
'cmdline': 'command-line',
'name': 'name',
'parentpid': 'parent-pid',
'pid': 'pid',
'path': 'current-directory'
}
protocols = {
'tcp': 4,
'udp': 4,
'icmp': 3,
'http': 7,
'https': 7,
'ftp': 7
}
registry_references_mapping = {
'keyValueCreated': 'creates',
'keyValueModified': 'modifies'
}
regkey_object_mapping = {
'name': {'type': 'text', 'object_relation': 'name'},
'newdata': {'type': 'text', 'object_relation': 'data'},
'path': {'type': 'regkey', 'object_relation': 'key'}
}
signerinfo_object_mapping = {
'sigissuer': {'type': 'text', 'object_relation': 'issuer'},
'version': {'type': 'text', 'object_relation': 'version'}
}

405
misp_modules/lib/joe_parser.py
View File

@ -1,53 +1,15 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import json
from collections import defaultdict from collections import defaultdict
from datetime import datetime from datetime import datetime
from pymisp import MISPAttribute, MISPEvent, MISPObject from pymisp import MISPAttribute, MISPEvent, MISPObject
import json from joe_mapping import (arch_type_mapping, domain_object_mapping,
dropped_file_mapping, dropped_hash_mapping, elf_object_mapping,
elf_section_flags_mapping, file_object_fields, file_object_mapping,
arch_type_mapping = {'ANDROID': 'parse_apk', 'LINUX': 'parse_elf', 'WINDOWS': 'parse_pe'} file_references_mapping, network_behavior_fields,
domain_object_mapping = {'@ip': ('ip-dst', 'ip'), '@name': ('domain', 'domain')} network_connection_object_mapping, pe_object_fields, pe_object_mapping,
dropped_file_mapping = {'@entropy': ('float', 'entropy'), pe_section_object_mapping, process_object_fields, protocols,
'@file': ('filename', 'filename'), registry_references_mapping, regkey_object_mapping, signerinfo_object_mapping)
'@size': ('size-in-bytes', 'size-in-bytes'),
'@type': ('mime-type', 'mimetype')}
dropped_hash_mapping = {'MD5': 'md5', 'SHA': 'sha1', 'SHA-256': 'sha256', 'SHA-512': 'sha512'}
elf_object_mapping = {'epaddr': 'entrypoint-address', 'machine': 'arch', 'osabi': 'os_abi'}
elf_section_flags_mapping = {'A': 'ALLOC', 'I': 'INFO_LINK', 'M': 'MERGE',
'S': 'STRINGS', 'T': 'TLS', 'W': 'WRITE',
'X': 'EXECINSTR'}
file_object_fields = ['filename', 'md5', 'sha1', 'sha256', 'sha512', 'ssdeep']
file_object_mapping = {'entropy': ('float', 'entropy'),
'filesize': ('size-in-bytes', 'size-in-bytes'),
'filetype': ('mime-type', 'mimetype')}
file_references_mapping = {'fileCreated': 'creates', 'fileDeleted': 'deletes',
'fileMoved': 'moves', 'fileRead': 'reads', 'fileWritten': 'writes'}
network_behavior_fields = ('srcip', 'dstip', 'srcport', 'dstport')
network_connection_object_mapping = {'srcip': ('ip-src', 'ip-src'), 'dstip': ('ip-dst', 'ip-dst'),
'srcport': ('port', 'src-port'), 'dstport': ('port', 'dst-port')}
pe_object_fields = {'entrypoint': ('text', 'entrypoint-address'),
'imphash': ('imphash', 'imphash')}
pe_object_mapping = {'CompanyName': 'company-name', 'FileDescription': 'file-description',
'FileVersion': 'file-version', 'InternalName': 'internal-filename',
'LegalCopyright': 'legal-copyright', 'OriginalFilename': 'original-filename',
'ProductName': 'product-filename', 'ProductVersion': 'product-version',
'Translation': 'lang-id'}
pe_section_object_mapping = {'characteristics': ('text', 'characteristic'),
'entropy': ('float', 'entropy'),
'name': ('text', 'name'), 'rawaddr': ('hex', 'offset'),
'rawsize': ('size-in-bytes', 'size-in-bytes'),
'virtaddr': ('hex', 'virtual_address'),
'virtsize': ('size-in-bytes', 'virtual_size')}
process_object_fields = {'cmdline': 'command-line', 'name': 'name',
'parentpid': 'parent-pid', 'pid': 'pid',
'path': 'current-directory'}
protocols = {'tcp': 4, 'udp': 4, 'icmp': 3,
'http': 7, 'https': 7, 'ftp': 7}
registry_references_mapping = {'keyValueCreated': 'creates', 'keyValueModified': 'modifies'}
regkey_object_mapping = {'name': ('text', 'name'), 'newdata': ('text', 'data'),
'path': ('regkey', 'key')}
signerinfo_object_mapping = {'sigissuer': ('text', 'issuer'),
'version': ('text', 'version')}
class JoeParser(): class JoeParser():
@ -57,7 +19,7 @@ class JoeParser():
self.attributes = defaultdict(lambda: defaultdict(set)) self.attributes = defaultdict(lambda: defaultdict(set))
self.process_references = {} self.process_references = {}
self.import_pe = config["import_pe"] self.import_executable = config["import_executable"]
self.create_mitre_attack = config["mitre_attack"] self.create_mitre_attack = config["mitre_attack"]
def parse_data(self, data): def parse_data(self, data):
@ -101,26 +63,46 @@ class JoeParser():
for droppedfile in droppedinfo['hash']: for droppedfile in droppedinfo['hash']:
file_object = MISPObject('file') file_object = MISPObject('file')
for key, mapping in dropped_file_mapping.items(): for key, mapping in dropped_file_mapping.items():
attribute_type, object_relation = mapping if droppedfile.get(key) is not None:
file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': droppedfile[key], 'to_ids': False}) attribute = {'value': droppedfile[key], 'to_ids': False}
attribute.update(mapping)
file_object.add_attribute(**attribute)
if droppedfile['@malicious'] == 'true': if droppedfile['@malicious'] == 'true':
file_object.add_attribute('state', **{'type': 'text', 'value': 'Malicious', 'to_ids': False}) file_object.add_attribute(
**{
'type': 'text',
'object_relation': 'state',
'value': 'Malicious',
'to_ids': False
}
)
for h in droppedfile['value']: for h in droppedfile['value']:
hash_type = dropped_hash_mapping[h['@algo']] hash_type = dropped_hash_mapping[h['@algo']]
file_object.add_attribute(hash_type, **{'type': hash_type, 'value': h['$'], 'to_ids': False}) file_object.add_attribute(
self.misp_event.add_object(**file_object) **{
self.references[self.process_references[(int(droppedfile['@targetid']), droppedfile['@process'])]].append({ 'type': hash_type,
'referenced_uuid': file_object.uuid, 'object_relation': hash_type,
'relationship_type': 'drops' 'value': h['$'],
}) 'to_ids': False
}
)
self.misp_event.add_object(file_object)
reference_key = (int(droppedfile['@targetid']), droppedfile['@process'])
if reference_key in self.process_references:
self.references[self.process_references[reference_key]].append(
{
'referenced_uuid': file_object.uuid,
'relationship_type': 'drops'
}
)
def parse_mitre_attack(self): def parse_mitre_attack(self):
mitreattack = self.data['mitreattack'] mitreattack = self.data.get('mitreattack', {})
if mitreattack: if mitreattack:
for tactic in mitreattack['tactic']: for tactic in mitreattack['tactic']:
if tactic.get('technique'): if tactic.get('technique'):
for technique in tactic['technique']: for technique in tactic['technique']:
self.misp_event.add_tag('misp-galaxy:mitre-attack-pattern="{} - {}"'.format(technique['name'], technique['id'])) self.misp_event.add_tag(f'misp-galaxy:mitre-attack-pattern="{technique["name"]} - {technique["id"]}"')
def parse_network_behavior(self): def parse_network_behavior(self):
network = self.data['behavior']['network'] network = self.data['behavior']['network']
@ -134,37 +116,65 @@ class JoeParser():
attributes = self.prefetch_attributes_data(connection) attributes = self.prefetch_attributes_data(connection)
if len(data.keys()) == len(set(protocols[protocol] for protocol in data.keys())): if len(data.keys()) == len(set(protocols[protocol] for protocol in data.keys())):
network_connection_object = MISPObject('network-connection') network_connection_object = MISPObject('network-connection')
for object_relation, attribute in attributes.items(): for attribute in attributes:
network_connection_object.add_attribute(object_relation, **attribute) network_connection_object.add_attribute(**attribute)
network_connection_object.add_attribute('first-packet-seen', network_connection_object.add_attribute(
**{'type': 'datetime', **{
'value': min(tuple(min(timestamp) for timestamp in data.values())), 'type': 'datetime',
'to_ids': False}) 'object_relation': 'first-packet-seen',
'value': min(tuple(min(timestamp) for timestamp in data.values())),
'to_ids': False
}
)
for protocol in data.keys(): for protocol in data.keys():
network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), network_connection_object.add_attribute(
**{'type': 'text', 'value': protocol, 'to_ids': False}) **{
self.misp_event.add_object(**network_connection_object) 'type': 'text',
'object_relation': f'layer{protocols[protocol]}-protocol',
'value': protocol,
'to_ids': False
}
)
self.misp_event.add_object(network_connection_object)
self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=network_connection_object.uuid, self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=network_connection_object.uuid,
relationship_type='initiates')) relationship_type='initiates'))
else: else:
for protocol, timestamps in data.items(): for protocol, timestamps in data.items():
network_connection_object = MISPObject('network-connection') network_connection_object = MISPObject('network-connection')
for object_relation, attribute in attributes.items(): for attribute in attributes:
network_connection_object.add_attribute(object_relation, **attribute) network_connection_object.add_attribute(**attribute)
network_connection_object.add_attribute('first-packet-seen', **{'type': 'datetime', 'value': min(timestamps), 'to_ids': False}) network_connection_object.add_attribute(
network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol, 'to_ids': False}) **{
self.misp_event.add_object(**network_connection_object) 'type': 'datetime',
'object_relation': 'first-packet-seen',
'value': min(timestamps),
'to_ids': False
}
)
network_connection_object.add_attribute(
**{
'type': 'text',
'object_relation': f'layer{protocols[protocol]}-protocol',
'value': protocol,
'to_ids': False
}
)
self.misp_event.add_object(network_connection_object)
self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=network_connection_object.uuid, self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=network_connection_object.uuid,
relationship_type='initiates')) relationship_type='initiates'))
def parse_screenshot(self): def parse_screenshot(self):
screenshotdata = self.data['behavior']['screenshotdata'] if self.data['behavior'].get('screenshotdata', {}).get('interesting') is not None:
if screenshotdata: screenshotdata = self.data['behavior']['screenshotdata']['interesting']['$']
screenshotdata = screenshotdata['interesting']['$'] self.misp_event.add_attribute(
attribute = {'type': 'attachment', 'value': 'screenshot.jpg', **{
'data': screenshotdata, 'disable_correlation': True, 'type': 'attachment',
'to_ids': False} 'value': 'screenshot.jpg',
self.misp_event.add_attribute(**attribute) 'data': screenshotdata,
'disable_correlation': True,
'to_ids': False
}
)
def parse_system_behavior(self): def parse_system_behavior(self):
if not 'system' in self.data['behavior']: if not 'system' in self.data['behavior']:
@ -177,10 +187,24 @@ class JoeParser():
general = process['general'] general = process['general']
process_object = MISPObject('process') process_object = MISPObject('process')
for feature, relation in process_object_fields.items(): for feature, relation in process_object_fields.items():
process_object.add_attribute(relation, **{'type': 'text', 'value': general[feature], 'to_ids': False}) process_object.add_attribute(
start_time = datetime.strptime('{} {}'.format(general['date'], general['time']), '%d/%m/%Y %H:%M:%S') **{
process_object.add_attribute('start-time', **{'type': 'datetime', 'value': start_time, 'to_ids': False}) 'type': 'text',
self.misp_event.add_object(**process_object) 'object_relation': relation,
'value': general[feature],
'to_ids': False
}
)
start_time = datetime.strptime(f"{general['date']} {general['time']}", '%d/%m/%Y %H:%M:%S')
process_object.add_attribute(
**{
'type': 'datetime',
'object_relation': 'start-time',
'value': start_time,
'to_ids': False
}
)
self.misp_event.add_object(process_object)
for field, to_call in process_activities.items(): for field, to_call in process_activities.items():
if process.get(field): if process.get(field):
to_call(process_object.uuid, process[field]) to_call(process_object.uuid, process[field])
@ -213,9 +237,15 @@ class JoeParser():
url_object = MISPObject("url") url_object = MISPObject("url")
self.analysisinfo_uuid = url_object.uuid self.analysisinfo_uuid = url_object.uuid
url_object.add_attribute(
url_object.add_attribute("url", generalinfo["target"]["url"], to_ids=False) **{
self.misp_event.add_object(**url_object) 'type': 'url',
'object_relation': 'url',
'value': generalinfo["target"]["url"],
'to_ids': False
}
)
self.misp_event.add_object(url_object)
def parse_fileinfo(self): def parse_fileinfo(self):
fileinfo = self.data['fileinfo'] fileinfo = self.data['fileinfo']
@ -224,20 +254,29 @@ class JoeParser():
self.analysisinfo_uuid = file_object.uuid self.analysisinfo_uuid = file_object.uuid
for field in file_object_fields: for field in file_object_fields:
file_object.add_attribute(field, **{'type': field, 'value': fileinfo[field], 'to_ids': False}) file_object.add_attribute(
**{
'type': field,
'object_relation': field,
'value': fileinfo[field],
'to_ids': False
}
)
for field, mapping in file_object_mapping.items(): for field, mapping in file_object_mapping.items():
attribute_type, object_relation = mapping if fileinfo.get(field) is not None:
file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': fileinfo[field], 'to_ids': False}) attribute = {'value': fileinfo[field], 'to_ids': False}
attribute.update(mapping)
file_object.add_attribute(**attribute)
arch = self.data['generalinfo']['arch'] arch = self.data['generalinfo']['arch']
if arch in arch_type_mapping: if self.import_executable and arch in arch_type_mapping:
to_call = arch_type_mapping[arch] to_call = arch_type_mapping[arch]
getattr(self, to_call)(fileinfo, file_object) getattr(self, to_call)(fileinfo, file_object)
else: else:
self.misp_event.add_object(**file_object) self.misp_event.add_object(file_object)
def parse_apk(self, fileinfo, file_object): def parse_apk(self, fileinfo, file_object):
apkinfo = fileinfo['apk'] apkinfo = fileinfo['apk']
self.misp_event.add_object(**file_object) self.misp_event.add_object(file_object)
permission_lists = defaultdict(list) permission_lists = defaultdict(list)
for permission in apkinfo['requiredpermissions']['permission']: for permission in apkinfo['requiredpermissions']['permission']:
permission = permission['@name'].split('.') permission = permission['@name'].split('.')
@ -245,16 +284,30 @@ class JoeParser():
attribute_type = 'text' attribute_type = 'text'
for comment, permissions in permission_lists.items(): for comment, permissions in permission_lists.items():
permission_object = MISPObject('android-permission') permission_object = MISPObject('android-permission')
permission_object.add_attribute('comment', **dict(type=attribute_type, value=comment, to_ids=False)) permission_object.add_attribute(
**{
'type': attribute_type,
'object_relation': 'comment',
'value': comment,
'to_ids': False
}
)
for permission in permissions: for permission in permissions:
permission_object.add_attribute('permission', **dict(type=attribute_type, value=permission, to_ids=False)) permission_object.add_attribute(
self.misp_event.add_object(**permission_object) **{
'type': attribute_type,
'object_relation': 'permission',
'value': permission,
'to_ids': False
}
)
self.misp_event.add_object(permission_object)
self.references[file_object.uuid].append(dict(referenced_uuid=permission_object.uuid, self.references[file_object.uuid].append(dict(referenced_uuid=permission_object.uuid,
relationship_type='grants')) relationship_type='grants'))
def parse_elf(self, fileinfo, file_object): def parse_elf(self, fileinfo, file_object):
elfinfo = fileinfo['elf'] elfinfo = fileinfo['elf']
self.misp_event.add_object(**file_object) self.misp_event.add_object(file_object)
attribute_type = 'text' attribute_type = 'text'
relationship = 'includes' relationship = 'includes'
size = 'size-in-bytes' size = 'size-in-bytes'
@ -266,47 +319,96 @@ class JoeParser():
if elf.get('type'): if elf.get('type'):
# Haven't seen anything but EXEC yet in the files I tested # Haven't seen anything but EXEC yet in the files I tested
attribute_value = "EXECUTABLE" if elf['type'] == "EXEC (Executable file)" else elf['type'] attribute_value = "EXECUTABLE" if elf['type'] == "EXEC (Executable file)" else elf['type']
elf_object.add_attribute('type', **dict(type=attribute_type, value=attribute_value, to_ids=False)) elf_object.add_attribute(
**{
'type': attribute_type,
'object_relation': 'type',
'value': attribute_value,
'to_ids': False
}
)
for feature, relation in elf_object_mapping.items(): for feature, relation in elf_object_mapping.items():
if elf.get(feature): if elf.get(feature):
elf_object.add_attribute(relation, **dict(type=attribute_type, value=elf[feature], to_ids=False)) elf_object.add_attribute(
**{
'type': attribute_type,
'object_relation': relation,
'value': elf[feature],
'to_ids': False
}
)
sections_number = len(fileinfo['sections']['section']) sections_number = len(fileinfo['sections']['section'])
elf_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number, 'to_ids': False}) elf_object.add_attribute(
self.misp_event.add_object(**elf_object) **{
'type': 'counter',
'object_relation': 'number-sections',
'value': sections_number,
'to_ids': False
}
)
self.misp_event.add_object(elf_object)
for section in fileinfo['sections']['section']: for section in fileinfo['sections']['section']:
section_object = MISPObject('elf-section') section_object = MISPObject('elf-section')
for feature in ('name', 'type'): for feature in ('name', 'type'):
if section.get(feature): if section.get(feature):
section_object.add_attribute(feature, **dict(type=attribute_type, value=section[feature], to_ids=False)) section_object.add_attribute(
**{
'type': attribute_type,
'object_relation': feature,
'value': section[feature],
'to_ids': False
}
)
if section.get('size'): if section.get('size'):
section_object.add_attribute(size, **dict(type=size, value=int(section['size'], 16), to_ids=False)) section_object.add_attribute(
**{
'type': size,
'object_relation': size,
'value': int(section['size'], 16),
'to_ids': False
}
)
for flag in section['flagsdesc']: for flag in section['flagsdesc']:
try: try:
attribute_value = elf_section_flags_mapping[flag] attribute_value = elf_section_flags_mapping[flag]
section_object.add_attribute('flag', **dict(type=attribute_type, value=attribute_value, to_ids=False)) section_object.add_attribute(
**{
'type': attribute_type,
'object_relation': 'flag',
'value': attribute_value,
'to_ids': False
}
)
except KeyError: except KeyError:
print(f'Unknown elf section flag: {flag}') print(f'Unknown elf section flag: {flag}')
continue continue
self.misp_event.add_object(**section_object) self.misp_event.add_object(section_object)
self.references[elf_object.uuid].append(dict(referenced_uuid=section_object.uuid, self.references[elf_object.uuid].append(dict(referenced_uuid=section_object.uuid,
relationship_type=relationship)) relationship_type=relationship))
def parse_pe(self, fileinfo, file_object): def parse_pe(self, fileinfo, file_object):
if not self.import_pe:
return
try: try:
peinfo = fileinfo['pe'] peinfo = fileinfo['pe']
except KeyError: except KeyError:
self.misp_event.add_object(**file_object) self.misp_event.add_object(file_object)
return return
pe_object = MISPObject('pe') pe_object = MISPObject('pe')
relationship = 'includes' relationship = 'includes'
file_object.add_reference(pe_object.uuid, relationship) file_object.add_reference(pe_object.uuid, relationship)
self.misp_event.add_object(**file_object) self.misp_event.add_object(file_object)
for field, mapping in pe_object_fields.items(): for field, mapping in pe_object_fields.items():
attribute_type, object_relation = mapping if peinfo.get(field) is not None:
pe_object.add_attribute(object_relation, **{'type': attribute_type, 'value': peinfo[field], 'to_ids': False}) attribute = {'value': peinfo[field], 'to_ids': False}
pe_object.add_attribute('compilation-timestamp', **{'type': 'datetime', 'value': int(peinfo['timestamp'].split()[0], 16), 'to_ids': False}) attribute.update(mapping)
pe_object.add_attribute(**attribute)
pe_object.add_attribute(
**{
'type': 'datetime',
'object_relation': 'compilation-timestamp',
'value': int(peinfo['timestamp'].split()[0], 16),
'to_ids': False
}
)
program_name = fileinfo['filename'] program_name = fileinfo['filename']
if peinfo['versions']: if peinfo['versions']:
for feature in peinfo['versions']['version']: for feature in peinfo['versions']['version']:
@ -314,33 +416,57 @@ class JoeParser():
if name == 'InternalName': if name == 'InternalName':
program_name = feature['value'] program_name = feature['value']
if name in pe_object_mapping: if name in pe_object_mapping:
pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value'], 'to_ids': False}) pe_object.add_attribute(
**{
'type': 'text',
'object_relation': pe_object_mapping[name],
'value': feature['value'],
'to_ids': False
}
)
sections_number = len(peinfo['sections']['section']) sections_number = len(peinfo['sections']['section'])
pe_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number, 'to_ids': False}) pe_object.add_attribute(
**{
'type': 'counter',
'object_relation': 'number-sections',
'value': sections_number,
'to_ids': False
}
)
signatureinfo = peinfo['signature'] signatureinfo = peinfo['signature']
if signatureinfo['signed']: if signatureinfo['signed']:
signerinfo_object = MISPObject('authenticode-signerinfo') signerinfo_object = MISPObject('authenticode-signerinfo')
pe_object.add_reference(signerinfo_object.uuid, 'signed-by') pe_object.add_reference(signerinfo_object.uuid, 'signed-by')
self.misp_event.add_object(**pe_object) self.misp_event.add_object(pe_object)
signerinfo_object.add_attribute('program-name', **{'type': 'text', 'value': program_name, 'to_ids': False}) signerinfo_object.add_attribute(
**{
'type': 'text',
'object_relation': 'program-name',
'value': program_name,
'to_ids': False
}
)
for feature, mapping in signerinfo_object_mapping.items(): for feature, mapping in signerinfo_object_mapping.items():
attribute_type, object_relation = mapping if signatureinfo.get(feature) is not None:
signerinfo_object.add_attribute(object_relation, **{'type': attribute_type, 'value': signatureinfo[feature], 'to_ids': False}) attribute = {'value': signatureinfo[feature], 'to_ids': False}
self.misp_event.add_object(**signerinfo_object) attribute.update(mapping)
signerinfo_object.add_attribute(**attribute)
self.misp_event.add_object(signerinfo_object)
else: else:
self.misp_event.add_object(**pe_object) self.misp_event.add_object(pe_object)
for section in peinfo['sections']['section']: for section in peinfo['sections']['section']:
section_object = self.parse_pe_section(section) section_object = self.parse_pe_section(section)
self.references[pe_object.uuid].append(dict(referenced_uuid=section_object.uuid, self.references[pe_object.uuid].append(dict(referenced_uuid=section_object.uuid,
relationship_type=relationship)) relationship_type=relationship))
self.misp_event.add_object(**section_object) self.misp_event.add_object(section_object)
def parse_pe_section(self, section): def parse_pe_section(self, section):
section_object = MISPObject('pe-section') section_object = MISPObject('pe-section')
for feature, mapping in pe_section_object_mapping.items(): for feature, mapping in pe_section_object_mapping.items():
if section.get(feature): if section.get(feature) is not None:
attribute_type, object_relation = mapping attribute = {'value': section[feature], 'to_ids': False}
section_object.add_attribute(object_relation, **{'type': attribute_type, 'value': section[feature], 'to_ids': False}) attribute.update(mapping)
section_object.add_attribute(**attribute)
return section_object return section_object
def parse_network_interactions(self): def parse_network_interactions(self):
@ -350,10 +476,11 @@ class JoeParser():
if domain['@ip'] != 'unknown': if domain['@ip'] != 'unknown':
domain_object = MISPObject('domain-ip') domain_object = MISPObject('domain-ip')
for key, mapping in domain_object_mapping.items(): for key, mapping in domain_object_mapping.items():
attribute_type, object_relation = mapping if domain.get(key) is not None:
domain_object.add_attribute(object_relation, attribute = {'value': domain[key], 'to_ids': False}
**{'type': attribute_type, 'value': domain[key], 'to_ids': False}) attribute.update(mapping)
self.misp_event.add_object(**domain_object) domain_object.add_attribute(**attribute)
self.misp_event.add_object(domain_object)
reference = dict(referenced_uuid=domain_object.uuid, relationship_type='contacts') reference = dict(referenced_uuid=domain_object.uuid, relationship_type='contacts')
self.add_process_reference(domain['@targetid'], domain['@currentpath'], reference) self.add_process_reference(domain['@targetid'], domain['@currentpath'], reference)
else: else:
@ -396,10 +523,19 @@ class JoeParser():
for call in registryactivities[feature]['call']: for call in registryactivities[feature]['call']:
registry_key = MISPObject('registry-key') registry_key = MISPObject('registry-key')
for field, mapping in regkey_object_mapping.items(): for field, mapping in regkey_object_mapping.items():
attribute_type, object_relation = mapping if call.get(field) is not None:
registry_key.add_attribute(object_relation, **{'type': attribute_type, 'value': call[field], 'to_ids': False}) attribute = {'value': call[field], 'to_ids': False}
registry_key.add_attribute('data-type', **{'type': 'text', 'value': 'REG_{}'.format(call['type'].upper()), 'to_ids': False}) attribute.update(mapping)
self.misp_event.add_object(**registry_key) registry_key.add_attribute(**attribute)
registry_key.add_attribute(
**{
'type': 'text',
'object_relation': 'data-type',
'value': f"REG_{call['type'].upper()}",
'to_ids': False
}
)
self.misp_event.add_object(registry_key)
self.references[process_uuid].append(dict(referenced_uuid=registry_key.uuid, self.references[process_uuid].append(dict(referenced_uuid=registry_key.uuid,
relationship_type=relationship)) relationship_type=relationship))
@ -429,8 +565,9 @@ class JoeParser():
@staticmethod @staticmethod
def prefetch_attributes_data(connection): def prefetch_attributes_data(connection):
attributes = {} attributes = []
for field, value in zip(network_behavior_fields, connection): for field, value in zip(network_behavior_fields, connection):
attribute_type, object_relation = network_connection_object_mapping[field] attribute = {'value': value, 'to_ids': False}
attributes[object_relation] = {'type': attribute_type, 'value': value, 'to_ids': False} attribute.update(network_connection_object_mapping[field])
attributes.append(attribute)
return attributes return attributes