From cba06ab3722fdb03658bbaca1b9765c68568a2e1 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 7 Mar 2022 17:53:43 +0100 Subject: [PATCH] fix: [joe parser] Some clean-up on the Joe parser --- misp_modules/lib/__init__.py | 1 + misp_modules/lib/joe_mapping.py | 114 +++++++++ misp_modules/lib/joe_parser.py | 405 +++++++++++++++++++++----------- 3 files changed, 386 insertions(+), 134 deletions(-) create mode 100644 misp_modules/lib/joe_mapping.py diff --git a/misp_modules/lib/__init__.py b/misp_modules/lib/__init__.py index 2939e75..dffa255 100644 --- a/misp_modules/lib/__init__.py +++ b/misp_modules/lib/__init__.py @@ -1,3 +1,4 @@ +import joe_mapping from .vt_graph_parser import * # noqa all = ['joe_parser', 'lastline_api', 'cof2misp', 'qintel_helper'] diff --git a/misp_modules/lib/joe_mapping.py b/misp_modules/lib/joe_mapping.py new file mode 100644 index 0000000..eda961e --- /dev/null +++ b/misp_modules/lib/joe_mapping.py @@ -0,0 +1,114 @@ +arch_type_mapping = { + 'ANDROID': 'parse_apk', + 'LINUX': 'parse_elf', + 'WINDOWS': 'parse_pe' +} +domain_object_mapping = { + '@ip': {'type': 'ip-dst', 'object_relation': 'ip'}, + '@name': {'type': 'domain', 'object_relation': 'domain'} +} +dropped_file_mapping = { + '@entropy': {'type': 'float', 'object_relation': 'entropy'}, + '@file': {'type': 'filename', 'object_relation': 'filename'}, + '@size': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'}, + '@type': {'type': 'mime-type', 'object_relation': 'mimetype'} +} +dropped_hash_mapping = { + 'MD5': 'md5', + 'SHA': 'sha1', + 'SHA-256': 'sha256', + 'SHA-512': 'sha512' +} +elf_object_mapping = { + 'epaddr': 'entrypoint-address', + 'machine': 'arch', + 'osabi': 'os_abi' +} +elf_section_flags_mapping = { + 'A': 'ALLOC', + 'I': 'INFO_LINK', + 'M': 'MERGE', + 'S': 'STRINGS', + 'T': 'TLS', + 'W': 'WRITE', + 'X': 'EXECINSTR' +} +file_object_fields = ( + 'filename', + 'md5', + 'sha1', + 'sha256', + 'sha512', + 'ssdeep' +) +file_object_mapping = { + 'entropy': {'type': 'float', 'object_relation': 'entropy'}, + 'filesize': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'}, + 'filetype': {'type': 'mime-type', 'object_relation': 'mimetype'} +} +file_references_mapping = { + 'fileCreated': 'creates', + 'fileDeleted': 'deletes', + 'fileMoved': 'moves', + 'fileRead': 'reads', + 'fileWritten': 'writes' +} +network_behavior_fields = ('srcip', 'dstip', 'srcport', 'dstport') +network_connection_object_mapping = { + 'srcip': {'type': 'ip-src', 'object_relation': 'ip-src'}, + 'dstip': {'type': 'ip-dst', 'object_relation': 'ip-dst'}, + 'srcport': {'type': 'port', 'object_relation': 'src-port'}, + 'dstport': {'type': 'port', 'object_relation': 'dst-port'} +} +pe_object_fields = { + 'entrypoint': {'type': 'text', 'object_relation': 'entrypoint-address'}, + 'imphash': {'type': 'imphash', 'object_relation': 'imphash'} +} +pe_object_mapping = { + 'CompanyName': 'company-name', + 'FileDescription': 'file-description', + 'FileVersion': 'file-version', + 'InternalName': 'internal-filename', + 'LegalCopyright': 'legal-copyright', + 'OriginalFilename': 'original-filename', + 'ProductName': 'product-filename', + 'ProductVersion': 'product-version', + 'Translation': 'lang-id' +} +pe_section_object_mapping = { + 'characteristics': {'type': 'text', 'object_relation': 'characteristic'}, + 'entropy': {'type': 'float', 'object_relation': 'entropy'}, + 'name': {'type': 'text', 'object_relation': 'name'}, + 'rawaddr': {'type': 'hex', 'object_relation': 'offset'}, + 'rawsize': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'}, + 'virtaddr': {'type': 'hex', 'object_relation': 'virtual_address'}, + 'virtsize': {'type': 'size-in-bytes', 'object_relation': 'virtual_size'} +} +process_object_fields = { + 'cmdline': 'command-line', + 'name': 'name', + 'parentpid': 'parent-pid', + 'pid': 'pid', + 'path': 'current-directory' +} +protocols = { + 'tcp': 4, + 'udp': 4, + 'icmp': 3, + 'http': 7, + 'https': 7, + 'ftp': 7 +} +registry_references_mapping = { + 'keyValueCreated': 'creates', + 'keyValueModified': 'modifies' +} +regkey_object_mapping = { + 'name': {'type': 'text', 'object_relation': 'name'}, + 'newdata': {'type': 'text', 'object_relation': 'data'}, + 'path': {'type': 'regkey', 'object_relation': 'key'} +} +signerinfo_object_mapping = { + 'sigissuer': {'type': 'text', 'object_relation': 'issuer'}, + 'version': {'type': 'text', 'object_relation': 'version'} +} diff --git a/misp_modules/lib/joe_parser.py b/misp_modules/lib/joe_parser.py index 8b400fe..8ae57a9 100644 --- a/misp_modules/lib/joe_parser.py +++ b/misp_modules/lib/joe_parser.py @@ -1,53 +1,15 @@ # -*- coding: utf-8 -*- +import json from collections import defaultdict from datetime import datetime from pymisp import MISPAttribute, MISPEvent, MISPObject -import json - - -arch_type_mapping = {'ANDROID': 'parse_apk', 'LINUX': 'parse_elf', 'WINDOWS': 'parse_pe'} -domain_object_mapping = {'@ip': ('ip-dst', 'ip'), '@name': ('domain', 'domain')} -dropped_file_mapping = {'@entropy': ('float', 'entropy'), - '@file': ('filename', 'filename'), - '@size': ('size-in-bytes', 'size-in-bytes'), - '@type': ('mime-type', 'mimetype')} -dropped_hash_mapping = {'MD5': 'md5', 'SHA': 'sha1', 'SHA-256': 'sha256', 'SHA-512': 'sha512'} -elf_object_mapping = {'epaddr': 'entrypoint-address', 'machine': 'arch', 'osabi': 'os_abi'} -elf_section_flags_mapping = {'A': 'ALLOC', 'I': 'INFO_LINK', 'M': 'MERGE', - 'S': 'STRINGS', 'T': 'TLS', 'W': 'WRITE', - 'X': 'EXECINSTR'} -file_object_fields = ['filename', 'md5', 'sha1', 'sha256', 'sha512', 'ssdeep'] -file_object_mapping = {'entropy': ('float', 'entropy'), - 'filesize': ('size-in-bytes', 'size-in-bytes'), - 'filetype': ('mime-type', 'mimetype')} -file_references_mapping = {'fileCreated': 'creates', 'fileDeleted': 'deletes', - 'fileMoved': 'moves', 'fileRead': 'reads', 'fileWritten': 'writes'} -network_behavior_fields = ('srcip', 'dstip', 'srcport', 'dstport') -network_connection_object_mapping = {'srcip': ('ip-src', 'ip-src'), 'dstip': ('ip-dst', 'ip-dst'), - 'srcport': ('port', 'src-port'), 'dstport': ('port', 'dst-port')} -pe_object_fields = {'entrypoint': ('text', 'entrypoint-address'), - 'imphash': ('imphash', 'imphash')} -pe_object_mapping = {'CompanyName': 'company-name', 'FileDescription': 'file-description', - 'FileVersion': 'file-version', 'InternalName': 'internal-filename', - 'LegalCopyright': 'legal-copyright', 'OriginalFilename': 'original-filename', - 'ProductName': 'product-filename', 'ProductVersion': 'product-version', - 'Translation': 'lang-id'} -pe_section_object_mapping = {'characteristics': ('text', 'characteristic'), - 'entropy': ('float', 'entropy'), - 'name': ('text', 'name'), 'rawaddr': ('hex', 'offset'), - 'rawsize': ('size-in-bytes', 'size-in-bytes'), - 'virtaddr': ('hex', 'virtual_address'), - 'virtsize': ('size-in-bytes', 'virtual_size')} -process_object_fields = {'cmdline': 'command-line', 'name': 'name', - 'parentpid': 'parent-pid', 'pid': 'pid', - 'path': 'current-directory'} -protocols = {'tcp': 4, 'udp': 4, 'icmp': 3, - 'http': 7, 'https': 7, 'ftp': 7} -registry_references_mapping = {'keyValueCreated': 'creates', 'keyValueModified': 'modifies'} -regkey_object_mapping = {'name': ('text', 'name'), 'newdata': ('text', 'data'), - 'path': ('regkey', 'key')} -signerinfo_object_mapping = {'sigissuer': ('text', 'issuer'), - 'version': ('text', 'version')} +from joe_mapping import (arch_type_mapping, domain_object_mapping, + dropped_file_mapping, dropped_hash_mapping, elf_object_mapping, + elf_section_flags_mapping, file_object_fields, file_object_mapping, + file_references_mapping, network_behavior_fields, + network_connection_object_mapping, pe_object_fields, pe_object_mapping, + pe_section_object_mapping, process_object_fields, protocols, + registry_references_mapping, regkey_object_mapping, signerinfo_object_mapping) class JoeParser(): @@ -57,7 +19,7 @@ class JoeParser(): self.attributes = defaultdict(lambda: defaultdict(set)) self.process_references = {} - self.import_pe = config["import_pe"] + self.import_executable = config["import_executable"] self.create_mitre_attack = config["mitre_attack"] def parse_data(self, data): @@ -101,26 +63,46 @@ class JoeParser(): for droppedfile in droppedinfo['hash']: file_object = MISPObject('file') for key, mapping in dropped_file_mapping.items(): - attribute_type, object_relation = mapping - file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': droppedfile[key], 'to_ids': False}) + if droppedfile.get(key) is not None: + attribute = {'value': droppedfile[key], 'to_ids': False} + attribute.update(mapping) + file_object.add_attribute(**attribute) if droppedfile['@malicious'] == 'true': - file_object.add_attribute('state', **{'type': 'text', 'value': 'Malicious', 'to_ids': False}) + file_object.add_attribute( + **{ + 'type': 'text', + 'object_relation': 'state', + 'value': 'Malicious', + 'to_ids': False + } + ) for h in droppedfile['value']: hash_type = dropped_hash_mapping[h['@algo']] - file_object.add_attribute(hash_type, **{'type': hash_type, 'value': h['$'], 'to_ids': False}) - self.misp_event.add_object(**file_object) - self.references[self.process_references[(int(droppedfile['@targetid']), droppedfile['@process'])]].append({ - 'referenced_uuid': file_object.uuid, - 'relationship_type': 'drops' - }) + file_object.add_attribute( + **{ + 'type': hash_type, + 'object_relation': hash_type, + 'value': h['$'], + 'to_ids': False + } + ) + self.misp_event.add_object(file_object) + reference_key = (int(droppedfile['@targetid']), droppedfile['@process']) + if reference_key in self.process_references: + self.references[self.process_references[reference_key]].append( + { + 'referenced_uuid': file_object.uuid, + 'relationship_type': 'drops' + } + ) def parse_mitre_attack(self): - mitreattack = self.data['mitreattack'] + mitreattack = self.data.get('mitreattack', {}) if mitreattack: for tactic in mitreattack['tactic']: if tactic.get('technique'): for technique in tactic['technique']: - self.misp_event.add_tag('misp-galaxy:mitre-attack-pattern="{} - {}"'.format(technique['name'], technique['id'])) + self.misp_event.add_tag(f'misp-galaxy:mitre-attack-pattern="{technique["name"]} - {technique["id"]}"') def parse_network_behavior(self): network = self.data['behavior']['network'] @@ -134,37 +116,65 @@ class JoeParser(): attributes = self.prefetch_attributes_data(connection) if len(data.keys()) == len(set(protocols[protocol] for protocol in data.keys())): network_connection_object = MISPObject('network-connection') - for object_relation, attribute in attributes.items(): - network_connection_object.add_attribute(object_relation, **attribute) - network_connection_object.add_attribute('first-packet-seen', - **{'type': 'datetime', - 'value': min(tuple(min(timestamp) for timestamp in data.values())), - 'to_ids': False}) + for attribute in attributes: + network_connection_object.add_attribute(**attribute) + network_connection_object.add_attribute( + **{ + 'type': 'datetime', + 'object_relation': 'first-packet-seen', + 'value': min(tuple(min(timestamp) for timestamp in data.values())), + 'to_ids': False + } + ) for protocol in data.keys(): - network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), - **{'type': 'text', 'value': protocol, 'to_ids': False}) - self.misp_event.add_object(**network_connection_object) + network_connection_object.add_attribute( + **{ + 'type': 'text', + 'object_relation': f'layer{protocols[protocol]}-protocol', + 'value': protocol, + 'to_ids': False + } + ) + self.misp_event.add_object(network_connection_object) self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=network_connection_object.uuid, relationship_type='initiates')) else: for protocol, timestamps in data.items(): network_connection_object = MISPObject('network-connection') - for object_relation, attribute in attributes.items(): - network_connection_object.add_attribute(object_relation, **attribute) - network_connection_object.add_attribute('first-packet-seen', **{'type': 'datetime', 'value': min(timestamps), 'to_ids': False}) - network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol, 'to_ids': False}) - self.misp_event.add_object(**network_connection_object) + for attribute in attributes: + network_connection_object.add_attribute(**attribute) + network_connection_object.add_attribute( + **{ + 'type': 'datetime', + 'object_relation': 'first-packet-seen', + 'value': min(timestamps), + 'to_ids': False + } + ) + network_connection_object.add_attribute( + **{ + 'type': 'text', + 'object_relation': f'layer{protocols[protocol]}-protocol', + 'value': protocol, + 'to_ids': False + } + ) + self.misp_event.add_object(network_connection_object) self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=network_connection_object.uuid, relationship_type='initiates')) def parse_screenshot(self): - screenshotdata = self.data['behavior']['screenshotdata'] - if screenshotdata: - screenshotdata = screenshotdata['interesting']['$'] - attribute = {'type': 'attachment', 'value': 'screenshot.jpg', - 'data': screenshotdata, 'disable_correlation': True, - 'to_ids': False} - self.misp_event.add_attribute(**attribute) + if self.data['behavior'].get('screenshotdata', {}).get('interesting') is not None: + screenshotdata = self.data['behavior']['screenshotdata']['interesting']['$'] + self.misp_event.add_attribute( + **{ + 'type': 'attachment', + 'value': 'screenshot.jpg', + 'data': screenshotdata, + 'disable_correlation': True, + 'to_ids': False + } + ) def parse_system_behavior(self): if not 'system' in self.data['behavior']: @@ -177,10 +187,24 @@ class JoeParser(): general = process['general'] process_object = MISPObject('process') for feature, relation in process_object_fields.items(): - process_object.add_attribute(relation, **{'type': 'text', 'value': general[feature], 'to_ids': False}) - start_time = datetime.strptime('{} {}'.format(general['date'], general['time']), '%d/%m/%Y %H:%M:%S') - process_object.add_attribute('start-time', **{'type': 'datetime', 'value': start_time, 'to_ids': False}) - self.misp_event.add_object(**process_object) + process_object.add_attribute( + **{ + 'type': 'text', + 'object_relation': relation, + 'value': general[feature], + 'to_ids': False + } + ) + start_time = datetime.strptime(f"{general['date']} {general['time']}", '%d/%m/%Y %H:%M:%S') + process_object.add_attribute( + **{ + 'type': 'datetime', + 'object_relation': 'start-time', + 'value': start_time, + 'to_ids': False + } + ) + self.misp_event.add_object(process_object) for field, to_call in process_activities.items(): if process.get(field): to_call(process_object.uuid, process[field]) @@ -213,9 +237,15 @@ class JoeParser(): url_object = MISPObject("url") self.analysisinfo_uuid = url_object.uuid - - url_object.add_attribute("url", generalinfo["target"]["url"], to_ids=False) - self.misp_event.add_object(**url_object) + url_object.add_attribute( + **{ + 'type': 'url', + 'object_relation': 'url', + 'value': generalinfo["target"]["url"], + 'to_ids': False + } + ) + self.misp_event.add_object(url_object) def parse_fileinfo(self): fileinfo = self.data['fileinfo'] @@ -224,20 +254,29 @@ class JoeParser(): self.analysisinfo_uuid = file_object.uuid for field in file_object_fields: - file_object.add_attribute(field, **{'type': field, 'value': fileinfo[field], 'to_ids': False}) + file_object.add_attribute( + **{ + 'type': field, + 'object_relation': field, + 'value': fileinfo[field], + 'to_ids': False + } + ) for field, mapping in file_object_mapping.items(): - attribute_type, object_relation = mapping - file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': fileinfo[field], 'to_ids': False}) + if fileinfo.get(field) is not None: + attribute = {'value': fileinfo[field], 'to_ids': False} + attribute.update(mapping) + file_object.add_attribute(**attribute) arch = self.data['generalinfo']['arch'] - if arch in arch_type_mapping: + if self.import_executable and arch in arch_type_mapping: to_call = arch_type_mapping[arch] getattr(self, to_call)(fileinfo, file_object) else: - self.misp_event.add_object(**file_object) + self.misp_event.add_object(file_object) def parse_apk(self, fileinfo, file_object): apkinfo = fileinfo['apk'] - self.misp_event.add_object(**file_object) + self.misp_event.add_object(file_object) permission_lists = defaultdict(list) for permission in apkinfo['requiredpermissions']['permission']: permission = permission['@name'].split('.') @@ -245,16 +284,30 @@ class JoeParser(): attribute_type = 'text' for comment, permissions in permission_lists.items(): permission_object = MISPObject('android-permission') - permission_object.add_attribute('comment', **dict(type=attribute_type, value=comment, to_ids=False)) + permission_object.add_attribute( + **{ + 'type': attribute_type, + 'object_relation': 'comment', + 'value': comment, + 'to_ids': False + } + ) for permission in permissions: - permission_object.add_attribute('permission', **dict(type=attribute_type, value=permission, to_ids=False)) - self.misp_event.add_object(**permission_object) + permission_object.add_attribute( + **{ + 'type': attribute_type, + 'object_relation': 'permission', + 'value': permission, + 'to_ids': False + } + ) + self.misp_event.add_object(permission_object) self.references[file_object.uuid].append(dict(referenced_uuid=permission_object.uuid, relationship_type='grants')) def parse_elf(self, fileinfo, file_object): elfinfo = fileinfo['elf'] - self.misp_event.add_object(**file_object) + self.misp_event.add_object(file_object) attribute_type = 'text' relationship = 'includes' size = 'size-in-bytes' @@ -266,47 +319,96 @@ class JoeParser(): if elf.get('type'): # Haven't seen anything but EXEC yet in the files I tested attribute_value = "EXECUTABLE" if elf['type'] == "EXEC (Executable file)" else elf['type'] - elf_object.add_attribute('type', **dict(type=attribute_type, value=attribute_value, to_ids=False)) + elf_object.add_attribute( + **{ + 'type': attribute_type, + 'object_relation': 'type', + 'value': attribute_value, + 'to_ids': False + } + ) for feature, relation in elf_object_mapping.items(): if elf.get(feature): - elf_object.add_attribute(relation, **dict(type=attribute_type, value=elf[feature], to_ids=False)) + elf_object.add_attribute( + **{ + 'type': attribute_type, + 'object_relation': relation, + 'value': elf[feature], + 'to_ids': False + } + ) sections_number = len(fileinfo['sections']['section']) - elf_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number, 'to_ids': False}) - self.misp_event.add_object(**elf_object) + elf_object.add_attribute( + **{ + 'type': 'counter', + 'object_relation': 'number-sections', + 'value': sections_number, + 'to_ids': False + } + ) + self.misp_event.add_object(elf_object) for section in fileinfo['sections']['section']: section_object = MISPObject('elf-section') for feature in ('name', 'type'): if section.get(feature): - section_object.add_attribute(feature, **dict(type=attribute_type, value=section[feature], to_ids=False)) + section_object.add_attribute( + **{ + 'type': attribute_type, + 'object_relation': feature, + 'value': section[feature], + 'to_ids': False + } + ) if section.get('size'): - section_object.add_attribute(size, **dict(type=size, value=int(section['size'], 16), to_ids=False)) + section_object.add_attribute( + **{ + 'type': size, + 'object_relation': size, + 'value': int(section['size'], 16), + 'to_ids': False + } + ) for flag in section['flagsdesc']: try: attribute_value = elf_section_flags_mapping[flag] - section_object.add_attribute('flag', **dict(type=attribute_type, value=attribute_value, to_ids=False)) + section_object.add_attribute( + **{ + 'type': attribute_type, + 'object_relation': 'flag', + 'value': attribute_value, + 'to_ids': False + } + ) except KeyError: print(f'Unknown elf section flag: {flag}') continue - self.misp_event.add_object(**section_object) + self.misp_event.add_object(section_object) self.references[elf_object.uuid].append(dict(referenced_uuid=section_object.uuid, relationship_type=relationship)) def parse_pe(self, fileinfo, file_object): - if not self.import_pe: - return try: peinfo = fileinfo['pe'] except KeyError: - self.misp_event.add_object(**file_object) + self.misp_event.add_object(file_object) return pe_object = MISPObject('pe') relationship = 'includes' file_object.add_reference(pe_object.uuid, relationship) - self.misp_event.add_object(**file_object) + self.misp_event.add_object(file_object) for field, mapping in pe_object_fields.items(): - attribute_type, object_relation = mapping - pe_object.add_attribute(object_relation, **{'type': attribute_type, 'value': peinfo[field], 'to_ids': False}) - pe_object.add_attribute('compilation-timestamp', **{'type': 'datetime', 'value': int(peinfo['timestamp'].split()[0], 16), 'to_ids': False}) + if peinfo.get(field) is not None: + attribute = {'value': peinfo[field], 'to_ids': False} + attribute.update(mapping) + pe_object.add_attribute(**attribute) + pe_object.add_attribute( + **{ + 'type': 'datetime', + 'object_relation': 'compilation-timestamp', + 'value': int(peinfo['timestamp'].split()[0], 16), + 'to_ids': False + } + ) program_name = fileinfo['filename'] if peinfo['versions']: for feature in peinfo['versions']['version']: @@ -314,33 +416,57 @@ class JoeParser(): if name == 'InternalName': program_name = feature['value'] if name in pe_object_mapping: - pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value'], 'to_ids': False}) + pe_object.add_attribute( + **{ + 'type': 'text', + 'object_relation': pe_object_mapping[name], + 'value': feature['value'], + 'to_ids': False + } + ) sections_number = len(peinfo['sections']['section']) - pe_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number, 'to_ids': False}) + pe_object.add_attribute( + **{ + 'type': 'counter', + 'object_relation': 'number-sections', + 'value': sections_number, + 'to_ids': False + } + ) signatureinfo = peinfo['signature'] if signatureinfo['signed']: signerinfo_object = MISPObject('authenticode-signerinfo') pe_object.add_reference(signerinfo_object.uuid, 'signed-by') - self.misp_event.add_object(**pe_object) - signerinfo_object.add_attribute('program-name', **{'type': 'text', 'value': program_name, 'to_ids': False}) + self.misp_event.add_object(pe_object) + signerinfo_object.add_attribute( + **{ + 'type': 'text', + 'object_relation': 'program-name', + 'value': program_name, + 'to_ids': False + } + ) for feature, mapping in signerinfo_object_mapping.items(): - attribute_type, object_relation = mapping - signerinfo_object.add_attribute(object_relation, **{'type': attribute_type, 'value': signatureinfo[feature], 'to_ids': False}) - self.misp_event.add_object(**signerinfo_object) + if signatureinfo.get(feature) is not None: + attribute = {'value': signatureinfo[feature], 'to_ids': False} + attribute.update(mapping) + signerinfo_object.add_attribute(**attribute) + self.misp_event.add_object(signerinfo_object) else: - self.misp_event.add_object(**pe_object) + self.misp_event.add_object(pe_object) for section in peinfo['sections']['section']: section_object = self.parse_pe_section(section) self.references[pe_object.uuid].append(dict(referenced_uuid=section_object.uuid, relationship_type=relationship)) - self.misp_event.add_object(**section_object) + self.misp_event.add_object(section_object) def parse_pe_section(self, section): section_object = MISPObject('pe-section') for feature, mapping in pe_section_object_mapping.items(): - if section.get(feature): - attribute_type, object_relation = mapping - section_object.add_attribute(object_relation, **{'type': attribute_type, 'value': section[feature], 'to_ids': False}) + if section.get(feature) is not None: + attribute = {'value': section[feature], 'to_ids': False} + attribute.update(mapping) + section_object.add_attribute(**attribute) return section_object def parse_network_interactions(self): @@ -350,10 +476,11 @@ class JoeParser(): if domain['@ip'] != 'unknown': domain_object = MISPObject('domain-ip') for key, mapping in domain_object_mapping.items(): - attribute_type, object_relation = mapping - domain_object.add_attribute(object_relation, - **{'type': attribute_type, 'value': domain[key], 'to_ids': False}) - self.misp_event.add_object(**domain_object) + if domain.get(key) is not None: + attribute = {'value': domain[key], 'to_ids': False} + attribute.update(mapping) + domain_object.add_attribute(**attribute) + self.misp_event.add_object(domain_object) reference = dict(referenced_uuid=domain_object.uuid, relationship_type='contacts') self.add_process_reference(domain['@targetid'], domain['@currentpath'], reference) else: @@ -396,10 +523,19 @@ class JoeParser(): for call in registryactivities[feature]['call']: registry_key = MISPObject('registry-key') for field, mapping in regkey_object_mapping.items(): - attribute_type, object_relation = mapping - registry_key.add_attribute(object_relation, **{'type': attribute_type, 'value': call[field], 'to_ids': False}) - registry_key.add_attribute('data-type', **{'type': 'text', 'value': 'REG_{}'.format(call['type'].upper()), 'to_ids': False}) - self.misp_event.add_object(**registry_key) + if call.get(field) is not None: + attribute = {'value': call[field], 'to_ids': False} + attribute.update(mapping) + registry_key.add_attribute(**attribute) + registry_key.add_attribute( + **{ + 'type': 'text', + 'object_relation': 'data-type', + 'value': f"REG_{call['type'].upper()}", + 'to_ids': False + } + ) + self.misp_event.add_object(registry_key) self.references[process_uuid].append(dict(referenced_uuid=registry_key.uuid, relationship_type=relationship)) @@ -429,8 +565,9 @@ class JoeParser(): @staticmethod def prefetch_attributes_data(connection): - attributes = {} + attributes = [] for field, value in zip(network_behavior_fields, connection): - attribute_type, object_relation = network_connection_object_mapping[field] - attributes[object_relation] = {'type': attribute_type, 'value': value, 'to_ids': False} + attribute = {'value': value, 'to_ids': False} + attribute.update(network_connection_object_mapping[field]) + attributes.append(attribute) return attributes