From 28cd4b0896d7dfdc1438b4f916fe41cbed46efa9 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Tue, 11 Dec 2018 12:07:08 +0100 Subject: [PATCH] chg: human friendly names for objects --- .../transforms/common/entities.py | 8 +- src/MISP_maltego/transforms/common/util.py | 78 +++++++++++++++---- .../transforms/eventtoattributes.py | 4 +- 3 files changed, 71 insertions(+), 19 deletions(-) diff --git a/src/MISP_maltego/transforms/common/entities.py b/src/MISP_maltego/transforms/common/entities.py index 7368739..3f75fb5 100644 --- a/src/MISP_maltego/transforms/common/entities.py +++ b/src/MISP_maltego/transforms/common/entities.py @@ -50,8 +50,8 @@ class MISPGalaxy(Entity): uuid = StringEntityField('uuid', display_name='UUID') name = StringEntityField('name', display_name='Name', is_value=True) - description = StringEntityField('description', display_name='Description') - cluster_type = StringEntityField('galaxy_type', display_name='Type') - cluster_value = StringEntityField('value', display_name='Value') - synonyms = StringEntityField('synonyms', display_name='Synonyms') + description = StringEntityField('description', display_name='Description', matching_rule=MatchingRule.Loose) + cluster_type = StringEntityField('galaxy_type', display_name='Type', matching_rule=MatchingRule.Loose) + cluster_value = StringEntityField('value', display_name='Value', matching_rule=MatchingRule.Loose) + synonyms = StringEntityField('synonyms', display_name='Synonyms', matching_rule=MatchingRule.Loose) tag_name = StringEntityField('tag_name', display_name='Tag') diff --git a/src/MISP_maltego/transforms/common/util.py b/src/MISP_maltego/transforms/common/util.py index 9d71345..92ac322 100644 --- a/src/MISP_maltego/transforms/common/util.py +++ b/src/MISP_maltego/transforms/common/util.py @@ -95,13 +95,20 @@ mapping_misp_to_maltego = { 'twitter-id': [Twitter], # object mappings 'nameserver': [NSRecord], - # FIXME add more object mappings + # TODO add more object mappings # custom types created internally for technical reasons # 'rekey_value': [Unknown] } +misp_connection = None -def get_misp_connection(config): + +def get_misp_connection(config=None): + global misp_connection + if misp_connection: + return misp_connection + if not config: + raise Exception("ERROR: MISP connection not yet established, and config not provided as parameter.") if config['MISP_maltego.local.misp_verify'] in ['True', 'true', 1, 'yes', 'Yes']: misp_verify = True else: @@ -110,12 +117,13 @@ def get_misp_connection(config): misp_debug = True else: misp_debug = False - return PyMISP(config['MISP_maltego.local.misp_url'], config['MISP_maltego.local.misp_key'], misp_verify, 'json', misp_debug) + misp_connection = PyMISP(config['MISP_maltego.local.misp_url'], config['MISP_maltego.local.misp_key'], misp_verify, 'json', misp_debug) + return misp_connection def entity_obj_to_entity(entity_obj, v, t, **kwargs): if entity_obj == Hash: - return entity_obj(v, _type=t, **kwargs) # FIXME type is conflicting with type of Entity, Report this as bug see line 326 /usr/local/lib/python3.5/dist-packages/canari/maltego/entities.py + return entity_obj(v, _type=t, **kwargs) # LATER type is conflicting with type of Entity, Report this as bug see line 326 /usr/local/lib/python3.5/dist-packages/canari/maltego/entities.py return entity_obj(v, **kwargs) @@ -125,7 +133,7 @@ def attribute_to_entity(a, link_label=None): a['data'] = None # empty the file content as we really don't need this here # FIXME feature request for misp.get_event() to not get attachment content if a['type'] == 'malware-sample': a['type'] = 'filename|md5' - if a['type'] == 'regkey|value': # FIXME regkey|value => needs to be a special non-combined object + if a['type'] == 'regkey|value': # LATER regkey|value => needs to be a special non-combined object a['type'] = 'regkey' # special cases @@ -180,12 +188,51 @@ def attribute_to_entity(a, link_label=None): if t['name'].startswith('misp-galaxy'): continue yield Hashtag(t['name']) - # TODO : relationships from attributes - not yet supported by MISP yet, but there are references in the datamodel + # LATER : relationships from attributes - not yet supported by MISP yet, but there are references in the datamodel def object_to_entity(o, link_label=None): + # Generate a human readable display-name: + # - find the first RequiredOneOf that exists + # - if none, use the first RequiredField + # LATER further finetune the human readable version of this object + misp = get_misp_connection() + o_template = misp.get_object_template_id(o['template_uuid']) + human_readable = None + try: + found = False + while not found: # the while loop is broken once something is found, or the requiredOneOf has no elements left + required_ote_type = o_template['ObjectTemplate']['requirements']['requiredOneOf'].pop(0) + for ote in o_template['ObjectTemplateElement']: + if ote['object_relation'] == required_ote_type: + required_a_type = ote['type'] + break + for a in o['Attribute']: + if a['type'] == required_a_type: + human_readable = '{}: {}'.format(o['name'], a['value']) + found = True + break + except Exception: + pass + if not human_readable: + try: + found = False + parts = [] + for required_ote_type in o_template['ObjectTemplate']['requirements']['required']: + for ote in o_template['ObjectTemplateElement']: + if ote['object_relation'] == required_ote_type: + required_a_type = ote['type'] + break + for a in o['Attribute']: + if a['type'] == required_a_type: + parts.append(a['value']) + break + human_readable = '{}: {}'.format(o['name'], '|'.join(parts)) + except Exception: + human_readable = o['name'] + pass return MISPObject( - o['name'], + human_readable, uuid=o['uuid'], event_id=int(o['event_id']), meta_category=o.get('meta_category'), @@ -195,7 +242,7 @@ def object_to_entity(o, link_label=None): ) -def object_to_attributes(o): +def object_to_attributes(o, e): # first process attributes from an object that belong together (eg: first-name + last-name), and remove them from the list if o['name'] == 'person': first_name = get_attribute_in_object(o, 'first-name', drop=True).get('value') @@ -212,15 +259,22 @@ def object_to_attributes(o): for ref in o['ObjectReference']: # the reference is an Object if ref.get('Object'): - ref['Object']['event_id'] = ref['event_id'] # FIXME remove this ugly workaround - object can't be requested directly from MISP, and to find a full object we need the event_id - yield object_to_entity(ref['Object'], link_label=ref['relationship_type']) + # get the full object in the event, as our objectReference included does not contain everything we need + sub_object = get_object_in_event(ref['Object']['uuid'], e) + yield object_to_entity(sub_object, link_label=ref['relationship_type']) # the reference is an Attribute if ref.get('Attribute'): - ref['Attribute']['event_id'] = ref['event_id'] # FIXME remove this ugly workaround - object can't be requested directly from MISP, and to find a full object we need the event_id + ref['Attribute']['event_id'] = ref['event_id'] # LATER remove this ugly workaround - object can't be requested directly from MISP using the uuid, and to find a full object we need the event_id for item in attribute_to_entity(ref['Attribute'], link_label=ref['relationship_type']): yield item +def get_object_in_event(uuid, e): + for o in e['Event']['Object']: + if o['uuid'] == uuid: + return o + + def get_attribute_in_object(o, attribute_type, drop=False): '''Gets the first attribute of a specific type within an object''' found_attribute = {'value': ''} @@ -238,7 +292,6 @@ def event_to_entity(e, link_style=LinkStyle.Normal): def galaxycluster_to_entity(c, link_label=None): - # print(json.dumps(c, sort_keys=True, indent=4)) if c['meta'].get('synonyms'): synonyms = ', '.join(c['meta']['synonyms']) else: @@ -302,7 +355,6 @@ def galaxy_update_local_copy(force=False): with open(fullPathClusters) as fp: galaxy = json.load(fp) for cluster in galaxy['values']: - # print(cluster['uuid']) if 'uuid' not in cluster: continue # keep track of the cluster, but also enhance it to look like the cluster we receive when accessing the web. diff --git a/src/MISP_maltego/transforms/eventtoattributes.py b/src/MISP_maltego/transforms/eventtoattributes.py index bcd6ff3..857b8f2 100644 --- a/src/MISP_maltego/transforms/eventtoattributes.py +++ b/src/MISP_maltego/transforms/eventtoattributes.py @@ -30,7 +30,7 @@ class EventToAttributes(Transform): maltego_misp_event = request.entity # print(dir(maltego_misp_event)) misp = get_misp_connection(config) - event_json = misp.get_event(maltego_misp_event.id) # FIXME get it without attachments + event_json = misp.get_event(maltego_misp_event.id) # FIXME get it without attachments # FIXME use search + includeAttachments:0, eventid: as request body # print(json.dumps(event_json, sort_keys=True, indent=4)) if not event_json.get('Event'): return response @@ -72,7 +72,7 @@ class ObjectToAttributes(Transform): event_json = misp.get_event(maltego_object.event_id) for o in event_json['Event']['Object']: if o['uuid'] == maltego_object.uuid: - for entity in object_to_attributes(o): + for entity in object_to_attributes(o, event_json): if entity: response += entity