diff --git a/README.md b/README.md index 60e3167..5740619 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,6 @@ Alternatively initiate a transform on an existing Maltego entity. The currently supported entities are: `AS`, `DNSName`, `Domain`, `EmailAddress`, `File`, `Hash`, `IPv4Address`, `NSRecord`, `Person`, `PhoneNumber`, `URL`, `Website` -Dependencies: -* [PyMISP](https://github.com/MISP/PyMISP) -* [Canari3](https://github.com/redcanari/canari3) - ## Installation and User Guide: Installation is fairly easy, just read the steps in the [documentation](https://github.com/MISP/MISP-maltego/blob/master/doc/README.md). diff --git a/doc/README.md b/doc/README.md index c2e874c..177cd04 100644 --- a/doc/README.md +++ b/doc/README.md @@ -36,7 +36,7 @@ In this use case we will be using already existing entities and will initiate a Example: * create an entity `domain` with the value `1dnscontrol.com`. * right click and choose *Local Transforms* > *MISP_maltego* > *Domain To Event* -![animated screenshot](https://github.com/MISP/MISP-maltego/blob/master/doc/img/usecase1-transform.gif) +![animated screenshot](https://raw.githubusercontent.com/MISP/MISP-maltego/master/doc/img/usecase1-transform.gif) * continue loading transforms on the *MISP Event* ## Transform from MISP Event ID @@ -45,14 +45,20 @@ While MISP already has a graphing capability we would like to use the power of M * One **manual** way is to right click and choose *Local Transforms* > *MISP_maltego* > *Event To Attributes* * Notice the event is transformed to *Attributes*, *Objects*, *Tags*, *Galaxies* and related *MISP Events* * You can now further transform on an *Object* > *Object To Attributes* and see the content of the object -![machine transforms](https://github.com/MISP/MISP-maltego/blob/master/doc/img/usecase2-manual.gif) +![machine transforms](https://raw.githubusercontent.com/MISP/MISP-maltego/master/doc/img/usecase2-manual.gif) * Alternatively you can also use the **Maltego Machine** to speed up things. * Click on the *MISP Event* and in the left menu choose *Event to All* in the *Machines* section. -![machine transforms](https://github.com/MISP/MISP-maltego/blob/master/doc/img/usecase2-machine-menu.png) +![machine transforms](https://raw.githubusercontent.com/MISP/MISP-maltego/master/doc/img/usecase2-machine-menu.png) * Notice that the whole event, objects and such will get expanded with data from your MISP instance. -![animated screenshot](https://github.com/MISP/MISP-maltego/blob/master/doc/img/usecase2-machine.gif) +![animated screenshot](https://raw.githubusercontent.com/MISP/MISP-maltego/master/doc/img/usecase2-machine.gif) * You can now further transform on any data. - + +## Which data is already in MISP? +If you use MISP as central database it can be quite convenient to know which data is present in MISP, and which data is not; especially after using a number of other transforms. +To permit this MISP-Maltego will always add a green bookmark to all the data that is present in MISP. +![green bookmark](https://raw.githubusercontent.com/MISP/MISP-maltego/master/doc/img/usecase3-bookmark.png) + + ## Transform from Galaxy TODO ## Visualise MITRE ATT&CK diff --git a/doc/img/usecase3-bookmark.png b/doc/img/usecase3-bookmark.png new file mode 100644 index 0000000..2dcaad9 Binary files /dev/null and b/doc/img/usecase3-bookmark.png differ diff --git a/setup.py b/setup.py index d31dee6..44e8a07 100755 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ setup( package_data={ '': ['*.gif', '*.png', '*.conf', '*.mtz', '*.machine'] # list of resources }, + python_requires='>=3.5', install_requires=[ 'canari>=3.3.9,<4', 'PyMISP' diff --git a/src/MISP_maltego/resources/maltego/entities.mtz b/src/MISP_maltego/resources/maltego/entities.mtz index d52217b..5d82027 100644 Binary files a/src/MISP_maltego/resources/maltego/entities.mtz and b/src/MISP_maltego/resources/maltego/entities.mtz differ diff --git a/src/MISP_maltego/resources/maltego/misp_MISPEventToAll.machine b/src/MISP_maltego/resources/maltego/misp_MISPEventToAll.machine index 30b7167..fcc10b1 100644 --- a/src/MISP_maltego/resources/maltego/misp_MISPEventToAll.machine +++ b/src/MISP_maltego/resources/maltego/misp_MISPEventToAll.machine @@ -1,12 +1,19 @@ -machine("misp.MISPEventToAll", - displayName:"Event to All", - author:"Christophe Vandeplas", - description: "Automatically expands MISP Objects to their attributes") { - start { - run("MISP_maltego.EventToAttributes") - paths { - run("MISP_maltego.ObjectToAttributes") - run("MISP_maltego.GalaxyToRelations") - } - } -} +machine("misp.MISPEventToAll", + displayName:"Event to All", + author:"Christophe Vandeplas", + description: "Automatically expands MISP Objects to their attributes") { + start { + run("MISP_maltego.EventToAttributes") + run("MISP_maltego.AttributeToEvent") + paths { + path { + run("MISP_maltego.ObjectToAttributes") + run("MISP_maltego.AttributeToEvent") + } + path { + run("MISP_maltego.GalaxyToRelations") + } + } + + } +} diff --git a/src/MISP_maltego/transforms/attributetoevent.py b/src/MISP_maltego/transforms/attributetoevent.py index f40e15b..cea3db4 100644 --- a/src/MISP_maltego/transforms/attributetoevent.py +++ b/src/MISP_maltego/transforms/attributetoevent.py @@ -1,8 +1,7 @@ -from canari.maltego.entities import Hash, Domain, IPv4Address, URL, DNSName, AS, Website, NSRecord, PhoneNumber, EmailAddress, File, Hashtag, Company, Alias, Twitter from canari.maltego.transform import Transform # from canari.framework import EnableDebugWindow -from MISP_maltego.transforms.common.entities import MISPEvent -from MISP_maltego.transforms.common.util import get_misp_connection, event_to_entity +from MISP_maltego.transforms.common.util import get_misp_connection, event_to_entity, get_attribute_in_event, attribute_to_entity +from MISP_maltego.transforms.common.entities import Unknown __author__ = 'Christophe Vandeplas' __copyright__ = 'Copyright 2018, MISP_maltego Project' @@ -16,86 +15,68 @@ __status__ = 'Development' # @EnableDebugWindow -class AttributeToEvent(Transform): - # The transform input entity type. - input_type = None +class AttributeInMISP(Transform): + """Green bookmark if known in MISP""" + display_name = 'in MISP?' + input_type = Unknown def do_transform(self, request, response, config): maltego_misp_attribute = request.entity - misp = get_misp_connection(config) - # misp. - events_json = misp.search(controller='events', values=maltego_misp_attribute.value, withAttachments=False) + # skip MISP Events (value = int) + try: + int(maltego_misp_attribute.value) + return response + except Exception: + pass + misp = get_misp_connection(config) + events_json = misp.search(controller='events', values=maltego_misp_attribute.value, withAttachments=False) + in_misp = False for e in events_json['response']: + in_misp = True + break + # find the object again, and bookmark it green + # we need to do really rebuild the Entity from scratch as request.entity is of type Unknown + if in_misp: + for e in events_json['response']: + attr = get_attribute_in_event(e, maltego_misp_attribute.value) + if attr: + for item in attribute_to_entity(attr, only_self=True): + response += item + return response + + +# @EnableDebugWindow +class AttributeToEvent(Transform): + display_name = 'to MISP Event' + input_type = Unknown + + def do_transform(self, request, response, config): + maltego_misp_attribute = request.entity + # skip MISP Events (value = int) + try: + int(maltego_misp_attribute.value) + return response + except Exception: + pass + + misp = get_misp_connection(config) + events_json = misp.search(controller='events', values=maltego_misp_attribute.value, withAttachments=False) + in_misp = False + for e in events_json['response']: + in_misp = True response += event_to_entity(e) + # find the object again, and bookmark it green + # we need to do really rebuild the Entity from scratch as request.entity is of type Unknown + if in_misp: + for e in events_json['response']: + attr = get_attribute_in_event(e, maltego_misp_attribute.value) + if attr: + for item in attribute_to_entity(attr, only_self=True): + response += item return response def on_terminate(self): """This method gets called when transform execution is prematurely terminated. It is only applicable for local transforms. It can be excluded if you don't need it.""" pass - - -class HashToEvent(AttributeToEvent): - input_type = Hash - - -class DomainToEvent(AttributeToEvent): - input_type = Domain - - -class IPv4AddressToEvent(AttributeToEvent): - display_name = 'IPv4AddressToEvent' - input_type = IPv4Address - - -class URLToEvent(AttributeToEvent): - display_name = 'URLToEvent' - input_type = URL - - -class DNSNameToEvent(AttributeToEvent): - display_name = 'DNSNameToEvent' - input_type = DNSName - - -class ASToEvent(AttributeToEvent): - display_name = 'ASToEvent' - input_type = AS - - -class WebsiteToEvent(AttributeToEvent): - input_type = Website - - -class NSRecordToEvent(AttributeToEvent): - display_name = 'NSRecordToEvent' - input_type = NSRecord - - -class PhoneNumberToEvent(AttributeToEvent): - input_type = PhoneNumber - - -class EmailAddressToEvent(AttributeToEvent): - input_type = EmailAddress - - -class FileToEvent(AttributeToEvent): - input_type = File - - -class HashtagToEvent(AttributeToEvent): - input_type = Hashtag - - -class AliasToEvent(AttributeToEvent): - input_type = Alias - - -class TwitterToEvent(AttributeToEvent): - input_type = Twitter - - -class CompanyToEvent(AttributeToEvent): - input_type = Company diff --git a/src/MISP_maltego/transforms/common/entities.py b/src/MISP_maltego/transforms/common/entities.py index 3f75fb5..c32f89d 100644 --- a/src/MISP_maltego/transforms/common/entities.py +++ b/src/MISP_maltego/transforms/common/entities.py @@ -13,10 +13,16 @@ __status__ = 'Development' __all__ = [ 'MISPEvent', 'MISPObject', - 'MISPGalaxy' + 'MISPGalaxy', + 'Unknown' ] +class Unknown(Entity): + _category_ = 'Unknown' + _namespace_ = 'maltego' + + class MISPEvent(Entity): _category_ = 'MISP' _namespace_ = 'misp' diff --git a/src/MISP_maltego/transforms/common/util.py b/src/MISP_maltego/transforms/common/util.py index d22bc40..703af50 100644 --- a/src/MISP_maltego/transforms/common/util.py +++ b/src/MISP_maltego/transforms/common/util.py @@ -1,6 +1,6 @@ -from canari.maltego.entities import Unknown, Hash, Domain, IPv4Address, URL, DNSName, AS, Website, NSRecord, PhoneNumber, EmailAddress, File, Person, Hashtag, Location, Company, Alias, Port, Twitter -from MISP_maltego.transforms.common.entities import MISPEvent, MISPObject, MISPGalaxy -from canari.maltego.message import UIMessageType, UIMessage, Label, LinkStyle +from canari.maltego.entities import Hash, Domain, IPv4Address, URL, DNSName, AS, Website, NSRecord, PhoneNumber, EmailAddress, File, Person, Hashtag, Location, Company, Alias, Port, Twitter +from MISP_maltego.transforms.common.entities import MISPEvent, MISPObject, MISPGalaxy, Unknown +from canari.maltego.message import UIMessageType, UIMessage, Label, LinkStyle, MaltegoException, Bookmark from pymisp import PyMISP import json import os @@ -110,7 +110,7 @@ def get_misp_connection(config=None): if misp_connection: return misp_connection if not config: - raise Exception("ERROR: MISP connection not yet established, and config not provided as parameter.") + raise MaltegoException("ERROR: MISP connection not yet established, and config not provided as parameter.") if config['MISP_maltego.local.misp_verify'] in ['True', 'true', 1, 'yes', 'Yes']: misp_verify = True else: @@ -119,7 +119,10 @@ def get_misp_connection(config=None): misp_debug = True else: misp_debug = False - misp_connection = PyMISP(config['MISP_maltego.local.misp_url'], config['MISP_maltego.local.misp_key'], misp_verify, 'json', misp_debug) + try: + misp_connection = PyMISP(config['MISP_maltego.local.misp_url'], config['MISP_maltego.local.misp_key'], misp_verify, 'json', misp_debug) + except Exception: + raise MaltegoException("ERROR: Cannot connect to MISP server. Please verify your MISP_Maltego.conf settings") return misp_connection @@ -130,7 +133,7 @@ def entity_obj_to_entity(entity_obj, v, t, **kwargs): return entity_obj(v, **kwargs) -def attribute_to_entity(a, link_label=None, event_tags=None): +def attribute_to_entity(a, link_label=None, event_tags=None, only_self=False): # prepare some attributes to a better form a['data'] = None # empty the file content as we really don't need this here if a['type'] == 'malware-sample': @@ -139,31 +142,34 @@ def attribute_to_entity(a, link_label=None, event_tags=None): a['type'] = 'regkey' combined_tags = event_tags - if 'Galaxy' in a: + if 'Galaxy' in a and not only_self: for g in a['Galaxy']: for c in g['GalaxyCluster']: yield galaxycluster_to_entity(c) # TODO today the tag is attached to the event, not the attribute, this is something we want to fix soon. - if 'Tag' in a: + if 'Tag' in a and not only_self: for t in a['Tag']: combined_tags.append(t['name']) # ignore all misp-galaxies if t['name'].startswith('misp-galaxy'): continue - yield Hashtag(t['name']) + # ignore all those we add as notes + if tag_matches_note_prefix(t['name']): + continue + yield Hashtag(t['name'], bookmark=Bookmark.Green) notes = convert_tags_to_note(combined_tags) # special cases if a['type'] in ('url', 'uri'): - yield(URL(url=a['value'], link_label=link_label, notes=notes)) + yield(URL(url=a['value'], link_label=link_label, notes=notes, bookmark=Bookmark.Green)) return # attribute is from an object, and a relation gives better understanding of the type of attribute if a.get('object_relation') and mapping_misp_to_maltego.get(a['object_relation']): entity_obj = mapping_misp_to_maltego[a['object_relation']][0] - yield entity_obj(a['value'], labels=[Label('comment', a.get('comment'))], link_label=link_label, notes=notes) + yield entity_obj(a['value'], labels=[Label('comment', a.get('comment'))], link_label=link_label, notes=notes, bookmark=Bookmark.Green) # combined attributes elif '|' in a['type']: @@ -174,7 +180,7 @@ def attribute_to_entity(a, link_label=None, event_tags=None): labels = [Label('comment', a.get('comment'))] if entity_obj == File: labels.append(Label('hash', v_2)) - yield entity_obj_to_entity(entity_obj, v_1, t_1, labels=labels, link_label=link_label, notes=notes) # LATER change the comment to include the second part of the regkey + yield entity_obj_to_entity(entity_obj, v_1, t_1, labels=labels, link_label=link_label, notes=notes, bookmark=Bookmark.Green) # LATER change the comment to include the second part of the regkey else: yield UIMessage("Type {} of combined type {} not supported for attribute: {}".format(t_1, a['type'], a), type=UIMessageType.Inform) if t_2 in mapping_misp_to_maltego: @@ -182,18 +188,18 @@ def attribute_to_entity(a, link_label=None, event_tags=None): labels = [Label('comment', a.get('comment'))] if entity_obj == Hash: labels.append(Label('filename', v_1)) - yield entity_obj_to_entity(entity_obj, v_2, t_2, labels=labels, link_label=link_label, notes=notes) # LATER change the comment to include the first part of the regkey + yield entity_obj_to_entity(entity_obj, v_2, t_2, labels=labels, link_label=link_label, notes=notes, bookmark=Bookmark.Green) # LATER change the comment to include the first part of the regkey else: yield UIMessage("Type {} of combined type {} not supported for attribute: {}".format(t_2, a['type'], a), type=UIMessageType.Inform) # normal attributes elif a['type'] in mapping_misp_to_maltego: entity_obj = mapping_misp_to_maltego[a['type']][0] - yield entity_obj_to_entity(entity_obj, a['value'], a['type'], labels=[Label('comment', a.get('comment'))], link_label=link_label, notes=notes) + yield entity_obj_to_entity(entity_obj, a['value'], a['type'], labels=[Label('comment', a.get('comment'))], link_label=link_label, notes=notes, bookmark=Bookmark.Green) # not supported in our maltego mapping else: - yield Unknown(a['value'], type=a['type'], labels=[Label('comment', a.get('comment'))], link_label=link_label, notes=notes) + yield Unknown(a['value'], type=a['type'], labels=[Label('comment', a.get('comment'))], link_label=link_label, notes=notes, bookmark=Bookmark.Green) yield UIMessage("Type {} not fully supported for attribute: {}".format(a['type'], a), type=UIMessageType.Inform) # LATER : relationships from attributes - not yet supported by MISP yet, but there are references in the datamodel @@ -246,7 +252,8 @@ def object_to_entity(o, link_label=None): meta_category=o.get('meta_category'), description=o.get('description'), comment=o.get('comment'), - link_label=link_label + link_label=link_label, + bookmark=Bookmark.Green ) @@ -255,7 +262,7 @@ def object_to_attributes(o, e): if o['name'] == 'person': first_name = get_attribute_in_object(o, 'first-name', drop=True).get('value') last_name = get_attribute_in_object(o, 'last-name', drop=True).get('value') - yield entity_obj_to_entity(Person, ' '.join([first_name, last_name]).strip(), 'person', lastname=last_name, firstnames=first_name) + yield entity_obj_to_entity(Person, ' '.join([first_name, last_name]).strip(), 'person', lastname=last_name, firstnames=first_name, bookmark=Bookmark.Green) # process normal attributes for a in o['Attribute']: @@ -295,6 +302,17 @@ def get_attribute_in_object(o, attribute_type, drop=False): return found_attribute +def get_attribute_in_event(e, attribute_value): + for a in e['Event']["Attribute"]: + if a['value'] == attribute_value: + return a + for o in e['Event']['Object']: + for a in o['Attribute']: + if a['value'] == attribute_value: + return a + return None + + def convert_tags_to_note(tags): if not tags: return None @@ -306,13 +324,20 @@ def convert_tags_to_note(tags): return '\n'.join(notes) +def tag_matches_note_prefix(tag): + for tag_note_prefix in tag_note_prefixes: + if tag.startswith(tag_note_prefix): + return True + return False + + def event_to_entity(e, link_style=LinkStyle.Normal): tags = [] if 'Tag' in e['Event']: for t in e['Event']['Tag']: tags.append(t['name']) notes = convert_tags_to_note(tags) - return MISPEvent(e['Event']['id'], uuid=e['Event']['uuid'], info=e['Event']['info'], link_style=link_style, notes=notes) + return MISPEvent(e['Event']['id'], uuid=e['Event']['uuid'], info=e['Event']['info'], link_style=link_style, notes=notes, bookmark=Bookmark.Green) def galaxycluster_to_entity(c, link_label=None): @@ -335,7 +360,8 @@ def galaxycluster_to_entity(c, link_label=None): synonyms=synonyms, tag_name=c['tag_name'], link_label=link_label, - icon_url=icon_url + icon_url=icon_url, + bookmark=Bookmark.Green ) diff --git a/src/MISP_maltego/transforms/eventtoattributes.py b/src/MISP_maltego/transforms/eventtoattributes.py index e144d2a..7e834bb 100644 --- a/src/MISP_maltego/transforms/eventtoattributes.py +++ b/src/MISP_maltego/transforms/eventtoattributes.py @@ -2,7 +2,7 @@ from canari.maltego.entities import Hashtag from canari.maltego.transform import Transform # from canari.framework import EnableDebugWindow from MISP_maltego.transforms.common.entities import MISPEvent, MISPObject -from MISP_maltego.transforms.common.util import get_misp_connection, attribute_to_entity, event_to_entity, galaxycluster_to_entity, object_to_entity, object_to_attributes +from MISP_maltego.transforms.common.util import get_misp_connection, attribute_to_entity, event_to_entity, galaxycluster_to_entity, object_to_entity, object_to_attributes, tag_matches_note_prefix from canari.maltego.message import LinkStyle import json @@ -52,7 +52,7 @@ class EventToAttributes(Transform): # The transform input entity type. input_type = MISPEvent - description = 'Expands an Event to Attributes, Tags, Galaxies and related events' + description = 'Expands an Event to Attributes, Tags, Galaxies' def do_transform(self, request, response, config): maltego_misp_event = request.entity @@ -61,6 +61,7 @@ class EventToAttributes(Transform): if not event_json.get('Event'): return response + response += event_to_entity(event_json) event_tags = [] if 'Tag' in event_json['Event']: for t in event_json['Event']['Tag']: @@ -68,13 +69,16 @@ class EventToAttributes(Transform): # ignore all misp-galaxies if t['name'].startswith('misp-galaxy'): continue + # ignore all those we add as notes + if tag_matches_note_prefix(t['name']): + continue response += Hashtag(t['name']) for g in event_json['Event']['Galaxy']: for c in g['GalaxyCluster']: response += galaxycluster_to_entity(c) - for e in event_json['Event']['RelatedEvent']: - response += event_to_entity(e, link_style=LinkStyle.DashDot) + # for e in event_json['Event']['RelatedEvent']: + # response += event_to_entity(e, link_style=LinkStyle.DashDot) for a in event_json['Event']["Attribute"]: for entity in attribute_to_entity(a, event_tags=event_tags):