From 84868d8cfbdacd2101a2ee4c3cf4288c464b3677 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Thu, 28 May 2020 10:25:35 +0200 Subject: [PATCH 1/4] fix: [pymisp] fixes #31 no timeout in establishing MISP connection --- setup.py | 2 +- src/MISP_maltego/transforms/common/util.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 75b82a6..d8b4730 100755 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ setup( python_requires='>=3.5', install_requires=[ 'canari>=3.3.10,<4', - 'PyMISP>=2.4.114' + 'PyMISP>=2.4.127' ], dependency_links=[ # custom links for the install_requires diff --git a/src/MISP_maltego/transforms/common/util.py b/src/MISP_maltego/transforms/common/util.py index 582df22..b49886b 100644 --- a/src/MISP_maltego/transforms/common/util.py +++ b/src/MISP_maltego/transforms/common/util.py @@ -85,7 +85,7 @@ class MISPConnection(): misp_key = parameters['mispkey'].value except AttributeError: raise MaltegoException("ERROR: mispurl and mispkey need to be set to something valid") - self.misp = PyMISP(misp_url, misp_key, misp_verify, 'json', misp_debug, tool='misp_maltego') + self.misp = PyMISP(misp_url, misp_key, misp_verify, 'json', misp_debug, tool='misp_maltego', timeout=(2, 60)) except Exception: if is_local_exec_mode(): raise MaltegoException("ERROR: Cannot connect to MISP server. Please verify your MISP_Maltego.conf settings.") @@ -219,6 +219,7 @@ def attribute_to_entity(a, link_label=None, event_tags=[], only_self=False): if a['type'] in ('url', 'uri'): yield(URL(url=a['value'], short_title=a['value'], link_label=link_label, notes=notes, bookmark=Bookmark.Green)) return + # FIXME implement attachment screenshot type # attribute is from an object, and a relation gives better understanding of the type of attribute if a.get('object_relation') and mapping_misp_to_maltego.get(a['object_relation']): From 0b71d8a4f43f2df27c9266b7a90f48d7b40d5a90 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Sun, 14 Jun 2020 19:54:49 +0200 Subject: [PATCH 2/4] fix: [transform] fixes #35 - slow Search in MISP --- .../transforms/attributetoevent.py | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/MISP_maltego/transforms/attributetoevent.py b/src/MISP_maltego/transforms/attributetoevent.py index 693b34b..2bbf629 100644 --- a/src/MISP_maltego/transforms/attributetoevent.py +++ b/src/MISP_maltego/transforms/attributetoevent.py @@ -82,19 +82,23 @@ class SearchInMISP(Transform): # for all other normal entities conn = MISPConnection(config, request.parameters) - events_json = conn.misp.search(controller='events', value=request.entity.value, with_attachments=False) + # we need to do really rebuild the Entity from scratch as request.entity is of type Unknown - for e in events_json: - # find the value as attribute - attr = get_attribute_in_event(e, request.entity.value, substring=True) - if attr: - for item in attribute_to_entity(attr, only_self=True): - response += item - # find the value as object, and return the object - if 'Object' in e['Event']: - for o in e['Event']['Object']: - if get_attribute_in_object(o, attribute_value=request.entity.value, substring=True).get('value'): - response += conn.object_to_entity(o, link_label=link_label) + # TODO First try to build the object, then only attributes (for those that are not in object, or for all?) + # obj_json = conn.misp.search(controller='objects', value=request.entity.value, with_attachments=False) + # for o in obj_json: + # for item in attribute_to_entity(attr, only_self=True, link_label=link_label): + # response += item + # # find the value as object, and return the object + # if 'Object' in e['Event']: + # for o in e['Event']['Object']: + # if get_attribute_in_object(o, attribute_value=request.entity.value, substring=True).get('value'): + # response += conn.object_to_entity(o, link_label=link_label) + + attr_json = conn.misp.search(controller='attributes', value=request.entity.value, with_attachments=False) + for a in attr_json['Attribute']: + for item in attribute_to_entity(a, only_self=True, link_label=link_label): + response += item return response From df4d92b447899f0d29b434c8cda2b65fae4dc033 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Sat, 4 Jul 2020 18:50:41 +0200 Subject: [PATCH 3/4] fix: [galaxies] fixes #37 #38 --- setup.py | 2 +- src/MISP_maltego/transforms/attributetoevent.py | 1 + src/MISP_maltego/transforms/common/util.py | 8 +++++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index d8b4730..1a72b40 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='MISP_maltego', author='Christophe Vandeplas', # also update version in util.py - version='1.4.5', + version='1.4.6', author_email='christophe@vandeplas.com', maintainer='Christophe Vandeplas', url='https://github.com/MISP/MISP-maltego', diff --git a/src/MISP_maltego/transforms/attributetoevent.py b/src/MISP_maltego/transforms/attributetoevent.py index 2bbf629..daa507e 100644 --- a/src/MISP_maltego/transforms/attributetoevent.py +++ b/src/MISP_maltego/transforms/attributetoevent.py @@ -85,6 +85,7 @@ class SearchInMISP(Transform): # we need to do really rebuild the Entity from scratch as request.entity is of type Unknown # TODO First try to build the object, then only attributes (for those that are not in object, or for all?) + # TODO check for the right version of MISP before, it needs to be 2.4.127 or higher. # obj_json = conn.misp.search(controller='objects', value=request.entity.value, with_attachments=False) # for o in obj_json: # for item in attribute_to_entity(attr, only_self=True, link_label=link_label): diff --git a/src/MISP_maltego/transforms/common/util.py b/src/MISP_maltego/transforms/common/util.py index b49886b..9ae73aa 100644 --- a/src/MISP_maltego/transforms/common/util.py +++ b/src/MISP_maltego/transforms/common/util.py @@ -12,7 +12,7 @@ import requests import tempfile import time -__version__ = '1.4.5' # also update version in setup.py +__version__ = '1.4.6' # also update version in setup.py tag_note_prefixes = ['tlp:', 'PAP:', 'de-vs:', 'euci:', 'fr-classif:', 'nato:'] @@ -445,9 +445,9 @@ def galaxycluster_to_entity(c, link_label=None, link_direction=LinkDirection.Inp # LATER this uses the galaxies from github as the MISP web UI does not fully support the Galaxies in the webui. # See https://github.com/MISP/MISP/issues/3801 -galaxy_archive_url = 'https://github.com/MISP/misp-galaxy/archive/master.zip' +galaxy_archive_url = 'https://github.com/MISP/misp-galaxy/archive/main.zip' local_path_uuid_mapping = os.path.join(local_path_root, 'MISP_maltego_galaxy_mapping.json') -local_path_clusters = os.path.join(local_path_root, 'misp-galaxy-master', 'clusters') +local_path_clusters = os.path.join(local_path_root, 'misp-galaxy-main', 'clusters') galaxy_cluster_uuids = None @@ -480,6 +480,8 @@ def galaxy_update_local_copy(force=False): zf.extractall(local_path_root) zf.close() except Exception: + # remove the lock + os.remove(lockfile) raise(MaltegoException("ERROR: Could not download Galaxy data from htts://github.com/MISP/MISP-galaxy/. Please check internet connectivity.")) # generate the uuid mapping and save it to a file From c11da6661e55266eb3d569d0dcd1c6e694e92dc1 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Tue, 14 Jul 2020 17:04:44 +0200 Subject: [PATCH 4/4] fix: [connection] fixes #39 thanks to @andurin --- src/MISP_maltego/transforms/common/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MISP_maltego/transforms/common/util.py b/src/MISP_maltego/transforms/common/util.py index 9ae73aa..a99b939 100644 --- a/src/MISP_maltego/transforms/common/util.py +++ b/src/MISP_maltego/transforms/common/util.py @@ -85,7 +85,7 @@ class MISPConnection(): misp_key = parameters['mispkey'].value except AttributeError: raise MaltegoException("ERROR: mispurl and mispkey need to be set to something valid") - self.misp = PyMISP(misp_url, misp_key, misp_verify, 'json', misp_debug, tool='misp_maltego', timeout=(2, 60)) + self.misp = PyMISP(url=misp_url, key=misp_key, ssl=misp_verify, debug=misp_debug, tool='misp_maltego', timeout=(2, 60)) except Exception: if is_local_exec_mode(): raise MaltegoException("ERROR: Cannot connect to MISP server. Please verify your MISP_Maltego.conf settings.")