From 3e44181aeda77c8298b4688d4054f0b61b139974 Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Tue, 29 Oct 2019 15:02:08 +0000 Subject: [PATCH 01/16] Added EQL export test module --- .../modules/export_mod/endgame_export.py | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 misp_modules/modules/export_mod/endgame_export.py diff --git a/misp_modules/modules/export_mod/endgame_export.py b/misp_modules/modules/export_mod/endgame_export.py new file mode 100644 index 0000000..5b0a05b --- /dev/null +++ b/misp_modules/modules/export_mod/endgame_export.py @@ -0,0 +1,108 @@ +""" +Export module for converting MISP events into Endgame EQL queries +""" +import base64 +import csv +import io +import json +import logging + +misperrors = {"error": "Error"} + +moduleinfo = { + "version": "0.1", + "author": "92 COS DOM", + "description": "Export MISP event in Event Query Language", + "module-type": ["export"] +} + +# config fields expected from the MISP administrator +# Default_Source: The source of the data. Typically this won't be changed from the default +moduleconfig = ["Default_Source"] + +# Map of MISP fields => ThreatConnect fields +fieldmap = { +# "domain": "Host", +# "domain|ip": "Host|Address", +# "hostname": "hostname", + "ip-src": "source_address", + "ip-dst": "destination_address", +# "ip-src|port": "Address", +# "ip-dst|port": "Address", +# "url": "URL", + "filename": "file_name" +} + +# Describe what events have what fields +event_types = { + "source_address": "network", + "destination_address": "network", + "file_name": "file" +} + +# combine all the MISP fields from fieldmap into one big list +mispattributes = { + "input": list(fieldmap.keys()) +} + + +def handler(q=False): + """ + Convert a MISP query into a CSV file matching the ThreatConnect Structured Import file format. + Input + q: Query dictionary + """ + if q is False or not q: + return False + + # Check if we were given a configuration + request = json.loads(q) + config = request.get("config", {"Default_Source": ""}) + logging.info("Setting config to: %s", config) + + response = io.StringIO() + + # start parsing MISP data + queryDict = {} + for event in request["data"]: + for attribute in event["Attribute"]: + if attribute["type"] in mispattributes["input"]: + logging.debug("Adding %s to EQL query", attribute["value"]) + event_type = event_types[fieldmap[attribute["type"]]] + if event_type not in queryDict.keys(): + queryDict[event_type] = {} + queryDict[event_type][fieldmap[attribute["type"]]] = attribute["value"] + + for query in queryDict.keys(): + response.write("{} where\n") + for field in query.keys(): + response.write("\t{} == \"{}\"\n") + + return {"response": [], "data": str(base64.b64encode(bytes(response.getvalue(), 'utf-8')), 'utf-8')} + + +def introspection(): + """ + Relay the supported attributes to MISP. + No Input + Output + Dictionary of supported MISP attributes + """ + modulesetup = { + "responseType": "application/txt", + "outputFileExtension": "txt", + "userConfig": {}, + "inputSource": [] + } + return modulesetup + + +def version(): + """ + Relay module version and associated metadata to MISP. + No Input + Output + moduleinfo: metadata output containing all potential configuration values + """ + moduleinfo["config"] = moduleconfig + return moduleinfo From 8ac4b610b8353ba99282b9e1f715d32e8b5e6d9d Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Tue, 29 Oct 2019 15:11:31 +0000 Subject: [PATCH 02/16] Added endgame export to __all__ --- misp_modules/modules/export_mod/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py index 1affbd2..b2c89b7 100644 --- a/misp_modules/modules/export_mod/__init__.py +++ b/misp_modules/modules/export_mod/__init__.py @@ -1,2 +1,2 @@ -__all__ = ['cef_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport', +__all__ = ['cef_export', 'liteexport', 'goamlexport', 'endgame_export', 'threat_connect_export', 'pdfexport', 'threatStream_misp_export', 'osqueryexport', 'nexthinkexport'] From c3ccc9c5773c8ecb861d59fb0d3afbbad1b70d98 Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Tue, 29 Oct 2019 15:52:49 +0000 Subject: [PATCH 03/16] Attempting to import endgame module --- misp_modules/modules/export_mod/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py index b2c89b7..c8afb65 100644 --- a/misp_modules/modules/export_mod/__init__.py +++ b/misp_modules/modules/export_mod/__init__.py @@ -1,2 +1,2 @@ -__all__ = ['cef_export', 'liteexport', 'goamlexport', 'endgame_export', 'threat_connect_export', 'pdfexport', +__all__ = ['cef_export', 'endgame_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport', 'threatStream_misp_export', 'osqueryexport', 'nexthinkexport'] From 3142b0ab0270fc853260fcf295662a594ba0db19 Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Tue, 29 Oct 2019 16:08:58 +0000 Subject: [PATCH 04/16] Fixed type error in JSON parsing --- misp_modules/modules/export_mod/endgame_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/endgame_export.py b/misp_modules/modules/export_mod/endgame_export.py index 5b0a05b..8f2816e 100644 --- a/misp_modules/modules/export_mod/endgame_export.py +++ b/misp_modules/modules/export_mod/endgame_export.py @@ -75,7 +75,7 @@ def handler(q=False): for query in queryDict.keys(): response.write("{} where\n") - for field in query.keys(): + for field in queryDict[query].keys(): response.write("\t{} == \"{}\"\n") return {"response": [], "data": str(base64.b64encode(bytes(response.getvalue(), 'utf-8')), 'utf-8')} From 5802575e4474fb05616f07675bc1d8be08d4555a Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Tue, 29 Oct 2019 16:29:36 +0000 Subject: [PATCH 05/16] Fixed string formatting --- misp_modules/modules/export_mod/endgame_export.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/export_mod/endgame_export.py b/misp_modules/modules/export_mod/endgame_export.py index 8f2816e..5ba7ea4 100644 --- a/misp_modules/modules/export_mod/endgame_export.py +++ b/misp_modules/modules/export_mod/endgame_export.py @@ -71,12 +71,12 @@ def handler(q=False): event_type = event_types[fieldmap[attribute["type"]]] if event_type not in queryDict.keys(): queryDict[event_type] = {} - queryDict[event_type][fieldmap[attribute["type"]]] = attribute["value"] + queryDict[event_type][attribute["value"]] = fieldmap[attribute["type"]] for query in queryDict.keys(): - response.write("{} where\n") - for field in queryDict[query].keys(): - response.write("\t{} == \"{}\"\n") + response.write("{} where\n".format(query)) + for value in queryDict[query].keys(): + response.write("\t{} == \"{}\"\n".format(queryDict[query][value], value)) return {"response": [], "data": str(base64.b64encode(bytes(response.getvalue(), 'utf-8')), 'utf-8')} From a426ad249d50081d68da94ca942bd70bd581dcad Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Tue, 29 Oct 2019 19:42:47 +0000 Subject: [PATCH 06/16] Added EQL enrichment module --- misp_modules/modules/expansion/__init__.py | 2 +- misp_modules/modules/expansion/eql.py | 105 +++++++++++++++++++++ 2 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 misp_modules/modules/expansion/eql.py diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index ef31ad9..77562ff 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -4,7 +4,7 @@ import sys sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3]))) __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl', - 'countrycode', 'cve', 'cve_advanced', 'dns', 'btc_steroids', 'domaintools', 'eupi', + 'countrycode', 'cve', 'cve_advanced', 'dns', 'btc_steroids', 'domaintools', 'eupi', 'eql', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', diff --git a/misp_modules/modules/expansion/eql.py b/misp_modules/modules/expansion/eql.py new file mode 100644 index 0000000..fcf0497 --- /dev/null +++ b/misp_modules/modules/expansion/eql.py @@ -0,0 +1,105 @@ +""" +Export module for converting MISP events into Endgame EQL queries +""" +import base64 +import csv +import io +import json +import logging + +misperrors = {"error": "Error"} + +moduleinfo = { + "version": "0.1", + "author": "92 COS DOM", + "description": "Generates EQL queries from events", + "module-type": ["expansion"] +} + +# Map of MISP fields => ThreatConnect fields +fieldmap = { +# "domain": "Host", +# "domain|ip": "Host|Address", +# "hostname": "hostname", + "ip-src": "source_address", + "ip-dst": "destination_address", +# "ip-src|port": "Address", +# "ip-dst|port": "Address", +# "url": "URL", + "filename": "file_name" +} + +# Describe what events have what fields +event_types = { + "source_address": "network", + "destination_address": "network", + "file_name": "file" +} + +# combine all the MISP fields from fieldmap into one big list +mispattributes = { + "input": list(fieldmap.keys()) +} + + +def handler(q=False): + """ + Convert a MISP query into a CSV file matching the ThreatConnect Structured Import file format. + Input + q: Query dictionary + """ + if q is False or not q: + return False + + # Check if we were given a configuration + request = json.loads(q) + config = request.get("config", {"Default_Source": ""}) + logging.info("Setting config to: %s", config) + + # start parsing MISP data + queryDict = {} + for event in request["data"]: + for attribute in event["Attribute"]: + if attribute["type"] in mispattributes["input"]: + logging.debug("Adding %s to EQL query", attribute["value"]) + event_type = event_types[fieldmap[attribute["type"]]] + if event_type not in queryDict.keys(): + queryDict[event_type] = {} + queryDict[event_type][attribute["value"]] = fieldmap[attribute["type"]] + + response = [] + fullEql = "" + for query in queryDict.keys(): + fullEql += "{} where\n".format(query) + for value in queryDict[query].keys(): + fullEql += "\t{} == \"{}\"\n".format(queryDict[query][value], value) + response.append({'types': ['comment'], 'categories': ['External analysis'], 'values': fullEql, 'comment': "Event EQL queries"}) + return {'results': response} + + +def introspection(): + """ + Relay the supported attributes to MISP. + No Input + Output + Dictionary of supported MISP attributes + """ +# modulesetup = { +# "responseType": "application/txt", +# "outputFileExtension": "txt", +# "userConfig": {}, +# "inputSource": [] +# } +# return modulesetup + return mispattributes + + +def version(): + """ + Relay module version and associated metadata to MISP. + No Input + Output + moduleinfo: metadata output containing all potential configuration values + """ + #moduleinfo["config"] = moduleconfig + return moduleinfo From c06ceedfb84c8776b3dec82b05ac3e5b10e1e456 Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Tue, 29 Oct 2019 20:11:35 +0000 Subject: [PATCH 07/16] Changed to single attribute EQL --- misp_modules/modules/expansion/eql.py | 28 ++++++++++++--------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/misp_modules/modules/expansion/eql.py b/misp_modules/modules/expansion/eql.py index fcf0497..fc64671 100644 --- a/misp_modules/modules/expansion/eql.py +++ b/misp_modules/modules/expansion/eql.py @@ -56,23 +56,19 @@ def handler(q=False): config = request.get("config", {"Default_Source": ""}) logging.info("Setting config to: %s", config) - # start parsing MISP data - queryDict = {} - for event in request["data"]: - for attribute in event["Attribute"]: - if attribute["type"] in mispattributes["input"]: - logging.debug("Adding %s to EQL query", attribute["value"]) - event_type = event_types[fieldmap[attribute["type"]]] - if event_type not in queryDict.keys(): - queryDict[event_type] = {} - queryDict[event_type][attribute["value"]] = fieldmap[attribute["type"]] - + for supportedType in fieldmap.keys(): + if request.get(supportedType): + attrType = supportedType + + if attrType: + eqlType = fieldmap[attrType] + event_type = event_type[eqlType] + fullEql = "{} where {} == \"{}\"".format(event_type, eqlType, request[attrType]) + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + response = [] - fullEql = "" - for query in queryDict.keys(): - fullEql += "{} where\n".format(query) - for value in queryDict[query].keys(): - fullEql += "\t{} == \"{}\"\n".format(queryDict[query][value], value) response.append({'types': ['comment'], 'categories': ['External analysis'], 'values': fullEql, 'comment': "Event EQL queries"}) return {'results': response} From c1ca9369104edc805801c198fb8d16193e4be83b Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Tue, 29 Oct 2019 20:14:07 +0000 Subject: [PATCH 08/16] Fixed syntax error --- misp_modules/modules/expansion/eql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/eql.py b/misp_modules/modules/expansion/eql.py index fc64671..1a7bc77 100644 --- a/misp_modules/modules/expansion/eql.py +++ b/misp_modules/modules/expansion/eql.py @@ -62,7 +62,7 @@ def handler(q=False): if attrType: eqlType = fieldmap[attrType] - event_type = event_type[eqlType] + event_type = event_types[eqlType] fullEql = "{} where {} == \"{}\"".format(event_type, eqlType, request[attrType]) else: misperrors['error'] = "Unsupported attributes type" From 2a4c7ff1502b8e42e07c296fd38c0a6ca74c83a5 Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Tue, 29 Oct 2019 20:22:41 +0000 Subject: [PATCH 09/16] Added ors for compound queries --- misp_modules/modules/export_mod/endgame_export.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/misp_modules/modules/export_mod/endgame_export.py b/misp_modules/modules/export_mod/endgame_export.py index 5ba7ea4..dab15f9 100644 --- a/misp_modules/modules/export_mod/endgame_export.py +++ b/misp_modules/modules/export_mod/endgame_export.py @@ -16,10 +16,6 @@ moduleinfo = { "module-type": ["export"] } -# config fields expected from the MISP administrator -# Default_Source: The source of the data. Typically this won't be changed from the default -moduleconfig = ["Default_Source"] - # Map of MISP fields => ThreatConnect fields fieldmap = { # "domain": "Host", @@ -72,11 +68,14 @@ def handler(q=False): if event_type not in queryDict.keys(): queryDict[event_type] = {} queryDict[event_type][attribute["value"]] = fieldmap[attribute["type"]] - + i = 0 for query in queryDict.keys(): response.write("{} where\n".format(query)) for value in queryDict[query].keys(): - response.write("\t{} == \"{}\"\n".format(queryDict[query][value], value)) + if i != 0: + response.write(" or\n") + response.write("\t{} == \"{}\"".format(queryDict[query][value], value)) + i += 1 return {"response": [], "data": str(base64.b64encode(bytes(response.getvalue(), 'utf-8')), 'utf-8')} @@ -104,5 +103,5 @@ def version(): Output moduleinfo: metadata output containing all potential configuration values """ - moduleinfo["config"] = moduleconfig +# moduleinfo["config"] = moduleconfig return moduleinfo From 08fc938acdc1b2f397de746cfaa280708a0f3489 Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Wed, 30 Oct 2019 13:41:40 +0000 Subject: [PATCH 10/16] Fixed comments --- misp_modules/modules/export_mod/endgame_export.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/misp_modules/modules/export_mod/endgame_export.py b/misp_modules/modules/export_mod/endgame_export.py index dab15f9..dbec4f3 100644 --- a/misp_modules/modules/export_mod/endgame_export.py +++ b/misp_modules/modules/export_mod/endgame_export.py @@ -16,16 +16,10 @@ moduleinfo = { "module-type": ["export"] } -# Map of MISP fields => ThreatConnect fields +# Map of MISP fields => Endgame fields fieldmap = { -# "domain": "Host", -# "domain|ip": "Host|Address", -# "hostname": "hostname", "ip-src": "source_address", "ip-dst": "destination_address", -# "ip-src|port": "Address", -# "ip-dst|port": "Address", -# "url": "URL", "filename": "file_name" } @@ -103,5 +97,4 @@ def version(): Output moduleinfo: metadata output containing all potential configuration values """ -# moduleinfo["config"] = moduleconfig return moduleinfo From 62d25b1f760fc5400dc66c7d5b16df1d2f9a182e Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Wed, 30 Oct 2019 13:46:52 +0000 Subject: [PATCH 11/16] Changed file name to mass eql export --- .../modules/export_mod/{endgame_export.py => mass_eql_export.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename misp_modules/modules/export_mod/{endgame_export.py => mass_eql_export.py} (100%) diff --git a/misp_modules/modules/export_mod/endgame_export.py b/misp_modules/modules/export_mod/mass_eql_export.py similarity index 100% rename from misp_modules/modules/export_mod/endgame_export.py rename to misp_modules/modules/export_mod/mass_eql_export.py From dc4c09f7511c1ae78a79723bb4848dea6473ba00 Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Wed, 30 Oct 2019 13:47:43 +0000 Subject: [PATCH 12/16] Fixed python links --- misp_modules/modules/export_mod/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py index c8afb65..77dec0d 100644 --- a/misp_modules/modules/export_mod/__init__.py +++ b/misp_modules/modules/export_mod/__init__.py @@ -1,2 +1,2 @@ -__all__ = ['cef_export', 'endgame_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport', +__all__ = ['cef_export', 'mass_eql_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport', 'threatStream_misp_export', 'osqueryexport', 'nexthinkexport'] From 393b33d02de08f49f85d08e01dbf02ee03eaccb5 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 30 Oct 2019 16:31:57 +0100 Subject: [PATCH 13/16] fix: Fixed config field parsing for various modules - Same as previous commit --- misp_modules/modules/expansion/securitytrails.py | 7 ++++--- misp_modules/modules/expansion/shodan.py | 4 ++-- misp_modules/modules/expansion/urlscan.py | 7 +++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 21ff089..cc6b8c7 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -37,14 +37,15 @@ def handler(q=False): request = json.loads(q) - if not request.get('config') and not (request['config'].get('apikey')): - misperrors['error'] = 'DNS authentication is missing' + if not request.get('config') or not (request['config'].get('apikey')): + misperrors['error'] = 'SecurityTrails authentication is missing' return misperrors api = DnsTrails(request['config'].get('apikey')) if not api: - misperrors['error'] = 'Onyphe Error instance api' + misperrors['error'] = 'SecurityTrails Error instance api' + return misperrors if request.get('ip-src'): ip = request['ip-src'] return handle_ip(api, ip, misperrors) diff --git a/misp_modules/modules/expansion/shodan.py b/misp_modules/modules/expansion/shodan.py index fbdf5cd..5a4b792 100755 --- a/misp_modules/modules/expansion/shodan.py +++ b/misp_modules/modules/expansion/shodan.py @@ -27,8 +27,8 @@ def handler(q=False): misperrors['error'] = "Unsupported attributes type" return misperrors - if not request.get('config') and not (request['config'].get('apikey')): - misperrors['error'] = 'shodan authentication is missing' + if not request.get('config') or not request['config'].get('apikey'): + misperrors['error'] = 'Shodan authentication is missing' return misperrors api = shodan.Shodan(request['config'].get('apikey')) diff --git a/misp_modules/modules/expansion/urlscan.py b/misp_modules/modules/expansion/urlscan.py index f8dccbb..302022e 100644 --- a/misp_modules/modules/expansion/urlscan.py +++ b/misp_modules/modules/expansion/urlscan.py @@ -31,10 +31,9 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if (request.get('config')): - if (request['config'].get('apikey') is None): - misperrors['error'] = 'urlscan apikey is missing' - return misperrors + if not request.get('config') or not request['config'].get('apikey'): + misperrors['error'] = 'Urlscan apikey is missing' + return misperrors client = urlscanAPI(request['config']['apikey']) r = {'results': []} From d4eb88c66a03389e9de73cbfbbcb332d98f7af8e Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 30 Oct 2019 16:34:15 +0100 Subject: [PATCH 14/16] fix: Avoiding various modules to fail with uncritical issues - Avoiding securitytrails to fail with an unavailable feature for free accounts - Avoiding urlhaus to fail with input attribute fields that are not critical for the query and results - Avoiding VT modules to fail when a certain resource does not exist in the dataset --- misp_modules/modules/expansion/securitytrails.py | 3 --- misp_modules/modules/expansion/urlhaus.py | 2 +- misp_modules/modules/expansion/virustotal.py | 13 +++++++------ misp_modules/modules/expansion/virustotal_public.py | 11 ++++++----- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index cc6b8c7..a88437b 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -93,9 +93,6 @@ def handle_domain(api, domain, misperrors): if status_ok: if r: result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + ' Error whois result' - return misperrors time.sleep(1) r, status_ok = expand_history_ipv4_ipv6(api, domain) diff --git a/misp_modules/modules/expansion/urlhaus.py b/misp_modules/modules/expansion/urlhaus.py index 21a3718..30b78ee 100644 --- a/misp_modules/modules/expansion/urlhaus.py +++ b/misp_modules/modules/expansion/urlhaus.py @@ -60,7 +60,7 @@ class PayloadQuery(URLhaus): def query_api(self): hash_type = self.attribute.type file_object = MISPObject('file') - if self.attribute.event_id != '0': + if hasattr(self.attribute, 'object_id') and hasattr(self.attribute, 'event_id') and self.attribute.event_id != '0': file_object.id = self.attribute.object_id response = requests.post(self.url, data={'{}_hash'.format(hash_type): self.attribute.value}).json() other_hash_type = 'md5' if hash_type == 'sha256' else 'sha256' diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index c6263fc..cd0e738 100644 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -172,12 +172,13 @@ class VirusTotalParser(object): return attribute.uuid def parse_vt_object(self, query_result): - vt_object = MISPObject('virustotal-report') - vt_object.add_attribute('permalink', type='link', value=query_result['permalink']) - detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total']) - vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio) - self.misp_event.add_object(**vt_object) - return vt_object.uuid + if query_result['response_code'] == 1: + vt_object = MISPObject('virustotal-report') + vt_object.add_attribute('permalink', type='link', value=query_result['permalink']) + detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total']) + vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio) + self.misp_event.add_object(**vt_object) + return vt_object.uuid def parse_error(status_code): diff --git a/misp_modules/modules/expansion/virustotal_public.py b/misp_modules/modules/expansion/virustotal_public.py index 7074826..69c2c85 100644 --- a/misp_modules/modules/expansion/virustotal_public.py +++ b/misp_modules/modules/expansion/virustotal_public.py @@ -56,11 +56,12 @@ class VirusTotalParser(): self.misp_event.add_object(**domain_ip_object) def parse_vt_object(self, query_result): - vt_object = MISPObject('virustotal-report') - vt_object.add_attribute('permalink', type='link', value=query_result['permalink']) - detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total']) - vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio) - self.misp_event.add_object(**vt_object) + if query_result['response_code'] == 1: + vt_object = MISPObject('virustotal-report') + vt_object.add_attribute('permalink', type='link', value=query_result['permalink']) + detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total']) + vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio) + self.misp_event.add_object(**vt_object) def get_query_result(self, query_type): params = {query_type: self.attribute.value, 'apikey': self.apikey} From b63a0d1eb8dcb286d2fe61d57fd7493996b7e2b8 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Wed, 30 Oct 2019 16:39:07 +0100 Subject: [PATCH 15/16] fix: Making urlscan module available in MISP for ip attributes - As expected in the the handler function --- misp_modules/modules/expansion/urlscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/urlscan.py b/misp_modules/modules/expansion/urlscan.py index 302022e..e6af7f6 100644 --- a/misp_modules/modules/expansion/urlscan.py +++ b/misp_modules/modules/expansion/urlscan.py @@ -22,7 +22,7 @@ moduleinfo = { moduleconfig = ['apikey'] misperrors = {'error': 'Error'} mispattributes = { - 'input': ['hostname', 'domain', 'url'], + 'input': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url'], 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url', 'text', 'link', 'hash'] } From 717be2b8599dfa7ee8154498b1fd8dc7a51bd2eb Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Wed, 30 Oct 2019 15:44:47 +0000 Subject: [PATCH 16/16] Removed extraneous comments and unused imports --- misp_modules/modules/expansion/eql.py | 19 +------------------ .../modules/export_mod/mass_eql_export.py | 1 - 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/misp_modules/modules/expansion/eql.py b/misp_modules/modules/expansion/eql.py index 1a7bc77..46cc05e 100644 --- a/misp_modules/modules/expansion/eql.py +++ b/misp_modules/modules/expansion/eql.py @@ -1,9 +1,6 @@ """ Export module for converting MISP events into Endgame EQL queries """ -import base64 -import csv -import io import json import logging @@ -16,16 +13,10 @@ moduleinfo = { "module-type": ["expansion"] } -# Map of MISP fields => ThreatConnect fields +# Map of MISP fields => Endgame fields fieldmap = { -# "domain": "Host", -# "domain|ip": "Host|Address", -# "hostname": "hostname", "ip-src": "source_address", "ip-dst": "destination_address", -# "ip-src|port": "Address", -# "ip-dst|port": "Address", -# "url": "URL", "filename": "file_name" } @@ -80,13 +71,6 @@ def introspection(): Output Dictionary of supported MISP attributes """ -# modulesetup = { -# "responseType": "application/txt", -# "outputFileExtension": "txt", -# "userConfig": {}, -# "inputSource": [] -# } -# return modulesetup return mispattributes @@ -97,5 +81,4 @@ def version(): Output moduleinfo: metadata output containing all potential configuration values """ - #moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/export_mod/mass_eql_export.py b/misp_modules/modules/export_mod/mass_eql_export.py index dbec4f3..f42874d 100644 --- a/misp_modules/modules/export_mod/mass_eql_export.py +++ b/misp_modules/modules/export_mod/mass_eql_export.py @@ -2,7 +2,6 @@ Export module for converting MISP events into Endgame EQL queries """ import base64 -import csv import io import json import logging