From 3e44181aeda77c8298b4688d4054f0b61b139974 Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Tue, 29 Oct 2019 15:02:08 +0000 Subject: [PATCH 01/13] Added EQL export test module --- .../modules/export_mod/endgame_export.py | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 misp_modules/modules/export_mod/endgame_export.py diff --git a/misp_modules/modules/export_mod/endgame_export.py b/misp_modules/modules/export_mod/endgame_export.py new file mode 100644 index 0000000..5b0a05b --- /dev/null +++ b/misp_modules/modules/export_mod/endgame_export.py @@ -0,0 +1,108 @@ +""" +Export module for converting MISP events into Endgame EQL queries +""" +import base64 +import csv +import io +import json +import logging + +misperrors = {"error": "Error"} + +moduleinfo = { + "version": "0.1", + "author": "92 COS DOM", + "description": "Export MISP event in Event Query Language", + "module-type": ["export"] +} + +# config fields expected from the MISP administrator +# Default_Source: The source of the data. Typically this won't be changed from the default +moduleconfig = ["Default_Source"] + +# Map of MISP fields => ThreatConnect fields +fieldmap = { +# "domain": "Host", +# "domain|ip": "Host|Address", +# "hostname": "hostname", + "ip-src": "source_address", + "ip-dst": "destination_address", +# "ip-src|port": "Address", +# "ip-dst|port": "Address", +# "url": "URL", + "filename": "file_name" +} + +# Describe what events have what fields +event_types = { + "source_address": "network", + "destination_address": "network", + "file_name": "file" +} + +# combine all the MISP fields from fieldmap into one big list +mispattributes = { + "input": list(fieldmap.keys()) +} + + +def handler(q=False): + """ + Convert a MISP query into a CSV file matching the ThreatConnect Structured Import file format. + Input + q: Query dictionary + """ + if q is False or not q: + return False + + # Check if we were given a configuration + request = json.loads(q) + config = request.get("config", {"Default_Source": ""}) + logging.info("Setting config to: %s", config) + + response = io.StringIO() + + # start parsing MISP data + queryDict = {} + for event in request["data"]: + for attribute in event["Attribute"]: + if attribute["type"] in mispattributes["input"]: + logging.debug("Adding %s to EQL query", attribute["value"]) + event_type = event_types[fieldmap[attribute["type"]]] + if event_type not in queryDict.keys(): + queryDict[event_type] = {} + queryDict[event_type][fieldmap[attribute["type"]]] = attribute["value"] + + for query in queryDict.keys(): + response.write("{} where\n") + for field in query.keys(): + response.write("\t{} == \"{}\"\n") + + return {"response": [], "data": str(base64.b64encode(bytes(response.getvalue(), 'utf-8')), 'utf-8')} + + +def introspection(): + """ + Relay the supported attributes to MISP. + No Input + Output + Dictionary of supported MISP attributes + """ + modulesetup = { + "responseType": "application/txt", + "outputFileExtension": "txt", + "userConfig": {}, + "inputSource": [] + } + return modulesetup + + +def version(): + """ + Relay module version and associated metadata to MISP. + No Input + Output + moduleinfo: metadata output containing all potential configuration values + """ + moduleinfo["config"] = moduleconfig + return moduleinfo From 8ac4b610b8353ba99282b9e1f715d32e8b5e6d9d Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Tue, 29 Oct 2019 15:11:31 +0000 Subject: [PATCH 02/13] Added endgame export to __all__ --- misp_modules/modules/export_mod/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py index 1affbd2..b2c89b7 100644 --- a/misp_modules/modules/export_mod/__init__.py +++ b/misp_modules/modules/export_mod/__init__.py @@ -1,2 +1,2 @@ -__all__ = ['cef_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport', +__all__ = ['cef_export', 'liteexport', 'goamlexport', 'endgame_export', 'threat_connect_export', 'pdfexport', 'threatStream_misp_export', 'osqueryexport', 'nexthinkexport'] From c3ccc9c5773c8ecb861d59fb0d3afbbad1b70d98 Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Tue, 29 Oct 2019 15:52:49 +0000 Subject: [PATCH 03/13] Attempting to import endgame module --- misp_modules/modules/export_mod/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py index b2c89b7..c8afb65 100644 --- a/misp_modules/modules/export_mod/__init__.py +++ b/misp_modules/modules/export_mod/__init__.py @@ -1,2 +1,2 @@ -__all__ = ['cef_export', 'liteexport', 'goamlexport', 'endgame_export', 'threat_connect_export', 'pdfexport', +__all__ = ['cef_export', 'endgame_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport', 'threatStream_misp_export', 'osqueryexport', 'nexthinkexport'] From 3142b0ab0270fc853260fcf295662a594ba0db19 Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Tue, 29 Oct 2019 16:08:58 +0000 Subject: [PATCH 04/13] Fixed type error in JSON parsing --- misp_modules/modules/export_mod/endgame_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/endgame_export.py b/misp_modules/modules/export_mod/endgame_export.py index 5b0a05b..8f2816e 100644 --- a/misp_modules/modules/export_mod/endgame_export.py +++ b/misp_modules/modules/export_mod/endgame_export.py @@ -75,7 +75,7 @@ def handler(q=False): for query in queryDict.keys(): response.write("{} where\n") - for field in query.keys(): + for field in queryDict[query].keys(): response.write("\t{} == \"{}\"\n") return {"response": [], "data": str(base64.b64encode(bytes(response.getvalue(), 'utf-8')), 'utf-8')} From 5802575e4474fb05616f07675bc1d8be08d4555a Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Tue, 29 Oct 2019 16:29:36 +0000 Subject: [PATCH 05/13] Fixed string formatting --- misp_modules/modules/export_mod/endgame_export.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/export_mod/endgame_export.py b/misp_modules/modules/export_mod/endgame_export.py index 8f2816e..5ba7ea4 100644 --- a/misp_modules/modules/export_mod/endgame_export.py +++ b/misp_modules/modules/export_mod/endgame_export.py @@ -71,12 +71,12 @@ def handler(q=False): event_type = event_types[fieldmap[attribute["type"]]] if event_type not in queryDict.keys(): queryDict[event_type] = {} - queryDict[event_type][fieldmap[attribute["type"]]] = attribute["value"] + queryDict[event_type][attribute["value"]] = fieldmap[attribute["type"]] for query in queryDict.keys(): - response.write("{} where\n") - for field in queryDict[query].keys(): - response.write("\t{} == \"{}\"\n") + response.write("{} where\n".format(query)) + for value in queryDict[query].keys(): + response.write("\t{} == \"{}\"\n".format(queryDict[query][value], value)) return {"response": [], "data": str(base64.b64encode(bytes(response.getvalue(), 'utf-8')), 'utf-8')} From a426ad249d50081d68da94ca942bd70bd581dcad Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Tue, 29 Oct 2019 19:42:47 +0000 Subject: [PATCH 06/13] Added EQL enrichment module --- misp_modules/modules/expansion/__init__.py | 2 +- misp_modules/modules/expansion/eql.py | 105 +++++++++++++++++++++ 2 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 misp_modules/modules/expansion/eql.py diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index ef31ad9..77562ff 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -4,7 +4,7 @@ import sys sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3]))) __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl', - 'countrycode', 'cve', 'cve_advanced', 'dns', 'btc_steroids', 'domaintools', 'eupi', + 'countrycode', 'cve', 'cve_advanced', 'dns', 'btc_steroids', 'domaintools', 'eupi', 'eql', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', diff --git a/misp_modules/modules/expansion/eql.py b/misp_modules/modules/expansion/eql.py new file mode 100644 index 0000000..fcf0497 --- /dev/null +++ b/misp_modules/modules/expansion/eql.py @@ -0,0 +1,105 @@ +""" +Export module for converting MISP events into Endgame EQL queries +""" +import base64 +import csv +import io +import json +import logging + +misperrors = {"error": "Error"} + +moduleinfo = { + "version": "0.1", + "author": "92 COS DOM", + "description": "Generates EQL queries from events", + "module-type": ["expansion"] +} + +# Map of MISP fields => ThreatConnect fields +fieldmap = { +# "domain": "Host", +# "domain|ip": "Host|Address", +# "hostname": "hostname", + "ip-src": "source_address", + "ip-dst": "destination_address", +# "ip-src|port": "Address", +# "ip-dst|port": "Address", +# "url": "URL", + "filename": "file_name" +} + +# Describe what events have what fields +event_types = { + "source_address": "network", + "destination_address": "network", + "file_name": "file" +} + +# combine all the MISP fields from fieldmap into one big list +mispattributes = { + "input": list(fieldmap.keys()) +} + + +def handler(q=False): + """ + Convert a MISP query into a CSV file matching the ThreatConnect Structured Import file format. + Input + q: Query dictionary + """ + if q is False or not q: + return False + + # Check if we were given a configuration + request = json.loads(q) + config = request.get("config", {"Default_Source": ""}) + logging.info("Setting config to: %s", config) + + # start parsing MISP data + queryDict = {} + for event in request["data"]: + for attribute in event["Attribute"]: + if attribute["type"] in mispattributes["input"]: + logging.debug("Adding %s to EQL query", attribute["value"]) + event_type = event_types[fieldmap[attribute["type"]]] + if event_type not in queryDict.keys(): + queryDict[event_type] = {} + queryDict[event_type][attribute["value"]] = fieldmap[attribute["type"]] + + response = [] + fullEql = "" + for query in queryDict.keys(): + fullEql += "{} where\n".format(query) + for value in queryDict[query].keys(): + fullEql += "\t{} == \"{}\"\n".format(queryDict[query][value], value) + response.append({'types': ['comment'], 'categories': ['External analysis'], 'values': fullEql, 'comment': "Event EQL queries"}) + return {'results': response} + + +def introspection(): + """ + Relay the supported attributes to MISP. + No Input + Output + Dictionary of supported MISP attributes + """ +# modulesetup = { +# "responseType": "application/txt", +# "outputFileExtension": "txt", +# "userConfig": {}, +# "inputSource": [] +# } +# return modulesetup + return mispattributes + + +def version(): + """ + Relay module version and associated metadata to MISP. + No Input + Output + moduleinfo: metadata output containing all potential configuration values + """ + #moduleinfo["config"] = moduleconfig + return moduleinfo From c06ceedfb84c8776b3dec82b05ac3e5b10e1e456 Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Tue, 29 Oct 2019 20:11:35 +0000 Subject: [PATCH 07/13] Changed to single attribute EQL --- misp_modules/modules/expansion/eql.py | 28 ++++++++++++--------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/misp_modules/modules/expansion/eql.py b/misp_modules/modules/expansion/eql.py index fcf0497..fc64671 100644 --- a/misp_modules/modules/expansion/eql.py +++ b/misp_modules/modules/expansion/eql.py @@ -56,23 +56,19 @@ def handler(q=False): config = request.get("config", {"Default_Source": ""}) logging.info("Setting config to: %s", config) - # start parsing MISP data - queryDict = {} - for event in request["data"]: - for attribute in event["Attribute"]: - if attribute["type"] in mispattributes["input"]: - logging.debug("Adding %s to EQL query", attribute["value"]) - event_type = event_types[fieldmap[attribute["type"]]] - if event_type not in queryDict.keys(): - queryDict[event_type] = {} - queryDict[event_type][attribute["value"]] = fieldmap[attribute["type"]] - + for supportedType in fieldmap.keys(): + if request.get(supportedType): + attrType = supportedType + + if attrType: + eqlType = fieldmap[attrType] + event_type = event_type[eqlType] + fullEql = "{} where {} == \"{}\"".format(event_type, eqlType, request[attrType]) + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + response = [] - fullEql = "" - for query in queryDict.keys(): - fullEql += "{} where\n".format(query) - for value in queryDict[query].keys(): - fullEql += "\t{} == \"{}\"\n".format(queryDict[query][value], value) response.append({'types': ['comment'], 'categories': ['External analysis'], 'values': fullEql, 'comment': "Event EQL queries"}) return {'results': response} From c1ca9369104edc805801c198fb8d16193e4be83b Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Tue, 29 Oct 2019 20:14:07 +0000 Subject: [PATCH 08/13] Fixed syntax error --- misp_modules/modules/expansion/eql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/eql.py b/misp_modules/modules/expansion/eql.py index fc64671..1a7bc77 100644 --- a/misp_modules/modules/expansion/eql.py +++ b/misp_modules/modules/expansion/eql.py @@ -62,7 +62,7 @@ def handler(q=False): if attrType: eqlType = fieldmap[attrType] - event_type = event_type[eqlType] + event_type = event_types[eqlType] fullEql = "{} where {} == \"{}\"".format(event_type, eqlType, request[attrType]) else: misperrors['error'] = "Unsupported attributes type" From 2a4c7ff1502b8e42e07c296fd38c0a6ca74c83a5 Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Tue, 29 Oct 2019 20:22:41 +0000 Subject: [PATCH 09/13] Added ors for compound queries --- misp_modules/modules/export_mod/endgame_export.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/misp_modules/modules/export_mod/endgame_export.py b/misp_modules/modules/export_mod/endgame_export.py index 5ba7ea4..dab15f9 100644 --- a/misp_modules/modules/export_mod/endgame_export.py +++ b/misp_modules/modules/export_mod/endgame_export.py @@ -16,10 +16,6 @@ moduleinfo = { "module-type": ["export"] } -# config fields expected from the MISP administrator -# Default_Source: The source of the data. Typically this won't be changed from the default -moduleconfig = ["Default_Source"] - # Map of MISP fields => ThreatConnect fields fieldmap = { # "domain": "Host", @@ -72,11 +68,14 @@ def handler(q=False): if event_type not in queryDict.keys(): queryDict[event_type] = {} queryDict[event_type][attribute["value"]] = fieldmap[attribute["type"]] - + i = 0 for query in queryDict.keys(): response.write("{} where\n".format(query)) for value in queryDict[query].keys(): - response.write("\t{} == \"{}\"\n".format(queryDict[query][value], value)) + if i != 0: + response.write(" or\n") + response.write("\t{} == \"{}\"".format(queryDict[query][value], value)) + i += 1 return {"response": [], "data": str(base64.b64encode(bytes(response.getvalue(), 'utf-8')), 'utf-8')} @@ -104,5 +103,5 @@ def version(): Output moduleinfo: metadata output containing all potential configuration values """ - moduleinfo["config"] = moduleconfig +# moduleinfo["config"] = moduleconfig return moduleinfo From 08fc938acdc1b2f397de746cfaa280708a0f3489 Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Wed, 30 Oct 2019 13:41:40 +0000 Subject: [PATCH 10/13] Fixed comments --- misp_modules/modules/export_mod/endgame_export.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/misp_modules/modules/export_mod/endgame_export.py b/misp_modules/modules/export_mod/endgame_export.py index dab15f9..dbec4f3 100644 --- a/misp_modules/modules/export_mod/endgame_export.py +++ b/misp_modules/modules/export_mod/endgame_export.py @@ -16,16 +16,10 @@ moduleinfo = { "module-type": ["export"] } -# Map of MISP fields => ThreatConnect fields +# Map of MISP fields => Endgame fields fieldmap = { -# "domain": "Host", -# "domain|ip": "Host|Address", -# "hostname": "hostname", "ip-src": "source_address", "ip-dst": "destination_address", -# "ip-src|port": "Address", -# "ip-dst|port": "Address", -# "url": "URL", "filename": "file_name" } @@ -103,5 +97,4 @@ def version(): Output moduleinfo: metadata output containing all potential configuration values """ -# moduleinfo["config"] = moduleconfig return moduleinfo From 62d25b1f760fc5400dc66c7d5b16df1d2f9a182e Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Wed, 30 Oct 2019 13:46:52 +0000 Subject: [PATCH 11/13] Changed file name to mass eql export --- .../modules/export_mod/{endgame_export.py => mass_eql_export.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename misp_modules/modules/export_mod/{endgame_export.py => mass_eql_export.py} (100%) diff --git a/misp_modules/modules/export_mod/endgame_export.py b/misp_modules/modules/export_mod/mass_eql_export.py similarity index 100% rename from misp_modules/modules/export_mod/endgame_export.py rename to misp_modules/modules/export_mod/mass_eql_export.py From dc4c09f7511c1ae78a79723bb4848dea6473ba00 Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Wed, 30 Oct 2019 13:47:43 +0000 Subject: [PATCH 12/13] Fixed python links --- misp_modules/modules/export_mod/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py index c8afb65..77dec0d 100644 --- a/misp_modules/modules/export_mod/__init__.py +++ b/misp_modules/modules/export_mod/__init__.py @@ -1,2 +1,2 @@ -__all__ = ['cef_export', 'endgame_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport', +__all__ = ['cef_export', 'mass_eql_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport', 'threatStream_misp_export', 'osqueryexport', 'nexthinkexport'] From 717be2b8599dfa7ee8154498b1fd8dc7a51bd2eb Mon Sep 17 00:00:00 2001 From: Braden Laverick Date: Wed, 30 Oct 2019 15:44:47 +0000 Subject: [PATCH 13/13] Removed extraneous comments and unused imports --- misp_modules/modules/expansion/eql.py | 19 +------------------ .../modules/export_mod/mass_eql_export.py | 1 - 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/misp_modules/modules/expansion/eql.py b/misp_modules/modules/expansion/eql.py index 1a7bc77..46cc05e 100644 --- a/misp_modules/modules/expansion/eql.py +++ b/misp_modules/modules/expansion/eql.py @@ -1,9 +1,6 @@ """ Export module for converting MISP events into Endgame EQL queries """ -import base64 -import csv -import io import json import logging @@ -16,16 +13,10 @@ moduleinfo = { "module-type": ["expansion"] } -# Map of MISP fields => ThreatConnect fields +# Map of MISP fields => Endgame fields fieldmap = { -# "domain": "Host", -# "domain|ip": "Host|Address", -# "hostname": "hostname", "ip-src": "source_address", "ip-dst": "destination_address", -# "ip-src|port": "Address", -# "ip-dst|port": "Address", -# "url": "URL", "filename": "file_name" } @@ -80,13 +71,6 @@ def introspection(): Output Dictionary of supported MISP attributes """ -# modulesetup = { -# "responseType": "application/txt", -# "outputFileExtension": "txt", -# "userConfig": {}, -# "inputSource": [] -# } -# return modulesetup return mispattributes @@ -97,5 +81,4 @@ def version(): Output moduleinfo: metadata output containing all potential configuration values """ - #moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/export_mod/mass_eql_export.py b/misp_modules/modules/export_mod/mass_eql_export.py index dbec4f3..f42874d 100644 --- a/misp_modules/modules/export_mod/mass_eql_export.py +++ b/misp_modules/modules/export_mod/mass_eql_export.py @@ -2,7 +2,6 @@ Export module for converting MISP events into Endgame EQL queries """ import base64 -import csv import io import json import logging