diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py index ee457cf..1b6d161 100644 --- a/misp_modules/modules/export_mod/__init__.py +++ b/misp_modules/modules/export_mod/__init__.py @@ -1 +1 @@ -__all__ = ['testexport','cef_export','liteexport'] +__all__ = ['testexport','cef_export','liteexport','threat_connect_export'] diff --git a/misp_modules/modules/export_mod/threat_connect_export.py b/misp_modules/modules/export_mod/threat_connect_export.py new file mode 100644 index 0000000..0b51fb7 --- /dev/null +++ b/misp_modules/modules/export_mod/threat_connect_export.py @@ -0,0 +1,118 @@ +""" +Export module for converting MISP events into ThreatConnect Structured Import files. This export data is meant to be used with the "Structured Import" ability of ThreatConnect. + +Source: http://kb.threatconnect.com/customer/en/portal/articles/1912599-using-structured-import/ +Source: http://kb.threatconnect.com/customer/en/portal/articles/2092925-the-threatconnect-data-model/ +""" +import base64 +import csv +import io +import json +import logging + +misperrors = {"error": "Error"} + +moduleinfo = { + "version": "0.1", + "author": "CenturyLink CIRT", + "description": "Export a structured CSV file for uploading to ThreatConnect", + "module-type": ["export"] +} + +# config fields expected from the MISP administrator +# Default_Source: The source of the data. Typically this won't be changed from the default +moduleconfig = ["Default_Source"] + +# Map of MISP fields => ThreatConnect fields +fieldmap = { + "domain": "Host", + "domain|ip": "Host|Address", + "hostname": "Host", + "ip-src": "Address", + "ip-dst": "Address", + "ip-src|port": "Address", + "ip-dst|port": "Address", + "whois-registrant-email": "EmailAddress", + "email-src": "EmailAddress", + "email-dst": "EmailAddress", + "url": "URL", + "md5": "File", + "filename|md5": "File" +} + +# combine all the MISP fields from fieldmap into one big list +mispattributes = { + "input": list(fieldmap.keys()) +} + + +def handler(q=False): + """ + Convert a MISP query into a CSV file matching the ThreatConnect Structured Import file format. + Input + q: Query dictionary + """ + if q is False or not q: + return False + + # Check if we were given a configuration + request = json.loads(q) + config = request.get("config", {"Default_Source": ""}) + logging.info("Setting config to: %s", config) + + response = io.StringIO() + writer = csv.DictWriter(response, fieldnames=["Type", "Value", "Source", "Description"]) + writer.writeheader() + + # start parsing MISP data + for event in request["data"]: + for attribute in event["Attribute"]: + if attribute["type"] in mispattributes["input"]: + logging.debug("Adding %s to structured CSV export of ThreatConnectExport", attribute["value"]) + if "|" in attribute["type"]: + # if the attribute type has multiple values, line it up with the corresponding ThreatConnect values in fieldmap + indicators = tuple(attribute["value"].split("|")) + tc_types = tuple(fieldmap[attribute["type"]].split("|")) + for i, indicator in enumerate(indicators): + writer.writerow({ + "Type": tc_types[i], + "Value": indicator, + "Source": config["Default_Source"], + "Description": attribute["comment"] + }) + else: + writer.writerow({ + "Type": fieldmap[attribute["type"]], + "Value": attribute["value"], + "Source": config["Default_Source"], + "Description": attribute["comment"] + }) + + return {"response": [], "data": str(base64.b64encode(bytes(response.getvalue(), 'utf-8')), 'utf-8')} + + +def introspection(): + """ + Relay the supported attributes to MISP. + No Input + Output + Dictionary of supported MISP attributes + """ + modulesetup = { + "responseType": "application/txt", + "outputFileExtension": "csv", + "userConfig": {}, + "inputSource": [] + } + return modulesetup + + +def version(): + """ + Relay module version and associated metadata to MISP. + No Input + Output + moduleinfo: metadata output containing all potential configuration values + """ + moduleinfo["config"] = moduleconfig + return moduleinfo diff --git a/tests/test_files/misp_event.json b/tests/test_files/misp_event.json new file mode 100644 index 0000000..ce0db0a --- /dev/null +++ b/tests/test_files/misp_event.json @@ -0,0 +1,297 @@ +{ + "Event": { + "id": "625", + "orgc_id": "2", + "org_id": "1", + "date": "2017-05-24", + "threat_level_id": "3", + "info": "M2M - Fwd: IMG_3428.pdf", + "published": false, + "uuid": "59259036-fcd0-4749-8a6c-4d88950d210f", + "attribute_count": "2", + "analysis": "1", + "timestamp": "1500496265", + "distribution": "3", + "proposal_email_lock": false, + "user_id": "1", + "locked": false, + "publish_timestamp": "0", + "sharing_group_id": "0", + "disable_correlation": false + }, + "User": { + "email": "admin@misp.training", + "id": "1" + }, + "ThreatLevel": { + "name": "Low", + "id": "3" + }, + "Org": { + "id": "1", + "name": "MISP", + "uuid": "56ef3277-1ad4-42f6-b90b-04e5c0a83832" + }, + "Orgc": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "Attribute": [{ + "id": "157835", + "type": "attachment", + "category": "Artifacts dropped", + "to_ids": false, + "uuid": "59259037-1014-4669-96b1-46af950d210f", + "event_id": "625", + "distribution": "5", + "timestamp": "1495633975", + "comment": "IMG_3428.pdf", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "value": "tmpzuni0skf", + "AttributeTag": [], + "ShadowAttribute": [] + }, { + "id": "164191", + "type": "domain|ip", + "category": "Network activity", + "to_ids": false, + "uuid": "59430251-e6a4-4900-b78b-060dc0a83832", + "event_id": "625", + "distribution": "5", + "timestamp": "1497563729", + "comment": "Test data", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "value": "google.com|127.0.0.1", + "AttributeTag": [], + "ShadowAttribute": [] + }], + "ShadowAttribute": [], + "EventTag": [{ + "id": "1482", + "event_id": "625", + "tag_id": "2", + "Tag": { + "id": "2", + "name": "tlp:white", + "colour": "#ffffff", + "exportable": true, + "org_id": "0", + "hide_tag": false + } + }], + "Galaxy": [], + "RelatedEvent": [{ + "Event": { + "id": "226", + "date": "2015-11-05", + "threat_level_id": "4", + "info": "OSINT Expansion on Systematic cyber attacks against Israeli and Palestinian targets going on for a year by Norman", + "published": true, + "uuid": "563b3ea6-b26c-401f-a68b-4d84950d210b", + "analysis": "2", + "timestamp": "1487757679", + "distribution": "3", + "org_id": "1", + "orgc_id": "3", + "Org": { + "id": "1", + "name": "MISP", + "uuid": "56ef3277-1ad4-42f6-b90b-04e5c0a83832" + }, + "Orgc": { + "id": "3", + "name": "CthulhuSPRL.be", + "uuid": "55f6ea5f-fd34-43b8-ac1d-40cb950d210f" + } + } + }, { + "Event": { + "id": "207", + "date": "2015-04-03", + "threat_level_id": "4", + "info": "OSINT The Dyre Wolf report from IBM", + "published": true, + "uuid": "551e8745-ace0-461c-b9eb-ce36950d210b", + "analysis": "2", + "timestamp": "1428070986", + "distribution": "3", + "org_id": "1", + "orgc_id": "3", + "Org": { + "id": "1", + "name": "MISP", + "uuid": "56ef3277-1ad4-42f6-b90b-04e5c0a83832" + }, + "Orgc": { + "id": "3", + "name": "CthulhuSPRL.be", + "uuid": "55f6ea5f-fd34-43b8-ac1d-40cb950d210f" + } + } + }, { + "Event": { + "id": "209", + "date": "2015-01-26", + "threat_level_id": "2", + "info": "OSINT I Know You Want Me - Unplugging PlugX from Takahiro Haruyama & Hiroshi Suzuki Black Hat Asia 2014 presentation", + "published": true, + "uuid": "54c60f43-b084-453a-a162-4e08950d210b", + "analysis": "2", + "timestamp": "1422356942", + "distribution": "3", + "org_id": "1", + "orgc_id": "3", + "Org": { + "id": "1", + "name": "MISP", + "uuid": "56ef3277-1ad4-42f6-b90b-04e5c0a83832" + }, + "Orgc": { + "id": "3", + "name": "CthulhuSPRL.be", + "uuid": "55f6ea5f-fd34-43b8-ac1d-40cb950d210f" + } + } + }, { + "Event": { + "id": "214", + "date": "2014-12-18", + "threat_level_id": "4", + "info": "Expansion on two IPs listed in OSINT IOCs from various campaigns listed in Detecting Bleeding Edge Malware presentation at hack.lu 2014", + "published": true, + "uuid": "54932a3e-7284-4753-b95c-4e08950d210b", + "analysis": "2", + "timestamp": "1442489489", + "distribution": "3", + "org_id": "1", + "orgc_id": "3", + "Org": { + "id": "1", + "name": "MISP", + "uuid": "56ef3277-1ad4-42f6-b90b-04e5c0a83832" + }, + "Orgc": { + "id": "3", + "name": "CthulhuSPRL.be", + "uuid": "55f6ea5f-fd34-43b8-ac1d-40cb950d210f" + } + } + }, { + "Event": { + "id": "208", + "date": "2014-11-20", + "threat_level_id": "4", + "info": "Import of CitizenLab public DB of malware indicators", + "published": true, + "uuid": "546e08ce-3134-4892-997b-73ff950d210b", + "analysis": "2", + "timestamp": "1487758220", + "distribution": "3", + "org_id": "1", + "orgc_id": "3", + "Org": { + "id": "1", + "name": "MISP", + "uuid": "56ef3277-1ad4-42f6-b90b-04e5c0a83832" + }, + "Orgc": { + "id": "3", + "name": "CthulhuSPRL.be", + "uuid": "55f6ea5f-fd34-43b8-ac1d-40cb950d210f" + } + } + }, { + "Event": { + "id": "373", + "date": "2014-11-18", + "threat_level_id": "4", + "info": "OSINT Expansion on Additional indicators relating to Sofacy (APT28) phishing blog post by PWC", + "published": true, + "uuid": "546bc3e8-d498-4e0c-b169-f2ea950d210b", + "analysis": "2", + "timestamp": "1487758281", + "distribution": "3", + "org_id": "1", + "orgc_id": "3", + "Org": { + "id": "1", + "name": "MISP", + "uuid": "56ef3277-1ad4-42f6-b90b-04e5c0a83832" + }, + "Orgc": { + "id": "3", + "name": "CthulhuSPRL.be", + "uuid": "55f6ea5f-fd34-43b8-ac1d-40cb950d210f" + } + } + }, { + "Event": { + "id": "230", + "date": "2014-10-02", + "threat_level_id": "3", + "info": "OSINT ShellShock scanning IPs from OpenDNS", + "published": true, + "uuid": "542e4c9c-cadc-4f8f-bb11-6d13950d210b", + "analysis": "2", + "timestamp": "1442489604", + "distribution": "3", + "org_id": "1", + "orgc_id": "3", + "Org": { + "id": "1", + "name": "MISP", + "uuid": "56ef3277-1ad4-42f6-b90b-04e5c0a83832" + }, + "Orgc": { + "id": "3", + "name": "CthulhuSPRL.be", + "uuid": "55f6ea5f-fd34-43b8-ac1d-40cb950d210f" + } + } + }], + "RelatedAttribute": { + "164191": [{ + "id": "207", + "org_id": "1", + "info": "OSINT The Dyre Wolf report from IBM", + "value": "google.com" + }, { + "id": "208", + "org_id": "1", + "info": "Import of CitizenLab public DB of malware indicators", + "value": "127.0.0.1" + }, { + "id": "209", + "org_id": "1", + "info": "OSINT I Know You Want Me - Unplugging PlugX from Takahiro Haruyama & Hiroshi Suzuki Black Hat Asia 2014 presentation", + "value": "127.0.0.1" + }, { + "id": "214", + "org_id": "1", + "info": "Expansion on two IPs listed in OSINT IOCs from various campaigns listed in Detecting Bleeding Edge Malware presentation at hack.lu 2014", + "value": "127.0.0.1" + }, { + "id": "226", + "org_id": "1", + "info": "OSINT Expansion on Systematic cyber attacks against Israeli and Palestinian targets going on for a year by Norman", + "value": "127.0.0.1" + }, { + "id": "230", + "org_id": "1", + "info": "OSINT ShellShock scanning IPs from OpenDNS", + "value": "127.0.0.1" + }, { + "id": "373", + "org_id": "1", + "info": "OSINT Expansion on Additional indicators relating to Sofacy (APT28) phishing blog post by PWC", + "value": "127.0.0.1" + }] + }, + "RelatedShadowAttribute": [], + "Sighting": [] +} \ No newline at end of file diff --git a/tests/threatconnect_export_test.py b/tests/threatconnect_export_test.py new file mode 100644 index 0000000..92a3c9a --- /dev/null +++ b/tests/threatconnect_export_test.py @@ -0,0 +1,60 @@ +"""Test module for the ThreatConnect Export module""" +import base64 +import csv +import io +import json +import os +import unittest +import requests + + +class TestModules(unittest.TestCase): + """Unittest module for threat_connect_export.py""" + def setUp(self): + self.headers = {'Content-Type': 'application/json'} + self.url = "http://127.0.0.1:6666/" + self.module = "threat_connect_export" + input_event_path = "%s/test_files/misp_event.json" % os.path.dirname(os.path.realpath(__file__)) + with open(input_event_path, "r") as ifile: + self.event = json.load(ifile) + + def test_01_introspection(self): + """Taken from test.py""" + try: + response = requests.get(self.url + "modules") + modules = [module["name"] for module in response.json()] + assert self.module in modules + finally: + response.connection.close() + + def test_02_export(self): + """Test an event export""" + test_source = "Test Export" + query = { + "module": self.module, + "data": [self.event], + "config": { + "Default_Source": test_source + } + } + + try: + response = requests.post(self.url + "query", headers=self.headers, data=json.dumps(query)) + data = base64.b64decode(response.json()["data"]).decode("utf-8") + csvfile = io.StringIO(data) + reader = csv.DictReader(csvfile) + + values = [field["Value"] for field in reader] + assert "google.com" in values + assert "127.0.0.1" in values + + # resetting file pointer to read through again and extract sources + csvfile.seek(0) + # use a set comprehension to deduplicate sources + sources = {field["Source"] for field in reader} + assert test_source in sources + finally: + response.connection.close() + +if __name__ == "__main__": + unittest.main()