From 50bae1f549c55a90ac7da0ef259d2eac05e70c29 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Wed, 28 Sep 2016 15:45:01 +0200 Subject: [PATCH 01/32] Simple import module to import MISP JSON format --- misp_modules/modules/import_mod/__init__.py | 2 +- misp_modules/modules/import_mod/mispjson.py | 61 +++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100755 misp_modules/modules/import_mod/mispjson.py diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py index 82505d9..fd5d539 100644 --- a/misp_modules/modules/import_mod/__init__.py +++ b/misp_modules/modules/import_mod/__init__.py @@ -1,3 +1,3 @@ from . import _vmray -__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport', 'email_import'] +__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport', 'email_import', 'mispjson'] diff --git a/misp_modules/modules/import_mod/mispjson.py b/misp_modules/modules/import_mod/mispjson.py new file mode 100755 index 0000000..628d3f0 --- /dev/null +++ b/misp_modules/modules/import_mod/mispjson.py @@ -0,0 +1,61 @@ +import json +import base64 + +misperrors = {'error': 'Error'} +userConfig = { }; + +inputSource = ['file'] + +moduleinfo = {'version': '0.1', 'author': 'Richard van den Berg', + 'description': 'MISP JSON format import module for MISP', + 'module-type': ['import']} + +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + r = {'results': []} + request = json.loads(q) + try: + mfile = base64.b64decode(request["data"]).decode('utf-8') + misp = json.loads(mfile) + event = misp['response'][0]['Event'] + for a in event["Attribute"]: + tmp = {} + tmp["values"] = a["value"] + tmp["categories"] = a["category"] + tmp["types"] = a["type"] + tmp["to_ids"] = a["to_ids"] + tmp["comment"] = a["comment"] + if a.get("data"): + tmp["data"] = a["data"] + r['results'].append(tmp) + except: + pass + return r + +def introspection(): + modulesetup = {} + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo + +if __name__ == '__main__': + x = open('test.json', 'r') + r = handler(q=x.read()) + print(json.dumps(r)) From 3a4c540a81bbb36c45b50e98cb71ce14a441fe5b Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Wed, 11 Jan 2017 09:45:57 +0100 Subject: [PATCH 02/32] Updated description to reflect merging use case --- misp_modules/modules/import_mod/mispjson.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/import_mod/mispjson.py b/misp_modules/modules/import_mod/mispjson.py index 628d3f0..f9d52ec 100755 --- a/misp_modules/modules/import_mod/mispjson.py +++ b/misp_modules/modules/import_mod/mispjson.py @@ -7,7 +7,7 @@ userConfig = { }; inputSource = ['file'] moduleinfo = {'version': '0.1', 'author': 'Richard van den Berg', - 'description': 'MISP JSON format import module for MISP', + 'description': 'MISP JSON format import module for merging MISP events', 'module-type': ['import']} moduleconfig = [] From 3590504821cf068301736b6dd89f5df044e18b73 Mon Sep 17 00:00:00 2001 From: Joerg Stephan Date: Sat, 21 Jan 2017 23:31:19 +0100 Subject: [PATCH 03/32] XForce Exchange v1 (alpha) --- .../modules/expansion/xforceexchange.py | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 misp_modules/modules/expansion/xforceexchange.py diff --git a/misp_modules/modules/expansion/xforceexchange.py b/misp_modules/modules/expansion/xforceexchange.py new file mode 100644 index 0000000..5d1a24a --- /dev/null +++ b/misp_modules/modules/expansion/xforceexchange.py @@ -0,0 +1,89 @@ +#!/usr/bin/python + +import urllib2 +import json + +import sys +BASEurl = "https://api.xforce.ibmcloud.com/" + +extensions = {"ip1": "ipr/%s", + "ip2": "ipr/malware/%s", + "url": "url/%s", + "hash": "malware/%s", + "vuln": "/vulnerabilities/search/%s"} + +sys.path.append('./') + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-src','ip-dst' 'vulnerability', 'md5', 'sha1', 'sha256'], + 'output': ['ip-src', 'ip-dst', 'text']} + +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '1', 'author': 'Joerg Stephan (@johest)', + 'description': 'IBM X-Force Exchange expansion module', + 'module-type': ['expansion', 'hover']} + +# config fields that your code expects from the site admin +moduleconfig = ["apikey", "event_limit"] +limit = 5000 #Default + + + +def MyHeader(key=False): + global limit + if key is False: + return None + + return {"Authorization": "Basic %s " % key, + "Accept": "application/json", + 'User-Agent': 'Mozilla 5.0'} + +def handler(q=False): + global limit + if q is False: + return False + + q = json.loads(q) + + key = q["config"]["apikey"] + limit = int(q["config"].get("event_limit", 5)) + + r = {"results": []} + + if "ip-src" in q: + r["results"] += dict( (apicall("ip1", q["ip-src"], key)).items() + (apicall("ip2", q["ip-src"], key)).items()) + if "ip-dst" in q: + r["results"] += dict( (apicall("ip1", q["ip-src"], key)).items() + (apicall("ip2", q["ip-src"], key)).items()) + if "md5" in q: + r["results"] += apicall("hash", q["md5"], key) + if "sha1" in q: + r["results"] += apicall("hash", q["sha1"], key) + if "sha256" in q: + r["results"] += apicall("hash", q["sha256"], key) + if 'vulnerability' in q: + r["results"] += apicall("vuln", q["vulnerability"], key) + + uniq = [] + for res in r["results"]: + if res not in uniq: + uniq.append(res) + r["results"] = uniq + return r + +def apicall(indicator_type, indicator, key=False): + try: + myURL = BASEurl + (extensions[str(indicator_type)])%indicator + request = urllib2.Request(myURL, None, MyHeader(key)) + data = urllib2.urlopen(request) + jsondata = json.loads(data.read()) + except: + return None + return jsondata + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo \ No newline at end of file From 03044e1e6aefaf290159d8499b314191f5399299 Mon Sep 17 00:00:00 2001 From: Joerg Stephan Date: Sun, 22 Jan 2017 00:00:15 +0100 Subject: [PATCH 04/32] merged xforce exchange --- REQUIREMENTS | 1 + 1 file changed, 1 insertion(+) diff --git a/REQUIREMENTS b/REQUIREMENTS index 6cda15a..2c845ec 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -19,3 +19,4 @@ pytesseract SPARQLWrapper domaintools_api pygeoip +urllib2 From dad73feaa41011795d50a9ff0b43ff2bb2454e7a Mon Sep 17 00:00:00 2001 From: Joerg Stephan Date: Tue, 31 Jan 2017 16:34:41 +0100 Subject: [PATCH 05/32] python3 changes --- .../modules/expansion/xforceexchange.py | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/misp_modules/modules/expansion/xforceexchange.py b/misp_modules/modules/expansion/xforceexchange.py index 5d1a24a..8a01e77 100644 --- a/misp_modules/modules/expansion/xforceexchange.py +++ b/misp_modules/modules/expansion/xforceexchange.py @@ -1,6 +1,4 @@ -#!/usr/bin/python - -import urllib2 +import requests import json import sys @@ -71,19 +69,17 @@ def handler(q=False): return r def apicall(indicator_type, indicator, key=False): - try: + #try: myURL = BASEurl + (extensions[str(indicator_type)])%indicator - request = urllib2.Request(myURL, None, MyHeader(key)) - data = urllib2.urlopen(request) - jsondata = json.loads(data.read()) - except: - return None - return jsondata + jsondata = requests.get(myURL, headers=MyHeader(key)).json() + #except: + #return None + return jsondata def introspection(): - return mispattributes + return mispattributes def version(): - moduleinfo['config'] = moduleconfig - return moduleinfo \ No newline at end of file + moduleinfo['config'] = moduleconfig + return moduleinfo \ No newline at end of file From 2651e68238b67cb95e97e5facf6e8fd9e7df5359 Mon Sep 17 00:00:00 2001 From: Joerg Stephan Date: Tue, 31 Jan 2017 16:54:53 +0100 Subject: [PATCH 06/32] removed urrlib2 --- REQUIREMENTS | 1 - 1 file changed, 1 deletion(-) diff --git a/REQUIREMENTS b/REQUIREMENTS index 2c845ec..6cda15a 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -19,4 +19,3 @@ pytesseract SPARQLWrapper domaintools_api pygeoip -urllib2 From 68250094ff0abe8abebccaf105a5889de1c9e8bc Mon Sep 17 00:00:00 2001 From: Joerg Stephan Date: Tue, 31 Jan 2017 16:57:16 +0100 Subject: [PATCH 07/32] v1 --- misp_modules/modules/expansion/xforceexchange.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/misp_modules/modules/expansion/xforceexchange.py b/misp_modules/modules/expansion/xforceexchange.py index 8a01e77..5d67a9b 100644 --- a/misp_modules/modules/expansion/xforceexchange.py +++ b/misp_modules/modules/expansion/xforceexchange.py @@ -1,7 +1,7 @@ import requests import json - import sys + BASEurl = "https://api.xforce.ibmcloud.com/" extensions = {"ip1": "ipr/%s", @@ -69,12 +69,12 @@ def handler(q=False): return r def apicall(indicator_type, indicator, key=False): - #try: + try: myURL = BASEurl + (extensions[str(indicator_type)])%indicator jsondata = requests.get(myURL, headers=MyHeader(key)).json() - #except: - #return None - return jsondata + except: + return None + return jsondata def introspection(): return mispattributes From de3495ea6cc8dcbc8d4404a081b55f0c5b7079b2 Mon Sep 17 00:00:00 2001 From: Joerg Stephan Date: Wed, 1 Feb 2017 14:05:29 +0100 Subject: [PATCH 08/32] passed local run check --- .../modules/expansion/xforceexchange.py | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/misp_modules/modules/expansion/xforceexchange.py b/misp_modules/modules/expansion/xforceexchange.py index 5d67a9b..459c69a 100644 --- a/misp_modules/modules/expansion/xforceexchange.py +++ b/misp_modules/modules/expansion/xforceexchange.py @@ -5,16 +5,17 @@ import sys BASEurl = "https://api.xforce.ibmcloud.com/" extensions = {"ip1": "ipr/%s", - "ip2": "ipr/malware/%s", - "url": "url/%s", - "hash": "malware/%s", - "vuln": "/vulnerabilities/search/%s"} + "ip2": "ipr/malware/%s", + "url": "url/%s", + "hash": "malware/%s", + "vuln": "/vulnerabilities/search/%s", + "dns": "resolve/%s"} sys.path.append('./') misperrors = {'error': 'Error'} mispattributes = {'input': ['ip-src','ip-dst' 'vulnerability', 'md5', 'sha1', 'sha256'], - 'output': ['ip-src', 'ip-dst', 'text']} + 'output': ['ip-src', 'ip-dst', 'text', 'domain']} # possible module-types: 'expansion', 'hover' or both moduleinfo = {'version': '1', 'author': 'Joerg Stephan (@johest)', @@ -49,9 +50,9 @@ def handler(q=False): r = {"results": []} if "ip-src" in q: - r["results"] += dict( (apicall("ip1", q["ip-src"], key)).items() + (apicall("ip2", q["ip-src"], key)).items()) + r["results"] += apicall("dns", q["ip-src"], key) if "ip-dst" in q: - r["results"] += dict( (apicall("ip1", q["ip-src"], key)).items() + (apicall("ip2", q["ip-src"], key)).items()) + r["results"] += apicall("dns", q["ip-dst"], key) if "md5" in q: r["results"] += apicall("hash", q["md5"], key) if "sha1" in q: @@ -60,6 +61,8 @@ def handler(q=False): r["results"] += apicall("hash", q["sha256"], key) if 'vulnerability' in q: r["results"] += apicall("vuln", q["vulnerability"], key) + if "domain" in q: + r["results"] += apicall("dns", q["domain"], key) uniq = [] for res in r["results"]: @@ -73,8 +76,21 @@ def apicall(indicator_type, indicator, key=False): myURL = BASEurl + (extensions[str(indicator_type)])%indicator jsondata = requests.get(myURL, headers=MyHeader(key)).json() except: - return None - return jsondata + jsondata = None + redata = [] + #print(jsondata) + if not jsondata is None: + if indicator_type is "hash": + if "malware" in jsondata: + lopointer = jsondata["malware"] + redata.append({"type": "text", "values": lopointer["risk"]}) + if indicator_type is "dns": + if "records" in str(jsondata): + lopointer = jsondata["Passive"]["records"] + for dataset in lopointer: + redata.append({"type":"domain", "values": dataset["value"]}) + + return redata def introspection(): return mispattributes @@ -82,4 +98,4 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig - return moduleinfo \ No newline at end of file + return moduleinfo From b5b7e09ef458b95069dd78ba93b6954abdb1d621 Mon Sep 17 00:00:00 2001 From: rmarsollier Date: Fri, 10 Feb 2017 14:16:39 +0100 Subject: [PATCH 09/32] Some improvements of virustotal plugin --- misp_modules/modules/expansion/virustotal.py | 27 +++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index c0c7715..a5ccd7d 100755 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -6,12 +6,12 @@ import base64 import os misperrors = {'error': 'Error'} -mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst"], - 'output':['domain', "ip-src", "ip-dst", "text"] +mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "sha512"], + 'output':['domain', "ip-src", "ip-dst", "text", "md5", "sha1", "sha256", "sha512", "ssdeep", "authentihash", "filename"] } # possible module-types: 'expansion', 'hover' or both -moduleinfo = {'version': '', 'author': 'Hannah Ward', +moduleinfo = {'version': '2', 'author': 'Hannah Ward', 'description': 'Get information from virustotal', 'module-type': ['expansion']} @@ -39,6 +39,14 @@ def handler(q=False): r["results"] += getDomain(q["domain"], key) if 'hostname' in q: r["results"] += getDomain(q['hostname'], key) + if 'md5' in q: + r["results"] += getHash(q['md5'], key) + if 'sha1' in q: + r["results"] += getHash(q['sha1'], key) + if 'sha256' in q: + r["results"] += getHash(q['sha256'], key) + if 'sha512' in q: + r["results"] += getHash(q['sha512'], key) uniq = [] for res in r["results"]: @@ -47,6 +55,19 @@ def handler(q=False): r["results"] = uniq return r +def getHash(hash, key, do_not_recurse = False): + global limit + toReturn = [] + req = requests.get("https://www.virustotal.com/vtapi/v2/file/report", + params = { "allinfo":1, "apikey":key, 'resource': hash} + ).json() + if req["response_code"] == 0: + #Nothing found + return [] + + toReturn += getMoreInfo(req, key) + return toReturn + def getIP(ip, key, do_not_recurse = False): global limit toReturn = [] From 6f378578dc6644a0c18249ccd277b5eb54e8b034 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 21 Feb 2017 11:27:59 +0100 Subject: [PATCH 10/32] fix: misp-modules are by default installed in /bin --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ed364ff..a2ca75c 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ cd misp-modules sudo pip3 install -I -r REQUIREMENTS sudo pip3 install -I . sudo vi /etc/rc.local, add this line: `sudo -u www-data misp-modules -s &` -/usr/local/bin/misp-modules #to start the modules +misp-modules #to start the modules ~~~~ ## How to add your own MISP modules? From dd2646a0f4399a14384552287530a6484b209900 Mon Sep 17 00:00:00 2001 From: Tristan METAYER Date: Tue, 21 Feb 2017 16:48:09 +0100 Subject: [PATCH 11/32] Add lite Export module --- misp_modules/modules/export_mod/__init__.py | 2 +- misp_modules/modules/export_mod/liteexport.py | 81 +++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) create mode 100755 misp_modules/modules/export_mod/liteexport.py diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py index 69f1c00..ee457cf 100644 --- a/misp_modules/modules/export_mod/__init__.py +++ b/misp_modules/modules/export_mod/__init__.py @@ -1 +1 @@ -__all__ = ['testexport','cef_export'] +__all__ = ['testexport','cef_export','liteexport'] diff --git a/misp_modules/modules/export_mod/liteexport.py b/misp_modules/modules/export_mod/liteexport.py new file mode 100755 index 0000000..c8e2251 --- /dev/null +++ b/misp_modules/modules/export_mod/liteexport.py @@ -0,0 +1,81 @@ +import json +import base64 + +misperrors = {'error': 'Error'} + +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '1', + 'author': 'TM', + 'description': 'export lite', + 'module-type': ['export']} + +# config fields that your code expects from the site admin +moduleconfig = ["indent_json_export"] + +#~ mispattributes = {'input':'all'} ? +mispattributes = {} +outputFileExtension = "json" +responseType = "application/json" + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if "config" in request: + config = request["config"] + else: + config = {"indent_json_export":None} + + if 'data' not in request: + return False + + liteEvent = {'Event':{}} + + for evt in request['data']: + rawEvent = evt['Event'] + liteEvent['Event']['info'] = rawEvent['info'] + liteEvent['Event']['Attribute'] = [] + + attrs = evt['Attribute'] + for attr in attrs: + liteAttr = {} + liteAttr['category'] = attr['category'] + liteAttr['type'] = attr['type'] + liteAttr['value'] = attr['value'] + liteEvent['Event']['Attribute'].append(liteAttr) + + return {"response":[], + 'data': str(base64.b64encode( + bytes( + json.dumps(liteEvent, indent=config['indent_json_export']), + 'utf-8')), + 'utf-8') + } + +def introspection(): + modulesetup = {} + try: + responseType + modulesetup['responseType'] = responseType + except NameError: + pass + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + outputFileExtension + modulesetup['outputFileExtension'] = outputFileExtension + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 20cb534203c2c02556bd349697433d0b56137267 Mon Sep 17 00:00:00 2001 From: Tristan METAYER Date: Tue, 21 Feb 2017 17:12:17 +0100 Subject: [PATCH 12/32] Exclude internal reference --- misp_modules/modules/export_mod/liteexport.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/misp_modules/modules/export_mod/liteexport.py b/misp_modules/modules/export_mod/liteexport.py index c8e2251..b22c350 100755 --- a/misp_modules/modules/export_mod/liteexport.py +++ b/misp_modules/modules/export_mod/liteexport.py @@ -3,14 +3,14 @@ import base64 misperrors = {'error': 'Error'} -# possible module-types: 'expansion', 'hover' or both moduleinfo = {'version': '1', 'author': 'TM', 'description': 'export lite', 'module-type': ['export']} -# config fields that your code expects from the site admin -moduleconfig = ["indent_json_export"] +#~ config form admin site but do not work +#~ moduleconfig = ["indent_json_export"] +moduleconfig = [] #~ mispattributes = {'input':'all'} ? mispattributes = {} @@ -29,6 +29,7 @@ def handler(q=False): if 'data' not in request: return False + #~ Misp json structur liteEvent = {'Event':{}} for evt in request['data']: @@ -38,11 +39,12 @@ def handler(q=False): attrs = evt['Attribute'] for attr in attrs: - liteAttr = {} - liteAttr['category'] = attr['category'] - liteAttr['type'] = attr['type'] - liteAttr['value'] = attr['value'] - liteEvent['Event']['Attribute'].append(liteAttr) + if 'Internal reference' not in attr['category']: + liteAttr = {} + liteAttr['category'] = attr['category'] + liteAttr['type'] = attr['type'] + liteAttr['value'] = attr['value'] + liteEvent['Event']['Attribute'].append(liteAttr) return {"response":[], 'data': str(base64.b64encode( From 6953b847e5be0e7a344db80839724f6daa919c21 Mon Sep 17 00:00:00 2001 From: truckydev Date: Fri, 24 Feb 2017 15:09:18 +0100 Subject: [PATCH 13/32] add information about offline installation --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index a2ca75c..84fd62d 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ ### Export modules * [CEF](misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF). +* [Lite Export](/misp-modules/blob/master/misp_modules/modules/export_mod/liteexport.py) module to export a lite event. ### Import modules @@ -352,6 +353,23 @@ Recommended Plugin.Import_ocr_enabled true Enable or disable the ocr In this same menu set any other plugin settings that are required for testing. +## Install misp-module on an offline instance. +First, you need to grab all necessery packages for example like this : + +Use pip wheel to create an archive +~~~ +mkdir misp-modules-offline +pip3 wheel -r REQUIREMENTS shodan --wheel-dir=./misp-modules-offline +tar -cjvf misp-module-bundeled.tar.bz2 ./misp-modules-offline/* +~~~ +On offline machine : +~~~ +mkdir misp-modules-bundle +tar xvf misp-module-bundeled.tar.bz2 -C misp-modules-bundle +cd misp-modules-bundle +ls -1|while read line; do sudo pip3 install --force-reinstall --ignore-installed --upgrade --no-index --no-deps ${line};done +~~~ +Next you can follow standard install procedure. ## How to contribute your own module? From c508e60f65cc113371b6e7b358da7a86a21e3ec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 27 Feb 2017 13:32:31 +0100 Subject: [PATCH 14/32] Add OpenIOC import module --- REQUIREMENTS | 1 + misp_modules/modules/import_mod/__init__.py | 3 +- .../modules/import_mod/openiocimport.py | 58 ++++++++++++ tests/openioc.xml | 91 +++++++++++++++++++ tests/test.py | 85 +++++++++-------- 5 files changed, 200 insertions(+), 38 deletions(-) create mode 100755 misp_modules/modules/import_mod/openiocimport.py create mode 100644 tests/openioc.xml diff --git a/REQUIREMENTS b/REQUIREMENTS index 6cda15a..b4e5ebc 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -19,3 +19,4 @@ pytesseract SPARQLWrapper domaintools_api pygeoip +bs4 diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py index fd5d539..6beeaa2 100644 --- a/misp_modules/modules/import_mod/__init__.py +++ b/misp_modules/modules/import_mod/__init__.py @@ -1,3 +1,4 @@ from . import _vmray -__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport', 'email_import', 'mispjson'] +__all__ = ['vmray_import', 'testimport', 'ocr', 'stiximport', 'cuckooimport', + 'email_import', 'mispjson', 'openiocimport'] diff --git a/misp_modules/modules/import_mod/openiocimport.py b/misp_modules/modules/import_mod/openiocimport.py new file mode 100755 index 0000000..27ef3f9 --- /dev/null +++ b/misp_modules/modules/import_mod/openiocimport.py @@ -0,0 +1,58 @@ +import json +import base64 + +from pymisp.tools import openioc + +misperrors = {'error': 'Error'} +userConfig = {} +inputSource = ['file'] + +moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot', + 'description': 'Import OpenIOC package', + 'module-type': ['import']} + +moduleconfig = [] + + +def handler(q=False): + # Just in case we have no data + if q is False: + return False + + # The return value + r = {'results': []} + + # Load up that JSON + q = json.loads(q) + + # It's b64 encoded, so decode that stuff + package = base64.b64decode(q.get("data")).decode('utf-8') + + # If something really weird happened + if not package: + return json.dumps({"success": 0}) + + pkg = openioc.load_openioc(package) + for attrib in pkg.attributes: + r["results"].append({"values": [attrib.value], "types": [attrib.type], "categories": [attrib.category]}) + return r + + +def introspection(): + modulesetup = {} + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/tests/openioc.xml b/tests/openioc.xml new file mode 100644 index 0000000..dc7858c --- /dev/null +++ b/tests/openioc.xml @@ -0,0 +1,91 @@ + + + STUXNET VIRUS (METHODOLOGY) + Generic indicator for the stuxnet virus. When loaded, stuxnet spawns lsass.exe in a suspended state. The malware then maps in its own executable section and fixes up the CONTEXT to point to the newly mapped in section. This is a common task performed by malware and allows the malware to execute under the pretense of a known and trusted process. + methodology + Mandiant + 0001-01-01T00:00:00 + + + + + + .stub + + + + mdmcpq3.PNF + + + + mdmeric3.PNF + + + + oem6C.PNF + + + + oem7A.PNF + + + + + fs_rec.sys + + + + mrxsmb.sys + + + + sr.sys + + + + fastfat.sys + + + + + + mrxcls.sys + + + + Realtek Semiconductor Corp + + + + + + mrxnet.sys + + + + Realtek Semiconductor Corp + + + + + + HKEY_LOCAL_MACHINE\SYSTEM\ControlSet001\Services\MRxCls\ImagePath + + + + mrxcls.sys + + + + + + HKEY_LOCAL_MACHINE\SYSTEM\ControlSet001\Services\MRxNet\ImagePath + + + + mrxnet.sys + + + + + \ No newline at end of file diff --git a/tests/test.py b/tests/test.py index a94bbdf..156846e 100644 --- a/tests/test.py +++ b/tests/test.py @@ -41,6 +41,20 @@ class TestModules(unittest.TestCase): print(response.json()) response.connection.close() + def test_openioc(self): + with open("tests/openioc.xml", "rb") as f: + content = base64.b64encode(f.read()) + data = json.dumps({"module": "openiocimport", + "data": content.decode(), + }) + response = requests.post(self.url + "query", data=data).json() + print(response) + + print("OpenIOC :: {}".format(response)) + values = [x["values"][0] for x in response["results"]] + assert("mrxcls.sys" in values) + assert("mdmcpq3.PNF" in values) + def test_stix(self): with open("tests/stix.xml", "rb") as f: content = base64.b64encode(f.read()) @@ -57,7 +71,7 @@ class TestModules(unittest.TestCase): assert("eu-society.com" in values) def test_email_headers(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": None, "guess_zip_attachment_passwords": None, "extract_urls": None} @@ -105,7 +119,7 @@ class TestModules(unittest.TestCase): self.assertIn("", values) def test_email_attachment_basic(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": None, "guess_zip_attachment_passwords": None, "extract_urls": None} @@ -128,9 +142,8 @@ class TestModules(unittest.TestCase): attch_data = base64.b64decode(i["data"]) self.assertEqual(attch_data, b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') - def test_email_attachment_unpack(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": None, "extract_urls": None} @@ -162,7 +175,7 @@ class TestModules(unittest.TestCase): def test_email_dont_unpack_compressed_doc_attachments(self): """Ensures that compressed """ - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": None, "extract_urls": None} @@ -192,9 +205,8 @@ class TestModules(unittest.TestCase): self.assertEqual(filesum.hexdigest(), '098da5381a90d4a51e6b844c18a0fecf2e364813c2f8b317cfdc51c21f2506a5') - def test_email_attachment_unpack_with_password(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": 'true', "extract_urls": None} @@ -221,9 +233,8 @@ class TestModules(unittest.TestCase): self.assertEqual(attch_data, b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') - def test_email_attachment_password_in_body(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": 'true', "extract_urls": None} @@ -246,7 +257,7 @@ class TestModules(unittest.TestCase): 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') def test_email_attachment_password_in_body_quotes(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": 'true', "extract_urls": None} @@ -274,7 +285,7 @@ class TestModules(unittest.TestCase): 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') def test_email_attachment_password_in_html_body(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": 'true', "extract_urls": None} @@ -304,7 +315,7 @@ class TestModules(unittest.TestCase): query['data'] = decode_email(message) data = json.dumps(query) response = requests.post(self.url + "query", data=data) - #print(response.json()) + # print(response.json()) values = [x["values"] for x in response.json()["results"]] self.assertIn('EICAR.com', values) for i in response.json()['results']: @@ -315,7 +326,7 @@ class TestModules(unittest.TestCase): 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') def test_email_attachment_password_in_subject(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", "guess_zip_attachment_passwords": 'true', "extract_urls": None} @@ -344,9 +355,8 @@ class TestModules(unittest.TestCase): self.assertEqual(attch_data, 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') - def test_email_extract_html_body_urls(self): - query = {"module":"email_import"} + query = {"module": "email_import"} query["config"] = {"unzip_attachments": None, "guess_zip_attachment_passwords": None, "extract_urls": "true"} @@ -374,12 +384,12 @@ without modifying core components. The API is available via a simple REST API wh query['data'] = decode_email(message) data = json.dumps(query) response = requests.post(self.url + "query", data=data) - #print(response.json()) + # print(response.json()) values = [x["values"] for x in response.json()["results"]] self.assertIn("https://github.com/MISP/MISP", values) self.assertIn("https://www.circl.lu/assets/files/misp-training/3.1-MISP-modules.pdf", values) - #def test_domaintools(self): + # def test_domaintools(self): # query = {'config': {'username': 'test_user', 'api_key': 'test_key'}, 'module': 'domaintools', 'domain': 'domaintools.com'} # try: # response = requests.post(self.url + "query", data=json.dumps(query)).json() @@ -388,33 +398,34 @@ without modifying core components. The API is available via a simple REST API wh # response = requests.post(self.url + "query", data=json.dumps(query)).json() # print(response) + def decode_email(message): message64 = base64.b64encode(message.as_bytes()).decode() return message64 def get_base_email(): - headers = {"Received":"via dmail-2008.19 for +INBOX; Tue, 3 Feb 2009 19:29:12 -0600 (CST)", - "Received":"from abc.luxsci.com ([10.10.10.10]) by xyz.luxsci.com (8.13.7/8.13.7) with ESMTP id n141TCa7022588 for ; Tue, 3 Feb 2009 19:29:12 -0600", - "Received":"from [192.168.0.3] (verizon.net [44.44.44.44]) (user=test@sender.com mech=PLAIN bits=2) by abc.luxsci.com (8.13.7/8.13.7) with ESMTP id n141SAfo021855 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=NOT) for ; Tue, 3 Feb 2009 19:28:10 -0600", - "X-Received":"by 192.168.0.45 with SMTP id q4mr156123401yw1g.911.1912342394963; Tue, 3 Feb 2009 19:32:15 -0600 (PST)", - "Message-ID":"<4988EF2D.40804@example.com>", - "Date":"Tue, 03 Feb 2009 20:28:13 -0500", - "From":'"Innocent Person" ', - "User-Agent":'Thunderbird 2.0.0.19 (Windows/20081209)', - "Sender":'"Malicious MailAgent" ', - "References":"", - "In-Reply-To":"", - "Accept-Language":'en-US', - "X-Mailer":'mlx 5.1.7', + headers = {"Received": "via dmail-2008.19 for +INBOX; Tue, 3 Feb 2009 19:29:12 -0600 (CST)", + "Received": "from abc.luxsci.com ([10.10.10.10]) by xyz.luxsci.com (8.13.7/8.13.7) with ESMTP id n141TCa7022588 for ; Tue, 3 Feb 2009 19:29:12 -0600", + "Received": "from [192.168.0.3] (verizon.net [44.44.44.44]) (user=test@sender.com mech=PLAIN bits=2) by abc.luxsci.com (8.13.7/8.13.7) with ESMTP id n141SAfo021855 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=NOT) for ; Tue, 3 Feb 2009 19:28:10 -0600", + "X-Received": "by 192.168.0.45 with SMTP id q4mr156123401yw1g.911.1912342394963; Tue, 3 Feb 2009 19:32:15 -0600 (PST)", + "Message-ID": "<4988EF2D.40804@example.com>", + "Date": "Tue, 03 Feb 2009 20:28:13 -0500", + "From": '"Innocent Person" ', + "User-Agent": 'Thunderbird 2.0.0.19 (Windows/20081209)', + "Sender": '"Malicious MailAgent" ', + "References": "", + "In-Reply-To": "", + "Accept-Language": 'en-US', + "X-Mailer": 'mlx 5.1.7', "Return-Path": "evil_spoofer@example.com", - "Thread-Topic":'This is a thread.', - "Thread-Index":'AQHSR8Us3H3SoaY1oUy9AAwZfMF922bnA9GAgAAi9s4AAGvxAA==', - "Content-Language":'en-US', - "To":'"Testy Testerson" ', - "Cc":'"Second Person" , "Other Friend" , "Last One" ', - "Subject":'Example Message', - "MIME-Version":'1.0'} + "Thread-Topic": 'This is a thread.', + "Thread-Index": 'AQHSR8Us3H3SoaY1oUy9AAwZfMF922bnA9GAgAAi9s4AAGvxAA==', + "Content-Language": 'en-US', + "To": '"Testy Testerson" ', + "Cc": '"Second Person" , "Other Friend" , "Last One" ', + "Subject": 'Example Message', + "MIME-Version": '1.0'} msg = MIMEMultipart() for key, val in headers.items(): msg.add_header(key, val) From 312f792b22172bab61fb1c9207828483e73ec885 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 27 Feb 2017 14:10:11 +0100 Subject: [PATCH 15/32] OpenIOC import module added --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 84fd62d..a4c7514 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [Cuckoo JSON](misp_modules/modules/import_mod/cuckooimport.py) Cuckoo JSON import. * [OCR](misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes. +* [OpenIOC](misp_modules/modules/import_mod/openiocimport.py) OpenIOC import based on PyMISP library. * [stiximport](misp_modules/modules/import_mod/stiximport.py) - An import module to process STIX xml/json. * [Email Import](misp_modules/modules/import_mod/email_import.py) Email import module for MISP to import basic metadata. * [VMRay](misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export. From bc1eab3520a605e5c059dc549acc3e345451ba70 Mon Sep 17 00:00:00 2001 From: kx499 Date: Tue, 28 Feb 2017 22:04:24 -0500 Subject: [PATCH 16/32] fixed spacing, addressed error handling for public api, added subdomains, and added context comment --- misp_modules/modules/expansion/virustotal.py | 199 +++++++++++-------- 1 file changed, 113 insertions(+), 86 deletions(-) diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index a5ccd7d..63a8720 100755 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -7,7 +7,8 @@ import os misperrors = {'error': 'Error'} mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "sha512"], - 'output':['domain', "ip-src", "ip-dst", "text", "md5", "sha1", "sha256", "sha512", "ssdeep", "authentihash", "filename"] + 'output': ['domain', "ip-src", "ip-dst", "text", "md5", "sha1", "sha256", "sha512", "ssdeep", + "authentihash", "filename"] } # possible module-types: 'expansion', 'hover' or both @@ -17,7 +18,9 @@ moduleinfo = {'version': '2', 'author': 'Hannah Ward', # config fields that your code expects from the site admin moduleconfig = ["apikey", "event_limit"] -limit = 5 #Default +limit = 5 # Default +comment = '%s: Enriched via VT' + def handler(q=False): global limit @@ -30,145 +33,169 @@ def handler(q=False): limit = int(q["config"].get("event_limit", 5)) r = {"results": []} - + if "ip-src" in q: - r["results"] += getIP(q["ip-src"], key) + r["results"] += getIP(q["ip-src"], key) if "ip-dst" in q: - r["results"] += getIP(q["ip-dst"], key) + r["results"] += getIP(q["ip-dst"], key) if "domain" in q: - r["results"] += getDomain(q["domain"], key) + r["results"] += getDomain(q["domain"], key) if 'hostname' in q: - r["results"] += getDomain(q['hostname'], key) + r["results"] += getDomain(q['hostname'], key) if 'md5' in q: - r["results"] += getHash(q['md5'], key) + r["results"] += getHash(q['md5'], key) if 'sha1' in q: - r["results"] += getHash(q['sha1'], key) + r["results"] += getHash(q['sha1'], key) if 'sha256' in q: - r["results"] += getHash(q['sha256'], key) + r["results"] += getHash(q['sha256'], key) if 'sha512' in q: - r["results"] += getHash(q['sha512'], key) + r["results"] += getHash(q['sha512'], key) uniq = [] for res in r["results"]: - if res not in uniq: - uniq.append(res) + if res not in uniq: + uniq.append(res) r["results"] = uniq return r -def getHash(hash, key, do_not_recurse = False): + +def getHash(hash, key, do_not_recurse=False): global limit toReturn = [] - req = requests.get("https://www.virustotal.com/vtapi/v2/file/report", - params = { "allinfo":1, "apikey":key, 'resource': hash} - ).json() + try: + req = requests.get("https://www.virustotal.com/vtapi/v2/file/report", + params={"allinfo": 1, "apikey": key, 'resource': hash} + ).json() + except: + return [] + if req["response_code"] == 0: - #Nothing found - return [] + # Nothing found + return [] toReturn += getMoreInfo(req, key) return toReturn -def getIP(ip, key, do_not_recurse = False): + +def getIP(ip, key, do_not_recurse=False): global limit toReturn = [] - req = requests.get("https://www.virustotal.com/vtapi/v2/ip-address/report", - params = {"ip":ip, "apikey":key} - ).json() + try: + req = requests.get("https://www.virustotal.com/vtapi/v2/ip-address/report", + params={"ip": ip, "apikey": key} + ).json() + except: + return [] + if req["response_code"] == 0: - #Nothing found - return [] - + # Nothing found + return [] + if "resolutions" in req: - for res in req["resolutions"][:limit]: - toReturn.append( {"types":["domain"], "values":[res["hostname"]]}) - #Pivot from here to find all domain info - if not do_not_recurse: - toReturn += getDomain(res["hostname"], key, True) + for res in req["resolutions"][:limit]: + toReturn.append({"types": ["domain"], "values": [res["hostname"]], "comment":comment % ip}) + # Pivot from here to find all domain info + if not do_not_recurse: + toReturn += getDomain(res["hostname"], key, True) toReturn += getMoreInfo(req, key) return toReturn - + + def getDomain(domain, key, do_not_recurse=False): global limit toReturn = [] - req = requests.get("https://www.virustotal.com/vtapi/v2/domain/report", - params = {"domain":domain, "apikey":key} - ).json() + try: + req = requests.get("https://www.virustotal.com/vtapi/v2/domain/report", + params={"domain": domain, "apikey": key} + ).json() + except: + return [] + if req["response_code"] == 0: - #Nothing found - return [] - + # Nothing found + return [] + if "resolutions" in req: - for res in req["resolutions"][:limit]: - toReturn.append( {"types":["ip-dst", "ip-src"], "values":[res["ip_address"]]}) - #Pivot from here to find all info on IPs - if not do_not_recurse: - toReturn += getIP(res["ip_address"], key, True) + for res in req["resolutions"][:limit]: + toReturn.append({"types": ["ip-dst", "ip-src"], "values": [res["ip_address"]], "comment":comment % domain}) + # Pivot from here to find all info on IPs + if not do_not_recurse: + toReturn += getIP(res["ip_address"], key, True) + if "subdomains" in req: + for subd in req["subdomains"]: + toReturn.append({"types": ["domain"], "values": [subd], "comment": comment % domain}) toReturn += getMoreInfo(req, key) return toReturn + def findAll(data, keys): - a = [] - if isinstance(data, dict): - for key in data.keys(): - if key in keys: - a.append(data[key]) - else: - if isinstance(data[key], (dict, list)): - a += findAll(data[key], keys) - if isinstance(data, list): - for i in data: - a += findAll(i, keys) - - return a + a = [] + if isinstance(data, dict): + for key in data.keys(): + if key in keys: + a.append(data[key]) + else: + if isinstance(data[key], (dict, list)): + a += findAll(data[key], keys) + if isinstance(data, list): + for i in data: + a += findAll(i, keys) + + return a + def getMoreInfo(req, key): global limit r = [] - #Get all hashes first + # Get all hashes first hashes = [] hashes = findAll(req, ["md5", "sha1", "sha256", "sha512"]) - r.append({"types":["md5", "sha1", "sha256", "sha512"], "values":hashes}) + r.append({"types": ["md5", "sha1", "sha256", "sha512"], "values": hashes}) for hsh in hashes[:limit]: - #Search VT for some juicy info - data = requests.get("http://www.virustotal.com/vtapi/v2/file/report", - params={"allinfo":1, "apikey":key, "resource":hsh} - ).json() + # Search VT for some juicy info + try: + data = requests.get("http://www.virustotal.com/vtapi/v2/file/report", + params={"allinfo": 1, "apikey": key, "resource": hsh} + ).json() + except: + continue - # Go through each key and check if it exists - if "submission_names" in data: - r.append({'types':["filename"], "values":data["submission_names"]}) + # Go through each key and check if it exists + if "submission_names" in data: + r.append({'types': ["filename"], "values": data["submission_names"], "comment":comment % hsh}) - if "ssdeep" in data: - r.append({'types':["ssdeep"], "values":[data["ssdeep"]]}) + if "ssdeep" in data: + r.append({'types': ["ssdeep"], "values": [data["ssdeep"]], "comment":comment % hsh}) - if "authentihash" in data: - r.append({"types":["authentihash"], "values":[data["authentihash"]]}) + if "authentihash" in data: + r.append({"types": ["authentihash"], "values": [data["authentihash"]], "comment":comment % hsh}) - if "ITW_urls" in data: - r.append({"types":["url"], "values":data["ITW_urls"]}) + if "ITW_urls" in data: + r.append({"types": ["url"], "values": data["ITW_urls"], "comment":comment % hsh}) - #Get the malware sample - sample = requests.get("https://www.virustotal.com/vtapi/v2/file/download", - params = {"hash":hsh, "apikey":key}) - - malsample = sample.content + # Get the malware sample + sample = requests.get("https://www.virustotal.com/vtapi/v2/file/download", + params={"hash": hsh, "apikey": key}) + + malsample = sample.content + + # It is possible for VT to not give us any submission names + if "submission_names" in data: + r.append({"types": ["malware-sample"], + "categories": ["Payload delivery"], + "values": data["submission_names"], + "data": str(base64.b64encode(malsample), 'utf-8') + } + ) - # It is possible for VT to not give us any submission names - if "submission_names" in data: - r.append({"types":["malware-sample"], - "categories":["Payload delivery"], - "values":data["submission_names"], - "data": str(base64.b64encode(malsample), 'utf-8') - } - ) - return r + def introspection(): return mispattributes + def version(): moduleinfo['config'] = moduleconfig return moduleinfo - From 01fdf3e52b63d0ab1d8d9b8af12234a71fc4f635 Mon Sep 17 00:00:00 2001 From: kx499 Date: Fri, 3 Mar 2017 15:55:52 -0500 Subject: [PATCH 17/32] Initial commit of IPRep module --- misp_modules/modules/expansion/iprep.py | 85 +++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100755 misp_modules/modules/expansion/iprep.py diff --git a/misp_modules/modules/expansion/iprep.py b/misp_modules/modules/expansion/iprep.py new file mode 100755 index 0000000..1dc6460 --- /dev/null +++ b/misp_modules/modules/expansion/iprep.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- + +import json +import requests + + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['freetext']} +moduleinfo = {'version': '1.0', 'author': 'Keith Faber', + 'description': 'Query IPRep Data for IP Address', + 'module-type': ['expansion']} + +moduleconfig = ['apikey'] + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if request.get('ip-src'): + toquery = request['ip-src'] + elif request.get('ip-dst'): + toquery = request['ip-dst'] + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + + if not request.get('config') and not request['config'].get('apikey'): + misperrors['error'] = 'IPRep api key is missing' + return misperrors + + err, rep = parse_iprep(toquery, request['config'].get('apikey')) + if len(err) > 0: + misperrors['error'] = ','.join(err) + return misperrors + return rep + + +def parse_iprep(ip, api): + meta_fields = ['origin', 'Query_Time', 'created_on', 'IP_Lookup_History', 'IPs_in_collection', '_id', 'disclaimer', + 'MaxMind_Free_GeoIP', 'Unique_Lookups', 'query_result'] + rep = [] + err = [] + url = 'https://www.packetmail.net/iprep.php/%s' % ip + try: + data = requests.get(url, params={'apikey': api}).json() + except: + return ['Error pulling data'], rep + # print '%s' % data + for name, val in data.iteritems(): + if name not in meta_fields: + try: + context = val['context'] + if type(context) is list: + if context[0].get('alert'): + context = ','.join([hit['alert']['signature'] for hit in context]) + elif context[0].get('signature'): + context = ','.join([hit['signature'] for hit in context]) + elif context[0].get('target_port') and context[0].get('protocol'): + context = ','.join( + ['Port Attacked: %s %s' % (hit['target_port'], hit['protocol']) for hit in context]) + elif context[0].get('phishing_kit') and context[0].get('url'): + context = ','.join(['%s (%s)' % (hit['phishing_kit'], hit['url']) for hit in context]) + else: + context = ';'.join(['%s: %s' % (k, v) for k, v in context[0].iteritems()]) + + if val.get('special_note'): + context += '; ' + val['special_note'] + + misp_val = context + misp_comment = 'IPRep Source %s: %s' % (name, val['last_seen']) + rep.append({'types': mispattributes['output'], 'values': misp_val, 'comment': misp_comment}) + except: + err.append('Error parsing source: %s' % name) + + return err, rep + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 3ecd095d1e8e1879001eb0e4c9c2b5d020ef8745 Mon Sep 17 00:00:00 2001 From: kx499 Date: Sat, 4 Mar 2017 03:10:45 +0100 Subject: [PATCH 18/32] bug fixes, tweaks, and python3 learning curve :) --- misp_modules/modules/expansion/__init__.py | 2 +- misp_modules/modules/expansion/iprep.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 914cb1c..0fa3791 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -2,4 +2,4 @@ from . import _vmray __all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'ipasn', 'passivetotal', 'sourcecache', - 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki'] + 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep'] diff --git a/misp_modules/modules/expansion/iprep.py b/misp_modules/modules/expansion/iprep.py index 1dc6460..6073052 100755 --- a/misp_modules/modules/expansion/iprep.py +++ b/misp_modules/modules/expansion/iprep.py @@ -5,7 +5,7 @@ import requests misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['freetext']} +mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['text']} moduleinfo = {'version': '1.0', 'author': 'Keith Faber', 'description': 'Query IPRep Data for IP Address', 'module-type': ['expansion']} @@ -33,7 +33,7 @@ def handler(q=False): if len(err) > 0: misperrors['error'] = ','.join(err) return misperrors - return rep + return {'results': rep} def parse_iprep(ip, api): @@ -41,13 +41,14 @@ def parse_iprep(ip, api): 'MaxMind_Free_GeoIP', 'Unique_Lookups', 'query_result'] rep = [] err = [] + full_text = '' url = 'https://www.packetmail.net/iprep.php/%s' % ip try: data = requests.get(url, params={'apikey': api}).json() except: return ['Error pulling data'], rep # print '%s' % data - for name, val in data.iteritems(): + for name, val in data.items(): if name not in meta_fields: try: context = val['context'] @@ -62,17 +63,19 @@ def parse_iprep(ip, api): elif context[0].get('phishing_kit') and context[0].get('url'): context = ','.join(['%s (%s)' % (hit['phishing_kit'], hit['url']) for hit in context]) else: - context = ';'.join(['%s: %s' % (k, v) for k, v in context[0].iteritems()]) + context = ';'.join(['%s: %s' % (k, v) for k, v in context[0].items()]) if val.get('special_note'): context += '; ' + val['special_note'] misp_val = context + full_text += '\n%s' % context misp_comment = 'IPRep Source %s: %s' % (name, val['last_seen']) - rep.append({'types': mispattributes['output'], 'values': misp_val, 'comment': misp_comment}) + rep.append({'types': mispattributes['output'], 'categories':['External analysis'], 'values': misp_val, 'comment': misp_comment}) except: err.append('Error parsing source: %s' % name) + rep.append({'types': ['freetext'], 'values': full_text , 'comment': 'Free text import of IPRep'}) return err, rep From 44867b2adc00777ecb44fb481ab58d88623d64f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sun, 5 Mar 2017 18:59:36 +0100 Subject: [PATCH 19/32] Cosmetic changes --- misp_modules/modules/expansion/virustotal.py | 57 ++++++++++---------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index 63a8720..44199a1 100755 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -1,9 +1,7 @@ import json import requests -import hashlib -import re +from requests import HTTPError import base64 -import os misperrors = {'error': 'Error'} mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "sha512"], @@ -60,32 +58,33 @@ def handler(q=False): def getHash(hash, key, do_not_recurse=False): - global limit - toReturn = [] + req = requests.get("https://www.virustotal.com/vtapi/v2/file/report", + params={"allinfo": 1, "apikey": key, 'resource': hash}) try: - req = requests.get("https://www.virustotal.com/vtapi/v2/file/report", - params={"allinfo": 1, "apikey": key, 'resource': hash} - ).json() - except: - return [] + req.raise_for_status() + req = req.json() + except HTTPError as e: + misperrors['error'] = str(e) + return misperrors if req["response_code"] == 0: # Nothing found return [] - toReturn += getMoreInfo(req, key) - return toReturn + return getMoreInfo(req, key) def getIP(ip, key, do_not_recurse=False): global limit toReturn = [] + req = requests.get("https://www.virustotal.com/vtapi/v2/ip-address/report", + params={"ip": ip, "apikey": key}) try: - req = requests.get("https://www.virustotal.com/vtapi/v2/ip-address/report", - params={"ip": ip, "apikey": key} - ).json() - except: - return [] + req.raise_for_status() + req = req.json() + except HTTPError as e: + misperrors['error'] = str(e) + return misperrors if req["response_code"] == 0: # Nothing found @@ -93,7 +92,7 @@ def getIP(ip, key, do_not_recurse=False): if "resolutions" in req: for res in req["resolutions"][:limit]: - toReturn.append({"types": ["domain"], "values": [res["hostname"]], "comment":comment % ip}) + toReturn.append({"types": ["domain"], "values": [res["hostname"]], "comment": comment % ip}) # Pivot from here to find all domain info if not do_not_recurse: toReturn += getDomain(res["hostname"], key, True) @@ -105,12 +104,14 @@ def getIP(ip, key, do_not_recurse=False): def getDomain(domain, key, do_not_recurse=False): global limit toReturn = [] + req = requests.get("https://www.virustotal.com/vtapi/v2/domain/report", + params={"domain": domain, "apikey": key}) try: - req = requests.get("https://www.virustotal.com/vtapi/v2/domain/report", - params={"domain": domain, "apikey": key} - ).json() - except: - return [] + req.raise_for_status() + req = req.json() + except HTTPError as e: + misperrors['error'] = str(e) + return misperrors if req["response_code"] == 0: # Nothing found @@ -118,7 +119,7 @@ def getDomain(domain, key, do_not_recurse=False): if "resolutions" in req: for res in req["resolutions"][:limit]: - toReturn.append({"types": ["ip-dst", "ip-src"], "values": [res["ip_address"]], "comment":comment % domain}) + toReturn.append({"types": ["ip-dst", "ip-src"], "values": [res["ip_address"]], "comment": comment % domain}) # Pivot from here to find all info on IPs if not do_not_recurse: toReturn += getIP(res["ip_address"], key, True) @@ -163,16 +164,16 @@ def getMoreInfo(req, key): # Go through each key and check if it exists if "submission_names" in data: - r.append({'types': ["filename"], "values": data["submission_names"], "comment":comment % hsh}) + r.append({'types': ["filename"], "values": data["submission_names"], "comment": comment % hsh}) if "ssdeep" in data: - r.append({'types': ["ssdeep"], "values": [data["ssdeep"]], "comment":comment % hsh}) + r.append({'types': ["ssdeep"], "values": [data["ssdeep"]], "comment": comment % hsh}) if "authentihash" in data: - r.append({"types": ["authentihash"], "values": [data["authentihash"]], "comment":comment % hsh}) + r.append({"types": ["authentihash"], "values": [data["authentihash"]], "comment": comment % hsh}) if "ITW_urls" in data: - r.append({"types": ["url"], "values": data["ITW_urls"], "comment":comment % hsh}) + r.append({"types": ["url"], "values": data["ITW_urls"], "comment": comment % hsh}) # Get the malware sample sample = requests.get("https://www.virustotal.com/vtapi/v2/file/download", From 31a8fb0fe46d5d56588d273a19af799f091a0c7f Mon Sep 17 00:00:00 2001 From: kx499 Date: Mon, 6 Mar 2017 21:36:00 -0500 Subject: [PATCH 20/32] threatminer initial commit --- misp_modules/modules/expansion/threatminer.py | 169 ++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100755 misp_modules/modules/expansion/threatminer.py diff --git a/misp_modules/modules/expansion/threatminer.py b/misp_modules/modules/expansion/threatminer.py new file mode 100755 index 0000000..c34034d --- /dev/null +++ b/misp_modules/modules/expansion/threatminer.py @@ -0,0 +1,169 @@ +import json +import requests +from requests import HTTPError +import base64 + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['hostname', 'domain', 'ip-src', 'ip-dst', 'md5', 'sha1', 'sha256', 'sha512'], + 'output': ['domain', 'ip-src', 'ip-dst', 'text', 'md5', 'sha1', 'sha256', 'sha512', 'ssdeep', + 'authentihash', 'filename', 'whois-registrant-email', 'url', 'link'] + } + +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '2', 'author': 'Hannah Ward', + 'description': 'Get information from virustotal', + 'module-type': ['expansion']} + +desc = '%s: %s Threatminer' + + +def handler(q=False): + if q is False: + return False + + q = json.loads(q) + + r = {'results': []} + + if 'ip-src' in q: + r['results'] += get_ip(q['ip-src']) + if 'ip-dst' in q: + r['results'] += get_ip(q['ip-dst']) + if 'domain' in q: + r['results'] += get_domain(q['domain']) + if 'hostname' in q: + r['results'] += get_domain(q['hostname']) + if 'md5' in q: + r['results'] += get_hash(q['md5']) + if 'sha1' in q: + r['results'] += get_hash(q['sha1']) + if 'sha256' in q: + r['results'] += get_hash(q['sha256']) + if 'sha512' in q: + r['results'] += get_hash(q['sha512']) + + uniq = [] + for res in r['results']: + if res not in uniq: + uniq.append(res) + r['results'] = uniq + return r + + +def get_domain(q): + ret = [] + for flag in [1, 2, 3, 4, 5, 6]: + req = requests.get('https://www.threatminer.org/domain.php', params={'q': q, 'api': 'True', 'rt': flag}) + if not req.status_code == 200: + return [] + results = req.json().get('results') + if not results: + return [] + + for result in results: + if flag == 1: #whois + emails = result.get('whois', {}).get('emails') + for em_type, email in emails.items(): + ret.append({'types': ['whois-registrant-email'], 'values': [email], 'comment': desc % (q, 'whois')}) + if flag == 2: #pdns + ip = result.get('ip') + if ip: + ret.append({'types': ['ip-src', 'ip-dst'], 'values': [ip], 'comment': desc % (q, 'pdns')}) + if flag == 3: #uri + uri = result.get('uri') + if uri: + ret.append({'types': ['url'], 'values': [uri], 'comment': desc % (q, 'uri')}) + if flag == 4: #samples + if type(result) is str: + ret.append({'types': ['sha256'], 'values': [result], 'comment': desc % (q, 'samples')}) + if flag == 5: #subdomains + if type(result) is str: + ret.append({'types': ['domain'], 'values': [result], 'comment': desc % (q, 'subdomain')}) + if flag == 6: #reports + link = result.get('URL') + if link: + ret.append({'types': ['url'], 'values': [link], 'comment': desc % (q, 'report')}) + + return ret + + +def get_ip(q): + ret = [] + for flag in [1, 2, 3, 4, 5, 6]: + req = requests.get('https://www.threatminer.org/host.php', params={'q': q, 'api': 'True', 'rt': flag}) + if not req.status_code == 200: + return [] + results = req.json().get('results') + if not results: + return [] + + for result in results: + if flag == 1: #whois + emails = result.get('whois', {}).get('emails') + for em_type, email in emails.items(): + ret.append({'types': ['whois-registrant-email'], 'values': [email], 'comment': desc % (q, 'whois')}) + if flag == 2: #pdns + ip = result.get('ip') + if ip: + ret.append({'types': ['ip-src', 'ip-dst'], 'values': [ip], 'comment': desc % (q, 'pdns')}) + if flag == 3: #uri + uri = result.get('uri') + if uri: + ret.append({'types': ['url'], 'values': [uri], 'comment': desc % (q, 'uri')}) + if flag == 4: #samples + if type(result) is str: + ret.append({'types': ['sha256'], 'values': [result], 'comment': desc % (q, 'samples')}) + if flag == 5: #ssl + if type(result) is str: + ret.append({'types': ['x509-fingerprint-sha1'], 'values': [result], 'comment': desc % (q, 'subdomain')}) + if flag == 6: #reports + link = result.get('URL') + if link: + ret.append({'types': ['url'], 'values': [link], 'comment': desc % (q, 'report')}) + + return ret + + +def get_hash(q): + ret = [] + for flag in [1, 3, 6, 7]: + req = requests.get('https://www.threatminer.org/sample.php', params={'q': q, 'api': 'True', 'rt': flag}) + if not req.status_code == 200: + return [] + results = req.json().get('results') + if not results: + return [] + + for result in results: + if flag == 1: #meta (filename) + name = result.get('file_name') + if name: + ret.append({'types': ['filename'], 'values': [name], 'comment': desc % (q, 'file')}) + if flag == 3: #network + domains = result.get('domains') + for dom in domains: + if dom.get('domain'): + ret.append({'types': ['domain'], 'values': [dom['domain']], 'comment': desc % (q, 'network')}) + + hosts = result.get('hosts') + for h in hosts: + if type(h) is str: + ret.append({'types': ['ip-src', 'ip-dst'], 'values': [h], 'comment': desc % (q, 'network')}) + if flag == 6: #detections + detections = result.get('av_detections') + for d in detections: + if d.get('detection'): + ret.append({'types': ['text'], 'values': [d['detection']], 'comment': desc % (q, 'detection')}) + if flag == 7: #report + if type(result) is str: + ret.append({'types': ['sha256'], 'values': [result], 'comment': desc % (q, 'report')}) + + return ret + + +def introspection(): + return mispattributes + + +def version(): + return moduleinfo From aa3a11cd5f43b8f6b53dbf2c73232d4905394ae2 Mon Sep 17 00:00:00 2001 From: kx499 Date: Wed, 8 Mar 2017 04:08:23 +0100 Subject: [PATCH 21/32] bug fixes --- misp_modules/modules/expansion/__init__.py | 2 +- misp_modules/modules/expansion/threatminer.py | 20 +++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 0fa3791..de3c5c3 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -2,4 +2,4 @@ from . import _vmray __all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'ipasn', 'passivetotal', 'sourcecache', - 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep'] + 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer'] diff --git a/misp_modules/modules/expansion/threatminer.py b/misp_modules/modules/expansion/threatminer.py index c34034d..9b41c79 100755 --- a/misp_modules/modules/expansion/threatminer.py +++ b/misp_modules/modules/expansion/threatminer.py @@ -14,7 +14,7 @@ moduleinfo = {'version': '2', 'author': 'Hannah Ward', 'description': 'Get information from virustotal', 'module-type': ['expansion']} -desc = '%s: %s Threatminer' +desc = '%s: Threatminer - %s' def handler(q=False): @@ -55,14 +55,16 @@ def get_domain(q): for flag in [1, 2, 3, 4, 5, 6]: req = requests.get('https://www.threatminer.org/domain.php', params={'q': q, 'api': 'True', 'rt': flag}) if not req.status_code == 200: - return [] + continue results = req.json().get('results') if not results: - return [] + continue for result in results: if flag == 1: #whois emails = result.get('whois', {}).get('emails') + if not emails: + continue for em_type, email in emails.items(): ret.append({'types': ['whois-registrant-email'], 'values': [email], 'comment': desc % (q, 'whois')}) if flag == 2: #pdns @@ -92,14 +94,16 @@ def get_ip(q): for flag in [1, 2, 3, 4, 5, 6]: req = requests.get('https://www.threatminer.org/host.php', params={'q': q, 'api': 'True', 'rt': flag}) if not req.status_code == 200: - return [] + continue results = req.json().get('results') if not results: - return [] + continue for result in results: if flag == 1: #whois emails = result.get('whois', {}).get('emails') + if not emails: + continue for em_type, email in emails.items(): ret.append({'types': ['whois-registrant-email'], 'values': [email], 'comment': desc % (q, 'whois')}) if flag == 2: #pdns @@ -115,7 +119,7 @@ def get_ip(q): ret.append({'types': ['sha256'], 'values': [result], 'comment': desc % (q, 'samples')}) if flag == 5: #ssl if type(result) is str: - ret.append({'types': ['x509-fingerprint-sha1'], 'values': [result], 'comment': desc % (q, 'subdomain')}) + ret.append({'types': ['x509-fingerprint-sha1'], 'values': [result], 'comment': desc % (q, 'ssl')}) if flag == 6: #reports link = result.get('URL') if link: @@ -129,10 +133,10 @@ def get_hash(q): for flag in [1, 3, 6, 7]: req = requests.get('https://www.threatminer.org/sample.php', params={'q': q, 'api': 'True', 'rt': flag}) if not req.status_code == 200: - return [] + continue results = req.json().get('results') if not results: - return [] + continue for result in results: if flag == 1: #meta (filename) From 9bf3346e88086801d5d2fc1db9ad1b1c4527fc68 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 8 Mar 2017 17:25:11 +0100 Subject: [PATCH 22/32] ThreatMiner added --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a4c7514..2ea435f 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address. * [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets. * [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance. +* [threatminer](misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/). * [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to. * [virustotal](misp_modules/modules/expansion/virustotal.py) - an expansion module to pull known resolutions and malware samples related with an IP/Domain from virusTotal (this modules require a VirusTotal private API key) From 648c6414c3fba06874231919d7d8074664019c99 Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Wed, 8 Mar 2017 16:35:03 +0000 Subject: [PATCH 23/32] fix: Use the proper formatting method and not the horrible % one --- misp_modules/modules/expansion/threatminer.py | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/misp_modules/modules/expansion/threatminer.py b/misp_modules/modules/expansion/threatminer.py index 9b41c79..1a4be50 100755 --- a/misp_modules/modules/expansion/threatminer.py +++ b/misp_modules/modules/expansion/threatminer.py @@ -10,11 +10,11 @@ mispattributes = {'input': ['hostname', 'domain', 'ip-src', 'ip-dst', 'md5', 'sh } # possible module-types: 'expansion', 'hover' or both -moduleinfo = {'version': '2', 'author': 'Hannah Ward', - 'description': 'Get information from virustotal', +moduleinfo = {'version': '1', 'author': 'KX499', + 'description': 'Get information from ThreatMiner', 'module-type': ['expansion']} -desc = '%s: Threatminer - %s' +desc = '{}: Threatminer - {}' def handler(q=False): @@ -66,25 +66,25 @@ def get_domain(q): if not emails: continue for em_type, email in emails.items(): - ret.append({'types': ['whois-registrant-email'], 'values': [email], 'comment': desc % (q, 'whois')}) + ret.append({'types': ['whois-registrant-email'], 'values': [email], 'comment': desc.format(q, 'whois')}) if flag == 2: #pdns ip = result.get('ip') if ip: - ret.append({'types': ['ip-src', 'ip-dst'], 'values': [ip], 'comment': desc % (q, 'pdns')}) + ret.append({'types': ['ip-src', 'ip-dst'], 'values': [ip], 'comment': desc.format(q, 'pdns')}) if flag == 3: #uri uri = result.get('uri') if uri: - ret.append({'types': ['url'], 'values': [uri], 'comment': desc % (q, 'uri')}) + ret.append({'types': ['url'], 'values': [uri], 'comment': desc.format(q, 'uri')}) if flag == 4: #samples if type(result) is str: - ret.append({'types': ['sha256'], 'values': [result], 'comment': desc % (q, 'samples')}) + ret.append({'types': ['sha256'], 'values': [result], 'comment': desc.format(q, 'samples')}) if flag == 5: #subdomains if type(result) is str: - ret.append({'types': ['domain'], 'values': [result], 'comment': desc % (q, 'subdomain')}) + ret.append({'types': ['domain'], 'values': [result], 'comment': desc.format(q, 'subdomain')}) if flag == 6: #reports link = result.get('URL') if link: - ret.append({'types': ['url'], 'values': [link], 'comment': desc % (q, 'report')}) + ret.append({'types': ['url'], 'values': [link], 'comment': desc.format(q, 'report')}) return ret @@ -105,25 +105,25 @@ def get_ip(q): if not emails: continue for em_type, email in emails.items(): - ret.append({'types': ['whois-registrant-email'], 'values': [email], 'comment': desc % (q, 'whois')}) + ret.append({'types': ['whois-registrant-email'], 'values': [email], 'comment': desc.format(q, 'whois')}) if flag == 2: #pdns ip = result.get('ip') if ip: - ret.append({'types': ['ip-src', 'ip-dst'], 'values': [ip], 'comment': desc % (q, 'pdns')}) + ret.append({'types': ['ip-src', 'ip-dst'], 'values': [ip], 'comment': desc.format(q, 'pdns')}) if flag == 3: #uri uri = result.get('uri') if uri: - ret.append({'types': ['url'], 'values': [uri], 'comment': desc % (q, 'uri')}) + ret.append({'types': ['url'], 'values': [uri], 'comment': desc.format(q, 'uri')}) if flag == 4: #samples if type(result) is str: - ret.append({'types': ['sha256'], 'values': [result], 'comment': desc % (q, 'samples')}) + ret.append({'types': ['sha256'], 'values': [result], 'comment': desc.format(q, 'samples')}) if flag == 5: #ssl if type(result) is str: - ret.append({'types': ['x509-fingerprint-sha1'], 'values': [result], 'comment': desc % (q, 'ssl')}) + ret.append({'types': ['x509-fingerprint-sha1'], 'values': [result], 'comment': desc.format(q, 'ssl')}) if flag == 6: #reports link = result.get('URL') if link: - ret.append({'types': ['url'], 'values': [link], 'comment': desc % (q, 'report')}) + ret.append({'types': ['url'], 'values': [link], 'comment': desc.format(q, 'report')}) return ret @@ -142,25 +142,25 @@ def get_hash(q): if flag == 1: #meta (filename) name = result.get('file_name') if name: - ret.append({'types': ['filename'], 'values': [name], 'comment': desc % (q, 'file')}) + ret.append({'types': ['filename'], 'values': [name], 'comment': desc.format(q, 'file')}) if flag == 3: #network domains = result.get('domains') for dom in domains: if dom.get('domain'): - ret.append({'types': ['domain'], 'values': [dom['domain']], 'comment': desc % (q, 'network')}) + ret.append({'types': ['domain'], 'values': [dom['domain']], 'comment': desc.format(q, 'network')}) hosts = result.get('hosts') for h in hosts: if type(h) is str: - ret.append({'types': ['ip-src', 'ip-dst'], 'values': [h], 'comment': desc % (q, 'network')}) + ret.append({'types': ['ip-src', 'ip-dst'], 'values': [h], 'comment': desc.format(q, 'network')}) if flag == 6: #detections detections = result.get('av_detections') for d in detections: if d.get('detection'): - ret.append({'types': ['text'], 'values': [d['detection']], 'comment': desc % (q, 'detection')}) + ret.append({'types': ['text'], 'values': [d['detection']], 'comment': desc.format(q, 'detection')}) if flag == 7: #report if type(result) is str: - ret.append({'types': ['sha256'], 'values': [result], 'comment': desc % (q, 'report')}) + ret.append({'types': ['sha256'], 'values': [result], 'comment': desc.format(q, 'report')}) return ret From cdc61c34bd2502eba9ad7508ebaebd443e304290 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 8 Mar 2017 17:37:28 +0100 Subject: [PATCH 24/32] Missing expansion modules added in README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 2ea435f..7d0c17f 100644 --- a/README.md +++ b/README.md @@ -26,11 +26,15 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en). * [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind. * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address. +* [iprep](misp-modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net. * [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets. +* [shodan](misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module. * [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance. * [threatminer](misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/). * [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to. * [virustotal](misp_modules/modules/expansion/virustotal.py) - an expansion module to pull known resolutions and malware samples related with an IP/Domain from virusTotal (this modules require a VirusTotal private API key) +* [wikidata](misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module. +* [xforce](misp_modules/modules/expansion/xforceexchange.py) - an IBM X-Force Exchange expansion module. ### Export modules From 24c51a6e214bf77f282f9d3c746f8cdd6425cdb4 Mon Sep 17 00:00:00 2001 From: Tristan METAYER Date: Fri, 21 Apr 2017 15:53:48 +0200 Subject: [PATCH 25/32] Add indent field for export --- misp_modules/modules/export_mod/liteexport.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/misp_modules/modules/export_mod/liteexport.py b/misp_modules/modules/export_mod/liteexport.py index b22c350..5d47489 100755 --- a/misp_modules/modules/export_mod/liteexport.py +++ b/misp_modules/modules/export_mod/liteexport.py @@ -8,11 +8,8 @@ moduleinfo = {'version': '1', 'description': 'export lite', 'module-type': ['export']} -#~ config form admin site but do not work -#~ moduleconfig = ["indent_json_export"] -moduleconfig = [] +moduleconfig = ["indent_json_export"] -#~ mispattributes = {'input':'all'} ? mispattributes = {} outputFileExtension = "json" responseType = "application/json" @@ -20,11 +17,20 @@ responseType = "application/json" def handler(q=False): if q is False: return False + request = json.loads(q) + + config = {} if "config" in request: - config = request["config"] + config = request["config"] else: - config = {"indent_json_export":None} + config = {"indent_json_export" : None} + + if config['indent_json_export'] is not None: + try: + config['indent_json_export'] = int(config['indent_json_export']) + except: + config['indent_json_export'] = None if 'data' not in request: return False @@ -46,8 +52,8 @@ def handler(q=False): liteAttr['value'] = attr['value'] liteEvent['Event']['Attribute'].append(liteAttr) - return {"response":[], - 'data': str(base64.b64encode( + return {'response' : [], + 'data' : str(base64.b64encode( bytes( json.dumps(liteEvent, indent=config['indent_json_export']), 'utf-8')), From 79f48eccfe675336e8f99fad81898d9fd735cd1d Mon Sep 17 00:00:00 2001 From: Tristan METAYER Date: Tue, 2 May 2017 14:41:22 +0200 Subject: [PATCH 26/32] If filename add iocfilename as attachment --- .../modules/import_mod/openiocimport.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/import_mod/openiocimport.py b/misp_modules/modules/import_mod/openiocimport.py index 27ef3f9..50bc8ba 100755 --- a/misp_modules/modules/import_mod/openiocimport.py +++ b/misp_modules/modules/import_mod/openiocimport.py @@ -33,8 +33,24 @@ def handler(q=False): return json.dumps({"success": 0}) pkg = openioc.load_openioc(package) + + # add origin file as attachment + if q.get("filename"): + r["results"].append({ + "values": [q.get('filename')], + "types": ['attachment'], + "categories": ['Support Tool'], + "data" : q.get('data'), + }) + + # return all attributes for attrib in pkg.attributes: - r["results"].append({"values": [attrib.value], "types": [attrib.type], "categories": [attrib.category]}) + r["results"].append({ + "values": [attrib.value], + "types": [attrib.type], + "categories": [attrib.category], + "comment":attrib.comment}) + return r From 4ef7261168e8368fe3cf7d47d994767cccbcf7ce Mon Sep 17 00:00:00 2001 From: Tristan METAYER Date: Tue, 2 May 2017 15:04:40 +0200 Subject: [PATCH 27/32] Add user config to not add file as attachement in a box --- .../modules/import_mod/openiocimport.py | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/misp_modules/modules/import_mod/openiocimport.py b/misp_modules/modules/import_mod/openiocimport.py index 50bc8ba..420940c 100755 --- a/misp_modules/modules/import_mod/openiocimport.py +++ b/misp_modules/modules/import_mod/openiocimport.py @@ -4,8 +4,12 @@ import base64 from pymisp.tools import openioc misperrors = {'error': 'Error'} -userConfig = {} -inputSource = ['file'] +userConfig = { + 'not save ioc': { + 'type': 'Boolean', + 'message': 'If you check this box, IOC file will not save as an attachment in MISP' + } + }inputSource = ['file'] moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot', 'description': 'Import OpenIOC package', @@ -34,14 +38,17 @@ def handler(q=False): pkg = openioc.load_openioc(package) - # add origin file as attachment - if q.get("filename"): - r["results"].append({ - "values": [q.get('filename')], - "types": ['attachment'], - "categories": ['Support Tool'], - "data" : q.get('data'), - }) + if q.get('config'): + if q['config'].get('not save ioc') == "0": + + # add origin file as attachment + if q.get("filename"): + r["results"].append({ + "values": [q.get('filename')], + "types": ['attachment'], + "categories": ['Support Tool'], + "data" : q.get('data'), + }) # return all attributes for attrib in pkg.attributes: From 96f9cb4699b1f6a9d579f61ff6475fd3c2197ea7 Mon Sep 17 00:00:00 2001 From: Tristan METAYER Date: Tue, 2 May 2017 15:07:33 +0200 Subject: [PATCH 28/32] typo correction --- misp_modules/modules/import_mod/openiocimport.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/import_mod/openiocimport.py b/misp_modules/modules/import_mod/openiocimport.py index 420940c..9dcec3c 100755 --- a/misp_modules/modules/import_mod/openiocimport.py +++ b/misp_modules/modules/import_mod/openiocimport.py @@ -9,7 +9,9 @@ userConfig = { 'type': 'Boolean', 'message': 'If you check this box, IOC file will not save as an attachment in MISP' } - }inputSource = ['file'] + } + +inputSource = ['file'] moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot', 'description': 'Import OpenIOC package', From ba1d715ad171288237f350448d4809c69d8f2566 Mon Sep 17 00:00:00 2001 From: Tristan METAYER Date: Thu, 11 May 2017 09:54:25 +0200 Subject: [PATCH 29/32] Add a field for user to add tag for this import --- .../modules/import_mod/openiocimport.py | 117 ++++++++++-------- 1 file changed, 65 insertions(+), 52 deletions(-) diff --git a/misp_modules/modules/import_mod/openiocimport.py b/misp_modules/modules/import_mod/openiocimport.py index 9dcec3c..b472fba 100755 --- a/misp_modules/modules/import_mod/openiocimport.py +++ b/misp_modules/modules/import_mod/openiocimport.py @@ -5,79 +5,92 @@ from pymisp.tools import openioc misperrors = {'error': 'Error'} userConfig = { - 'not save ioc': { - 'type': 'Boolean', - 'message': 'If you check this box, IOC file will not save as an attachment in MISP' - } - } + 'not save ioc': { + 'type': 'Boolean', + 'message': 'If you check this box, IOC file will not save as an attachment in MISP' + }, + 'default tag': { + 'type': 'String', + 'message': 'Add tags spaced by a comma (tlp:white,misp:threat-level="no-risk")', + 'validation' : '0' + } + } inputSource = ['file'] moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot', - 'description': 'Import OpenIOC package', - 'module-type': ['import']} + 'description': 'Import OpenIOC package', + 'module-type': ['import']} moduleconfig = [] def handler(q=False): - # Just in case we have no data - if q is False: - return False + # Just in case we have no data + if q is False: + return False - # The return value - r = {'results': []} + # The return value + r = {'results': []} - # Load up that JSON - q = json.loads(q) + # Load up that JSON + q = json.loads(q) - # It's b64 encoded, so decode that stuff - package = base64.b64decode(q.get("data")).decode('utf-8') + # It's b64 encoded, so decode that stuff + package = base64.b64decode(q.get("data")).decode('utf-8') - # If something really weird happened - if not package: - return json.dumps({"success": 0}) + # If something really weird happened + if not package: + return json.dumps({"success": 0}) - pkg = openioc.load_openioc(package) + pkg = openioc.load_openioc(package) - if q.get('config'): - if q['config'].get('not save ioc') == "0": + if q.get('config'): + if q['config'].get('not save ioc') == "0": + addFile = { + "values": [q.get('filename')], + "types": ['attachment'], + "categories": ['Support Tool'], + "data" : q.get('data'), + } + # add tag + if q['config'].get('default tag') is not None: + addFile["tags"] = q['config']['default tag'].split(",") + # add file as attachment + r["results"].append(addFile) - # add origin file as attachment - if q.get("filename"): - r["results"].append({ - "values": [q.get('filename')], - "types": ['attachment'], - "categories": ['Support Tool'], - "data" : q.get('data'), - }) - # return all attributes - for attrib in pkg.attributes: - r["results"].append({ - "values": [attrib.value], - "types": [attrib.type], - "categories": [attrib.category], - "comment":attrib.comment}) + # return all attributes + for attrib in pkg.attributes: + toAppend = { + "values": [attrib.value], + "types": [attrib.type], + "categories": [attrib.category], + "comment":attrib.comment + } + # add tag + if q['config'].get('default tag') is not None: + toAppend["tags"] = q['config']['default tag'].split(",") - return r + r["results"].append(toAppend) + return r def introspection(): - modulesetup = {} - try: - userConfig - modulesetup['userConfig'] = userConfig - except NameError: - pass - try: - inputSource - modulesetup['inputSource'] = inputSource - except NameError: - pass - return modulesetup + modulesetup = {} + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup def version(): - moduleinfo['config'] = moduleconfig - return moduleinfo + moduleinfo['config'] = moduleconfig + return moduleinfo From 75c02058e6dee4620d0e42db2fd7984312fd770d Mon Sep 17 00:00:00 2001 From: Tristan METAYER Date: Thu, 11 May 2017 09:56:43 +0200 Subject: [PATCH 30/32] replace tab by space --- .../modules/import_mod/openiocimport.py | 130 +++++++++--------- 1 file changed, 65 insertions(+), 65 deletions(-) diff --git a/misp_modules/modules/import_mod/openiocimport.py b/misp_modules/modules/import_mod/openiocimport.py index b472fba..bf7a428 100755 --- a/misp_modules/modules/import_mod/openiocimport.py +++ b/misp_modules/modules/import_mod/openiocimport.py @@ -5,92 +5,92 @@ from pymisp.tools import openioc misperrors = {'error': 'Error'} userConfig = { - 'not save ioc': { - 'type': 'Boolean', - 'message': 'If you check this box, IOC file will not save as an attachment in MISP' - }, - 'default tag': { - 'type': 'String', - 'message': 'Add tags spaced by a comma (tlp:white,misp:threat-level="no-risk")', - 'validation' : '0' - } - } + 'not save ioc': { + 'type': 'Boolean', + 'message': 'If you check this box, IOC file will not save as an attachment in MISP' + }, + 'default tag': { + 'type': 'String', + 'message': 'Add tags spaced by a comma (tlp:white,misp:threat-level="no-risk")', + 'validation' : '0' + } + } inputSource = ['file'] moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot', - 'description': 'Import OpenIOC package', - 'module-type': ['import']} + 'description': 'Import OpenIOC package', + 'module-type': ['import']} moduleconfig = [] def handler(q=False): - # Just in case we have no data - if q is False: - return False + # Just in case we have no data + if q is False: + return False - # The return value - r = {'results': []} + # The return value + r = {'results': []} - # Load up that JSON - q = json.loads(q) + # Load up that JSON + q = json.loads(q) - # It's b64 encoded, so decode that stuff - package = base64.b64decode(q.get("data")).decode('utf-8') + # It's b64 encoded, so decode that stuff + package = base64.b64decode(q.get("data")).decode('utf-8') - # If something really weird happened - if not package: - return json.dumps({"success": 0}) + # If something really weird happened + if not package: + return json.dumps({"success": 0}) - pkg = openioc.load_openioc(package) + pkg = openioc.load_openioc(package) - if q.get('config'): - if q['config'].get('not save ioc') == "0": - addFile = { - "values": [q.get('filename')], - "types": ['attachment'], - "categories": ['Support Tool'], - "data" : q.get('data'), - } - # add tag - if q['config'].get('default tag') is not None: - addFile["tags"] = q['config']['default tag'].split(",") - # add file as attachment - r["results"].append(addFile) + if q.get('config'): + if q['config'].get('not save ioc') == "0": + addFile = { + "values": [q.get('filename')], + "types": ['attachment'], + "categories": ['Support Tool'], + "data" : q.get('data'), + } + # add tag + if q['config'].get('default tag') is not None: + addFile["tags"] = q['config']['default tag'].split(",") + # add file as attachment + r["results"].append(addFile) - # return all attributes - for attrib in pkg.attributes: - toAppend = { - "values": [attrib.value], - "types": [attrib.type], - "categories": [attrib.category], - "comment":attrib.comment - } - # add tag - if q['config'].get('default tag') is not None: - toAppend["tags"] = q['config']['default tag'].split(",") + # return all attributes + for attrib in pkg.attributes: + toAppend = { + "values": [attrib.value], + "types": [attrib.type], + "categories": [attrib.category], + "comment":attrib.comment + } + # add tag + if q['config'].get('default tag') is not None: + toAppend["tags"] = q['config']['default tag'].split(",") - r["results"].append(toAppend) - return r + r["results"].append(toAppend) + return r def introspection(): - modulesetup = {} - try: - userConfig - modulesetup['userConfig'] = userConfig - except NameError: - pass - try: - inputSource - modulesetup['inputSource'] = inputSource - except NameError: - pass - return modulesetup + modulesetup = {} + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup def version(): - moduleinfo['config'] = moduleconfig - return moduleinfo + moduleinfo['config'] = moduleconfig + return moduleinfo From c42c8a800ec78b02a8ca22ee15ea17d1a3c4629b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 24 May 2017 07:39:18 +0200 Subject: [PATCH 31/32] Update travis, fix open ioc import --- .travis.yml | 7 +++---- misp_modules/modules/import_mod/openiocimport.py | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index ec9b16f..c70bed3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,12 +9,12 @@ python: - "3.4" - "3.5" - "3.5-dev" + - "3.6" + - "3.6-dev" - "nightly" install: - - pip install -U nose - - pip install coveralls - - pip install codecov + - pip install -U nose codecov pytest - pip install -U -r REQUIREMENTS - pip install . @@ -35,4 +35,3 @@ script: after_success: - coverage combine .coverage* - codecov - - coveralls diff --git a/misp_modules/modules/import_mod/openiocimport.py b/misp_modules/modules/import_mod/openiocimport.py index bf7a428..d5e39a0 100755 --- a/misp_modules/modules/import_mod/openiocimport.py +++ b/misp_modules/modules/import_mod/openiocimport.py @@ -68,9 +68,9 @@ def handler(q=False): "categories": [attrib.category], "comment":attrib.comment } - # add tag - if q['config'].get('default tag') is not None: - toAppend["tags"] = q['config']['default tag'].split(",") + # add tag + if q.get('config') and q['config'].get('default tag') is not None: + toAppend["tags"] = q['config']['default tag'].split(",") r["results"].append(toAppend) return r From 8ed344778c0e0d2a848840ad1823a8f1b617057a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 24 May 2017 07:52:31 +0200 Subject: [PATCH 32/32] Use proper version of PyMISP --- REQUIREMENTS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REQUIREMENTS b/REQUIREMENTS index b4e5ebc..0665aac 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -13,7 +13,7 @@ ipasn-redis asnhistory git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client git+https://github.com/MISP/MISP-STIX-Converter.git#egg=misp_stix_converter -git+https://github.com/CIRCL/PyMISP.git#egg=pymisp +git+https://github.com/MISP/PyMISP.git#egg=pymisp pillow pytesseract SPARQLWrapper