From 4f5059fca43000a19f1aaf90b27ee934976774ab Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Fri, 12 Aug 2016 14:45:28 +0100 Subject: [PATCH 1/5] Added lookup by country code --- misp_modules/modules/expansion/countrycode.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100755 misp_modules/modules/expansion/countrycode.py diff --git a/misp_modules/modules/expansion/countrycode.py b/misp_modules/modules/expansion/countrycode.py new file mode 100755 index 0000000..9fc0bfd --- /dev/null +++ b/misp_modules/modules/expansion/countrycode.py @@ -0,0 +1,59 @@ +import json +import requests + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['hostname', 'domain'], 'output': ['ip-src', + 'ip-dst']} + +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '1', 'author': 'Hannah Ward', + 'description': 'Expand Country Codes', + 'module-type': ['expansion', 'hover']} + +# config fields that your code expects from the site admin +moduleconfig = [] + +common_tlds = {"com":"Commercial (Worldwide)", + "org":"Organisation (Worldwide)", + "net":"Network (Worldwide)", + "int":"International (Worldwide)", + "edu":"Education (Usually USA)", + "gov":"Government (USA)" + } + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + domain = request["domain"] + + # Get the extension + ext = domain.split(".")[-1] + + # Check if if's a common, non country one + if ext in common_tlds.keys(): + val = common_tlds[ext] + else: + # Retrieve a json full of country info + codes = requests.get("http://www.geognos.com/api/en/countries/info/all.json").json() + + if not codes["StatusMsg"] == "OK": + val = "Unknown" + else: + # Find our code based on TLD + codes = codes["Results"] + for code in codes.keys(): + if codes[code]["CountryCodes"]["tld"] == ext: + val = codes[code]["Name"] + r = {'results': [{'types':['text'], 'values':[val]}]} + return r + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo + From 917c95cad5a7b5b1c1e42113e8e6f4a8cd3bcf93 Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Fri, 12 Aug 2016 17:40:00 +0100 Subject: [PATCH 2/5] Added countrycode, working on virustotal --- misp_modules/modules/expansion/countrycode.py | 3 +- misp_modules/modules/expansion/virustotal.py | 125 ++++++++++++++++++ 2 files changed, 126 insertions(+), 2 deletions(-) create mode 100755 misp_modules/modules/expansion/virustotal.py diff --git a/misp_modules/modules/expansion/countrycode.py b/misp_modules/modules/expansion/countrycode.py index 9fc0bfd..df43a88 100755 --- a/misp_modules/modules/expansion/countrycode.py +++ b/misp_modules/modules/expansion/countrycode.py @@ -2,8 +2,7 @@ import json import requests misperrors = {'error': 'Error'} -mispattributes = {'input': ['hostname', 'domain'], 'output': ['ip-src', - 'ip-dst']} +mispattributes = {'input': ['hostname', 'domain']} # possible module-types: 'expansion', 'hover' or both moduleinfo = {'version': '1', 'author': 'Hannah Ward', diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py new file mode 100755 index 0000000..fcbf4f6 --- /dev/null +++ b/misp_modules/modules/expansion/virustotal.py @@ -0,0 +1,125 @@ +import json +import requests +import hashlib +import re +import base64 + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['domain', "ip-src", "ip-dst"], + 'output':['domain', "ip-src", "ip-dst", "text"] + } + +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '1', 'author': 'Hannah Ward', + 'description': 'Get information from virustotal', + 'module-type': ['expansion', 'hover']} + +# config fields that your code expects from the site admin +moduleconfig = ["apikey"] + + +def handler(q=False): + if q is False: + return False + + q = json.loads(q) + + key = q["config"]["apikey"] + + r = {"results": []} + + if "ip-src" in q: + r["results"] += getIP(q["ip-src"], key) + if "ip-dst" in q: + r["results"] += getIP(q["ip-dst"], key) + if "domain" in q: + r["results"] += getDomain(q["domain"], key) + + + return r + +def getIP(ip, key): + toReturn = [] + req = requests.get("https://www.virustotal.com/vtapi/v2/ip-address/report", + params = {"ip":ip, "apikey":key} + ).json() + if req["response_code"] == 0: + #Nothing found + return [] + + if "resolutions" in req: + for res in req["resolutions"]: + toReturn.append( {"types":["domain"], "values":[res["hostname"]]}) + + toReturn += getMoreInfo(req, key) + return toReturn + +def getDomain(ip, key): + toReturn = [] + req = requests.get("https://www.virustotal.com/vtapi/v2/domain/report", + params = {"domain":ip, "apikey":key} + ).json() + if req["response_code"] == 0: + #Nothing found + return [] + + if "resolutions" in req: + for res in req["resolutions"]: + toReturn.append( {"types":["ip-dst", "ip-src"], "values":[res["ip_address"]]}) + + toReturn += getMoreInfo(req, key) + + return toReturn + +def findAll(data, keys): + a = [] + if isinstance(data, dict): + for key in data.keys(): + if key in keys: + a.append(data[key]) + else: + if isinstance(data[key], (dict, list)): + a += findAll(data[key], keys) + if isinstance(data, list): + for i in data: + a += findAll(i, keys) + + return a + +def getMoreInfo(req, key): + r = [] + #Get all hashes first + hashes = [] + hashes = findAll(req, ["md5", "sha1", "sha256", "sha512"]) + r.append({"types":["md5", "sha1", "sha256", "sha512"], "values":hashes}) + for hsh in hashes: + #Search VT for some juicy info + data = requests.get("http://www.virustotal.com/vtapi/v2/file/report", + params={"allinfo":1, "apikey":key, "resource":hsh} + ).json() + if "submission_names" in data: + r.append({'types':["filename"], "values":data["submission_names"]}) + + if "ssdeep" in data: + r.append({'types':["ssdeep"], "values":[data["ssdeep"]]}) + + if "authentihash" in data: + r.append({"types":["authentihash"], "values":[data["authentihash"]]}) + + if "ITW_urls" in data: + r.append({"types":["url"], "values":data["ITW_urls"]}) + + #Get the malware sample + sample = requests.get("https://www.virustotal.com/vtapi/v2/file/download", + params = {"hash":hsh, "apikey":key}) + r.append({'types':['malware-sample'], 'values':[str(base64.b64encode(sample.content), 'utf-8')]}) + return r + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo + From 0f9221229a6b643e36d509d45f2174bdff3cc828 Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Mon, 15 Aug 2016 11:09:40 +0100 Subject: [PATCH 3/5] Improved virustotal module --- misp_modules/modules/expansion/countrycode.py | 2 +- misp_modules/modules/expansion/virustotal.py | 50 +++++++++++++------ 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/misp_modules/modules/expansion/countrycode.py b/misp_modules/modules/expansion/countrycode.py index df43a88..039da17 100755 --- a/misp_modules/modules/expansion/countrycode.py +++ b/misp_modules/modules/expansion/countrycode.py @@ -7,7 +7,7 @@ mispattributes = {'input': ['hostname', 'domain']} # possible module-types: 'expansion', 'hover' or both moduleinfo = {'version': '1', 'author': 'Hannah Ward', 'description': 'Expand Country Codes', - 'module-type': ['expansion', 'hover']} + 'module-type': ['hover']} # config fields that your code expects from the site admin moduleconfig = [] diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index fcbf4f6..474b604 100755 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -3,6 +3,7 @@ import requests import hashlib import re import base64 +import os misperrors = {'error': 'Error'} mispattributes = {'input': ['domain', "ip-src", "ip-dst"], @@ -12,7 +13,7 @@ mispattributes = {'input': ['domain', "ip-src", "ip-dst"], # possible module-types: 'expansion', 'hover' or both moduleinfo = {'version': '1', 'author': 'Hannah Ward', 'description': 'Get information from virustotal', - 'module-type': ['expansion', 'hover']} + 'module-type': ['expansion']} # config fields that your code expects from the site admin moduleconfig = ["apikey"] @@ -25,7 +26,6 @@ def handler(q=False): q = json.loads(q) key = q["config"]["apikey"] - r = {"results": []} if "ip-src" in q: @@ -35,10 +35,12 @@ def handler(q=False): if "domain" in q: r["results"] += getDomain(q["domain"], key) - + with open("/home/hward/debug.txt", "w") as f: + f.write(json.dumps(r)) return r -def getIP(ip, key): +def getIP(ip, key, do_not_recurse = False): + print("Getting info for {}".format(ip)) toReturn = [] req = requests.get("https://www.virustotal.com/vtapi/v2/ip-address/report", params = {"ip":ip, "apikey":key} @@ -50,14 +52,18 @@ def getIP(ip, key): if "resolutions" in req: for res in req["resolutions"]: toReturn.append( {"types":["domain"], "values":[res["hostname"]]}) + #Pivot from here to find all domain info + if not do_not_recurse: + toReturn += getDomain(res["hostname"], key, True) toReturn += getMoreInfo(req, key) return toReturn -def getDomain(ip, key): +def getDomain(domain, key, do_not_recurse=False): + print("Getting info for {}".format(domain)) toReturn = [] req = requests.get("https://www.virustotal.com/vtapi/v2/domain/report", - params = {"domain":ip, "apikey":key} + params = {"domain":domain, "apikey":key} ).json() if req["response_code"] == 0: #Nothing found @@ -66,9 +72,10 @@ def getDomain(ip, key): if "resolutions" in req: for res in req["resolutions"]: toReturn.append( {"types":["ip-dst", "ip-src"], "values":[res["ip_address"]]}) - + #Pivot from here to find all info on IPs + if not do_not_recurse: + toReturn += getIP(res["ip_address"], key, True) toReturn += getMoreInfo(req, key) - return toReturn def findAll(data, keys): @@ -86,33 +93,48 @@ def findAll(data, keys): return a +def isset(d, key): + if key in d: + if d[key] not in [None, '', ' ']: + return True + return False + def getMoreInfo(req, key): + print("Getting extra info for {}".format(req)) r = [] #Get all hashes first hashes = [] hashes = findAll(req, ["md5", "sha1", "sha256", "sha512"]) r.append({"types":["md5", "sha1", "sha256", "sha512"], "values":hashes}) - for hsh in hashes: + for hsh in hashes[:5]: #Search VT for some juicy info data = requests.get("http://www.virustotal.com/vtapi/v2/file/report", params={"allinfo":1, "apikey":key, "resource":hsh} ).json() - if "submission_names" in data: + if isset(data, "submission_names"): r.append({'types':["filename"], "values":data["submission_names"]}) - if "ssdeep" in data: + if isset(data, "ssdeep"): r.append({'types':["ssdeep"], "values":[data["ssdeep"]]}) - if "authentihash" in data: + if isset(data, "authentihash"): r.append({"types":["authentihash"], "values":[data["authentihash"]]}) - if "ITW_urls" in data: + if isset(data, "ITW_urls"): r.append({"types":["url"], "values":data["ITW_urls"]}) #Get the malware sample sample = requests.get("https://www.virustotal.com/vtapi/v2/file/download", params = {"hash":hsh, "apikey":key}) - r.append({'types':['malware-sample'], 'values':[str(base64.b64encode(sample.content), 'utf-8')]}) + + print(sample) + malsample = sample.content + r.append({"types":["malware-sample"], + "categories":["Payload delivery"], + "values":data["submission_names"], + "data": str(base64.b64encode(malsample), 'utf-8') + } + ) return r def introspection(): From 042bf2bb2f8c90393d61f9e8fbaf777676a16366 Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Wed, 17 Aug 2016 09:30:15 +0100 Subject: [PATCH 4/5] Added virustotal module --- misp_modules/modules/expansion/virustotal.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index 474b604..930872e 100755 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -35,8 +35,11 @@ def handler(q=False): if "domain" in q: r["results"] += getDomain(q["domain"], key) - with open("/home/hward/debug.txt", "w") as f: - f.write(json.dumps(r)) + uniq = [] + for res in r["results"]: + if res not in uniq: + uniq.append(res) + r["results"] = uniq return r def getIP(ip, key, do_not_recurse = False): From 317f820bbf2db31cd53e375273b24ac4a1fffcf4 Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Wed, 17 Aug 2016 09:33:49 +0100 Subject: [PATCH 5/5] Modified readme with virustotal/countrycode --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 9ff2460..af86bb5 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,9 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address. * [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets. * [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance. +* [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to. +* [virustotal](misp_modules/modules/expansion/virustotal.py) - an expansion module to pull known resolutions and malware samples related with an IP/Domain from virustotal + * (Probably requires private API key) ### Import modules