From 4ba86d4fa3c390d52a68d29be3df43df33ec21fc Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Wed, 17 Aug 2016 09:51:16 +0100 Subject: [PATCH 1/7] CountryCode JSON now is only grabbed once per server run --- misp_modules/modules/expansion/countrycode.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/countrycode.py b/misp_modules/modules/expansion/countrycode.py index 039da17..af58fc6 100755 --- a/misp_modules/modules/expansion/countrycode.py +++ b/misp_modules/modules/expansion/countrycode.py @@ -20,7 +20,10 @@ common_tlds = {"com":"Commercial (Worldwide)", "gov":"Government (USA)" } +codes = requests.get("http://www.geognos.com/api/en/countries/info/all.json").json() + def handler(q=False): + global codes if q is False: return False request = json.loads(q) @@ -34,8 +37,6 @@ def handler(q=False): val = common_tlds[ext] else: # Retrieve a json full of country info - codes = requests.get("http://www.geognos.com/api/en/countries/info/all.json").json() - if not codes["StatusMsg"] == "OK": val = "Unknown" else: From 232014f2215de65b5ebbba3ab80588b321b01de2 Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Wed, 17 Aug 2016 13:01:11 +0100 Subject: [PATCH 2/7] Added virustotal tests --- .gitignore | 1 + misp_modules/modules/expansion/virustotal.py | 21 ++++++++++++++------ tests/bodyvirustotal.json.sample | 1 + tests/test.py | 13 ++++++++++++ 4 files changed, 30 insertions(+), 6 deletions(-) create mode 100644 tests/bodyvirustotal.json.sample diff --git a/.gitignore b/.gitignore index 9676396..abab012 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.pyc *.swp +test/bodyvirustotal.json __pycache__ build/ dist/ diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index 930872e..df7b8b4 100755 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -6,7 +6,7 @@ import base64 import os misperrors = {'error': 'Error'} -mispattributes = {'input': ['domain', "ip-src", "ip-dst"], +mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst"], 'output':['domain', "ip-src", "ip-dst", "text"] } @@ -16,16 +16,19 @@ moduleinfo = {'version': '1', 'author': 'Hannah Ward', 'module-type': ['expansion']} # config fields that your code expects from the site admin -moduleconfig = ["apikey"] - +moduleconfig = ["apikey", "event_limit"] +limit = 5 #Default def handler(q=False): + global limit if q is False: return False q = json.loads(q) key = q["config"]["apikey"] + limit = int(q["config"].get("event_limit", 5)) + r = {"results": []} if "ip-src" in q: @@ -34,6 +37,8 @@ def handler(q=False): r["results"] += getIP(q["ip-dst"], key) if "domain" in q: r["results"] += getDomain(q["domain"], key) + if 'hostname' in q: + r["results"] += getDomain(q['hostname'], key) uniq = [] for res in r["results"]: @@ -43,6 +48,7 @@ def handler(q=False): return r def getIP(ip, key, do_not_recurse = False): + global limit print("Getting info for {}".format(ip)) toReturn = [] req = requests.get("https://www.virustotal.com/vtapi/v2/ip-address/report", @@ -53,7 +59,7 @@ def getIP(ip, key, do_not_recurse = False): return [] if "resolutions" in req: - for res in req["resolutions"]: + for res in req["resolutions"][:limit]: toReturn.append( {"types":["domain"], "values":[res["hostname"]]}) #Pivot from here to find all domain info if not do_not_recurse: @@ -63,6 +69,8 @@ def getIP(ip, key, do_not_recurse = False): return toReturn def getDomain(domain, key, do_not_recurse=False): + global limit + print("Getting info for {}".format(domain)) toReturn = [] req = requests.get("https://www.virustotal.com/vtapi/v2/domain/report", @@ -73,7 +81,7 @@ def getDomain(domain, key, do_not_recurse=False): return [] if "resolutions" in req: - for res in req["resolutions"]: + for res in req["resolutions"][:limit]: toReturn.append( {"types":["ip-dst", "ip-src"], "values":[res["ip_address"]]}) #Pivot from here to find all info on IPs if not do_not_recurse: @@ -103,13 +111,14 @@ def isset(d, key): return False def getMoreInfo(req, key): + global limit print("Getting extra info for {}".format(req)) r = [] #Get all hashes first hashes = [] hashes = findAll(req, ["md5", "sha1", "sha256", "sha512"]) r.append({"types":["md5", "sha1", "sha256", "sha512"], "values":hashes}) - for hsh in hashes[:5]: + for hsh in hashes[:limit]: #Search VT for some juicy info data = requests.get("http://www.virustotal.com/vtapi/v2/file/report", params={"allinfo":1, "apikey":key, "resource":hsh} diff --git a/tests/bodyvirustotal.json.sample b/tests/bodyvirustotal.json.sample new file mode 100644 index 0000000..43cad6b --- /dev/null +++ b/tests/bodyvirustotal.json.sample @@ -0,0 +1 @@ +{"module": "virustotal", "ip-dst": "5.104.106.190", "config": {"api_key": "deadbeef"} } diff --git a/tests/test.py b/tests/test.py index 1314479..a869338 100644 --- a/tests/test.py +++ b/tests/test.py @@ -5,6 +5,7 @@ import unittest import requests import base64 import json +import os class TestModules(unittest.TestCase): @@ -36,5 +37,17 @@ class TestModules(unittest.TestCase): response = requests.post(self.url + "query", data=data) print(response.json()) + def test_virustotal(self): + # This can't actually be tested without disclosing a private + # API key. This will attempt to run with a .gitignored keyfile + # and pass if it can't find one + + if not os.path.exists("tests/bodyvirustotal.json"): + return + + with open("tests/bodyvirustotal.json", "r") as f: + response = requests.post(self.url + "query", data=f.read()).json() + assert(response) + if __name__ == '__main__': unittest.main() From a53c43701ae01290112d2242eeed9b37f6e8b120 Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Wed, 17 Aug 2016 13:01:41 +0100 Subject: [PATCH 3/7] Added body.json to gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index abab012..cde1a60 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ *.pyc *.swp -test/bodyvirustotal.json +bodyvirustotal.json __pycache__ build/ dist/ From 9db9247e55f5f581dce62435bfd046cd47d20af1 Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Wed, 17 Aug 2016 13:04:30 +0100 Subject: [PATCH 4/7] Removed calls to print --- misp_modules/modules/expansion/virustotal.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index df7b8b4..4048942 100755 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -49,7 +49,6 @@ def handler(q=False): def getIP(ip, key, do_not_recurse = False): global limit - print("Getting info for {}".format(ip)) toReturn = [] req = requests.get("https://www.virustotal.com/vtapi/v2/ip-address/report", params = {"ip":ip, "apikey":key} @@ -70,8 +69,6 @@ def getIP(ip, key, do_not_recurse = False): def getDomain(domain, key, do_not_recurse=False): global limit - - print("Getting info for {}".format(domain)) toReturn = [] req = requests.get("https://www.virustotal.com/vtapi/v2/domain/report", params = {"domain":domain, "apikey":key} @@ -112,7 +109,6 @@ def isset(d, key): def getMoreInfo(req, key): global limit - print("Getting extra info for {}".format(req)) r = [] #Get all hashes first hashes = [] @@ -139,7 +135,6 @@ def getMoreInfo(req, key): sample = requests.get("https://www.virustotal.com/vtapi/v2/file/download", params = {"hash":hsh, "apikey":key}) - print(sample) malsample = sample.content r.append({"types":["malware-sample"], "categories":["Payload delivery"], From a492d975c414f24c724c25a5001d7aa1b720aff8 Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Fri, 19 Aug 2016 17:21:12 +0100 Subject: [PATCH 5/7] Now searches within observable_compositions --- .../modules/expansion/find_subdomains.py | 62 +++++++++++++++++++ misp_modules/modules/import_mod/stiximport.py | 37 ++++++++--- 2 files changed, 90 insertions(+), 9 deletions(-) create mode 100755 misp_modules/modules/expansion/find_subdomains.py diff --git a/misp_modules/modules/expansion/find_subdomains.py b/misp_modules/modules/expansion/find_subdomains.py new file mode 100755 index 0000000..4d40acb --- /dev/null +++ b/misp_modules/modules/expansion/find_subdomains.py @@ -0,0 +1,62 @@ +import json +import os +import subprocess +import requests +import tempfile + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['domain'], 'output': ['domain']} + +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '1', 'author': 'Hannah Ward', + 'description': 'Attempt to brute force subdomains', + 'module-type': ['expansion']} + +# config fields that your code expects from the site admin +moduleconfig = ["use_top_n_subdomains"] + +domains = requests.get("http://hannah-ward.uk/Subdomain_List.txt").text.split("\n")[:-1] + +def handler(q=False): + global domains + + if q is False: + return False + request = json.loads(q) + + r = {"results": []} + + f = tempfile.NamedTemporaryFile(delete=False, prefix="domains", mode="w") + print("Saving domains to {}".format(f.name)) + f.write("\n".join(domains[:int(request["config"]["use_top_n_subdomains"])])) + + f.close() + + print("Searching for subdomains of {}".format(request["domain"])) + print("Using {} domains".format(request["config"]["use_top_n_subdomains"])) + + print("Turning on tor...") + #subprocess.call([".","torsocks","on"]) + proc = subprocess.Popen(["knockpy", "-w", f.name, request["domain"]], + stdout=subprocess.PIPE + ) + + os.remove(f.name) + + print("Turning off tor...") + #subprocess.call([".","torsocks","off"]) + out,err = proc.communicate() + + r["results"] = {'values':out.split("\n"), "types":'domain'} + + return r + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo + diff --git a/misp_modules/modules/import_mod/stiximport.py b/misp_modules/modules/import_mod/stiximport.py index 0c39288..d734f3d 100755 --- a/misp_modules/modules/import_mod/stiximport.py +++ b/misp_modules/modules/import_mod/stiximport.py @@ -4,6 +4,7 @@ import re import base64 import hashlib import tempfile +import pickle misperrors = {'error': 'Error'} userConfig = {} @@ -49,6 +50,9 @@ def handler(q=False): # Load up the package into STIX package = loadPackage(package, memsize) + # Hash it + with open("/home/hward/tmp.dat", "wb") as f: + pickle.dump( package, f) # Build all the observables if package.observables: for obs in package.observables: @@ -62,7 +66,7 @@ def handler(q=False): # Aaaand the indicators if package.indicators: for ind in package.indicators: - r["results"].append(buildIndicator(ind)) + r["results"] += buildIndicator(ind) # Are you seeing a pattern? if package.exploit_targets: @@ -76,7 +80,7 @@ def handler(q=False): # Clean up results # Don't send on anything that didn't have a value - r["results"] = [x for x in r["results"] if len(x["values"]) != 0] + r["results"] = [x for x in r["results"] if isinstance(x, dict) and len(x["values"]) != 0] return r # Quick and dirty regex for IP addresses @@ -126,11 +130,14 @@ def buildIndicator(ind): and other fun things like that """ - r = {"values": [], "types": []} - + r = [] # Try to get hashes. I hate stix - if ind.observable: - return buildObservable(ind.observable) + if ind.observables: + for i in ind.observables: + if i.observable_composition: + for j in i.observable_composition.observables: + r.append(buildObservable(j)) + r.append(buildObservable(i)) return r @@ -152,7 +159,6 @@ def buildObservable(o): and extract the value and category """ - # Life is easier with json if not isinstance(o, dict): o = json.loads(o.to_json()) @@ -168,7 +174,6 @@ def buildObservable(o): props = o["object"]["properties"] # If it has an address_value field, it's gonna be an address - # print(props) # Kinda obvious really if "address_value" in props: @@ -193,7 +198,21 @@ def buildObservable(o): for hsh in props["hashes"]: r["values"].append(hsh["simple_hash_value"]["value"]) r["types"] = identifyHash(hsh["simple_hash_value"]["value"]) - return r + + elif "xsi:type" in props: + # Cybox. Ew. + try: + type_ = props["xsi:type"] + val = props["value"] + + if type_ == "LinkObjectType": + r["types"] = ["link"] + r["values"].append(val) + else: + print("Ignoring {}".format(type_)) + except: + pass + return r def loadPackage(data, memsize=1024): From 4e3300d66c4b49ca1258e6ab8e12d099a7c554bb Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Mon, 22 Aug 2016 14:18:19 +0100 Subject: [PATCH 6/7] Added CEF export module --- misp_modules/modules/export_mod/cef_export.py | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100755 misp_modules/modules/export_mod/cef_export.py diff --git a/misp_modules/modules/export_mod/cef_export.py b/misp_modules/modules/export_mod/cef_export.py new file mode 100755 index 0000000..3f2ff61 --- /dev/null +++ b/misp_modules/modules/export_mod/cef_export.py @@ -0,0 +1,81 @@ +import json +import base64 +import datetime + +misperrors = {'error': 'Error'} + +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '1', 'author': 'Hannah Ward', + 'description': 'Export a module in CEF format', + 'module-type': ['export']} + +# config fields that your code expects from the site admin +moduleconfig = ["Default_Severity", "Device_Vendor", "Device_Product", "Device_Version"] + +cefmapping = {"ip-src":"src", "ip-dst":"dst", "hostname":"dhost", "domain":"dhost", + "md5":"fileHash", "sha1":"fileHash", "sha256":"fileHash", + "url":"request"} + +mispattributes = {'input':list(cefmapping.keys())} +outputFileExtension = "cef" +responseType = "application/txt" + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if "config" in request: + config = request["config"] + else: + config = {"Default_Severity":1, "Device_Vendor":"MISP", "Device_Product":"MISP", "Device_Version":1} + + data = request["data"] + response = "" + for ev in data: + event = ev["Attribute"] + for attr in event: + if attr["type"] in cefmapping: + response += "{} host CEF:0|{}|{}|{}|{}|{}|{}|{}={}\n".format( + datetime.datetime.fromtimestamp(int(attr["timestamp"])).strftime("%b %d %H:%M:%S"), + config["Device_Vendor"], + config["Device_Product"], + config["Device_Version"], + attr["category"], + attr["category"], + config["Default_Severity"], + cefmapping[attr["type"]], + attr["value"], + ) + + r = {"response":[], "data":str(base64.b64encode(bytes(response, 'utf-8')), 'utf-8')} + return r + + +def introspection(): + modulesetup = {} + try: + responseType + modulesetup['responseType'] = responseType + except NameError: + pass + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + outputFileExtension + modulesetup['outputFileExtension'] = outputFileExtension + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo + From 4f923d66069a6cd24e82be32cf2dd6b94ad790d6 Mon Sep 17 00:00:00 2001 From: Hannah Ward Date: Thu, 1 Sep 2016 16:14:25 +0100 Subject: [PATCH 7/7] Removed silly subdomain module --- .../modules/expansion/find_subdomains.py | 62 ------------------- 1 file changed, 62 deletions(-) delete mode 100755 misp_modules/modules/expansion/find_subdomains.py diff --git a/misp_modules/modules/expansion/find_subdomains.py b/misp_modules/modules/expansion/find_subdomains.py deleted file mode 100755 index 4d40acb..0000000 --- a/misp_modules/modules/expansion/find_subdomains.py +++ /dev/null @@ -1,62 +0,0 @@ -import json -import os -import subprocess -import requests -import tempfile - -misperrors = {'error': 'Error'} -mispattributes = {'input': ['domain'], 'output': ['domain']} - -# possible module-types: 'expansion', 'hover' or both -moduleinfo = {'version': '1', 'author': 'Hannah Ward', - 'description': 'Attempt to brute force subdomains', - 'module-type': ['expansion']} - -# config fields that your code expects from the site admin -moduleconfig = ["use_top_n_subdomains"] - -domains = requests.get("http://hannah-ward.uk/Subdomain_List.txt").text.split("\n")[:-1] - -def handler(q=False): - global domains - - if q is False: - return False - request = json.loads(q) - - r = {"results": []} - - f = tempfile.NamedTemporaryFile(delete=False, prefix="domains", mode="w") - print("Saving domains to {}".format(f.name)) - f.write("\n".join(domains[:int(request["config"]["use_top_n_subdomains"])])) - - f.close() - - print("Searching for subdomains of {}".format(request["domain"])) - print("Using {} domains".format(request["config"]["use_top_n_subdomains"])) - - print("Turning on tor...") - #subprocess.call([".","torsocks","on"]) - proc = subprocess.Popen(["knockpy", "-w", f.name, request["domain"]], - stdout=subprocess.PIPE - ) - - os.remove(f.name) - - print("Turning off tor...") - #subprocess.call([".","torsocks","off"]) - out,err = proc.communicate() - - r["results"] = {'values':out.split("\n"), "types":'domain'} - - return r - - -def introspection(): - return mispattributes - - -def version(): - moduleinfo['config'] = moduleconfig - return moduleinfo -