MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #222 from chrisr3d/master 

Clean up + fix of some modules
pull/226/head
Christian Studer 2018-09-07 18:16:15 +02:00 committed by GitHub
parent 5107e78f5d cfbd63f14e
commit de846733bc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 161 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
misp_modules/modules/expansion/otx.py
View File

@ -32,16 +32,15 @@ def valid_ip(ip):
def findAll(data, keys):
a = []
if isinstance(data, dict):
for key in data.keys():
for key, value in data.items():
if key == keys:
a.append(data[key])
a.append(value)
else:
if isinstance(data[key], (dict, list)):
a += findAll(data[key], keys)
if isinstance(value, (dict, list)):
a.extend(findAll(value, keys))
if isinstance(data, list):
for i in data:
a += findAll(i, keys)
a.extend(findAll(i, keys))
return a
def valid_email(email):
@ -82,10 +81,10 @@ def handler(q=False):
return r
def getHash(hash, key):
def getHash(_hash, key):
ret = []
req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/file/analysis/" + hash).text)
req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/file/analysis/" + _hash).text)
for ip in findAll(req, "dst"):
if not isBlacklisted(ip) and valid_ip(ip):
@ -102,8 +101,8 @@ def getIP(ip, key):
ret = []
req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/ip/malware/" + ip + "?limit=1000").text )
for hash in findAll(req, "hash"):
ret.append({"types": ["sha256"], "values": [hash]})
for _hash in findAll(req, "hash"):
ret.append({"types": ["sha256"], "values": [_hash]})
req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/ip/passive_dns/" + ip).text )
@ -122,21 +121,21 @@ def getDomain(domain, key):
req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/domain/malware/" + domain + "?limit=1000").text )
for hash in findAll(req, "hash"):
ret.append({"types": ["sha256"], "values": [hash]})
for _hash in findAll(req, "hash"):
ret.append({"types": ["sha256"], "values": [_hash]})
req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/domain/whois/" + domain).text)
for domain in findAll(req, "domain"):
ret.append({"types": ["hostname"], "values": [domain]})
for _domain in findAll(req, "domain"):
ret.append({"types": ["hostname"], "values": [_domain]})
for email in findAll(req, "value"):
if valid_email(email):
ret.append({"types": ["email"], "values": [domain]})
ret.append({"types": ["email"], "values": [email]})
for domain in findAll(req, "hostname"):
if "." in domain and not isBlacklisted(domain):
ret.append({"types": ["hostname"], "values": [domain]})
for _domain in findAll(req, "hostname"):
if "." in _domain and not isBlacklisted(_domain):
ret.append({"types": ["hostname"], "values": [_domain]})
req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/hostname/passive_dns/" + domain).text)
for ip in findAll(req, "address"):

7
misp_modules/modules/expansion/rbl.py
View File

@ -6,7 +6,7 @@ try:
resolver = dns.resolver.Resolver()
resolver.timeout = 0.2
resolver.lifetime = 0.2
except:
except ModuleNotFoundError:
print("dnspython3 is missing, use 'pip install dnspython3' to install it.")
sys.exit(0)
@ -96,13 +96,12 @@ def handler(q=False):
txt = resolver.query(query,'TXT')
listed.append(query)
info.append(str(txt[0]))
except:
except Exception:
continue
result = {}
for l, i in zip(listed, info):
result[l] = i
r = {'results': [{'types': mispattributes.get('output'), 'values': json.dumps(result)}]}
return r
return {'results': [{'types': mispattributes.get('output'), 'values': json.dumps(result)}]}
def introspection():
return mispattributes

2
misp_modules/modules/expansion/sigma_syntax_validator.py
View File

@ -21,7 +21,7 @@ def handler(q=False):
return misperrors
config = SigmaConfiguration()
try:
parser = SigmaParser(yaml.load(request.get('sigma')), config)
parser = SigmaParser(yaml.safe_load(request.get('sigma')), config)
result = ("Syntax valid: {}".format(parser.values))
except Exception as e:
result = ("Syntax error: {}".format(str(e)))

308
misp_modules/modules/expansion/virustotal.py Executable file → Normal file
View File

@ -2,201 +2,165 @@ import json
import requests
from requests import HTTPError
import base64
from collections import defaultdict
misperrors = {'error': 'Error'}
mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "sha512"],
'output': ['domain', "ip-src", "ip-dst", "text", "md5", "sha1", "sha256", "sha512", "ssdeep",
"authentihash", "filename"]
}
"authentihash", "filename"]}
# possible module-types: 'expansion', 'hover' or both
moduleinfo = {'version': '2', 'author': 'Hannah Ward',
moduleinfo = {'version': '3', 'author': 'Hannah Ward',
'description': 'Get information from virustotal',
'module-type': ['expansion']}
# config fields that your code expects from the site admin
moduleconfig = ["apikey", "event_limit"]
limit = 5 # Default
comment = '%s: Enriched via VT'
comment = '{}: Enriched via VirusTotal'
hash_types = ["md5", "sha1", "sha256", "sha512"]
class VirusTotalRequest(object):
def __init__(self, config):
self.apikey = config['apikey']
self.limit = int(config.get('event_limit', 5))
self.base_url = "https://www.virustotal.com/vtapi/v2/{}/report"
self.results = defaultdict(set)
self.to_return = []
self.input_types_mapping = {'ip-src': self.get_ip, 'ip-dst': self.get_ip,
'domain': self.get_domain, 'hostname': self.get_domain,
'md5': self.get_hash, 'sha1': self.get_hash,
'sha256': self.get_hash, 'sha512': self.get_hash}
self.output_types_mapping = {'submission_names': 'filename', 'ssdeep': 'ssdeep',
'authentihash': 'authentihash', 'ITW_urls': 'url'}
def parse_request(self, q):
req_values = set()
for attribute_type, attribute_value in q.items():
req_values.add(attribute_value)
try:
error = self.input_types_mapping[attribute_type](attribute_value)
except KeyError:
continue
if error is not None:
return error
for key, values in self.results.items():
values = values.difference(req_values)
if values:
if isinstance(key, tuple):
types, comment = key
self.to_return.append({'types': list(types), 'values': list(values), 'comment': comment})
else:
self.to_return.append({'types': key, 'values': list(values)})
return self.to_return
def get_domain(self, domain, do_not_recurse=False):
req = requests.get(self.base_url.format('domain'), params={'domain': domain, 'apikey': self.apikey})
try:
req.raise_for_status()
req = req.json()
except HTTPError as e:
return str(e)
if req["response_code"] == 0:
# Nothing found
return []
if "resolutions" in req:
for res in req["resolutions"][:self.limit]:
ip_address = res["ip_address"]
self.results[(("ip-dst", "ip-src"), comment.format(domain))].add(ip_address)
# Pivot from here to find all domain info
if not do_not_recurse:
error = self.get_ip(ip_address, True)
if error is not None:
return error
self.get_more_info(req)
def get_hash(self, _hash):
req = requests.get(self.base_url.format('file'), params={'resource': _hash, 'apikey': self.apikey, 'allinfo': 1})
try:
req.raise_for_status()
req = req.json()
except HTTPError as e:
return str(e)
if req["response_code"] == 0:
# Nothing found
return []
self.get_more_info(req)
def get_ip(self, ip, do_not_recurse=False):
req = requests.get(self.base_url.format('ip-address'), params={'ip': ip, 'apikey': self.apikey})
try:
req.raise_for_status()
req = req.json()
except HTTPError as e:
return str(e)
if req["response_code"] == 0:
# Nothing found
return []
if "resolutions" in req:
for res in req["resolutions"][:self.limit]:
hostname = res["hostname"]
self.results[(("domain",), comment.format(ip))].add(hostname)
# Pivot from here to find all domain info
if not do_not_recurse:
error = self.get_domain(hostname, True)
if error is not None:
return error
self.get_more_info(req)
def find_all(self, data):
hashes = []
if isinstance(data, dict):
for key, value in data.items():
if key in hash_types:
self.results[key].add(value)
hashes.append(value)
else:
if isinstance(value, (dict, list)):
hashes.extend(self.find_all(value))
elif isinstance(data, list):
for d in data:
hashes.extend(self.find_all(d))
return hashes
def get_more_info(self, req):
# Get all hashes first
hashes = self.find_all(req)
for h in hashes[:self.limit]:
# Search VT for some juicy info
try:
data = requests.get(self.base_url.format('file'), params={'resource': h, 'apikey': self.apikey, 'allinfo': 1}).json()
except Exception:
continue
# Go through euch key and check if it exists
for VT_type, MISP_type in self.output_types_mapping.items():
if VT_type in data:
self.results[((MISP_type,), comment.format(h))].add(data[VT_type])
# Get the malware sample
sample = requests.get(self.base_url[:-6].format('file/download'), params={'hash': h, 'apikey': self.apikey})
malsample = sample.content
# It is possible for VT to not give us any submission names
if "submission_names" in data:
self.to_return.append({"types": ["malware-sample"], "categories": ["Payload delivery"],
"values": data["submimssion_names"], "data": str(base64.b64encore(malsample), 'utf-8')})
def handler(q=False):
global limit
if q is False:
return False
q = json.loads(q)
key = q["config"]["apikey"]
limit = int(q["config"].get("event_limit", 5))
r = {"results": []}
if "ip-src" in q:
r["results"] += getIP(q["ip-src"], key)
if "ip-dst" in q:
r["results"] += getIP(q["ip-dst"], key)
if "domain" in q:
r["results"] += getDomain(q["domain"], key)
if 'hostname' in q:
r["results"] += getDomain(q['hostname'], key)
if 'md5' in q:
r["results"] += getHash(q['md5'], key)
if 'sha1' in q:
r["results"] += getHash(q['sha1'], key)
if 'sha256' in q:
r["results"] += getHash(q['sha256'], key)
if 'sha512' in q:
r["results"] += getHash(q['sha512'], key)
uniq = []
for res in r["results"]:
if res not in uniq:
uniq.append(res)
r["results"] = uniq
return r
def getHash(hash, key, do_not_recurse=False):
req = requests.get("https://www.virustotal.com/vtapi/v2/file/report",
params={"allinfo": 1, "apikey": key, 'resource': hash})
try:
req.raise_for_status()
req = req.json()
except HTTPError as e:
misperrors['error'] = str(e)
if not q.get('config') or not q['config'].get('apikey'):
misperrors['error'] = "A VirusTotal api key is required for this module."
return misperrors
if req["response_code"] == 0:
# Nothing found
return []
return getMoreInfo(req, key)
def getIP(ip, key, do_not_recurse=False):
global limit
toReturn = []
req = requests.get("https://www.virustotal.com/vtapi/v2/ip-address/report",
params={"ip": ip, "apikey": key})
try:
req.raise_for_status()
req = req.json()
except HTTPError as e:
misperrors['error'] = str(e)
del q['module']
query = VirusTotalRequest(q.pop('config'))
r = query.parse_request(q)
if isinstance(r, str):
misperrors['error'] = r
return misperrors
if req["response_code"] == 0:
# Nothing found
return []
if "resolutions" in req:
for res in req["resolutions"][:limit]:
toReturn.append({"types": ["domain"], "values": [res["hostname"]], "comment": comment % ip})
# Pivot from here to find all domain info
if not do_not_recurse:
toReturn += getDomain(res["hostname"], key, True)
toReturn += getMoreInfo(req, key)
return toReturn
def getDomain(domain, key, do_not_recurse=False):
global limit
toReturn = []
req = requests.get("https://www.virustotal.com/vtapi/v2/domain/report",
params={"domain": domain, "apikey": key})
try:
req.raise_for_status()
req = req.json()
except HTTPError as e:
misperrors['error'] = str(e)
return misperrors
if req["response_code"] == 0:
# Nothing found
return []
if "resolutions" in req:
for res in req["resolutions"][:limit]:
toReturn.append({"types": ["ip-dst", "ip-src"], "values": [res["ip_address"]], "comment": comment % domain})
# Pivot from here to find all info on IPs
if not do_not_recurse:
toReturn += getIP(res["ip_address"], key, True)
if "subdomains" in req:
for subd in req["subdomains"]:
toReturn.append({"types": ["domain"], "values": [subd], "comment": comment % domain})
toReturn += getMoreInfo(req, key)
return toReturn
def findAll(data, keys):
a = []
if isinstance(data, dict):
for key in data.keys():
if key in keys:
a.append(data[key])
else:
if isinstance(data[key], (dict, list)):
a += findAll(data[key], keys)
if isinstance(data, list):
for i in data:
a += findAll(i, keys)
return a
def getMoreInfo(req, key):
global limit
r = []
# Get all hashes first
hashes = []
hashes = findAll(req, ["md5", "sha1", "sha256", "sha512"])
r.append({"types": ["freetext"], "values": hashes})
for hsh in hashes[:limit]:
# Search VT for some juicy info
try:
data = requests.get("http://www.virustotal.com/vtapi/v2/file/report",
params={"allinfo": 1, "apikey": key, "resource": hsh}
).json()
except:
continue
# Go through each key and check if it exists
if "submission_names" in data:
r.append({'types': ["filename"], "values": data["submission_names"], "comment": comment % hsh})
if "ssdeep" in data:
r.append({'types': ["ssdeep"], "values": [data["ssdeep"]], "comment": comment % hsh})
if "authentihash" in data:
r.append({"types": ["authentihash"], "values": [data["authentihash"]], "comment": comment % hsh})
if "ITW_urls" in data:
r.append({"types": ["url"], "values": data["ITW_urls"], "comment": comment % hsh})
# Get the malware sample
sample = requests.get("https://www.virustotal.com/vtapi/v2/file/download",
params={"hash": hsh, "apikey": key})
malsample = sample.content
# It is possible for VT to not give us any submission names
if "submission_names" in data:
r.append({"types": ["malware-sample"],
"categories": ["Payload delivery"],
"values": data["submission_names"],
"data": str(base64.b64encode(malsample), 'utf-8')
}
)
return r
return {'results': r}
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

2
misp_modules/modules/expansion/yara_syntax_validator.py
View File

@ -2,7 +2,7 @@ import json
import requests
try:
import yara
except:
except (OSError, ImportError):
print("yara is missing, use 'pip3 install yara' to install it.")
misperrors = {'error': 'Error'}

5
misp_modules/modules/export_mod/goamlexport.py
View File

@ -1,4 +1,5 @@
import json, datetime, base64
import json
import base64
from pymisp import MISPEvent
from collections import defaultdict, Counter
@ -67,7 +68,7 @@ class GoAmlGeneration(object):
try:
report_code.append(obj.get_attributes_by_relation('report-code')[0].value.split(' ')[0])
currency_code.append(obj.get_attributes_by_relation('currency-code')[0].value)
except:
except IndexError:
print('report_code or currency_code error')
self.uuids, self.report_codes, self.currency_codes = uuids, report_code, currency_code