MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #222 from chrisr3d/master

Browse Source 
Clean up + fix of some modules
pull/226/head
Christian Studer 5 years ago committed by GitHub
parent 5107e78f5d cfbd63f14e
commit de846733bc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 162 additions and 199 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 35
      misp_modules/modules/expansion/otx.py
  2. 7
      misp_modules/modules/expansion/rbl.py
  3. 2
      misp_modules/modules/expansion/sigma_syntax_validator.py
  4. 310
      misp_modules/modules/expansion/virustotal.py
  5. 2
      misp_modules/modules/expansion/yara_syntax_validator.py
  6. 5
      misp_modules/modules/export_mod/goamlexport.py

35
misp_modules/modules/expansion/otx.py
Unescape View File

@ -32,16 +32,15 @@ def valid_ip(ip):
def findAll(data, keys):
a = []
if isinstance(data, dict):
for key in data.keys():
for key, value in data.items():
if key == keys:
a.append(data[key])
a.append(value)
else:
if isinstance(data[key], (dict, list)):
a += findAll(data[key], keys)
if isinstance(value, (dict, list)):
a.extend(findAll(value, keys))
if isinstance(data, list):
for i in data:
a += findAll(i, keys)
a.extend(findAll(i, keys))
return a
def valid_email(email):
@ -82,10 +81,10 @@ def handler(q=False):
return r
def getHash(hash, key):
def getHash(_hash, key):
ret = []
req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/file/analysis/" + hash).text)
req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/file/analysis/" + _hash).text)
for ip in findAll(req, "dst"):
if not isBlacklisted(ip) and valid_ip(ip):
@ -102,8 +101,8 @@ def getIP(ip, key):
ret = []
req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/ip/malware/" + ip + "?limit=1000").text )
for hash in findAll(req, "hash"):
ret.append({"types": ["sha256"], "values": [hash]})
for _hash in findAll(req, "hash"):
ret.append({"types": ["sha256"], "values": [_hash]})
req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/ip/passive_dns/" + ip).text )
@ -122,21 +121,21 @@ def getDomain(domain, key):
req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/domain/malware/" + domain + "?limit=1000").text )
for hash in findAll(req, "hash"):
ret.append({"types": ["sha256"], "values": [hash]})
for _hash in findAll(req, "hash"):
ret.append({"types": ["sha256"], "values": [_hash]})
req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/domain/whois/" + domain).text)
for domain in findAll(req, "domain"):
ret.append({"types": ["hostname"], "values": [domain]})
for _domain in findAll(req, "domain"):
ret.append({"types": ["hostname"], "values": [_domain]})
for email in findAll(req, "value"):
if valid_email(email):
ret.append({"types": ["email"], "values": [domain]})
ret.append({"types": ["email"], "values": [email]})
for domain in findAll(req, "hostname"):
if "." in domain and not isBlacklisted(domain):
ret.append({"types": ["hostname"], "values": [domain]})
for _domain in findAll(req, "hostname"):
if "." in _domain and not isBlacklisted(_domain):
ret.append({"types": ["hostname"], "values": [_domain]})
req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/hostname/passive_dns/" + domain).text)
for ip in findAll(req, "address"):

7
misp_modules/modules/expansion/rbl.py
Unescape View File

@ -6,7 +6,7 @@ try:
resolver = dns.resolver.Resolver()
resolver.timeout = 0.2
resolver.lifetime = 0.2
except:
except ModuleNotFoundError:
print("dnspython3 is missing, use 'pip install dnspython3' to install it.")
sys.exit(0)
@ -96,13 +96,12 @@ def handler(q=False):
txt = resolver.query(query,'TXT')
listed.append(query)
info.append(str(txt[0]))
except:
except Exception:
continue
result = {}
for l, i in zip(listed, info):
result[l] = i
r = {'results': [{'types': mispattributes.get('output'), 'values': json.dumps(result)}]}
return r
return {'results': [{'types': mispattributes.get('output'), 'values': json.dumps(result)}]}
def introspection():
return mispattributes

2
misp_modules/modules/expansion/sigma_syntax_validator.py
Unescape View File

@ -21,7 +21,7 @@ def handler(q=False):
return misperrors
config = SigmaConfiguration()
try:
parser = SigmaParser(yaml.load(request.get('sigma')), config)
parser = SigmaParser(yaml.safe_load(request.get('sigma')), config)
result = ("Syntax valid: {}".format(parser.values))
except Exception as e:
result = ("Syntax error: {}".format(str(e)))

310
misp_modules/modules/expansion/virustotal.py
Unescape View File

@ -2,201 +2,165 @@ import json
import requests
from requests import HTTPError
import base64
from collections import defaultdict
misperrors = {'error': 'Error'}
mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "sha512"],
'output': ['domain', "ip-src", "ip-dst", "text", "md5", "sha1", "sha256", "sha512", "ssdeep",
"authentihash", "filename"]
}
"authentihash", "filename"]}
# possible module-types: 'expansion', 'hover' or both
moduleinfo = {'version': '2', 'author': 'Hannah Ward',
moduleinfo = {'version': '3', 'author': 'Hannah Ward',
'description': 'Get information from virustotal',
'module-type': ['expansion']}
# config fields that your code expects from the site admin
moduleconfig = ["apikey", "event_limit"]
limit = 5 # Default
comment = '%s: Enriched via VT'
comment = '{}: Enriched via VirusTotal'
hash_types = ["md5", "sha1", "sha256", "sha512"]
class VirusTotalRequest(object):
def __init__(self, config):
self.apikey = config['apikey']
self.limit = int(config.get('event_limit', 5))
self.base_url = "https://www.virustotal.com/vtapi/v2/{}/report"
self.results = defaultdict(set)
self.to_return = []
self.input_types_mapping = {'ip-src': self.get_ip, 'ip-dst': self.get_ip,
'domain': self.get_domain, 'hostname': self.get_domain,
'md5': self.get_hash, 'sha1': self.get_hash,
'sha256': self.get_hash, 'sha512': self.get_hash}
self.output_types_mapping = {'submission_names': 'filename', 'ssdeep': 'ssdeep',
'authentihash': 'authentihash', 'ITW_urls': 'url'}
def parse_request(self, q):
req_values = set()
for attribute_type, attribute_value in q.items():
req_values.add(attribute_value)
try:
error = self.input_types_mapping[attribute_type](attribute_value)
except KeyError:
continue
if error is not None:
return error
for key, values in self.results.items():
values = values.difference(req_values)
if values:
if isinstance(key, tuple):
types, comment = key
self.to_return.append({'types': list(types), 'values': list(values), 'comment': comment})
else:
self.to_return.append({'types': key, 'values': list(values)})
return self.to_return
def get_domain(self, domain, do_not_recurse=False):
req = requests.get(self.base_url.format('domain'), params={'domain': domain, 'apikey': self.apikey})
try:
req.raise_for_status()
req = req.json()
except HTTPError as e:
return str(e)
if req["response_code"] == 0:
# Nothing found
return []
if "resolutions" in req:
for res in req["resolutions"][:self.limit]:
ip_address = res["ip_address"]
self.results[(("ip-dst", "ip-src"), comment.format(domain))].add(ip_address)
# Pivot from here to find all domain info
if not do_not_recurse:
error = self.get_ip(ip_address, True)
if error is not None:
return error
self.get_more_info(req)
def get_hash(self, _hash):
req = requests.get(self.base_url.format('file'), params={'resource': _hash, 'apikey': self.apikey, 'allinfo': 1})
try:
req.raise_for_status()
req = req.json()
except HTTPError as e:
return str(e)
if req["response_code"] == 0:
# Nothing found
return []
self.get_more_info(req)
def get_ip(self, ip, do_not_recurse=False):
req = requests.get(self.base_url.format('ip-address'), params={'ip': ip, 'apikey': self.apikey})
try:
req.raise_for_status()
req = req.json()
except HTTPError as e:
return str(e)
if req["response_code"] == 0:
# Nothing found
return []
if "resolutions" in req:
for res in req["resolutions"][:self.limit]:
hostname = res["hostname"]
self.results[(("domain",), comment.format(ip))].add(hostname)
# Pivot from here to find all domain info
if not do_not_recurse:
error = self.get_domain(hostname, True)
if error is not None:
return error
self.get_more_info(req)
def find_all(self, data):
hashes = []
if isinstance(data, dict):
for key, value in data.items():
if key in hash_types:
self.results[key].add(value)
hashes.append(value)
else:
if isinstance(value, (dict, list)):
hashes.extend(self.find_all(value))
elif isinstance(data, list):
for d in data:
hashes.extend(self.find_all(d))
return hashes
def get_more_info(self, req):
# Get all hashes first
hashes = self.find_all(req)
for h in hashes[:self.limit]:
# Search VT for some juicy info
try:
data = requests.get(self.base_url.format('file'), params={'resource': h, 'apikey': self.apikey, 'allinfo': 1}).json()
except Exception:
continue
# Go through euch key and check if it exists
for VT_type, MISP_type in self.output_types_mapping.items():
if VT_type in data:
self.results[((MISP_type,), comment.format(h))].add(data[VT_type])
# Get the malware sample
sample = requests.get(self.base_url[:-6].format('file/download'), params={'hash': h, 'apikey': self.apikey})
malsample = sample.content
# It is possible for VT to not give us any submission names
if "submission_names" in data:
self.to_return.append({"types": ["malware-sample"], "categories": ["Payload delivery"],
"values": data["submimssion_names"], "data": str(base64.b64encore(malsample), 'utf-8')})
def handler(q=False):
global limit
if q is False:
return False
q = json.loads(q)
key = q["config"]["apikey"]
limit = int(q["config"].get("event_limit", 5))
r = {"results": []}
if "ip-src" in q:
r["results"] += getIP(q["ip-src"], key)
if "ip-dst" in q:
r["results"] += getIP(q["ip-dst"], key)
if "domain" in q:
r["results"] += getDomain(q["domain"], key)
if 'hostname' in q:
r["results"] += getDomain(q['hostname'], key)
if 'md5' in q:
r["results"] += getHash(q['md5'], key)
if 'sha1' in q:
r["results"] += getHash(q['sha1'], key)
if 'sha256' in q:
r["results"] += getHash(q['sha256'], key)
if 'sha512' in q:
r["results"] += getHash(q['sha512'], key)
uniq = []
for res in r["results"]:
if res not in uniq:
uniq.append(res)
r["results"] = uniq
return r
def getHash(hash, key, do_not_recurse=False):
req = requests.get("https://www.virustotal.com/vtapi/v2/file/report",
params={"allinfo": 1, "apikey": key, 'resource': hash})
try:
req.raise_for_status()
req = req.json()
except HTTPError as e:
misperrors['error'] = str(e)
if not q.get('config') or not q['config'].get('apikey'):
misperrors['error'] = "A VirusTotal api key is required for this module."
return misperrors
if req["response_code"] == 0:
# Nothing found
return []
return getMoreInfo(req, key)
def getIP(ip, key, do_not_recurse=False):
global limit
toReturn = []
req = requests.get("https://www.virustotal.com/vtapi/v2/ip-address/report",
params={"ip": ip, "apikey": key})
try:
req.raise_for_status()
req = req.json()
except HTTPError as e:
misperrors['error'] = str(e)
del q['module']
query = VirusTotalRequest(q.pop('config'))
r = query.parse_request(q)
if isinstance(r, str):
misperrors['error'] = r
return misperrors
if req["response_code"] == 0:
# Nothing found
return []
if "resolutions" in req:
for res in req["resolutions"][:limit]:
toReturn.append({"types": ["domain"], "values": [res["hostname"]], "comment": comment % ip})
# Pivot from here to find all domain info
if not do_not_recurse:
toReturn += getDomain(res["hostname"], key, True)
toReturn += getMoreInfo(req, key)
return toReturn
def getDomain(domain, key, do_not_recurse=False):
global limit
toReturn = []
req = requests.get("https://www.virustotal.com/vtapi/v2/domain/report",
params={"domain": domain, "apikey": key})
try:
req.raise_for_status()
req = req.json()
except HTTPError as e:
misperrors['error'] = str(e)
return misperrors
if req["response_code"] == 0:
# Nothing found
return []
if "resolutions" in req:
for res in req["resolutions"][:limit]:
toReturn.append({"types": ["ip-dst", "ip-src"], "values": [res["ip_address"]], "comment": comment % domain})
# Pivot from here to find all info on IPs
if not do_not_recurse:
toReturn += getIP(res["ip_address"], key, True)
if "subdomains" in req:
for subd in req["subdomains"]:
toReturn.append({"types": ["domain"], "values": [subd], "comment": comment % domain})
toReturn += getMoreInfo(req, key)
return toReturn
def findAll(data, keys):
a = []
if isinstance(data, dict):
for key in data.keys():
if key in keys:
a.append(data[key])
else:
if isinstance(data[key], (dict, list)):
a += findAll(data[key], keys)
if isinstance(data, list):
for i in data:
a += findAll(i, keys)
return a
def getMoreInfo(req, key):
global limit
r = []
# Get all hashes first
hashes = []
hashes = findAll(req, ["md5", "sha1", "sha256", "sha512"])
r.append({"types": ["freetext"], "values": hashes})
for hsh in hashes[:limit]:
# Search VT for some juicy info
try:
data = requests.get("http://www.virustotal.com/vtapi/v2/file/report",
params={"allinfo": 1, "apikey": key, "resource": hsh}
).json()
except:
continue
# Go through each key and check if it exists
if "submission_names" in data:
r.append({'types': ["filename"], "values": data["submission_names"], "comment": comment % hsh})
if "ssdeep" in data:
r.append({'types': ["ssdeep"], "values": [data["ssdeep"]], "comment": comment % hsh})
if "authentihash" in data:
r.append({"types": ["authentihash"], "values": [data["authentihash"]], "comment": comment % hsh})
if "ITW_urls" in data:
r.append({"types": ["url"], "values": data["ITW_urls"], "comment": comment % hsh})
# Get the malware sample
sample = requests.get("https://www.virustotal.com/vtapi/v2/file/download",
params={"hash": hsh, "apikey": key})
malsample = sample.content
# It is possible for VT to not give us any submission names
if "submission_names" in data:
r.append({"types": ["malware-sample"],
"categories": ["Payload delivery"],
"values": data["submission_names"],
"data": str(base64.b64encode(malsample), 'utf-8')
}
)
return r
return {'results': r}
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

2
misp_modules/modules/expansion/yara_syntax_validator.py
Unescape View File

@ -2,7 +2,7 @@ import json
import requests
try:
import yara
except:
except (OSError, ImportError):
print("yara is missing, use 'pip3 install yara' to install it.")
misperrors = {'error': 'Error'}

5
misp_modules/modules/export_mod/goamlexport.py
Unescape View File

@ -1,4 +1,5 @@
import json, datetime, base64
import json
import base64
from pymisp import MISPEvent
from collections import defaultdict, Counter
@ -67,7 +68,7 @@ class GoAmlGeneration(object):
try:
report_code.append(obj.get_attributes_by_relation('report-code')[0].value.split(' ')[0])
currency_code.append(obj.get_attributes_by_relation('currency-code')[0].value)
except:
except IndexError:
print('report_code or currency_code error')
self.uuids, self.report_codes, self.currency_codes = uuids, report_code, currency_code

Write Preview
Loading…
Cancel
Save