diff --git a/misp_modules/modules/expansion/otx.py b/misp_modules/modules/expansion/otx.py index 214e7f0..86685eb 100755 --- a/misp_modules/modules/expansion/otx.py +++ b/misp_modules/modules/expansion/otx.py @@ -32,16 +32,15 @@ def valid_ip(ip): def findAll(data, keys): a = [] if isinstance(data, dict): - for key in data.keys(): + for key, value in data.items(): if key == keys: - a.append(data[key]) + a.append(value) else: - if isinstance(data[key], (dict, list)): - a += findAll(data[key], keys) + if isinstance(value, (dict, list)): + a.extend(findAll(value, keys)) if isinstance(data, list): for i in data: - a += findAll(i, keys) - + a.extend(findAll(i, keys)) return a def valid_email(email): @@ -82,10 +81,10 @@ def handler(q=False): return r -def getHash(hash, key): +def getHash(_hash, key): ret = [] - req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/file/analysis/" + hash).text) + req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/file/analysis/" + _hash).text) for ip in findAll(req, "dst"): if not isBlacklisted(ip) and valid_ip(ip): @@ -102,8 +101,8 @@ def getIP(ip, key): ret = [] req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/ip/malware/" + ip + "?limit=1000").text ) - for hash in findAll(req, "hash"): - ret.append({"types": ["sha256"], "values": [hash]}) + for _hash in findAll(req, "hash"): + ret.append({"types": ["sha256"], "values": [_hash]}) req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/ip/passive_dns/" + ip).text ) @@ -122,21 +121,21 @@ def getDomain(domain, key): req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/domain/malware/" + domain + "?limit=1000").text ) - for hash in findAll(req, "hash"): - ret.append({"types": ["sha256"], "values": [hash]}) + for _hash in findAll(req, "hash"): + ret.append({"types": ["sha256"], "values": [_hash]}) req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/domain/whois/" + domain).text) - for domain in findAll(req, "domain"): - ret.append({"types": ["hostname"], "values": [domain]}) + for _domain in findAll(req, "domain"): + ret.append({"types": ["hostname"], "values": [_domain]}) for email in findAll(req, "value"): if valid_email(email): - ret.append({"types": ["email"], "values": [domain]}) + ret.append({"types": ["email"], "values": [email]}) - for domain in findAll(req, "hostname"): - if "." in domain and not isBlacklisted(domain): - ret.append({"types": ["hostname"], "values": [domain]}) + for _domain in findAll(req, "hostname"): + if "." in _domain and not isBlacklisted(_domain): + ret.append({"types": ["hostname"], "values": [_domain]}) req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/hostname/passive_dns/" + domain).text) for ip in findAll(req, "address"): diff --git a/misp_modules/modules/expansion/rbl.py b/misp_modules/modules/expansion/rbl.py index da8c5fb..7aac103 100644 --- a/misp_modules/modules/expansion/rbl.py +++ b/misp_modules/modules/expansion/rbl.py @@ -6,7 +6,7 @@ try: resolver = dns.resolver.Resolver() resolver.timeout = 0.2 resolver.lifetime = 0.2 -except: +except ModuleNotFoundError: print("dnspython3 is missing, use 'pip install dnspython3' to install it.") sys.exit(0) @@ -96,13 +96,12 @@ def handler(q=False): txt = resolver.query(query,'TXT') listed.append(query) info.append(str(txt[0])) - except: + except Exception: continue result = {} for l, i in zip(listed, info): result[l] = i - r = {'results': [{'types': mispattributes.get('output'), 'values': json.dumps(result)}]} - return r + return {'results': [{'types': mispattributes.get('output'), 'values': json.dumps(result)}]} def introspection(): return mispattributes diff --git a/misp_modules/modules/expansion/sigma_syntax_validator.py b/misp_modules/modules/expansion/sigma_syntax_validator.py index 0d5226f..6452654 100644 --- a/misp_modules/modules/expansion/sigma_syntax_validator.py +++ b/misp_modules/modules/expansion/sigma_syntax_validator.py @@ -21,7 +21,7 @@ def handler(q=False): return misperrors config = SigmaConfiguration() try: - parser = SigmaParser(yaml.load(request.get('sigma')), config) + parser = SigmaParser(yaml.safe_load(request.get('sigma')), config) result = ("Syntax valid: {}".format(parser.values)) except Exception as e: result = ("Syntax error: {}".format(str(e))) diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py old mode 100755 new mode 100644 index 3997ee6..524bc49 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -2,201 +2,165 @@ import json import requests from requests import HTTPError import base64 +from collections import defaultdict misperrors = {'error': 'Error'} mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "sha512"], 'output': ['domain', "ip-src", "ip-dst", "text", "md5", "sha1", "sha256", "sha512", "ssdeep", - "authentihash", "filename"] - } + "authentihash", "filename"]} # possible module-types: 'expansion', 'hover' or both -moduleinfo = {'version': '2', 'author': 'Hannah Ward', +moduleinfo = {'version': '3', 'author': 'Hannah Ward', 'description': 'Get information from virustotal', 'module-type': ['expansion']} # config fields that your code expects from the site admin moduleconfig = ["apikey", "event_limit"] -limit = 5 # Default -comment = '%s: Enriched via VT' +comment = '{}: Enriched via VirusTotal' +hash_types = ["md5", "sha1", "sha256", "sha512"] +class VirusTotalRequest(object): + def __init__(self, config): + self.apikey = config['apikey'] + self.limit = int(config.get('event_limit', 5)) + self.base_url = "https://www.virustotal.com/vtapi/v2/{}/report" + self.results = defaultdict(set) + self.to_return = [] + self.input_types_mapping = {'ip-src': self.get_ip, 'ip-dst': self.get_ip, + 'domain': self.get_domain, 'hostname': self.get_domain, + 'md5': self.get_hash, 'sha1': self.get_hash, + 'sha256': self.get_hash, 'sha512': self.get_hash} + self.output_types_mapping = {'submission_names': 'filename', 'ssdeep': 'ssdeep', + 'authentihash': 'authentihash', 'ITW_urls': 'url'} + + def parse_request(self, q): + req_values = set() + for attribute_type, attribute_value in q.items(): + req_values.add(attribute_value) + try: + error = self.input_types_mapping[attribute_type](attribute_value) + except KeyError: + continue + if error is not None: + return error + for key, values in self.results.items(): + values = values.difference(req_values) + if values: + if isinstance(key, tuple): + types, comment = key + self.to_return.append({'types': list(types), 'values': list(values), 'comment': comment}) + else: + self.to_return.append({'types': key, 'values': list(values)}) + return self.to_return + + def get_domain(self, domain, do_not_recurse=False): + req = requests.get(self.base_url.format('domain'), params={'domain': domain, 'apikey': self.apikey}) + try: + req.raise_for_status() + req = req.json() + except HTTPError as e: + return str(e) + if req["response_code"] == 0: + # Nothing found + return [] + if "resolutions" in req: + for res in req["resolutions"][:self.limit]: + ip_address = res["ip_address"] + self.results[(("ip-dst", "ip-src"), comment.format(domain))].add(ip_address) + # Pivot from here to find all domain info + if not do_not_recurse: + error = self.get_ip(ip_address, True) + if error is not None: + return error + self.get_more_info(req) + + def get_hash(self, _hash): + req = requests.get(self.base_url.format('file'), params={'resource': _hash, 'apikey': self.apikey, 'allinfo': 1}) + try: + req.raise_for_status() + req = req.json() + except HTTPError as e: + return str(e) + if req["response_code"] == 0: + # Nothing found + return [] + self.get_more_info(req) + + def get_ip(self, ip, do_not_recurse=False): + req = requests.get(self.base_url.format('ip-address'), params={'ip': ip, 'apikey': self.apikey}) + try: + req.raise_for_status() + req = req.json() + except HTTPError as e: + return str(e) + if req["response_code"] == 0: + # Nothing found + return [] + if "resolutions" in req: + for res in req["resolutions"][:self.limit]: + hostname = res["hostname"] + self.results[(("domain",), comment.format(ip))].add(hostname) + # Pivot from here to find all domain info + if not do_not_recurse: + error = self.get_domain(hostname, True) + if error is not None: + return error + self.get_more_info(req) + + def find_all(self, data): + hashes = [] + if isinstance(data, dict): + for key, value in data.items(): + if key in hash_types: + self.results[key].add(value) + hashes.append(value) + else: + if isinstance(value, (dict, list)): + hashes.extend(self.find_all(value)) + elif isinstance(data, list): + for d in data: + hashes.extend(self.find_all(d)) + return hashes + + def get_more_info(self, req): + # Get all hashes first + hashes = self.find_all(req) + for h in hashes[:self.limit]: + # Search VT for some juicy info + try: + data = requests.get(self.base_url.format('file'), params={'resource': h, 'apikey': self.apikey, 'allinfo': 1}).json() + except Exception: + continue + # Go through euch key and check if it exists + for VT_type, MISP_type in self.output_types_mapping.items(): + if VT_type in data: + self.results[((MISP_type,), comment.format(h))].add(data[VT_type]) + # Get the malware sample + sample = requests.get(self.base_url[:-6].format('file/download'), params={'hash': h, 'apikey': self.apikey}) + malsample = sample.content + # It is possible for VT to not give us any submission names + if "submission_names" in data: + self.to_return.append({"types": ["malware-sample"], "categories": ["Payload delivery"], + "values": data["submimssion_names"], "data": str(base64.b64encore(malsample), 'utf-8')}) def handler(q=False): - global limit if q is False: return False - q = json.loads(q) - - key = q["config"]["apikey"] - limit = int(q["config"].get("event_limit", 5)) - - r = {"results": []} - - if "ip-src" in q: - r["results"] += getIP(q["ip-src"], key) - if "ip-dst" in q: - r["results"] += getIP(q["ip-dst"], key) - if "domain" in q: - r["results"] += getDomain(q["domain"], key) - if 'hostname' in q: - r["results"] += getDomain(q['hostname'], key) - if 'md5' in q: - r["results"] += getHash(q['md5'], key) - if 'sha1' in q: - r["results"] += getHash(q['sha1'], key) - if 'sha256' in q: - r["results"] += getHash(q['sha256'], key) - if 'sha512' in q: - r["results"] += getHash(q['sha512'], key) - - uniq = [] - for res in r["results"]: - if res not in uniq: - uniq.append(res) - r["results"] = uniq - return r - - -def getHash(hash, key, do_not_recurse=False): - req = requests.get("https://www.virustotal.com/vtapi/v2/file/report", - params={"allinfo": 1, "apikey": key, 'resource': hash}) - try: - req.raise_for_status() - req = req.json() - except HTTPError as e: - misperrors['error'] = str(e) + if not q.get('config') or not q['config'].get('apikey'): + misperrors['error'] = "A VirusTotal api key is required for this module." return misperrors - - if req["response_code"] == 0: - # Nothing found - return [] - - return getMoreInfo(req, key) - - -def getIP(ip, key, do_not_recurse=False): - global limit - toReturn = [] - req = requests.get("https://www.virustotal.com/vtapi/v2/ip-address/report", - params={"ip": ip, "apikey": key}) - try: - req.raise_for_status() - req = req.json() - except HTTPError as e: - misperrors['error'] = str(e) + del q['module'] + query = VirusTotalRequest(q.pop('config')) + r = query.parse_request(q) + if isinstance(r, str): + misperrors['error'] = r return misperrors - - if req["response_code"] == 0: - # Nothing found - return [] - - if "resolutions" in req: - for res in req["resolutions"][:limit]: - toReturn.append({"types": ["domain"], "values": [res["hostname"]], "comment": comment % ip}) - # Pivot from here to find all domain info - if not do_not_recurse: - toReturn += getDomain(res["hostname"], key, True) - - toReturn += getMoreInfo(req, key) - return toReturn - - -def getDomain(domain, key, do_not_recurse=False): - global limit - toReturn = [] - req = requests.get("https://www.virustotal.com/vtapi/v2/domain/report", - params={"domain": domain, "apikey": key}) - try: - req.raise_for_status() - req = req.json() - except HTTPError as e: - misperrors['error'] = str(e) - return misperrors - - if req["response_code"] == 0: - # Nothing found - return [] - - if "resolutions" in req: - for res in req["resolutions"][:limit]: - toReturn.append({"types": ["ip-dst", "ip-src"], "values": [res["ip_address"]], "comment": comment % domain}) - # Pivot from here to find all info on IPs - if not do_not_recurse: - toReturn += getIP(res["ip_address"], key, True) - if "subdomains" in req: - for subd in req["subdomains"]: - toReturn.append({"types": ["domain"], "values": [subd], "comment": comment % domain}) - toReturn += getMoreInfo(req, key) - return toReturn - - -def findAll(data, keys): - a = [] - if isinstance(data, dict): - for key in data.keys(): - if key in keys: - a.append(data[key]) - else: - if isinstance(data[key], (dict, list)): - a += findAll(data[key], keys) - if isinstance(data, list): - for i in data: - a += findAll(i, keys) - - return a - - -def getMoreInfo(req, key): - global limit - r = [] - # Get all hashes first - hashes = [] - hashes = findAll(req, ["md5", "sha1", "sha256", "sha512"]) - r.append({"types": ["freetext"], "values": hashes}) - for hsh in hashes[:limit]: - # Search VT for some juicy info - try: - data = requests.get("http://www.virustotal.com/vtapi/v2/file/report", - params={"allinfo": 1, "apikey": key, "resource": hsh} - ).json() - except: - continue - - # Go through each key and check if it exists - if "submission_names" in data: - r.append({'types': ["filename"], "values": data["submission_names"], "comment": comment % hsh}) - - if "ssdeep" in data: - r.append({'types': ["ssdeep"], "values": [data["ssdeep"]], "comment": comment % hsh}) - - if "authentihash" in data: - r.append({"types": ["authentihash"], "values": [data["authentihash"]], "comment": comment % hsh}) - - if "ITW_urls" in data: - r.append({"types": ["url"], "values": data["ITW_urls"], "comment": comment % hsh}) - - # Get the malware sample - sample = requests.get("https://www.virustotal.com/vtapi/v2/file/download", - params={"hash": hsh, "apikey": key}) - - malsample = sample.content - - # It is possible for VT to not give us any submission names - if "submission_names" in data: - r.append({"types": ["malware-sample"], - "categories": ["Payload delivery"], - "values": data["submission_names"], - "data": str(base64.b64encode(malsample), 'utf-8') - } - ) - - return r - + return {'results': r} def introspection(): return mispattributes - def version(): moduleinfo['config'] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/yara_syntax_validator.py b/misp_modules/modules/expansion/yara_syntax_validator.py index 4953c41..c68d934 100644 --- a/misp_modules/modules/expansion/yara_syntax_validator.py +++ b/misp_modules/modules/expansion/yara_syntax_validator.py @@ -2,7 +2,7 @@ import json import requests try: import yara -except: +except (OSError, ImportError): print("yara is missing, use 'pip3 install yara' to install it.") misperrors = {'error': 'Error'} diff --git a/misp_modules/modules/export_mod/goamlexport.py b/misp_modules/modules/export_mod/goamlexport.py index 961a11b..f09f2e6 100644 --- a/misp_modules/modules/export_mod/goamlexport.py +++ b/misp_modules/modules/export_mod/goamlexport.py @@ -1,4 +1,5 @@ -import json, datetime, base64 +import json +import base64 from pymisp import MISPEvent from collections import defaultdict, Counter @@ -67,7 +68,7 @@ class GoAmlGeneration(object): try: report_code.append(obj.get_attributes_by_relation('report-code')[0].value.split(' ')[0]) currency_code.append(obj.get_attributes_by_relation('currency-code')[0].value) - except: + except IndexError: print('report_code or currency_code error') self.uuids, self.report_codes, self.currency_codes = uuids, report_code, currency_code