diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py old mode 100755 new mode 100644 index 3997ee6..404f621 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -2,201 +2,163 @@ import json import requests from requests import HTTPError import base64 +from collections import defaultdict misperrors = {'error': 'Error'} mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "sha512"], 'output': ['domain', "ip-src", "ip-dst", "text", "md5", "sha1", "sha256", "sha512", "ssdeep", - "authentihash", "filename"] - } + "authentihash", "filename"]} # possible module-types: 'expansion', 'hover' or both -moduleinfo = {'version': '2', 'author': 'Hannah Ward', +moduleinfo = {'version': '3', 'author': 'Hannah Ward', 'description': 'Get information from virustotal', 'module-type': ['expansion']} # config fields that your code expects from the site admin moduleconfig = ["apikey", "event_limit"] -limit = 5 # Default -comment = '%s: Enriched via VT' +comment = '{}: Enriched via VirusTotal' +hash_types = ["md5", "sha1", "sha256", "sha512"] +class VirusTotalRequest(object): + def __init__(self, config): + self.apikey = config['apikey'] + self.limit = int(config.get('event_limit', 5)) + self.base_url = "https://www.virustotal.com/vtapi/v2/{}/report" + self.results = defaultdict(set) + self.to_return = [] + self.input_types_mapping = {'ip-src': self.get_ip, 'ip-dst': self.get_ip, + 'domain': self.get_domain, 'hostname': self.get_domain, + 'md5': self.get_hash, 'sha1': self.get_hash, + 'sha256': self.get_hash, 'sha512': self.get_hash} + self.output_types_mapping = {'submission_names': 'filename', 'ssdeep': 'ssdeep', + 'authentihash': 'authentihash', 'ITW_urls': 'url'} + + def parse_request(self, attribute_type, attribute_value): + error = self.input_types_mapping[attribute_type](attribute_value) + if error is not None: + return error + for key, values in self.results.items(): + if isinstance(key, tuple): + types, comment = key + self.to_return.append({'types': list(types), 'values': list(values), 'comment': comment}) + else: + self.to_return.append({'types': key, 'values': list(values)}) + return self.to_return + + def get_domain(self, domain, do_not_recurse=False): + req = requests.get(self.base_url.format('domain'), params={'domain': domain, 'apikey': self.apikey}) + try: + req.raise_for_status() + req = req.json() + except HTTPError as e: + return str(e) + if req["response_code"] == 0: + # Nothing found + return [] + if "resolutions" in req: + for res in req["resolutions"][:self.limit]: + ip_address = res["ip_address"] + self.results[(("ip-dst", "ip-src"), comment.format(domain))].add(ip_address) + # Pivot from here to find all domain info + if not do_not_recurse: + error = self.get_ip(ip_address, True) + if error is not None: + return error + self.get_more_info(req) + + def get_hash(self, _hash): + req = requests.get(self.base_url.format('file'), params={'resource': _hash, 'apikey': self.apikey, 'allinfo': 1}) + try: + req.raise_for_status() + req = req.json() + except HTTPError as e: + return str(e) + if req["response_code"] == 0: + # Nothing found + return [] + self.get_more_info(req) + + def get_ip(self, ip, do_not_recurse=False): + req = requests.get(self.base_url.format('ip-address'), params={'ip': ip, 'apikey': self.apikey}) + try: + req.raise_for_status() + req = req.json() + except HTTPError as e: + return str(e) + if req["response_code"] == 0: + # Nothing found + return [] + if "resolutions" in req: + for res in req["resolutions"][:self.limit]: + hostname = res["hostname"] + self.results[(("domain",), comment.format(ip))].add(hostname) + # Pivot from here to find all domain info + if not do_not_recurse: + error = self.get_domain(hostname, True) + if error is not None: + return error + self.get_more_info(req) + + def find_all(self, data): + hashes = [] + if isinstance(data, dict): + for key, value in data.items(): + if key in hash_types: + print(key) + self.results[key].add(value) + hashes.append(value) + else: + if isinstance(value, (dict, list)): + hashes.extend(self.find_all(value)) + elif isinstance(data, list): + for d in data: + hashes.extend(self.find_all(d)) + return hashes + + def get_more_info(self, req): + # Get all hashes first + hashes = self.find_all(req) + for h in hashes[:self.limit]: + # Search VT for some juicy info + try: + data = requests.get(self.base_url.format('file'), params={'resource': h, 'apikey': apikey, 'allinfo': 1}).json() + except Exception: + continue + # Go through euch key and check if it exists + for VT_type, MISP_type in self.output_types_mapping.items(): + if VT_type in data: + self.results[((MISP_type,), comment.format(h))].add(data[VT_type]) + # Get the malware sample + sample = requests.get(self.base_url[:-6].format('file/download'), params={'hash': h, 'apikey': apikey}) + malsample = sample.content + # It is possible for VT to not give us any submission names + if "submission_names" in data: + self.to_return.append({"types": ["malware-sample"], "categories": ["Payload delivery"], + "values": data["submimssion_names"], "data": str(base64.b64encore(malsample), 'utf-8')}) def handler(q=False): - global limit if q is False: return False - q = json.loads(q) - - key = q["config"]["apikey"] - limit = int(q["config"].get("event_limit", 5)) - - r = {"results": []} - - if "ip-src" in q: - r["results"] += getIP(q["ip-src"], key) - if "ip-dst" in q: - r["results"] += getIP(q["ip-dst"], key) - if "domain" in q: - r["results"] += getDomain(q["domain"], key) - if 'hostname' in q: - r["results"] += getDomain(q['hostname'], key) - if 'md5' in q: - r["results"] += getHash(q['md5'], key) - if 'sha1' in q: - r["results"] += getHash(q['sha1'], key) - if 'sha256' in q: - r["results"] += getHash(q['sha256'], key) - if 'sha512' in q: - r["results"] += getHash(q['sha512'], key) - - uniq = [] - for res in r["results"]: - if res not in uniq: - uniq.append(res) - r["results"] = uniq - return r - - -def getHash(hash, key, do_not_recurse=False): - req = requests.get("https://www.virustotal.com/vtapi/v2/file/report", - params={"allinfo": 1, "apikey": key, 'resource': hash}) - try: - req.raise_for_status() - req = req.json() - except HTTPError as e: - misperrors['error'] = str(e) + if not q.get('config') or not q['config'].get('apikey'): + misperrors['error']: "A VirusTotal api key is required for this module." return misperrors - - if req["response_code"] == 0: - # Nothing found - return [] - - return getMoreInfo(req, key) - - -def getIP(ip, key, do_not_recurse=False): - global limit - toReturn = [] - req = requests.get("https://www.virustotal.com/vtapi/v2/ip-address/report", - params={"ip": ip, "apikey": key}) - try: - req.raise_for_status() - req = req.json() - except HTTPError as e: - misperrors['error'] = str(e) + del q['module'] + query = VirusTotalRequest(q.pop('config')) + r = query.parse_request(*list(q.items())[0]) + if isinstance(r, str): + misperrors['error'] = r return misperrors + return {'results': r} - if req["response_code"] == 0: - # Nothing found - return [] - - if "resolutions" in req: - for res in req["resolutions"][:limit]: - toReturn.append({"types": ["domain"], "values": [res["hostname"]], "comment": comment % ip}) - # Pivot from here to find all domain info - if not do_not_recurse: - toReturn += getDomain(res["hostname"], key, True) - - toReturn += getMoreInfo(req, key) - return toReturn - - -def getDomain(domain, key, do_not_recurse=False): - global limit - toReturn = [] - req = requests.get("https://www.virustotal.com/vtapi/v2/domain/report", - params={"domain": domain, "apikey": key}) - try: - req.raise_for_status() - req = req.json() - except HTTPError as e: - misperrors['error'] = str(e) - return misperrors - - if req["response_code"] == 0: - # Nothing found - return [] - - if "resolutions" in req: - for res in req["resolutions"][:limit]: - toReturn.append({"types": ["ip-dst", "ip-src"], "values": [res["ip_address"]], "comment": comment % domain}) - # Pivot from here to find all info on IPs - if not do_not_recurse: - toReturn += getIP(res["ip_address"], key, True) - if "subdomains" in req: - for subd in req["subdomains"]: - toReturn.append({"types": ["domain"], "values": [subd], "comment": comment % domain}) - toReturn += getMoreInfo(req, key) - return toReturn - - -def findAll(data, keys): - a = [] - if isinstance(data, dict): - for key in data.keys(): - if key in keys: - a.append(data[key]) - else: - if isinstance(data[key], (dict, list)): - a += findAll(data[key], keys) - if isinstance(data, list): - for i in data: - a += findAll(i, keys) - - return a - - -def getMoreInfo(req, key): - global limit - r = [] - # Get all hashes first - hashes = [] - hashes = findAll(req, ["md5", "sha1", "sha256", "sha512"]) - r.append({"types": ["freetext"], "values": hashes}) - for hsh in hashes[:limit]: - # Search VT for some juicy info - try: - data = requests.get("http://www.virustotal.com/vtapi/v2/file/report", - params={"allinfo": 1, "apikey": key, "resource": hsh} - ).json() - except: - continue - - # Go through each key and check if it exists - if "submission_names" in data: - r.append({'types': ["filename"], "values": data["submission_names"], "comment": comment % hsh}) - - if "ssdeep" in data: - r.append({'types': ["ssdeep"], "values": [data["ssdeep"]], "comment": comment % hsh}) - - if "authentihash" in data: - r.append({"types": ["authentihash"], "values": [data["authentihash"]], "comment": comment % hsh}) - - if "ITW_urls" in data: - r.append({"types": ["url"], "values": data["ITW_urls"], "comment": comment % hsh}) - - # Get the malware sample - sample = requests.get("https://www.virustotal.com/vtapi/v2/file/download", - params={"hash": hsh, "apikey": key}) - - malsample = sample.content - - # It is possible for VT to not give us any submission names - if "submission_names" in data: - r.append({"types": ["malware-sample"], - "categories": ["Payload delivery"], - "values": data["submission_names"], - "data": str(base64.b64encode(malsample), 'utf-8') - } - ) - - return r - +def get_ip(ip, key): + params = {'ip': ip, 'apikey': key} + req = requests.get('https://www.virustotal.com/vtapi/v2/ip-address/report', params=params) + return json.dumps(req.json(), indent=2) def introspection(): return mispattributes - def version(): moduleinfo['config'] = moduleconfig return moduleinfo