from urllib.parse import urlparse import vt from . import check_input_attribute, standard_error_message from pymisp import MISPAttribute, MISPEvent, MISPObject misperrors = {'error': 'Error'} mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "url", "ip-src|port", "ip-dst|port"], 'format': 'misp_standard'} # possible module-types: 'expansion', 'hover' or both moduleinfo = { 'version': '6', 'author': 'Hannah Ward', 'description': 'Enrich observables with the VirusTotal v3 API', 'module-type': ['expansion'], 'name': 'VirusTotal v3 Lookup', 'logo': 'virustotal.png', 'requirements': ['An access to the VirusTotal API (apikey), with a high request rate limit.'], 'features': 'New format of modules able to return attributes and objects.\n\nA module to take a MISP attribute as input and query the VirusTotal API to get additional data about it.\n\nCompared to the [standard VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/main/misp_modules/modules/expansion/virustotal_public.py), this module is made for advanced parsing of VirusTotal report, with a recursive analysis of the elements found after the first request.\n\nThus, it requires a higher request rate limit to avoid the API to return a 204 error (Request rate limit exceeded), and the data parsed from the different requests are returned as MISP attributes and objects, with the corresponding relations between each one of them.', 'references': ['https://www.virustotal.com/', 'https://docs.virustotal.com/reference/overview'], 'input': 'A domain, hash (md5, sha1, sha256 or sha512), hostname or IP address attribute.', 'output': 'MISP attributes and objects resulting from the parsing of the VirusTotal report concerning the input attribute.', } # config fields that your code expects from the site admin moduleconfig = ["apikey", "event_limit", 'proxy_host', 'proxy_port', 'proxy_username', 'proxy_password'] DEFAULT_RESULTS_LIMIT = 10 class VirusTotalParser: def __init__(self, client: vt.Client, limit: int) -> None: self.client = client self.limit = limit or DEFAULT_RESULTS_LIMIT self.misp_event = MISPEvent() self.attribute = MISPAttribute() self.parsed_objects = {} self.input_types_mapping = {'ip-src': self.parse_ip, 'ip-dst': self.parse_ip, 'domain': self.parse_domain, 'hostname': self.parse_domain, 'md5': self.parse_hash, 'sha1': self.parse_hash, 'sha256': self.parse_hash, 'url': self.parse_url, 'ip-src|port': self.parse_ip_port, 'ip-dst|port': self.parse_ip_port} self.proxies = None @staticmethod def get_total_analysis(analysis: dict, known_distributors: dict = None) -> int: if not analysis: return 0 count = sum([analysis['undetected'], analysis['suspicious'], analysis['harmless']]) return count if known_distributors else count + analysis['malicious'] def query_api(self, attribute: dict) -> None: self.attribute.from_dict(**attribute) self.input_types_mapping[self.attribute.type](self.attribute.value) def get_result(self) -> dict: event = self.misp_event.to_dict() results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} return {'results': results} def add_vt_report(self, report: vt.Object) -> str: analysis = report.get('last_analysis_stats') total = self.get_total_analysis(analysis, report.get('known_distributors')) if report.type == 'ip_address': rtype = 'ip-address' else: rtype = report.type permalink = f'https://www.virustotal.com/gui/{rtype}/{report.id}' vt_object = MISPObject('virustotal-report') vt_object.add_attribute('permalink', type='link', value=permalink) detection_ratio = f"{analysis['malicious']}/{total}" if analysis else '-/-' vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio, disable_correlation=True) self.misp_event.add_object(**vt_object) return vt_object.uuid def create_misp_object(self, report: vt.Object) -> MISPObject: misp_object = None vt_uuid = self.add_vt_report(report) if report.type == 'file': misp_object = MISPObject('file') for hash_type in ('md5', 'sha1', 'sha256', 'tlsh', 'vhash', 'ssdeep', 'imphash'): misp_object.add_attribute(hash_type, **{'type': hash_type, 'value': report.get(hash_type)}) elif report.type == 'domain': misp_object = MISPObject('domain-ip') misp_object.add_attribute('domain', type='domain', value=report.id) elif report.type == 'ip_address': misp_object = MISPObject('domain-ip') misp_object.add_attribute('ip', type='ip-dst', value=report.id) elif report.type == 'url': misp_object = MISPObject('url') misp_object.add_attribute('url', type='url', value=report.id) misp_object.add_reference(vt_uuid, 'analyzed-with') return misp_object ################################################################################ #### Main parsing functions #### # noqa ################################################################################ def parse_domain(self, domain: str) -> str: domain_report = self.client.get_object(f'/domains/{domain}') # DOMAIN domain_object = self.create_misp_object(domain_report) # WHOIS if domain_report.whois: whois_object = MISPObject('whois') whois_object.add_attribute('text', type='text', value=domain_report.whois) self.misp_event.add_object(**whois_object) # SIBLINGS AND SUBDOMAINS for relationship_name, misp_name in [('siblings', 'sibling-of'), ('subdomains', 'subdomain')]: rel_iterator = self.client.iterator(f'/domains/{domain_report.id}/{relationship_name}', limit=self.limit) for item in rel_iterator: attr = MISPAttribute() attr.from_dict(**dict(type='domain', value=item.id)) self.misp_event.add_attribute(**attr) domain_object.add_reference(attr.uuid, misp_name) # RESOLUTIONS resolutions_iterator = self.client.iterator(f'/domains/{domain_report.id}/resolutions', limit=self.limit) for resolution in resolutions_iterator: domain_object.add_attribute('ip', type='ip-dst', value=resolution.ip_address) # COMMUNICATING, DOWNLOADED AND REFERRER FILES for relationship_name, misp_name in [ ('communicating_files', 'communicates-with'), ('downloaded_files', 'downloaded-from'), ('referrer_files', 'referring') ]: files_iterator = self.client.iterator(f'/domains/{domain_report.id}/{relationship_name}', limit=self.limit) for file in files_iterator: file_object = self.create_misp_object(file) file_object.add_reference(domain_object.uuid, misp_name) self.misp_event.add_object(**file_object) # URLS urls_iterator = self.client.iterator(f'/domains/{domain_report.id}/urls', limit=self.limit) for url in urls_iterator: url_object = self.create_misp_object(url) url_object.add_reference(domain_object.uuid, 'hosted-in') self.misp_event.add_object(**url_object) self.misp_event.add_object(**domain_object) return domain_object.uuid def parse_hash(self, file_hash: str) -> str: file_report = self.client.get_object(f'/files/{file_hash}') file_object = self.create_misp_object(file_report) # ITW URLS urls_iterator = self.client.iterator(f'/files/{file_report.id}/itw_urls', limit=self.limit) for url in urls_iterator: url_object = self.create_misp_object(url) url_object.add_reference(file_object.uuid, 'downloaded') self.misp_event.add_object(**url_object) # COMMUNICATING, DOWNLOADED AND REFERRER FILES for relationship_name, misp_name in [ ('contacted_urls', 'communicates-with'), ('contacted_domains', 'communicates-with'), ('contacted_ips', 'communicates-with') ]: related_files_iterator = self.client.iterator(f'/files/{file_report.id}/{relationship_name}', limit=self.limit) for related_file in related_files_iterator: related_file_object = self.create_misp_object(related_file) related_file_object.add_reference(file_object.uuid, misp_name) self.misp_event.add_object(**related_file_object) self.misp_event.add_object(**file_object) return file_object.uuid def parse_ip_port(self, ipport: str) -> str: ip = ipport.split('|')[0] self.parse_ip(ip) def parse_ip(self, ip: str) -> str: ip_report = self.client.get_object(f'/ip_addresses/{ip}') # IP ip_object = self.create_misp_object(ip_report) # ASN asn_object = MISPObject('asn') asn_object.add_attribute('asn', type='AS', value=ip_report.asn) asn_object.add_attribute('subnet-announced', type='ip-src', value=ip_report.network) asn_object.add_attribute('country', type='text', value=ip_report.country) self.misp_event.add_object(**asn_object) # RESOLUTIONS resolutions_iterator = self.client.iterator(f'/ip_addresses/{ip_report.id}/resolutions', limit=self.limit) for resolution in resolutions_iterator: ip_object.add_attribute('domain', type='domain', value=resolution.host_name) # URLS urls_iterator = self.client.iterator(f'/ip_addresses/{ip_report.id}/urls', limit=self.limit) for url in urls_iterator: url_object = self.create_misp_object(url) url_object.add_reference(ip_object.uuid, 'hosted-in') self.misp_event.add_object(**url_object) self.misp_event.add_object(**ip_object) return ip_object.uuid def parse_url(self, url: str) -> str: url_id = vt.url_id(url) url_report = self.client.get_object(f'/urls/{url_id}') url_object = self.create_misp_object(url_report) # COMMUNICATING, DOWNLOADED AND REFERRER FILES for relationship_name, misp_name in [ ('communicating_files', 'communicates-with'), ('downloaded_files', 'downloaded-from'), ('referrer_files', 'referring') ]: files_iterator = self.client.iterator(f'/urls/{url_report.id}/{relationship_name}', limit=self.limit) for file in files_iterator: file_object = self.create_misp_object(file) file_object.add_reference(url_object.uuid, misp_name) self.misp_event.add_object(**file_object) self.misp_event.add_object(**url_object) return url_object.uuid def get_proxy_settings(config: dict) -> dict: """Returns proxy settings in the requests format. If no proxy settings are set, return None.""" proxies = None host = config.get('proxy_host') port = config.get('proxy_port') username = config.get('proxy_username') password = config.get('proxy_password') if host: if not port: misperrors['error'] = 'The virustotal_proxy_host config is set, ' \ 'please also set the virustotal_proxy_port.' raise KeyError parsed = urlparse(host) if 'http' in parsed.scheme: scheme = 'http' else: scheme = parsed.scheme netloc = parsed.netloc host = f'{netloc}:{port}' if username: if not password: misperrors['error'] = 'The virustotal_proxy_username config is set, ' \ 'please also set the virustotal_proxy_password.' raise KeyError auth = f'{username}:{password}' host = auth + '@' + host proxies = { 'http': f'{scheme}://{host}', 'https': f'{scheme}://{host}' } return proxies def parse_error(status_code: int) -> str: status_mapping = {204: 'VirusTotal request rate limit exceeded.', 400: 'Incorrect request, please check the arguments.', 403: 'You don\'t have enough privileges to make the request.'} if status_code in status_mapping: return status_mapping[status_code] return "VirusTotal may not be accessible." def dict_handler(request: dict): if not request.get('config') or not request['config'].get('apikey'): misperrors['error'] = 'A VirusTotal api key is required for this module.' return misperrors if not request.get('attribute') or not check_input_attribute(request['attribute']): return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} if request['attribute']['type'] not in mispattributes['input']: return {'error': 'Unsupported attribute type.'} event_limit = request['config'].get('event_limit') attribute = request['attribute'] proxy_settings = get_proxy_settings(request.get('config')) try: client = vt.Client(request['config']['apikey'], headers={ 'x-tool': 'MISPModuleVirusTotalExpansion', }, proxy=proxy_settings['http'] if proxy_settings else None) parser = VirusTotalParser(client, int(event_limit) if event_limit else None) parser.query_api(attribute) except vt.APIError as ex: misperrors['error'] = ex.message return misperrors return parser.get_result() def introspection(): return mispattributes def version(): moduleinfo['config'] = moduleconfig return moduleinfo