Merge pull request #426 from hildenjohannes/main

Recorded Future module: Add proxy support and User-Agent header

misp_modules/modules/expansion/recordedfuture.py

@@ -1,36 +1,89 @@
 import json
 import logging
 import requests
+from requests.exceptions import HTTPError, ProxyError,\
+    InvalidURL, ConnectTimeout, ConnectionError
 from . import check_input_attribute, checking_error, standard_error_message
-from urllib.parse import quote
+import platform
+import os
+from urllib.parse import quote, urlparse
 from pymisp import MISPAttribute, MISPEvent, MISPTag, MISPObject
 
-moduleinfo = {'version': '1.0', 'author': 'Recorded Future',
+moduleinfo = {
-              'description': 'Module to retrieve data from Recorded Future',
+    'version': '1.0.1',
-              'module-type': ['expansion', 'hover']}
+    'author': 'Recorded Future',
+    'description': 'Module to retrieve data from Recorded Future',
+    'module-type': ['expansion', 'hover']
+}
 
-moduleconfig = ['token']
+moduleconfig = ['token', 'proxy_host', 'proxy_port', 'proxy_username', 'proxy_password']
 
 misperrors = {'error': 'Error'}
 
-mispattributes = {'input': ['ip', 'ip-src', 'ip-dst', 'domain', 'hostname', 'md5', 'sha1', 'sha256',
+ATTRIBUTES = [
-                            'uri', 'url', 'vulnerability', 'weakness'],
+    'ip',
-                  'output': ['ip', 'ip-src', 'ip-dst', 'domain', 'hostname', 'md5', 'sha1', 'sha256',
+    'ip-src',
-                             'uri', 'url', 'vulnerability', 'weakness', 'email-src', 'text'],
+    'ip-dst',
-                  'format': 'misp_standard'}
+    'domain',
+    'hostname',
+    'md5',
+    'sha1',
+    'sha256',
+    'uri',
+    'url',
+    'vulnerability',
+    'weakness'
+]
+
+mispattributes = {
+    'input': ATTRIBUTES,
+    'output': ATTRIBUTES + ['email-src', 'text'],
+    'format': 'misp_standard'
+}
 
 LOGGER = logging.getLogger('recorded_future')
 LOGGER.setLevel(logging.INFO)
 
 
-def rf_lookup(api_token: str, category: str, ioc: str) -> requests.Response:
+class RequestHandler:
-    """Do a lookup call using Recorded Future's ConnectAPI."""
+    """A class for handling any outbound requests from this module."""
-    auth_header = {"X-RFToken": api_token}
+    def __init__(self):
-    parsed_ioc = quote(ioc, safe='')
+        self.session = requests.Session()
-    url = f'https://api.recordedfuture.com/v2/{category}/{parsed_ioc}?fields=risk%2CrelatedEntities'
+        self.app_id = f'{os.path.basename(__file__)}/{moduleinfo["version"]} ({platform.platform()}) ' \
-    response = requests.get(url, headers=auth_header)
+                      f'misp_enrichment/{moduleinfo["version"]} python-requests/{requests.__version__}'
-    response.raise_for_status()
+        self.proxies = None
-    return response
+        self.rf_token = None
+
+    def get(self, url: str, headers: dict = None) -> requests.Response:
+        """General get method with proxy error handling."""
+        try:
+            timeout = 7 if self.proxies else None
+            response = self.session.get(url, headers=headers, proxies=self.proxies, timeout=timeout)
+            response.raise_for_status()
+            return response
+        except (ConnectTimeout, ProxyError, InvalidURL) as error:
+            msg = f'Error connecting with proxy, please check the Recorded Future app proxy settings.'
+            LOGGER.error(f'{msg} Error: {error}')
+            misperrors['error'] = msg
+            raise
+
+    def rf_lookup(self, category: str, ioc: str) -> requests.Response:
+        """Do a lookup call using Recorded Future's ConnectAPI."""
+        parsed_ioc = quote(ioc, safe='')
+        url = f'https://api.recordedfuture.com/v2/{category}/{parsed_ioc}?fields=risk%2CrelatedEntities'
+        headers = {'X-RFToken': self.rf_token,
+                   'User-Agent': self.app_id}
+        try:
+            response = self.get(url, headers)
+        except HTTPError as error:
+            msg = f'Error when requesting data from Recorded Future. {error.response}: {error.response.reason}'
+            LOGGER.error(msg)
+            misperrors['error'] = msg
+            raise
+        return response
+
+
+GLOBAL_REQUEST_HANDLER = RequestHandler()
 
 
 class GalaxyFinder:
@@ -38,46 +91,45 @@ class GalaxyFinder:
     def __init__(self):
         self.session = requests.Session()
         self.sources = {
-            'RelatedThreatActor': ['https://raw.githubusercontent.com/MISP/misp-galaxy/'
+            'RelatedThreatActor': [
-                                   'main/clusters/threat-actor.json'],
+                'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/threat-actor.json'
-            'RelatedMalware': ['https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/banker.json',
+            ],
-                               'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/botnet.json',
+            'RelatedMalware': [
-                               'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/exploit-kit.json',
+                'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/banker.json',
-                               'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/rat.json',
+                'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/botnet.json',
-                               'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/ransomware.json',
+                'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/exploit-kit.json',
-                               'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/malpedia.json']
+                'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/rat.json',
+                'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/ransomware.json',
+                'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/malpedia.json'
+            ]
         }
         self.galaxy_clusters = {}
 
-    def pull_galaxy_cluster(self, related_type: str):
+    def pull_galaxy_cluster(self, related_type: str) -> None:
         """Fetches galaxy clusters for the related_type from the remote json files specified as self.sources."""
         # Only fetch clusters if not fetched previously
         if not self.galaxy_clusters.get(related_type):
             for source in self.sources.get(related_type):
-                response = self.session.get(source)
+                try:
-                if response.ok:
+                    response = GLOBAL_REQUEST_HANDLER.get(source)
                     name = source.split('/')[-1].split('.')[0]
                     self.galaxy_clusters[related_type] = {name: response.json()}
-                else:
+                except ConnectionError as error:
-                    LOGGER.info(f'pull_galaxy_cluster failed for source: {source},'
+                    LOGGER.warning(f'pull_galaxy_cluster failed for source: {source}, with error: {error}.')
-                                f' got response: {response}, {response.reason}.')
 
     def find_galaxy_match(self, indicator: str, related_type: str) -> str:
         """Searches the clusters of the related_type for a match with the indicator.
            :returns the first matching galaxy string or an empty string if no galaxy match is found.
         """
         self.pull_galaxy_cluster(related_type)
-        try:
+        for cluster_name, cluster in self.galaxy_clusters.get(related_type, {}).items():
-            for cluster_name, cluster in self.galaxy_clusters[related_type].items():
+            for value in cluster['values']:
-                for value in cluster['values']:
+                try:
-                    try:
+                    if indicator in value['meta']['synonyms'] or indicator in value['value']:
-                        if indicator in value['meta']['synonyms'] or indicator in value['value']:
+                        value = value['value']
-                            value = value['value']
+                        return f'misp-galaxy:{cluster_name}="{value}"'
-                            return f'misp-galaxy:{cluster_name}="{value}"'
+                except KeyError:
-                    except KeyError:
+                    pass
-                        pass
-        except KeyError:
-            pass
         return ''
 
 
@@ -113,57 +165,70 @@ class RFEnricher:
     """Class for enriching an attribute with data from Recorded Future.
        The enrichment data is returned as a custom MISP object.
     """
-    def __init__(self, api_token: str, attribute_props: dict):
+    def __init__(self, attribute_props: dict):
-        self.api_token = api_token
         self.event = MISPEvent()
         self.enrichment_object = MISPObject('Recorded Future Enrichment')
-        self.enrichment_object.from_dict(**{'meta-category': 'misc',
+        description = (
-                                            'description': 'An object containing the enriched attribute and related '
+            'An object containing the enriched attribute and '
-                                                           'entities from Recorded Future.',
+            'related entities from Recorded Future.'
-                                            'distribution': 0})
+        )
+        self.enrichment_object.from_dict(**{
+            'meta-category': 'misc',
+            'description': description,
+            'distribution': 0
+        })
 
         # Create a copy of enriched attribute to add tags to
         temp_attr = MISPAttribute()
         temp_attr.from_dict(**attribute_props)
         self.enriched_attribute = MISPAttribute()
-        self.enriched_attribute.from_dict(**{'value': temp_attr.value, 'type': temp_attr.type, 'distribution': 0})
+        self.enriched_attribute.from_dict(**{
+            'value': temp_attr.value,
+            'type': temp_attr.type,
+            'distribution': 0
+        })
 
         self.related_attributes = []
         self.color_picker = RFColors()
         self.galaxy_finder = GalaxyFinder()
 
         # Mapping from MISP-type to RF-type
-        self.type_to_rf_category = {'ip': 'ip', 'ip-src': 'ip', 'ip-dst': 'ip',
+        self.type_to_rf_category = {
-                                    'domain': 'domain', 'hostname': 'domain',
+            'ip': 'ip',
-                                    'md5': 'hash', 'sha1': 'hash', 'sha256': 'hash',
+            'ip-src': 'ip',
-                                    'uri': 'url', 'url': 'url',
+            'ip-dst': 'ip',
-                                    'vulnerability': 'vulnerability', 'weakness': 'vulnerability'}
+            'domain': 'domain',
+            'hostname': 'domain',
+            'md5': 'hash',
+            'sha1': 'hash',
+            'sha256': 'hash',
+            'uri': 'url',
+            'url': 'url',
+            'vulnerability': 'vulnerability',
+            'weakness': 'vulnerability'
+        }
 
         # Related entities from RF portrayed as related attributes in MISP
-        self.related_attribute_types = ['RelatedIpAddress', 'RelatedInternetDomainName', 'RelatedHash',
+        self.related_attribute_types = [
-                                        'RelatedEmailAddress', 'RelatedCyberVulnerability']
+            'RelatedIpAddress', 'RelatedInternetDomainName', 'RelatedHash',
+            'RelatedEmailAddress', 'RelatedCyberVulnerability'
+        ]
         # Related entities from RF portrayed as tags in MISP
         self.galaxy_tag_types = ['RelatedMalware', 'RelatedThreatActor']
 
-    def enrich(self):
+    def enrich(self) -> None:
         """Run the enrichment."""
         category = self.type_to_rf_category.get(self.enriched_attribute.type)
-
+        json_response = GLOBAL_REQUEST_HANDLER.rf_lookup(category, self.enriched_attribute.value)
-        try:
+        response = json.loads(json_response.content)
-            response = rf_lookup(self.api_token, category, self.enriched_attribute.value)
-            json_response = json.loads(response.content)
-        except requests.HTTPError as error:
-            misperrors['error'] = f'Error when requesting data from Recorded Future. ' \
-                                  f'{error.response} : {error.response.reason}'
-            raise error
 
         try:
             # Add risk score and risk rules as tags to the enriched attribute
-            risk_score = json_response['data']['risk']['score']
+            risk_score = response['data']['risk']['score']
             hex_color = self.color_picker.riskscore_color(risk_score)
             tag_name = f'recorded-future:risk-score="{risk_score}"'
             self.add_tag(tag_name, hex_color)
-            for evidence in json_response['data']['risk']['evidenceDetails']:
+            for evidence in response['data']['risk']['evidenceDetails']:
                 risk_rule = evidence['rule']
                 criticality = evidence['criticality']
                 hex_color = self.color_picker.riskrule_color(criticality)
@@ -171,7 +236,7 @@ class RFEnricher:
                 self.add_tag(tag_name, hex_color)
 
             # Retrieve related entities
-            for related_entity in json_response['data']['relatedEntities']:
+            for related_entity in response['data']['relatedEntities']:
                 related_type = related_entity['type']
                 if related_type in self.related_attribute_types:
                     # Related entities returned as additional attributes
@@ -191,9 +256,9 @@ class RFEnricher:
                                 galaxy_tags.append(galaxy)
                     for galaxy in galaxy_tags:
                         self.add_tag(galaxy)
-        except KeyError as error:
+        except KeyError:
             misperrors['error'] = 'Unexpected format in Recorded Future api response.'
-            raise error
+            raise
 
     def add_related_attribute(self, indicator: str, related_type: str) -> None:
         """Helper method for adding an indicator to the related attribute list."""
@@ -247,14 +312,54 @@ class RFEnricher:
         return {'results': result}
 
 
+def get_proxy_settings(config: dict) -> dict:
+    """Returns proxy settings in the requests format.
+       If no proxy settings are set, return None."""
+    proxies = None
+    host = config.get('proxy_host')
+    port = config.get('proxy_port')
+    username = config.get('proxy_username')
+    password = config.get('proxy_password')
+
+    if host:
+        if not port:
+            misperrors['error'] = 'The recordedfuture_proxy_host config is set, ' \
+                                  'please also set the recordedfuture_proxy_port.'
+            raise KeyError
+        parsed = urlparse(host)
+        if 'http' in parsed.scheme:
+            scheme = 'http'
+        else:
+            scheme = parsed.scheme
+        netloc = parsed.netloc
+        host = f'{netloc}:{port}'
+
+        if username:
+            if not password:
+                misperrors['error'] = 'The recordedfuture_proxy_username config is set, ' \
+                                      'please also set the recordedfuture_proxy_password.'
+                raise KeyError
+            auth = f'{username}:{password}'
+            host = auth + '@' + host
+
+        proxies = {
+            'http': f'{scheme}://{host}',
+            'https': f'{scheme}://{host}'
+        }
+
+    LOGGER.info(f'Proxy settings: {proxies}')
+    return proxies
+
+
 def handler(q=False):
     """Handle enrichment."""
     if q is False:
         return False
     request = json.loads(q)
 
-    if request.get('config') and request['config'].get('token'):
+    config = request.get('config')
-        token = request['config'].get('token')
+    if config and config.get('token'):
+        GLOBAL_REQUEST_HANDLER.rf_token = config.get('token')
     else:
         misperrors['error'] = 'Missing Recorded Future token.'
         return misperrors
@@ -263,11 +368,17 @@ def handler(q=False):
     if request['attribute']['type'] not in mispattributes['input']:
         return {'error': 'Unsupported attribute type.'}
 
+    try:
+        GLOBAL_REQUEST_HANDLER.proxies = get_proxy_settings(config)
+    except KeyError:
+        return misperrors
+
     input_attribute = request.get('attribute')
-    rf_enricher = RFEnricher(token, input_attribute)
+    rf_enricher = RFEnricher(input_attribute)
+
     try:
         rf_enricher.enrich()
-    except (requests.HTTPError, KeyError):
+    except (HTTPError, ConnectTimeout, ProxyError, InvalidURL, KeyError):
         return misperrors
 
     return rf_enricher.get_results()