feature: add qintel qsentry expansion module

misp_modules/lib/__init__.py

@@ -1,3 +1,3 @@
 from .vt_graph_parser import *  # noqa
 
-all = ['joe_parser', 'lastline_api', 'cof2misp']
+all = ['joe_parser', 'lastline_api', 'cof2misp', 'qintel_helper']

misp_modules/lib/qintel_helper.py

@@ -0,0 +1,263 @@
+# Copyright (c) 2009-2021 Qintel, LLC
+# Licensed under Apache 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt)
+
+from urllib.request import Request, urlopen
+from urllib.parse import urlencode
+from urllib.error import HTTPError
+from time import sleep
+from json import loads
+import os
+from copy import deepcopy
+from datetime import datetime, timedelta
+from gzip import GzipFile
+
+VERSION = '1.0.1'
+USER_AGENT = 'integrations-helper'
+MAX_RETRY_ATTEMPTS = 5
+
+DEFAULT_HEADERS = {
+    'User-Agent': f'{USER_AGENT}/{VERSION}'
+}
+
+REMOTE_MAP = {
+    'pmi': 'https://api.pmi.qintel.com',
+    'qwatch': 'https://api.qwatch.qintel.com',
+    'qauth': 'https://api.qauth.qintel.com',
+    'qsentry_feed': 'https://qsentry.qintel.com',
+    'qsentry': 'https://api.qsentry.qintel.com'
+}
+
+ENDPOINT_MAP = {
+    'pmi': {
+        'ping': '/users/me',
+        'cve': 'cves'
+    },
+    'qsentry_feed': {
+        'anon': '/files/anonymization',
+        'mal_hosting': '/files/malicious_hosting'
+    },
+    'qsentry': {},
+    'qwatch': {
+        'ping': '/users/me',
+        'exposures': 'exposures'
+    },
+    'qauth': {}
+}
+
+
+def _get_request_wait_time(attempts):
+    """ Use Fibonacci numbers for determining the time to wait when rate limits
+    have been encountered.
+    """
+
+    n = attempts + 3
+    a, b = 1, 0
+    for _ in range(n):
+        a, b = a + b, a
+
+    return a
+
+
+def _search(**kwargs):
+    remote = kwargs.get('remote')
+    max_retries = int(kwargs.get('max_retries', MAX_RETRY_ATTEMPTS))
+    params = kwargs.get('params', {})
+    headers = _set_headers(**kwargs)
+
+    logger = kwargs.get('logger')
+
+    params = urlencode(params)
+    url = remote + "?" + params
+    req = Request(url, headers=headers)
+
+    request_attempts = 1
+    while request_attempts < max_retries:
+        try:
+            return urlopen(req)
+
+        except HTTPError as e:
+            response = e
+
+        except Exception as e:
+            raise Exception('API connection error') from e
+
+        if response.code not in [429, 504]:
+            raise Exception(f'API connection error: {response}')
+
+        if request_attempts < max_retries:
+            wait_time = _get_request_wait_time(request_attempts)
+
+            if response.code == 429:
+                msg = 'rate limit reached on attempt {request_attempts}, ' \
+                      'waiting {wait_time} seconds'
+
+                if logger:
+                    logger(msg)
+
+            else:
+                msg = f'connection timed out, retrying in {wait_time} seconds'
+                if logger:
+                    logger(msg)
+
+            sleep(wait_time)
+
+        else:
+            raise Exception('Max API retries exceeded')
+
+        request_attempts += 1
+
+
+def _set_headers(**kwargs):
+    headers = deepcopy(DEFAULT_HEADERS)
+
+    if kwargs.get('user_agent'):
+        headers['User-Agent'] = \
+            f"{kwargs['user_agent']}/{USER_AGENT}/{VERSION}"
+
+    # TODO: deprecate
+    if kwargs.get('client_id') or kwargs.get('client_secret'):
+        try:
+            headers['Cf-Access-Client-Id'] = kwargs['client_id']
+            headers['Cf-Access-Client-Secret'] = kwargs['client_secret']
+        except KeyError:
+            raise Exception('missing client_id or client_secret')
+
+    if kwargs.get('token'):
+        headers['x-api-key'] = kwargs['token']
+
+    return headers
+
+
+def _set_remote(product, query_type, **kwargs):
+    remote = kwargs.get('remote')
+    endpoint = kwargs.get('endpoint', ENDPOINT_MAP[product].get(query_type))
+
+    if not remote:
+        remote = REMOTE_MAP[product]
+
+    if not endpoint:
+        raise Exception('invalid search type')
+
+    remote = remote.rstrip('/')
+    endpoint = endpoint.lstrip('/')
+
+    return f'{remote}/{endpoint}'
+
+
+def _process_qsentry(resp):
+    if resp.getheader('Content-Encoding', '') == 'gzip':
+        with GzipFile(fileobj=resp) as file:
+            for line in file.readlines():
+                yield loads(line)
+
+
+def search_pmi(search_term, query_type, **kwargs):
+    """
+    Search PMI
+
+    :param str search_term: Search term
+    :param str query_type: Query type [cve|ping]
+    :param dict kwargs: extra client args [remote|token|params]
+    :return: API JSON response object
+    :rtype: dict
+    """
+
+    kwargs['remote'] = _set_remote('pmi', query_type, **kwargs)
+    kwargs['token'] = kwargs.get('token', os.getenv('PMI_TOKEN'))
+
+    params = kwargs.get('params', {})
+    params.update({'identifier': search_term})
+    kwargs['params'] = params
+
+    return loads(_search(**kwargs).read())
+
+
+def search_qwatch(search_term, search_type, query_type, **kwargs):
+    """
+    Search QWatch for exposed credentials
+
+    :param str search_term: Search term
+    :param str search_type: Search term type [domain|email]
+    :param str query_type: Query type [exposures]
+    :param dict kwargs: extra client args [remote|token|params]
+    :return: API JSON response object
+    :rtype: dict
+    """
+
+    kwargs['remote'] = _set_remote('qwatch', query_type, **kwargs)
+    kwargs['token'] = kwargs.get('token', os.getenv('QWATCH_TOKEN'))
+
+    params = kwargs.get('params', {})
+    if search_type:
+        params.update({search_type: search_term})
+    kwargs['params'] = params
+
+    return loads(_search(**kwargs).read())
+
+
+def search_qauth(search_term, **kwargs):
+    """
+    Search QAuth
+
+    :param str search_term: Search term
+    :param dict kwargs: extra client args [remote|token|params]
+    :return: API JSON response object
+    :rtype: dict
+    """
+
+    if not kwargs.get('endpoint'):
+        kwargs['endpoint'] = '/'
+
+    kwargs['remote'] = _set_remote('qauth', None, **kwargs)
+    kwargs['token'] = kwargs.get('token', os.getenv('QAUTH_TOKEN'))
+
+    params = kwargs.get('params', {})
+    params.update({'q': search_term})
+    kwargs['params'] = params
+
+    return loads(_search(**kwargs).read())
+
+
+def search_qsentry(search_term, **kwargs):
+    """
+    Search QSentry
+
+    :param str search_term: Search term
+    :param dict kwargs: extra client args [remote|token|params]
+    :return: API JSON response object
+    :rtype: dict
+    """
+
+    if not kwargs.get('endpoint'):
+        kwargs['endpoint'] = '/'
+
+    kwargs['remote'] = _set_remote('qsentry', None, **kwargs)
+    kwargs['token'] = kwargs.get('token', os.getenv('QSENTRY_TOKEN'))
+
+    params = kwargs.get('params', {})
+    params.update({'q': search_term})
+    kwargs['params'] = params
+
+    return loads(_search(**kwargs).read())
+
+
+def qsentry_feed(query_type='anon', feed_date=datetime.today(), **kwargs):
+    """
+    Fetch the most recent QSentry Feed
+
+    :param str query_type: Feed type [anon|mal_hosting]
+    :param dict kwargs: extra client args [remote|token|params]
+    :param datetime feed_date: feed date to fetch
+    :return: API JSON response object
+    :rtype: Iterator[dict]
+    """
+
+    remote = _set_remote('qsentry_feed', query_type, **kwargs)
+    kwargs['token'] = kwargs.get('token', os.getenv('QSENTRY_TOKEN'))
+
+    feed_date = (feed_date - timedelta(days=1)).strftime('%Y%m%d')
+    kwargs['remote'] = f'{remote}/{feed_date}'
+
+    resp = _search(**kwargs)
+    for r in _process_qsentry(resp):
+        yield r

misp_modules/modules/expansion/__init__.py

@@ -17,7 +17,8 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c
            'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid',
            'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', 'malwarebazaar',
            'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion', 'censys_enrich',
-           'trustar_enrich', 'recordedfuture', 'html_to_markdown', 'socialscan', 'passive-ssh']
+           'trustar_enrich', 'recordedfuture', 'html_to_markdown', 'socialscan', 'passive-ssh',
+           'qintel_qsentry']
 
 
 minimum_required_fields = ('type', 'uuid', 'value')

misp_modules/modules/expansion/qintel_qsentry.py

@@ -0,0 +1,221 @@
+import logging
+import json
+
+from pymisp import MISPAttribute, MISPEvent, MISPTag,  MISPObject
+from . import check_input_attribute, checking_error, standard_error_message
+
+from qintel_helper import search_qsentry
+
+logger = logging.getLogger('qintel_qsentry')
+logger.setLevel(logging.DEBUG)
+
+moduleinfo = {
+    'version': '1.0',
+    'author': 'Qintel, LLC',
+    'description': 'Query Qintel QSentry for ip intelligence',
+    'module-type': ['hover', 'expansion']
+}
+
+moduleconfig = ['token', 'remote']
+
+misperrors = {'error': 'Error'}
+
+mispattributes = {
+    'input': ['ip-src', 'ip-dst'],
+    'output': ['ip-src', 'ip-dst', 'AS', 'freetext'],
+    'format': 'misp_standard'
+}
+
+TAG_COLOR = {
+    'benign': '#27ae60',
+    'suspicious': '#e6a902',
+    'malicious': '#c0392b'
+}
+
+CLIENT_HEADERS = {
+        'User-Agent': f"MISP/{moduleinfo['version']}",
+}
+
+
+def _return_error(message):
+    misperrors['error'] = message
+    return misperrors
+
+
+def _make_tags(enriched_attr, result):
+
+    for tag in result['tags']:
+        color = TAG_COLOR['suspicious']
+        if tag == 'criminal':
+            color = TAG_COLOR['malicious']
+
+        t = MISPTag()
+        t.from_dict(**{
+            'name': f'qintel:tag="{tag}"',
+            'colour': color
+        })
+        enriched_attr.add_tag(**t)
+
+    return enriched_attr
+
+
+def _make_enriched_attr(event, result, orig_attr):
+
+    enriched_object = MISPObject('Qintel Threat Enrichment')
+    enriched_object.add_reference(orig_attr.uuid, 'related-to')
+
+    enriched_attr = MISPAttribute()
+    enriched_attr.from_dict(**{
+        'value': orig_attr.value,
+        'type': orig_attr.type,
+        'distribution': 0,
+        'object_relation': 'enriched-attr',
+        'to_ids': orig_attr.to_ids
+    })
+
+    enriched_attr = _make_tags(enriched_attr, result)
+    enriched_object.add_attribute(**enriched_attr)
+
+    comment_attr = MISPAttribute()
+    comment_attr.from_dict(**{
+        'value': '\n'.join(result.get('descriptions', [])),
+        'type': 'text',
+        'object_relation': 'descriptions',
+        'distribution': 0
+    })
+    enriched_object.add_attribute(**comment_attr)
+
+    last_seen = MISPAttribute()
+    last_seen.from_dict(**{
+        'value': result.get('last_seen'),
+        'type': 'datetime',
+        'object_relation': 'last-seen',
+        'distribution': 0
+    })
+    enriched_object.add_attribute(**last_seen)
+
+    event.add_attribute(**orig_attr)
+    event.add_object(**enriched_object)
+
+    return event
+
+
+def _make_asn_attr(event, result, orig_attr):
+
+    asn_object = MISPObject('asn')
+    asn_object.add_reference(orig_attr.uuid, 'related-to')
+
+    asn_attr = MISPAttribute()
+    asn_attr.from_dict(**{
+        'type': 'AS',
+        'value': result.get('asn'),
+        'object_relation': 'asn',
+        'distribution': 0
+    })
+    asn_object.add_attribute(**asn_attr)
+
+    org_attr = MISPAttribute()
+    org_attr.from_dict(**{
+        'type': 'text',
+        'value': result.get('asn_name', 'unknown').title(),
+        'object_relation': 'description',
+        'distribution': 0
+    })
+    asn_object.add_attribute(**org_attr)
+
+    event.add_object(**asn_object)
+
+    return event
+
+
+def _format_hover(event, result):
+
+    enriched_object = event.get_objects_by_name('Qintel Threat Enrichment')[0]
+
+    tags = ', '.join(result.get('tags'))
+    enriched_object.add_attribute('Tags', type='text', value=tags)
+
+    return event
+
+
+def _format_result(attribute, result):
+
+    event = MISPEvent()
+
+    orig_attr = MISPAttribute()
+    orig_attr.from_dict(**attribute)
+
+    event = _make_enriched_attr(event, result, orig_attr)
+    event = _make_asn_attr(event, result, orig_attr)
+
+    return event
+
+
+def _check_config(config):
+    if not config:
+        return False
+
+    if not isinstance(config, dict):
+        return False
+
+    if config.get('token', '') == '':
+        return False
+
+    return True
+
+
+def _check_request(request):
+    if not request.get('attribute'):
+        return f'{standard_error_message}, {checking_error}'
+
+    check_reqs = ('type', 'value')
+    if not check_input_attribute(request['attribute'],
+                                 requirements=check_reqs):
+        return f'{standard_error_message}, {checking_error}'
+
+    if request['attribute']['type'] not in mispattributes['input']:
+        return 'Unsupported attribute type'
+
+
+def handler(q=False):
+    if not q:
+        return False
+
+    request = json.loads(q)
+    config = request.get('config')
+
+    if not _check_config(config):
+        return _return_error('Missing Qintel token')
+
+    check_request_error = _check_request(request)
+    if check_request_error:
+        return _return_error(check_request_error)
+
+    search_args = {
+        'token': config['token'],
+        'remote': config.get('remote')
+    }
+
+    try:
+        result = search_qsentry(request['attribute']['value'], **search_args)
+    except Exception as e:
+        return _return_error(str(e))
+
+    event = _format_result(request['attribute'], result)
+    if not request.get('event_id'):
+        event = _format_hover(event, result)
+
+    event = json.loads(event.to_json())
+
+    ret_result = {key: event[key] for key in ('Attribute', 'Object') if key
+                  in event}
+    return {'results': ret_result}
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo