From dc0660acd0df74e874be4cf81c9b5d4381fb594f Mon Sep 17 00:00:00 2001 From: Calvin Krzywiec Date: Mon, 22 Nov 2021 15:46:46 -0500 Subject: [PATCH] feature: add qintel qsentry expansion module --- misp_modules/lib/__init__.py | 2 +- misp_modules/lib/qintel_helper.py | 263 ++++++++++++++++++ misp_modules/modules/expansion/__init__.py | 3 +- .../modules/expansion/qintel_qsentry.py | 221 +++++++++++++++ 4 files changed, 487 insertions(+), 2 deletions(-) create mode 100644 misp_modules/lib/qintel_helper.py create mode 100644 misp_modules/modules/expansion/qintel_qsentry.py diff --git a/misp_modules/lib/__init__.py b/misp_modules/lib/__init__.py index d92e9890..2939e75b 100644 --- a/misp_modules/lib/__init__.py +++ b/misp_modules/lib/__init__.py @@ -1,3 +1,3 @@ from .vt_graph_parser import * # noqa -all = ['joe_parser', 'lastline_api', 'cof2misp'] +all = ['joe_parser', 'lastline_api', 'cof2misp', 'qintel_helper'] diff --git a/misp_modules/lib/qintel_helper.py b/misp_modules/lib/qintel_helper.py new file mode 100644 index 00000000..47106f78 --- /dev/null +++ b/misp_modules/lib/qintel_helper.py @@ -0,0 +1,263 @@ +# Copyright (c) 2009-2021 Qintel, LLC +# Licensed under Apache 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) + +from urllib.request import Request, urlopen +from urllib.parse import urlencode +from urllib.error import HTTPError +from time import sleep +from json import loads +import os +from copy import deepcopy +from datetime import datetime, timedelta +from gzip import GzipFile + +VERSION = '1.0.1' +USER_AGENT = 'integrations-helper' +MAX_RETRY_ATTEMPTS = 5 + +DEFAULT_HEADERS = { + 'User-Agent': f'{USER_AGENT}/{VERSION}' +} + +REMOTE_MAP = { + 'pmi': 'https://api.pmi.qintel.com', + 'qwatch': 'https://api.qwatch.qintel.com', + 'qauth': 'https://api.qauth.qintel.com', + 'qsentry_feed': 'https://qsentry.qintel.com', + 'qsentry': 'https://api.qsentry.qintel.com' +} + +ENDPOINT_MAP = { + 'pmi': { + 'ping': '/users/me', + 'cve': 'cves' + }, + 'qsentry_feed': { + 'anon': '/files/anonymization', + 'mal_hosting': '/files/malicious_hosting' + }, + 'qsentry': {}, + 'qwatch': { + 'ping': '/users/me', + 'exposures': 'exposures' + }, + 'qauth': {} +} + + +def _get_request_wait_time(attempts): + """ Use Fibonacci numbers for determining the time to wait when rate limits + have been encountered. + """ + + n = attempts + 3 + a, b = 1, 0 + for _ in range(n): + a, b = a + b, a + + return a + + +def _search(**kwargs): + remote = kwargs.get('remote') + max_retries = int(kwargs.get('max_retries', MAX_RETRY_ATTEMPTS)) + params = kwargs.get('params', {}) + headers = _set_headers(**kwargs) + + logger = kwargs.get('logger') + + params = urlencode(params) + url = remote + "?" + params + req = Request(url, headers=headers) + + request_attempts = 1 + while request_attempts < max_retries: + try: + return urlopen(req) + + except HTTPError as e: + response = e + + except Exception as e: + raise Exception('API connection error') from e + + if response.code not in [429, 504]: + raise Exception(f'API connection error: {response}') + + if request_attempts < max_retries: + wait_time = _get_request_wait_time(request_attempts) + + if response.code == 429: + msg = 'rate limit reached on attempt {request_attempts}, ' \ + 'waiting {wait_time} seconds' + + if logger: + logger(msg) + + else: + msg = f'connection timed out, retrying in {wait_time} seconds' + if logger: + logger(msg) + + sleep(wait_time) + + else: + raise Exception('Max API retries exceeded') + + request_attempts += 1 + + +def _set_headers(**kwargs): + headers = deepcopy(DEFAULT_HEADERS) + + if kwargs.get('user_agent'): + headers['User-Agent'] = \ + f"{kwargs['user_agent']}/{USER_AGENT}/{VERSION}" + + # TODO: deprecate + if kwargs.get('client_id') or kwargs.get('client_secret'): + try: + headers['Cf-Access-Client-Id'] = kwargs['client_id'] + headers['Cf-Access-Client-Secret'] = kwargs['client_secret'] + except KeyError: + raise Exception('missing client_id or client_secret') + + if kwargs.get('token'): + headers['x-api-key'] = kwargs['token'] + + return headers + + +def _set_remote(product, query_type, **kwargs): + remote = kwargs.get('remote') + endpoint = kwargs.get('endpoint', ENDPOINT_MAP[product].get(query_type)) + + if not remote: + remote = REMOTE_MAP[product] + + if not endpoint: + raise Exception('invalid search type') + + remote = remote.rstrip('/') + endpoint = endpoint.lstrip('/') + + return f'{remote}/{endpoint}' + + +def _process_qsentry(resp): + if resp.getheader('Content-Encoding', '') == 'gzip': + with GzipFile(fileobj=resp) as file: + for line in file.readlines(): + yield loads(line) + + +def search_pmi(search_term, query_type, **kwargs): + """ + Search PMI + + :param str search_term: Search term + :param str query_type: Query type [cve|ping] + :param dict kwargs: extra client args [remote|token|params] + :return: API JSON response object + :rtype: dict + """ + + kwargs['remote'] = _set_remote('pmi', query_type, **kwargs) + kwargs['token'] = kwargs.get('token', os.getenv('PMI_TOKEN')) + + params = kwargs.get('params', {}) + params.update({'identifier': search_term}) + kwargs['params'] = params + + return loads(_search(**kwargs).read()) + + +def search_qwatch(search_term, search_type, query_type, **kwargs): + """ + Search QWatch for exposed credentials + + :param str search_term: Search term + :param str search_type: Search term type [domain|email] + :param str query_type: Query type [exposures] + :param dict kwargs: extra client args [remote|token|params] + :return: API JSON response object + :rtype: dict + """ + + kwargs['remote'] = _set_remote('qwatch', query_type, **kwargs) + kwargs['token'] = kwargs.get('token', os.getenv('QWATCH_TOKEN')) + + params = kwargs.get('params', {}) + if search_type: + params.update({search_type: search_term}) + kwargs['params'] = params + + return loads(_search(**kwargs).read()) + + +def search_qauth(search_term, **kwargs): + """ + Search QAuth + + :param str search_term: Search term + :param dict kwargs: extra client args [remote|token|params] + :return: API JSON response object + :rtype: dict + """ + + if not kwargs.get('endpoint'): + kwargs['endpoint'] = '/' + + kwargs['remote'] = _set_remote('qauth', None, **kwargs) + kwargs['token'] = kwargs.get('token', os.getenv('QAUTH_TOKEN')) + + params = kwargs.get('params', {}) + params.update({'q': search_term}) + kwargs['params'] = params + + return loads(_search(**kwargs).read()) + + +def search_qsentry(search_term, **kwargs): + """ + Search QSentry + + :param str search_term: Search term + :param dict kwargs: extra client args [remote|token|params] + :return: API JSON response object + :rtype: dict + """ + + if not kwargs.get('endpoint'): + kwargs['endpoint'] = '/' + + kwargs['remote'] = _set_remote('qsentry', None, **kwargs) + kwargs['token'] = kwargs.get('token', os.getenv('QSENTRY_TOKEN')) + + params = kwargs.get('params', {}) + params.update({'q': search_term}) + kwargs['params'] = params + + return loads(_search(**kwargs).read()) + + +def qsentry_feed(query_type='anon', feed_date=datetime.today(), **kwargs): + """ + Fetch the most recent QSentry Feed + + :param str query_type: Feed type [anon|mal_hosting] + :param dict kwargs: extra client args [remote|token|params] + :param datetime feed_date: feed date to fetch + :return: API JSON response object + :rtype: Iterator[dict] + """ + + remote = _set_remote('qsentry_feed', query_type, **kwargs) + kwargs['token'] = kwargs.get('token', os.getenv('QSENTRY_TOKEN')) + + feed_date = (feed_date - timedelta(days=1)).strftime('%Y%m%d') + kwargs['remote'] = f'{remote}/{feed_date}' + + resp = _search(**kwargs) + for r in _process_qsentry(resp): + yield r diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index a817c2af..1666ffea 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -17,7 +17,8 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid', 'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', 'malwarebazaar', 'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion', 'censys_enrich', - 'trustar_enrich', 'recordedfuture', 'html_to_markdown', 'socialscan', 'passive-ssh'] + 'trustar_enrich', 'recordedfuture', 'html_to_markdown', 'socialscan', 'passive-ssh', + 'qintel_qsentry'] minimum_required_fields = ('type', 'uuid', 'value') diff --git a/misp_modules/modules/expansion/qintel_qsentry.py b/misp_modules/modules/expansion/qintel_qsentry.py new file mode 100644 index 00000000..6733b93f --- /dev/null +++ b/misp_modules/modules/expansion/qintel_qsentry.py @@ -0,0 +1,221 @@ +import logging +import json + +from pymisp import MISPAttribute, MISPEvent, MISPTag, MISPObject +from . import check_input_attribute, checking_error, standard_error_message + +from qintel_helper import search_qsentry + +logger = logging.getLogger('qintel_qsentry') +logger.setLevel(logging.DEBUG) + +moduleinfo = { + 'version': '1.0', + 'author': 'Qintel, LLC', + 'description': 'Query Qintel QSentry for ip intelligence', + 'module-type': ['hover', 'expansion'] +} + +moduleconfig = ['token', 'remote'] + +misperrors = {'error': 'Error'} + +mispattributes = { + 'input': ['ip-src', 'ip-dst'], + 'output': ['ip-src', 'ip-dst', 'AS', 'freetext'], + 'format': 'misp_standard' +} + +TAG_COLOR = { + 'benign': '#27ae60', + 'suspicious': '#e6a902', + 'malicious': '#c0392b' +} + +CLIENT_HEADERS = { + 'User-Agent': f"MISP/{moduleinfo['version']}", +} + + +def _return_error(message): + misperrors['error'] = message + return misperrors + + +def _make_tags(enriched_attr, result): + + for tag in result['tags']: + color = TAG_COLOR['suspicious'] + if tag == 'criminal': + color = TAG_COLOR['malicious'] + + t = MISPTag() + t.from_dict(**{ + 'name': f'qintel:tag="{tag}"', + 'colour': color + }) + enriched_attr.add_tag(**t) + + return enriched_attr + + +def _make_enriched_attr(event, result, orig_attr): + + enriched_object = MISPObject('Qintel Threat Enrichment') + enriched_object.add_reference(orig_attr.uuid, 'related-to') + + enriched_attr = MISPAttribute() + enriched_attr.from_dict(**{ + 'value': orig_attr.value, + 'type': orig_attr.type, + 'distribution': 0, + 'object_relation': 'enriched-attr', + 'to_ids': orig_attr.to_ids + }) + + enriched_attr = _make_tags(enriched_attr, result) + enriched_object.add_attribute(**enriched_attr) + + comment_attr = MISPAttribute() + comment_attr.from_dict(**{ + 'value': '\n'.join(result.get('descriptions', [])), + 'type': 'text', + 'object_relation': 'descriptions', + 'distribution': 0 + }) + enriched_object.add_attribute(**comment_attr) + + last_seen = MISPAttribute() + last_seen.from_dict(**{ + 'value': result.get('last_seen'), + 'type': 'datetime', + 'object_relation': 'last-seen', + 'distribution': 0 + }) + enriched_object.add_attribute(**last_seen) + + event.add_attribute(**orig_attr) + event.add_object(**enriched_object) + + return event + + +def _make_asn_attr(event, result, orig_attr): + + asn_object = MISPObject('asn') + asn_object.add_reference(orig_attr.uuid, 'related-to') + + asn_attr = MISPAttribute() + asn_attr.from_dict(**{ + 'type': 'AS', + 'value': result.get('asn'), + 'object_relation': 'asn', + 'distribution': 0 + }) + asn_object.add_attribute(**asn_attr) + + org_attr = MISPAttribute() + org_attr.from_dict(**{ + 'type': 'text', + 'value': result.get('asn_name', 'unknown').title(), + 'object_relation': 'description', + 'distribution': 0 + }) + asn_object.add_attribute(**org_attr) + + event.add_object(**asn_object) + + return event + + +def _format_hover(event, result): + + enriched_object = event.get_objects_by_name('Qintel Threat Enrichment')[0] + + tags = ', '.join(result.get('tags')) + enriched_object.add_attribute('Tags', type='text', value=tags) + + return event + + +def _format_result(attribute, result): + + event = MISPEvent() + + orig_attr = MISPAttribute() + orig_attr.from_dict(**attribute) + + event = _make_enriched_attr(event, result, orig_attr) + event = _make_asn_attr(event, result, orig_attr) + + return event + + +def _check_config(config): + if not config: + return False + + if not isinstance(config, dict): + return False + + if config.get('token', '') == '': + return False + + return True + + +def _check_request(request): + if not request.get('attribute'): + return f'{standard_error_message}, {checking_error}' + + check_reqs = ('type', 'value') + if not check_input_attribute(request['attribute'], + requirements=check_reqs): + return f'{standard_error_message}, {checking_error}' + + if request['attribute']['type'] not in mispattributes['input']: + return 'Unsupported attribute type' + + +def handler(q=False): + if not q: + return False + + request = json.loads(q) + config = request.get('config') + + if not _check_config(config): + return _return_error('Missing Qintel token') + + check_request_error = _check_request(request) + if check_request_error: + return _return_error(check_request_error) + + search_args = { + 'token': config['token'], + 'remote': config.get('remote') + } + + try: + result = search_qsentry(request['attribute']['value'], **search_args) + except Exception as e: + return _return_error(str(e)) + + event = _format_result(request['attribute'], result) + if not request.get('event_id'): + event = _format_hover(event, result) + + event = json.loads(event.to_json()) + + ret_result = {key: event[key] for key in ('Attribute', 'Object') if key + in event} + return {'results': ret_result} + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo