Merge pull request #536 from qintel/feat/qintel_qsentry

New Module: Qintel QSentry
pull/541/head
Alexandre Dulaunoy 2021-11-23 18:47:42 +01:00 committed by GitHub
commit 06d62d101c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 500 additions and 2 deletions

Binary file not shown.

After

Width:  |  Height:  |  Size: 46 KiB

View File

@ -0,0 +1,13 @@
{
"description": "A hover and expansion module which queries Qintel QSentry for ip reputation data",
"logo": "qintel.png",
"requirements": [
"A Qintel API token"
],
"input": "ip address attribute",
"ouput": "Objects containing the enriched IP, threat tags, last seen attributes and associated Autonomous System information",
"features": "This module takes an ip-address (ip-src or ip-dst) attribute as input, and queries the Qintel QSentry API to retrieve ip reputation data",
"references": [
"https://www.qintel.com/products/qsentry/"
]
}

View File

@ -1,3 +1,3 @@
from .vt_graph_parser import * # noqa from .vt_graph_parser import * # noqa
all = ['joe_parser', 'lastline_api', 'cof2misp'] all = ['joe_parser', 'lastline_api', 'cof2misp', 'qintel_helper']

View File

@ -0,0 +1,263 @@
# Copyright (c) 2009-2021 Qintel, LLC
# Licensed under Apache 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt)
from urllib.request import Request, urlopen
from urllib.parse import urlencode
from urllib.error import HTTPError
from time import sleep
from json import loads
import os
from copy import deepcopy
from datetime import datetime, timedelta
from gzip import GzipFile
VERSION = '1.0.1'
USER_AGENT = 'integrations-helper'
MAX_RETRY_ATTEMPTS = 5
DEFAULT_HEADERS = {
'User-Agent': f'{USER_AGENT}/{VERSION}'
}
REMOTE_MAP = {
'pmi': 'https://api.pmi.qintel.com',
'qwatch': 'https://api.qwatch.qintel.com',
'qauth': 'https://api.qauth.qintel.com',
'qsentry_feed': 'https://qsentry.qintel.com',
'qsentry': 'https://api.qsentry.qintel.com'
}
ENDPOINT_MAP = {
'pmi': {
'ping': '/users/me',
'cve': 'cves'
},
'qsentry_feed': {
'anon': '/files/anonymization',
'mal_hosting': '/files/malicious_hosting'
},
'qsentry': {},
'qwatch': {
'ping': '/users/me',
'exposures': 'exposures'
},
'qauth': {}
}
def _get_request_wait_time(attempts):
""" Use Fibonacci numbers for determining the time to wait when rate limits
have been encountered.
"""
n = attempts + 3
a, b = 1, 0
for _ in range(n):
a, b = a + b, a
return a
def _search(**kwargs):
remote = kwargs.get('remote')
max_retries = int(kwargs.get('max_retries', MAX_RETRY_ATTEMPTS))
params = kwargs.get('params', {})
headers = _set_headers(**kwargs)
logger = kwargs.get('logger')
params = urlencode(params)
url = remote + "?" + params
req = Request(url, headers=headers)
request_attempts = 1
while request_attempts < max_retries:
try:
return urlopen(req)
except HTTPError as e:
response = e
except Exception as e:
raise Exception('API connection error') from e
if response.code not in [429, 504]:
raise Exception(f'API connection error: {response}')
if request_attempts < max_retries:
wait_time = _get_request_wait_time(request_attempts)
if response.code == 429:
msg = 'rate limit reached on attempt {request_attempts}, ' \
'waiting {wait_time} seconds'
if logger:
logger(msg)
else:
msg = f'connection timed out, retrying in {wait_time} seconds'
if logger:
logger(msg)
sleep(wait_time)
else:
raise Exception('Max API retries exceeded')
request_attempts += 1
def _set_headers(**kwargs):
headers = deepcopy(DEFAULT_HEADERS)
if kwargs.get('user_agent'):
headers['User-Agent'] = \
f"{kwargs['user_agent']}/{USER_AGENT}/{VERSION}"
# TODO: deprecate
if kwargs.get('client_id') or kwargs.get('client_secret'):
try:
headers['Cf-Access-Client-Id'] = kwargs['client_id']
headers['Cf-Access-Client-Secret'] = kwargs['client_secret']
except KeyError:
raise Exception('missing client_id or client_secret')
if kwargs.get('token'):
headers['x-api-key'] = kwargs['token']
return headers
def _set_remote(product, query_type, **kwargs):
remote = kwargs.get('remote')
endpoint = kwargs.get('endpoint', ENDPOINT_MAP[product].get(query_type))
if not remote:
remote = REMOTE_MAP[product]
if not endpoint:
raise Exception('invalid search type')
remote = remote.rstrip('/')
endpoint = endpoint.lstrip('/')
return f'{remote}/{endpoint}'
def _process_qsentry(resp):
if resp.getheader('Content-Encoding', '') == 'gzip':
with GzipFile(fileobj=resp) as file:
for line in file.readlines():
yield loads(line)
def search_pmi(search_term, query_type, **kwargs):
"""
Search PMI
:param str search_term: Search term
:param str query_type: Query type [cve|ping]
:param dict kwargs: extra client args [remote|token|params]
:return: API JSON response object
:rtype: dict
"""
kwargs['remote'] = _set_remote('pmi', query_type, **kwargs)
kwargs['token'] = kwargs.get('token', os.getenv('PMI_TOKEN'))
params = kwargs.get('params', {})
params.update({'identifier': search_term})
kwargs['params'] = params
return loads(_search(**kwargs).read())
def search_qwatch(search_term, search_type, query_type, **kwargs):
"""
Search QWatch for exposed credentials
:param str search_term: Search term
:param str search_type: Search term type [domain|email]
:param str query_type: Query type [exposures]
:param dict kwargs: extra client args [remote|token|params]
:return: API JSON response object
:rtype: dict
"""
kwargs['remote'] = _set_remote('qwatch', query_type, **kwargs)
kwargs['token'] = kwargs.get('token', os.getenv('QWATCH_TOKEN'))
params = kwargs.get('params', {})
if search_type:
params.update({search_type: search_term})
kwargs['params'] = params
return loads(_search(**kwargs).read())
def search_qauth(search_term, **kwargs):
"""
Search QAuth
:param str search_term: Search term
:param dict kwargs: extra client args [remote|token|params]
:return: API JSON response object
:rtype: dict
"""
if not kwargs.get('endpoint'):
kwargs['endpoint'] = '/'
kwargs['remote'] = _set_remote('qauth', None, **kwargs)
kwargs['token'] = kwargs.get('token', os.getenv('QAUTH_TOKEN'))
params = kwargs.get('params', {})
params.update({'q': search_term})
kwargs['params'] = params
return loads(_search(**kwargs).read())
def search_qsentry(search_term, **kwargs):
"""
Search QSentry
:param str search_term: Search term
:param dict kwargs: extra client args [remote|token|params]
:return: API JSON response object
:rtype: dict
"""
if not kwargs.get('endpoint'):
kwargs['endpoint'] = '/'
kwargs['remote'] = _set_remote('qsentry', None, **kwargs)
kwargs['token'] = kwargs.get('token', os.getenv('QSENTRY_TOKEN'))
params = kwargs.get('params', {})
params.update({'q': search_term})
kwargs['params'] = params
return loads(_search(**kwargs).read())
def qsentry_feed(query_type='anon', feed_date=datetime.today(), **kwargs):
"""
Fetch the most recent QSentry Feed
:param str query_type: Feed type [anon|mal_hosting]
:param dict kwargs: extra client args [remote|token|params]
:param datetime feed_date: feed date to fetch
:return: API JSON response object
:rtype: Iterator[dict]
"""
remote = _set_remote('qsentry_feed', query_type, **kwargs)
kwargs['token'] = kwargs.get('token', os.getenv('QSENTRY_TOKEN'))
feed_date = (feed_date - timedelta(days=1)).strftime('%Y%m%d')
kwargs['remote'] = f'{remote}/{feed_date}'
resp = _search(**kwargs)
for r in _process_qsentry(resp):
yield r

View File

@ -17,7 +17,8 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c
'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid', 'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid',
'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', 'malwarebazaar', 'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', 'malwarebazaar',
'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion', 'censys_enrich', 'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion', 'censys_enrich',
'trustar_enrich', 'recordedfuture', 'html_to_markdown', 'socialscan', 'passive-ssh'] 'trustar_enrich', 'recordedfuture', 'html_to_markdown', 'socialscan', 'passive-ssh',
'qintel_qsentry']
minimum_required_fields = ('type', 'uuid', 'value') minimum_required_fields = ('type', 'uuid', 'value')

View File

@ -0,0 +1,221 @@
import logging
import json
from pymisp import MISPAttribute, MISPEvent, MISPTag, MISPObject
from . import check_input_attribute, checking_error, standard_error_message
from qintel_helper import search_qsentry
logger = logging.getLogger('qintel_qsentry')
logger.setLevel(logging.DEBUG)
moduleinfo = {
'version': '1.0',
'author': 'Qintel, LLC',
'description': 'Query Qintel QSentry for ip intelligence',
'module-type': ['hover', 'expansion']
}
moduleconfig = ['token', 'remote']
misperrors = {'error': 'Error'}
mispattributes = {
'input': ['ip-src', 'ip-dst'],
'output': ['ip-src', 'ip-dst', 'AS', 'freetext'],
'format': 'misp_standard'
}
TAG_COLOR = {
'benign': '#27ae60',
'suspicious': '#e6a902',
'malicious': '#c0392b'
}
CLIENT_HEADERS = {
'User-Agent': f"MISP/{moduleinfo['version']}",
}
def _return_error(message):
misperrors['error'] = message
return misperrors
def _make_tags(enriched_attr, result):
for tag in result['tags']:
color = TAG_COLOR['suspicious']
if tag == 'criminal':
color = TAG_COLOR['malicious']
t = MISPTag()
t.from_dict(**{
'name': f'qintel:tag="{tag}"',
'colour': color
})
enriched_attr.add_tag(**t)
return enriched_attr
def _make_enriched_attr(event, result, orig_attr):
enriched_object = MISPObject('Qintel Threat Enrichment')
enriched_object.add_reference(orig_attr.uuid, 'related-to')
enriched_attr = MISPAttribute()
enriched_attr.from_dict(**{
'value': orig_attr.value,
'type': orig_attr.type,
'distribution': 0,
'object_relation': 'enriched-attr',
'to_ids': orig_attr.to_ids
})
enriched_attr = _make_tags(enriched_attr, result)
enriched_object.add_attribute(**enriched_attr)
comment_attr = MISPAttribute()
comment_attr.from_dict(**{
'value': '\n'.join(result.get('descriptions', [])),
'type': 'text',
'object_relation': 'descriptions',
'distribution': 0
})
enriched_object.add_attribute(**comment_attr)
last_seen = MISPAttribute()
last_seen.from_dict(**{
'value': result.get('last_seen'),
'type': 'datetime',
'object_relation': 'last-seen',
'distribution': 0
})
enriched_object.add_attribute(**last_seen)
event.add_attribute(**orig_attr)
event.add_object(**enriched_object)
return event
def _make_asn_attr(event, result, orig_attr):
asn_object = MISPObject('asn')
asn_object.add_reference(orig_attr.uuid, 'related-to')
asn_attr = MISPAttribute()
asn_attr.from_dict(**{
'type': 'AS',
'value': result.get('asn'),
'object_relation': 'asn',
'distribution': 0
})
asn_object.add_attribute(**asn_attr)
org_attr = MISPAttribute()
org_attr.from_dict(**{
'type': 'text',
'value': result.get('asn_name', 'unknown').title(),
'object_relation': 'description',
'distribution': 0
})
asn_object.add_attribute(**org_attr)
event.add_object(**asn_object)
return event
def _format_hover(event, result):
enriched_object = event.get_objects_by_name('Qintel Threat Enrichment')[0]
tags = ', '.join(result.get('tags'))
enriched_object.add_attribute('Tags', type='text', value=tags)
return event
def _format_result(attribute, result):
event = MISPEvent()
orig_attr = MISPAttribute()
orig_attr.from_dict(**attribute)
event = _make_enriched_attr(event, result, orig_attr)
event = _make_asn_attr(event, result, orig_attr)
return event
def _check_config(config):
if not config:
return False
if not isinstance(config, dict):
return False
if config.get('token', '') == '':
return False
return True
def _check_request(request):
if not request.get('attribute'):
return f'{standard_error_message}, {checking_error}'
check_reqs = ('type', 'value')
if not check_input_attribute(request['attribute'],
requirements=check_reqs):
return f'{standard_error_message}, {checking_error}'
if request['attribute']['type'] not in mispattributes['input']:
return 'Unsupported attribute type'
def handler(q=False):
if not q:
return False
request = json.loads(q)
config = request.get('config')
if not _check_config(config):
return _return_error('Missing Qintel token')
check_request_error = _check_request(request)
if check_request_error:
return _return_error(check_request_error)
search_args = {
'token': config['token'],
'remote': config.get('remote')
}
try:
result = search_qsentry(request['attribute']['value'], **search_args)
except Exception as e:
return _return_error(str(e))
event = _format_result(request['attribute'], result)
if not request.get('event_id'):
event = _format_hover(event, result)
event = json.loads(event.to_json())
ret_result = {key: event[key] for key in ('Attribute', 'Object') if key
in event}
return {'results': ret_result}
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo