mirror of https://github.com/MISP/misp-modules
feature: add qintel qsentry expansion module
Calvin Krzywiec
parent
d4856a0c68
commit
dc0660acd0
|
|
@ -1,3 +1,3 @@
|
|||
from .vt_graph_parser import * # noqa
|
||||
|
||||
all = ['joe_parser', 'lastline_api', 'cof2misp']
|
||||
all = ['joe_parser', 'lastline_api', 'cof2misp', 'qintel_helper']
|
||||
|
|
|
|||
|
|
@ -0,0 +1,263 @@
|
|||
# Copyright (c) 2009-2021 Qintel, LLC
|
||||
# Licensed under Apache 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt)
|
||||
|
||||
from urllib.request import Request, urlopen
|
||||
from urllib.parse import urlencode
|
||||
from urllib.error import HTTPError
|
||||
from time import sleep
|
||||
from json import loads
|
||||
import os
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timedelta
|
||||
from gzip import GzipFile
|
||||
|
||||
VERSION = '1.0.1'
|
||||
USER_AGENT = 'integrations-helper'
|
||||
MAX_RETRY_ATTEMPTS = 5
|
||||
|
||||
DEFAULT_HEADERS = {
|
||||
'User-Agent': f'{USER_AGENT}/{VERSION}'
|
||||
}
|
||||
|
||||
REMOTE_MAP = {
|
||||
'pmi': 'https://api.pmi.qintel.com',
|
||||
'qwatch': 'https://api.qwatch.qintel.com',
|
||||
'qauth': 'https://api.qauth.qintel.com',
|
||||
'qsentry_feed': 'https://qsentry.qintel.com',
|
||||
'qsentry': 'https://api.qsentry.qintel.com'
|
||||
}
|
||||
|
||||
ENDPOINT_MAP = {
|
||||
'pmi': {
|
||||
'ping': '/users/me',
|
||||
'cve': 'cves'
|
||||
},
|
||||
'qsentry_feed': {
|
||||
'anon': '/files/anonymization',
|
||||
'mal_hosting': '/files/malicious_hosting'
|
||||
},
|
||||
'qsentry': {},
|
||||
'qwatch': {
|
||||
'ping': '/users/me',
|
||||
'exposures': 'exposures'
|
||||
},
|
||||
'qauth': {}
|
||||
}
|
||||
|
||||
|
||||
def _get_request_wait_time(attempts):
|
||||
""" Use Fibonacci numbers for determining the time to wait when rate limits
|
||||
have been encountered.
|
||||
"""
|
||||
|
||||
n = attempts + 3
|
||||
a, b = 1, 0
|
||||
for _ in range(n):
|
||||
a, b = a + b, a
|
||||
|
||||
return a
|
||||
|
||||
|
||||
def _search(**kwargs):
|
||||
remote = kwargs.get('remote')
|
||||
max_retries = int(kwargs.get('max_retries', MAX_RETRY_ATTEMPTS))
|
||||
params = kwargs.get('params', {})
|
||||
headers = _set_headers(**kwargs)
|
||||
|
||||
logger = kwargs.get('logger')
|
||||
|
||||
params = urlencode(params)
|
||||
url = remote + "?" + params
|
||||
req = Request(url, headers=headers)
|
||||
|
||||
request_attempts = 1
|
||||
while request_attempts < max_retries:
|
||||
try:
|
||||
return urlopen(req)
|
||||
|
||||
except HTTPError as e:
|
||||
response = e
|
||||
|
||||
except Exception as e:
|
||||
raise Exception('API connection error') from e
|
||||
|
||||
if response.code not in [429, 504]:
|
||||
raise Exception(f'API connection error: {response}')
|
||||
|
||||
if request_attempts < max_retries:
|
||||
wait_time = _get_request_wait_time(request_attempts)
|
||||
|
||||
if response.code == 429:
|
||||
msg = 'rate limit reached on attempt {request_attempts}, ' \
|
||||
'waiting {wait_time} seconds'
|
||||
|
||||
if logger:
|
||||
logger(msg)
|
||||
|
||||
else:
|
||||
msg = f'connection timed out, retrying in {wait_time} seconds'
|
||||
if logger:
|
||||
logger(msg)
|
||||
|
||||
sleep(wait_time)
|
||||
|
||||
else:
|
||||
raise Exception('Max API retries exceeded')
|
||||
|
||||
request_attempts += 1
|
||||
|
||||
|
||||
def _set_headers(**kwargs):
|
||||
headers = deepcopy(DEFAULT_HEADERS)
|
||||
|
||||
if kwargs.get('user_agent'):
|
||||
headers['User-Agent'] = \
|
||||
f"{kwargs['user_agent']}/{USER_AGENT}/{VERSION}"
|
||||
|
||||
# TODO: deprecate
|
||||
if kwargs.get('client_id') or kwargs.get('client_secret'):
|
||||
try:
|
||||
headers['Cf-Access-Client-Id'] = kwargs['client_id']
|
||||
headers['Cf-Access-Client-Secret'] = kwargs['client_secret']
|
||||
except KeyError:
|
||||
raise Exception('missing client_id or client_secret')
|
||||
|
||||
if kwargs.get('token'):
|
||||
headers['x-api-key'] = kwargs['token']
|
||||
|
||||
return headers
|
||||
|
||||
|
||||
def _set_remote(product, query_type, **kwargs):
|
||||
remote = kwargs.get('remote')
|
||||
endpoint = kwargs.get('endpoint', ENDPOINT_MAP[product].get(query_type))
|
||||
|
||||
if not remote:
|
||||
remote = REMOTE_MAP[product]
|
||||
|
||||
if not endpoint:
|
||||
raise Exception('invalid search type')
|
||||
|
||||
remote = remote.rstrip('/')
|
||||
endpoint = endpoint.lstrip('/')
|
||||
|
||||
return f'{remote}/{endpoint}'
|
||||
|
||||
|
||||
def _process_qsentry(resp):
|
||||
if resp.getheader('Content-Encoding', '') == 'gzip':
|
||||
with GzipFile(fileobj=resp) as file:
|
||||
for line in file.readlines():
|
||||
yield loads(line)
|
||||
|
||||
|
||||
def search_pmi(search_term, query_type, **kwargs):
|
||||
"""
|
||||
Search PMI
|
||||
|
||||
:param str search_term: Search term
|
||||
:param str query_type: Query type [cve|ping]
|
||||
:param dict kwargs: extra client args [remote|token|params]
|
||||
:return: API JSON response object
|
||||
:rtype: dict
|
||||
"""
|
||||
|
||||
kwargs['remote'] = _set_remote('pmi', query_type, **kwargs)
|
||||
kwargs['token'] = kwargs.get('token', os.getenv('PMI_TOKEN'))
|
||||
|
||||
params = kwargs.get('params', {})
|
||||
params.update({'identifier': search_term})
|
||||
kwargs['params'] = params
|
||||
|
||||
return loads(_search(**kwargs).read())
|
||||
|
||||
|
||||
def search_qwatch(search_term, search_type, query_type, **kwargs):
|
||||
"""
|
||||
Search QWatch for exposed credentials
|
||||
|
||||
:param str search_term: Search term
|
||||
:param str search_type: Search term type [domain|email]
|
||||
:param str query_type: Query type [exposures]
|
||||
:param dict kwargs: extra client args [remote|token|params]
|
||||
:return: API JSON response object
|
||||
:rtype: dict
|
||||
"""
|
||||
|
||||
kwargs['remote'] = _set_remote('qwatch', query_type, **kwargs)
|
||||
kwargs['token'] = kwargs.get('token', os.getenv('QWATCH_TOKEN'))
|
||||
|
||||
params = kwargs.get('params', {})
|
||||
if search_type:
|
||||
params.update({search_type: search_term})
|
||||
kwargs['params'] = params
|
||||
|
||||
return loads(_search(**kwargs).read())
|
||||
|
||||
|
||||
def search_qauth(search_term, **kwargs):
|
||||
"""
|
||||
Search QAuth
|
||||
|
||||
:param str search_term: Search term
|
||||
:param dict kwargs: extra client args [remote|token|params]
|
||||
:return: API JSON response object
|
||||
:rtype: dict
|
||||
"""
|
||||
|
||||
if not kwargs.get('endpoint'):
|
||||
kwargs['endpoint'] = '/'
|
||||
|
||||
kwargs['remote'] = _set_remote('qauth', None, **kwargs)
|
||||
kwargs['token'] = kwargs.get('token', os.getenv('QAUTH_TOKEN'))
|
||||
|
||||
params = kwargs.get('params', {})
|
||||
params.update({'q': search_term})
|
||||
kwargs['params'] = params
|
||||
|
||||
return loads(_search(**kwargs).read())
|
||||
|
||||
|
||||
def search_qsentry(search_term, **kwargs):
|
||||
"""
|
||||
Search QSentry
|
||||
|
||||
:param str search_term: Search term
|
||||
:param dict kwargs: extra client args [remote|token|params]
|
||||
:return: API JSON response object
|
||||
:rtype: dict
|
||||
"""
|
||||
|
||||
if not kwargs.get('endpoint'):
|
||||
kwargs['endpoint'] = '/'
|
||||
|
||||
kwargs['remote'] = _set_remote('qsentry', None, **kwargs)
|
||||
kwargs['token'] = kwargs.get('token', os.getenv('QSENTRY_TOKEN'))
|
||||
|
||||
params = kwargs.get('params', {})
|
||||
params.update({'q': search_term})
|
||||
kwargs['params'] = params
|
||||
|
||||
return loads(_search(**kwargs).read())
|
||||
|
||||
|
||||
def qsentry_feed(query_type='anon', feed_date=datetime.today(), **kwargs):
|
||||
"""
|
||||
Fetch the most recent QSentry Feed
|
||||
|
||||
:param str query_type: Feed type [anon|mal_hosting]
|
||||
:param dict kwargs: extra client args [remote|token|params]
|
||||
:param datetime feed_date: feed date to fetch
|
||||
:return: API JSON response object
|
||||
:rtype: Iterator[dict]
|
||||
"""
|
||||
|
||||
remote = _set_remote('qsentry_feed', query_type, **kwargs)
|
||||
kwargs['token'] = kwargs.get('token', os.getenv('QSENTRY_TOKEN'))
|
||||
|
||||
feed_date = (feed_date - timedelta(days=1)).strftime('%Y%m%d')
|
||||
kwargs['remote'] = f'{remote}/{feed_date}'
|
||||
|
||||
resp = _search(**kwargs)
|
||||
for r in _process_qsentry(resp):
|
||||
yield r
|
||||
|
|
@ -17,7 +17,8 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c
|
|||
'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid',
|
||||
'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', 'malwarebazaar',
|
||||
'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion', 'censys_enrich',
|
||||
'trustar_enrich', 'recordedfuture', 'html_to_markdown', 'socialscan', 'passive-ssh']
|
||||
'trustar_enrich', 'recordedfuture', 'html_to_markdown', 'socialscan', 'passive-ssh',
|
||||
'qintel_qsentry']
|
||||
|
||||
|
||||
minimum_required_fields = ('type', 'uuid', 'value')
|
||||
|
|
|
|||
|
|
@ -0,0 +1,221 @@
|
|||
import logging
|
||||
import json
|
||||
|
||||
from pymisp import MISPAttribute, MISPEvent, MISPTag, MISPObject
|
||||
from . import check_input_attribute, checking_error, standard_error_message
|
||||
|
||||
from qintel_helper import search_qsentry
|
||||
|
||||
logger = logging.getLogger('qintel_qsentry')
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
moduleinfo = {
|
||||
'version': '1.0',
|
||||
'author': 'Qintel, LLC',
|
||||
'description': 'Query Qintel QSentry for ip intelligence',
|
||||
'module-type': ['hover', 'expansion']
|
||||
}
|
||||
|
||||
moduleconfig = ['token', 'remote']
|
||||
|
||||
misperrors = {'error': 'Error'}
|
||||
|
||||
mispattributes = {
|
||||
'input': ['ip-src', 'ip-dst'],
|
||||
'output': ['ip-src', 'ip-dst', 'AS', 'freetext'],
|
||||
'format': 'misp_standard'
|
||||
}
|
||||
|
||||
TAG_COLOR = {
|
||||
'benign': '#27ae60',
|
||||
'suspicious': '#e6a902',
|
||||
'malicious': '#c0392b'
|
||||
}
|
||||
|
||||
CLIENT_HEADERS = {
|
||||
'User-Agent': f"MISP/{moduleinfo['version']}",
|
||||
}
|
||||
|
||||
|
||||
def _return_error(message):
|
||||
misperrors['error'] = message
|
||||
return misperrors
|
||||
|
||||
|
||||
def _make_tags(enriched_attr, result):
|
||||
|
||||
for tag in result['tags']:
|
||||
color = TAG_COLOR['suspicious']
|
||||
if tag == 'criminal':
|
||||
color = TAG_COLOR['malicious']
|
||||
|
||||
t = MISPTag()
|
||||
t.from_dict(**{
|
||||
'name': f'qintel:tag="{tag}"',
|
||||
'colour': color
|
||||
})
|
||||
enriched_attr.add_tag(**t)
|
||||
|
||||
return enriched_attr
|
||||
|
||||
|
||||
def _make_enriched_attr(event, result, orig_attr):
|
||||
|
||||
enriched_object = MISPObject('Qintel Threat Enrichment')
|
||||
enriched_object.add_reference(orig_attr.uuid, 'related-to')
|
||||
|
||||
enriched_attr = MISPAttribute()
|
||||
enriched_attr.from_dict(**{
|
||||
'value': orig_attr.value,
|
||||
'type': orig_attr.type,
|
||||
'distribution': 0,
|
||||
'object_relation': 'enriched-attr',
|
||||
'to_ids': orig_attr.to_ids
|
||||
})
|
||||
|
||||
enriched_attr = _make_tags(enriched_attr, result)
|
||||
enriched_object.add_attribute(**enriched_attr)
|
||||
|
||||
comment_attr = MISPAttribute()
|
||||
comment_attr.from_dict(**{
|
||||
'value': '\n'.join(result.get('descriptions', [])),
|
||||
'type': 'text',
|
||||
'object_relation': 'descriptions',
|
||||
'distribution': 0
|
||||
})
|
||||
enriched_object.add_attribute(**comment_attr)
|
||||
|
||||
last_seen = MISPAttribute()
|
||||
last_seen.from_dict(**{
|
||||
'value': result.get('last_seen'),
|
||||
'type': 'datetime',
|
||||
'object_relation': 'last-seen',
|
||||
'distribution': 0
|
||||
})
|
||||
enriched_object.add_attribute(**last_seen)
|
||||
|
||||
event.add_attribute(**orig_attr)
|
||||
event.add_object(**enriched_object)
|
||||
|
||||
return event
|
||||
|
||||
|
||||
def _make_asn_attr(event, result, orig_attr):
|
||||
|
||||
asn_object = MISPObject('asn')
|
||||
asn_object.add_reference(orig_attr.uuid, 'related-to')
|
||||
|
||||
asn_attr = MISPAttribute()
|
||||
asn_attr.from_dict(**{
|
||||
'type': 'AS',
|
||||
'value': result.get('asn'),
|
||||
'object_relation': 'asn',
|
||||
'distribution': 0
|
||||
})
|
||||
asn_object.add_attribute(**asn_attr)
|
||||
|
||||
org_attr = MISPAttribute()
|
||||
org_attr.from_dict(**{
|
||||
'type': 'text',
|
||||
'value': result.get('asn_name', 'unknown').title(),
|
||||
'object_relation': 'description',
|
||||
'distribution': 0
|
||||
})
|
||||
asn_object.add_attribute(**org_attr)
|
||||
|
||||
event.add_object(**asn_object)
|
||||
|
||||
return event
|
||||
|
||||
|
||||
def _format_hover(event, result):
|
||||
|
||||
enriched_object = event.get_objects_by_name('Qintel Threat Enrichment')[0]
|
||||
|
||||
tags = ', '.join(result.get('tags'))
|
||||
enriched_object.add_attribute('Tags', type='text', value=tags)
|
||||
|
||||
return event
|
||||
|
||||
|
||||
def _format_result(attribute, result):
|
||||
|
||||
event = MISPEvent()
|
||||
|
||||
orig_attr = MISPAttribute()
|
||||
orig_attr.from_dict(**attribute)
|
||||
|
||||
event = _make_enriched_attr(event, result, orig_attr)
|
||||
event = _make_asn_attr(event, result, orig_attr)
|
||||
|
||||
return event
|
||||
|
||||
|
||||
def _check_config(config):
|
||||
if not config:
|
||||
return False
|
||||
|
||||
if not isinstance(config, dict):
|
||||
return False
|
||||
|
||||
if config.get('token', '') == '':
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _check_request(request):
|
||||
if not request.get('attribute'):
|
||||
return f'{standard_error_message}, {checking_error}'
|
||||
|
||||
check_reqs = ('type', 'value')
|
||||
if not check_input_attribute(request['attribute'],
|
||||
requirements=check_reqs):
|
||||
return f'{standard_error_message}, {checking_error}'
|
||||
|
||||
if request['attribute']['type'] not in mispattributes['input']:
|
||||
return 'Unsupported attribute type'
|
||||
|
||||
|
||||
def handler(q=False):
|
||||
if not q:
|
||||
return False
|
||||
|
||||
request = json.loads(q)
|
||||
config = request.get('config')
|
||||
|
||||
if not _check_config(config):
|
||||
return _return_error('Missing Qintel token')
|
||||
|
||||
check_request_error = _check_request(request)
|
||||
if check_request_error:
|
||||
return _return_error(check_request_error)
|
||||
|
||||
search_args = {
|
||||
'token': config['token'],
|
||||
'remote': config.get('remote')
|
||||
}
|
||||
|
||||
try:
|
||||
result = search_qsentry(request['attribute']['value'], **search_args)
|
||||
except Exception as e:
|
||||
return _return_error(str(e))
|
||||
|
||||
event = _format_result(request['attribute'], result)
|
||||
if not request.get('event_id'):
|
||||
event = _format_hover(event, result)
|
||||
|
||||
event = json.loads(event.to_json())
|
||||
|
||||
ret_result = {key: event[key] for key in ('Attribute', 'Object') if key
|
||||
in event}
|
||||
return {'results': ret_result}
|
||||
|
||||
|
||||
def introspection():
|
||||
return mispattributes
|
||||
|
||||
|
||||
def version():
|
||||
moduleinfo['config'] = moduleconfig
|
||||
return moduleinfo
|
||||
Loading…
Reference in New Issue