misp-modules/misp_modules/modules/expansion/securitytrails.py

562 lines
18 KiB
Python
Raw Normal View History

2018-06-29 15:50:26 +02:00
import json
2018-06-29 11:27:36 +02:00
import logging
import sys
2018-07-10 16:41:44 +02:00
import time
2018-06-29 17:22:19 +02:00
2018-06-29 17:19:21 +02:00
from dnstrails import APIError
2018-06-29 17:22:19 +02:00
from dnstrails import DnsTrails
2018-06-29 11:27:36 +02:00
log = logging.getLogger('dnstrails')
log.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
2018-07-10 16:31:39 +02:00
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
2018-06-29 11:27:36 +02:00
ch.setFormatter(formatter)
log.addHandler(ch)
misperrors = {'error': 'Error'}
mispattributes = {
'input': ['hostname', 'domain', 'ip-src', 'ip-dst'],
2018-06-29 17:57:11 +02:00
'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'dns-soa-email',
'whois-registrant-email', 'whois-registrant-phone',
'whois-registrant-name',
'whois-registrar', 'whois-creation-date', 'domain']
2018-06-29 11:27:36 +02:00
}
moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven',
'description': 'Query on securitytrails.com',
2018-07-11 13:03:47 +02:00
'module-type': ['expansion', 'hover']}
2018-06-29 11:27:36 +02:00
# config fields that your code expects from the site admin
moduleconfig = ['apikey']
2018-06-29 15:50:26 +02:00
def handler(q=False):
if q:
request = json.loads(q)
if not request.get('config') or not (request['config'].get('apikey')):
misperrors['error'] = 'SecurityTrails authentication is missing'
2018-06-29 15:50:26 +02:00
return misperrors
api = DnsTrails(request['config'].get('apikey'))
if not api:
misperrors['error'] = 'SecurityTrails Error instance api'
return misperrors
2018-06-29 15:50:26 +02:00
if request.get('ip-src'):
ip = request['ip-src']
return handle_ip(api, ip, misperrors)
elif request.get('ip-dst'):
ip = request['ip-dst']
return handle_ip(api, ip, misperrors)
elif request.get('domain'):
domain = request['domain']
return handle_domain(api, domain, misperrors)
elif request.get('hostname'):
hostname = request['hostname']
return handle_domain(api, hostname, misperrors)
else:
2018-06-29 16:20:35 +02:00
misperrors['error'] = "Unsupported attributes types"
2018-06-29 15:50:26 +02:00
return misperrors
else:
return False
def handle_domain(api, domain, misperrors):
result_filtered = {"results": []}
2018-07-18 10:47:42 +02:00
r, status_ok = expand_domain_info(api, misperrors, domain)
2018-07-12 14:47:34 +02:00
2018-07-18 10:47:42 +02:00
if status_ok:
if r:
result_filtered['results'].extend(r)
else:
misperrors['error'] = misperrors['error'] + ' Error DNS result'
return misperrors
2018-07-17 17:09:01 +02:00
2018-07-18 10:47:42 +02:00
time.sleep(1)
r, status_ok = expand_subdomains(api, domain)
2018-06-29 15:50:26 +02:00
2018-07-18 10:47:42 +02:00
if status_ok:
if r:
result_filtered['results'].extend(r)
else:
misperrors['error'] = misperrors['error'] + ' Error subdomains result'
return misperrors
2018-07-10 16:31:39 +02:00
2018-07-18 10:47:42 +02:00
time.sleep(1)
r, status_ok = expand_whois(api, domain)
2018-07-17 17:09:01 +02:00
2018-07-18 10:47:42 +02:00
if status_ok:
if r:
result_filtered['results'].extend(r)
time.sleep(1)
r, status_ok = expand_history_ipv4_ipv6(api, domain)
2018-07-17 17:09:01 +02:00
if status_ok:
if r:
2018-07-17 17:22:48 +02:00
result_filtered['results'].extend(r)
2018-07-17 17:09:01 +02:00
else:
2018-07-18 10:47:42 +02:00
misperrors['error'] = misperrors['error'] + ' Error history ipv4'
2018-07-17 17:09:01 +02:00
return misperrors
2018-07-17 17:20:30 +02:00
2018-07-17 18:42:01 +02:00
time.sleep(1)
2018-07-18 10:47:42 +02:00
r, status_ok = expand_history_dns(api, domain)
if status_ok:
if r:
result_filtered['results'].extend(r)
else:
2018-12-11 15:29:09 +01:00
misperrors['error'] = misperrors['error'] + ' Error in expand History DNS'
2018-07-18 10:47:42 +02:00
return misperrors
r, status_ok = expand_history_whois(api, domain)
2018-07-17 18:42:01 +02:00
if status_ok:
if r:
result_filtered['results'].extend(r)
else:
2018-12-11 15:29:09 +01:00
misperrors['error'] = misperrors['error'] + ' Error in expand History Whois'
2018-07-18 10:47:42 +02:00
return misperrors
return result_filtered
def handle_ip(api, ip, misperrors):
result_filtered = {"results": []}
r, status_ok = expand_searching_domain(api, ip)
if status_ok:
if r:
result_filtered['results'].extend(r)
else:
misperrors['error'] += ' Error in expand searching domain'
2018-07-17 18:42:01 +02:00
return misperrors
2018-07-17 17:09:01 +02:00
return result_filtered
2018-06-29 15:50:26 +02:00
2018-07-10 16:31:39 +02:00
def expand_domain_info(api, misperror, domain):
2018-06-29 15:50:26 +02:00
r = []
status_ok = False
ns_servers = []
list_ipv4 = []
list_ipv6 = []
servers_mx = []
soa_hostnames = []
try:
results = api.domain(domain)
except APIError as e:
misperrors['error'] = e.value
return [], False
2018-06-29 15:50:26 +02:00
if results:
2018-06-29 16:17:32 +02:00
status_ok = True
2018-06-29 15:50:26 +02:00
if 'current_dns' in results:
if 'values' in results['current_dns']['ns']:
ns_servers = [ns_entry['nameserver'] for ns_entry in
results['current_dns']['ns']['values']
if 'nameserver' in ns_entry]
if 'values' in results['current_dns']['a']:
list_ipv4 = [a_entry['ip'] for a_entry in
results['current_dns']['a']['values'] if
'ip' in a_entry]
if 'values' in results['current_dns']['aaaa']:
list_ipv6 = [ipv6_entry['ipv6'] for ipv6_entry in
2018-07-11 08:49:59 +02:00
results['current_dns']['aaaa']['values'] if
2018-06-29 15:50:26 +02:00
'ipv6' in ipv6_entry]
if 'values' in results['current_dns']['mx']:
servers_mx = [mx_entry['hostname'] for mx_entry in
2018-07-10 16:31:39 +02:00
results['current_dns']['mx']['values'] if
'hostname' in mx_entry]
2018-06-29 15:50:26 +02:00
if 'values' in results['current_dns']['soa']:
soa_hostnames = [soa_entry['email'] for soa_entry in
2018-07-10 16:31:39 +02:00
results['current_dns']['soa']['values'] if
'email' in soa_entry]
2018-06-29 15:50:26 +02:00
if ns_servers:
2018-06-29 16:20:35 +02:00
r.append({'types': ['domain'],
2018-06-29 15:50:26 +02:00
'values': ns_servers,
2018-06-29 16:30:47 +02:00
'categories': ['Network activity'],
2018-06-29 15:50:26 +02:00
'comment': 'List of name servers of %s first seen %s ' %
2018-07-10 16:31:39 +02:00
(domain,
results['current_dns']['ns']['first_seen'])
})
2018-06-29 15:50:26 +02:00
if list_ipv4:
2018-06-29 16:20:35 +02:00
r.append({'types': ['domain|ip'],
2018-07-10 16:31:39 +02:00
'values': ['%s|%s' % (domain, ipv4) for ipv4 in
list_ipv4],
2018-06-29 16:30:47 +02:00
'categories': ['Network activity'],
2018-06-29 15:50:26 +02:00
'comment': ' List ipv4 of %s first seen %s' %
(domain,
results['current_dns']['a']['first_seen'])
2018-07-10 16:31:39 +02:00
})
2018-06-29 15:50:26 +02:00
if list_ipv6:
2018-06-29 16:20:35 +02:00
r.append({'types': ['domain|ip'],
2018-06-29 15:50:26 +02:00
'values': ['%s|%s' % (domain, ipv6) for ipv6 in
list_ipv6],
2018-06-29 16:30:47 +02:00
'categories': ['Network activity'],
2018-06-29 15:50:26 +02:00
'comment': ' List ipv6 of %s first seen %s' %
(domain,
results['current_dns']['aaaa']['first_seen'])
})
if servers_mx:
2018-06-29 16:20:35 +02:00
r.append({'types': ['domain'],
2018-06-29 15:50:26 +02:00
'values': servers_mx,
2018-06-29 16:30:47 +02:00
'categories': ['Network activity'],
2018-06-29 15:50:26 +02:00
'comment': ' List mx of %s first seen %s' %
(domain,
results['current_dns']['mx']['first_seen'])
})
if soa_hostnames:
2018-06-29 16:20:35 +02:00
r.append({'types': ['domain'],
2018-06-29 15:50:26 +02:00
'values': soa_hostnames,
2018-06-29 16:30:47 +02:00
'categories': ['Network activity'],
2018-06-29 15:50:26 +02:00
'comment': ' List soa of %s first seen %s' %
(domain,
results['current_dns']['soa']['first_seen'])
})
2018-06-29 17:19:21 +02:00
return r, status_ok
def expand_subdomains(api, domain):
r = []
status_ok = False
try:
results = api.subdomains(domain)
if results:
status_ok = True
if 'subdomains' in results:
r.append({
2018-06-29 17:26:56 +02:00
'types': ['domain'],
2018-07-10 16:31:39 +02:00
'values': ['%s.%s' % (sub, domain)
for sub in results['subdomains']],
2018-06-29 17:25:37 +02:00
'categories': ['Network activity'],
'comment': 'subdomains of %s' % domain
2018-06-29 17:19:21 +02:00
}
)
except APIError as e:
2018-07-18 10:47:42 +02:00
misperrors['error'] = e.value
return [], False
2018-07-10 15:17:37 +02:00
2018-06-29 15:50:26 +02:00
return r, status_ok
2018-06-29 16:11:24 +02:00
2018-06-29 17:19:21 +02:00
2018-06-29 17:57:11 +02:00
def expand_whois(api, domain):
r = []
status_ok = False
try:
results = api.whois(domain)
if results:
status_ok = True
item_registrant = __select_registrant_item(results)
2018-07-10 16:35:01 +02:00
if item_registrant:
2018-07-18 10:51:47 +02:00
2018-07-12 14:57:37 +02:00
if 'email' in item_registrant[0]:
2018-07-11 09:00:23 +02:00
r.append(
{
'types': ['whois-registrant-email'],
2018-07-12 14:32:56 +02:00
'values': [item_registrant[0]['email']],
2018-07-11 09:00:23 +02:00
'categories': ['Attribution'],
'comment': 'Whois information of %s by securitytrails'
% domain
}
)
2018-07-12 14:57:37 +02:00
if 'telephone' in item_registrant[0]:
2018-07-11 09:00:23 +02:00
r.append(
{
'types': ['whois-registrant-phone'],
2018-07-12 14:32:56 +02:00
'values': [item_registrant[0]['telephone']],
2018-07-11 09:00:23 +02:00
'categories': ['Attribution'],
'comment': 'Whois information of %s by securitytrails'
% domain
}
)
2018-07-12 14:57:37 +02:00
if 'name' in item_registrant[0]:
2018-07-11 09:00:23 +02:00
r.append(
{
'types': ['whois-registrant-name'],
2018-07-12 14:32:56 +02:00
'values': [item_registrant[0]['name']],
2018-07-11 09:00:23 +02:00
'categories': ['Attribution'],
'comment': 'Whois information of %s by securitytrails'
% domain
}
)
2018-07-12 14:57:37 +02:00
if 'registrarName' in item_registrant[0]:
2018-07-11 09:00:23 +02:00
r.append(
{
'types': ['whois-registrar'],
2018-07-12 14:32:56 +02:00
'values': [item_registrant[0]['registrarName']],
2018-07-11 09:00:23 +02:00
'categories': ['Attribution'],
'comment': 'Whois information of %s by securitytrails'
% domain
}
)
2018-07-12 14:57:37 +02:00
if 'createdDate' in item_registrant[0]:
2018-07-11 09:00:23 +02:00
r.append(
{
'types': ['whois-creation-date'],
2018-07-12 14:32:56 +02:00
'values': [item_registrant[0]['createdDate']],
2018-07-11 09:00:23 +02:00
'categories': ['Attribution'],
'comment': 'Whois information of %s by securitytrails'
% domain
}
)
2018-06-29 17:57:11 +02:00
except APIError as e:
2018-07-18 10:47:42 +02:00
misperrors['error'] = e.value
return [], False
2018-06-29 17:57:11 +02:00
return r, status_ok
2018-07-10 16:31:39 +02:00
2018-07-11 08:43:23 +02:00
def expand_history_ipv4_ipv6(api, domain):
2018-07-10 16:31:39 +02:00
r = []
status_ok = False
try:
results = api.history_dns_ipv4(domain)
if results:
status_ok = True
2018-07-11 08:43:23 +02:00
r.extend(__history_ip(results, domain))
time.sleep(1)
results = api.history_dns_aaaa(domain)
if results:
status_ok = True
2018-07-11 09:02:47 +02:00
r.extend(__history_ip(results, domain, type_ip='ipv6'))
2018-07-10 16:31:39 +02:00
except APIError as e:
2018-07-18 10:47:42 +02:00
misperrors['error'] = e.value
2018-07-12 13:40:51 +02:00
return [], False
2018-07-10 16:31:39 +02:00
return r, status_ok
2018-07-11 09:39:09 +02:00
def expand_history_dns(api, domain):
r = []
status_ok = False
try:
results = api.history_dns_ns(domain)
if results:
2018-07-11 11:24:49 +02:00
r.extend(__history_dns(results, domain, 'nameserver', 'ns'))
time.sleep(1)
2018-07-11 11:37:07 +02:00
results = api.history_dns_soa(domain)
2018-07-11 11:24:49 +02:00
if results:
r.extend(__history_dns(results, domain, 'email', 'soa'))
time.sleep(1)
results = api.history_dns_mx(domain)
if results:
status_ok = True
r.extend(__history_dns(results, domain, 'host', 'mx'))
2018-07-11 09:39:09 +02:00
except APIError as e:
2018-07-18 10:47:42 +02:00
misperrors['error'] = e.value
2018-07-12 13:40:51 +02:00
return [], False
2018-07-11 09:39:09 +02:00
2018-07-11 11:52:10 +02:00
status_ok = True
2018-07-11 09:39:09 +02:00
return r, status_ok
2018-07-12 13:40:51 +02:00
def expand_history_whois(api, domain):
r = []
status_ok = False
try:
results = api.history_whois(domain)
if results:
2018-07-12 14:14:43 +02:00
if 'items' in results['result']:
for item in results['result']['items']:
2018-07-12 13:40:51 +02:00
item_registrant = __select_registrant_item(item)
2018-07-12 14:47:34 +02:00
2018-07-12 14:52:01 +02:00
r.append(
2018-07-12 14:32:56 +02:00
{
2018-07-12 14:52:01 +02:00
'types': ['domain'],
2018-07-12 14:32:56 +02:00
'values': item['nameServers'],
'categories': ['Network activity'],
'comment': 'Whois history Name Servers of %s '
'Status: %s ' % (
2018-07-18 10:58:51 +02:00
domain, ' '.join(item['status']))
2018-07-12 13:40:51 +02:00
2018-07-12 14:32:56 +02:00
}
)
if item_registrant:
if 'email' in item_registrant[0]:
2018-07-12 14:27:59 +02:00
r.append(
{
'types': ['whois-registrant-email'],
2018-07-12 14:32:56 +02:00
'values': [item_registrant[0]['email']],
2018-07-12 14:27:59 +02:00
'categories': ['Attribution'],
'comment': 'Whois history registrant email of %s'
'Status: %s' % (
2018-07-18 10:58:51 +02:00
domain,
' '.join(item['status']))
2018-07-12 14:27:59 +02:00
}
)
2018-07-12 14:55:56 +02:00
if 'telephone' in item_registrant[0]:
2018-07-12 14:27:59 +02:00
r.append(
{
'types': ['whois-registrant-phone'],
2018-07-12 14:32:56 +02:00
'values': [item_registrant[0]['telephone']],
2018-07-12 14:27:59 +02:00
'categories': ['Attribution'],
'comment': 'Whois history registrant phone of %s'
'Status: %s' % (
2018-07-18 10:58:51 +02:00
domain,
' '.join(item['status']))
2018-07-12 14:27:59 +02:00
}
)
2018-07-12 13:40:51 +02:00
except APIError as e:
2018-07-18 10:47:42 +02:00
misperrors['error'] = e.value
2018-07-12 13:40:51 +02:00
return [], False
2018-07-12 14:47:34 +02:00
status_ok = True
2018-07-17 15:05:15 +02:00
2018-07-12 13:40:51 +02:00
return r, status_ok
2018-07-11 09:02:47 +02:00
def __history_ip(results, domain, type_ip='ip'):
2018-07-11 08:43:23 +02:00
r = []
if 'records' in results:
for record in results['records']:
if 'values' in record:
for item in record['values']:
r.append(
{'types': ['domain|ip'],
2018-07-11 09:02:47 +02:00
'values': ['%s|%s' % (domain, item[type_ip])],
2018-07-11 08:43:23 +02:00
'categories': ['Network activity'],
2018-07-11 09:39:09 +02:00
'comment': 'History IP on securitytrails %s '
'last seen: %s first seen: %s' %
(domain, record['last_seen'],
2018-07-11 08:43:23 +02:00
record['first_seen'])
}
)
return r
2018-07-11 09:39:09 +02:00
2018-07-11 11:24:49 +02:00
def __history_dns(results, domain, type_serv, service):
r = []
if 'records' in results:
for record in results['records']:
if 'values' in record:
2018-07-11 11:47:10 +02:00
values = record['values']
if type(values) is list:
for item in record['values']:
r.append(
{'types': ['domain|ip'],
'values': [item[type_serv]],
'categories': ['Network activity'],
'comment': 'history %s of %s last seen: %s first seen: %s' %
(service, domain, record['last_seen'],
record['first_seen'])
}
)
else:
2018-07-11 11:24:49 +02:00
r.append(
{'types': ['domain|ip'],
2018-07-11 11:47:10 +02:00
'values': [values[type_serv]],
2018-07-11 11:24:49 +02:00
'categories': ['Network activity'],
'comment': 'history %s of %s last seen: %s first seen: %s' %
(service, domain, record['last_seen'],
record['first_seen'])
}
)
return r
2018-07-17 15:05:15 +02:00
def expand_searching_domain(api, ip):
r = []
status_ok = False
try:
2018-07-17 17:20:30 +02:00
results = api.searching_domains(ipv4=ip)
2018-07-17 15:05:15 +02:00
if results:
if 'records' in results:
res = [(r['host_provider'], r['hostname'], r['whois'])
for r in results['records']]
for host_provider, hostname, whois in res:
comment = 'domain for %s by %s' % (ip, host_provider[0])
2018-07-17 17:21:38 +02:00
if whois['registrar']:
2018-07-17 15:05:15 +02:00
comment = comment + ' registrar %s' % whois['registrar']
r.append(
{
'types': ['domain'],
'category': ['Network activity'],
'values': [hostname],
'comment': comment
}
)
status_ok = True
2018-07-17 18:32:50 +02:00
except APIError as e:
2018-07-17 18:43:52 +02:00
misperrors['error'] = e.value
2018-07-17 18:32:50 +02:00
return [], False
return r, status_ok
2018-06-29 16:11:24 +02:00
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
2018-06-29 17:57:11 +02:00
return moduleinfo
def __select_registrant_item(entry):
2018-07-17 15:05:15 +02:00
res = None
2018-06-29 17:57:11 +02:00
if 'contacts' in entry:
2018-07-12 14:59:50 +02:00
res = list(filter(lambda x: x['type'] == 'registrant',
entry['contacts']))
2018-07-17 15:05:15 +02:00
2018-07-12 14:27:59 +02:00
if 'contact' in entry:
2018-07-12 14:38:38 +02:00
res = list(filter(lambda x: x['type'] == 'registrant',
entry['contact']))
2018-07-12 14:47:34 +02:00
2018-07-17 15:05:15 +02:00
return res