From 60f772b9050ccd9ae3678dd38bbd9003de2df3c1 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 11:27:36 +0200 Subject: [PATCH 01/78] add new module dnstrails --- misp_modules/modules/expansion/dnstrails.py | 25 +++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 misp_modules/modules/expansion/dnstrails.py diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py new file mode 100644 index 0000000..d77c8d1 --- /dev/null +++ b/misp_modules/modules/expansion/dnstrails.py @@ -0,0 +1,25 @@ +import logging +import sys + +log = logging.getLogger('dnstrails') +log.setLevel(logging.DEBUG) +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +log.addHandler(ch) + +misperrors = {'error': 'Error'} +mispattributes = { + 'input': ['hostname', 'domain', 'ip-src', 'ip-dst'], + 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'dns-soa-email'] +} + +moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', + 'description': 'Query on securitytrails.com', + 'module-type': ['expansion', 'hover']} + +# config fields that your code expects from the site admin +moduleconfig = ['apikey'] + + From 035606a21acfed52a951c56c3ffe5a551d3d4d74 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 11:47:11 +0200 Subject: [PATCH 02/78] add link pydnstrain in requirements --- REQUIREMENTS | 1 + 1 file changed, 1 insertion(+) diff --git a/REQUIREMENTS b/REQUIREMENTS index a8baf52..11393b0 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -12,6 +12,7 @@ asnhistory git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client git+https://github.com/MISP/PyMISP.git#egg=pymisp git+https://github.com/sebdraven/pyonyphe#egg=pyonyphe +git+https://github.com/sebdraven/pydnstrails#egg=pydnstrails pillow pytesseract SPARQLWrapper From cfe971a27185a1f64fc2927f3ffbe9b74a98ffcc Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 15:50:26 +0200 Subject: [PATCH 03/78] add expand domains --- misp_modules/modules/expansion/dnstrails.py | 140 ++++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index d77c8d1..3655269 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -1,5 +1,7 @@ +import json import logging import sys +from dnstrails import DnsTrails log = logging.getLogger('dnstrails') log.setLevel(logging.DEBUG) @@ -23,3 +25,141 @@ moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', moduleconfig = ['apikey'] +def handler(q=False): + if q: + + request = json.loads(q) + + if not request.get('config') and not (request['config'].get('apikey')): + misperrors['error'] = 'DNS authentication is missing' + return misperrors + + api = DnsTrails(request['config'].get('apikey')) + + if not api: + misperrors['error'] = 'Onyphe Error instance api' + + ip = "" + dns_name = "" + + ip = '' + if request.get('ip-src'): + ip = request['ip-src'] + return handle_ip(api, ip, misperrors) + elif request.get('ip-dst'): + ip = request['ip-dst'] + return handle_ip(api, ip, misperrors) + elif request.get('domain'): + domain = request['domain'] + return handle_domain(api, domain, misperrors) + elif request.get('hostname'): + hostname = request['hostname'] + return handle_domain(api, hostname, misperrors) + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + else: + return False + + +def handle_domain(api, domain, misperrors): + result_filtered = {"results": []} + + r, status_ok = expand_domain_info(api, misperrors, domain) + + if status_ok: + result_filtered['results'].extend(r) + else: + misperrors['error'] = 'Error pastries result' + return misperrors + + return result_filtered + +def handle_ip(api, ip, misperrors): + pass + + +def expand_domain_info(api, misperror,domain): + r = [] + status_ok = False + ns_servers = [] + list_ipv4 = [] + list_ipv6 = [] + servers_mx = [] + soa_hostnames = [] + + results = api.domain(domain) + + if results: + if 'current_dns' in results: + if 'values' in results['current_dns']['ns']: + ns_servers = [ns_entry['nameserver'] for ns_entry in + results['current_dns']['ns']['values'] + if 'nameserver' in ns_entry] + if 'values' in results['current_dns']['a']: + list_ipv4 = [a_entry['ip'] for a_entry in + results['current_dns']['a']['values'] if + 'ip' in a_entry] + + if 'values' in results['current_dns']['aaaa']: + list_ipv6 = [ipv6_entry['ipv6'] for ipv6_entry in + results['current_ns']['aaaa']['values'] if + 'ipv6' in ipv6_entry] + + if 'values' in results['current_dns']['mx']: + servers_mx = [mx_entry['hostname'] for mx_entry in + results['current_dns']['mx']['values'] if + 'hostname' in mx_entry] + if 'values' in results['current_dns']['soa']: + soa_hostnames = [soa_entry['email'] for soa_entry in + results['current_dns']['soa']['values'] if + 'email' in soa_entry] + + if ns_servers: + r.append({'type': ['domain'], + 'values': ns_servers, + 'Category': ['Network Activity'], + 'comment': 'List of name servers of %s first seen %s ' % + (domain, results['current_dns']['ns']['first_seen']) + }) + + if list_ipv4: + r.append({'type': ['domain|ip'], + 'values': ['%s|%s' % (domain, ipv4) for ipv4 in list_ipv4], + 'Category': ['Network Activity'], + 'comment': ' List ipv4 of %s first seen %s' % + (domain, + results['current_dns']['a']['first_seen']) + + }) + if list_ipv6: + r.append({'type': ['domain|ip'], + 'values': ['%s|%s' % (domain, ipv6) for ipv6 in + list_ipv6], + 'Category': ['Network Activity'], + 'comment': ' List ipv6 of %s first seen %s' % + (domain, + results['current_dns']['aaaa']['first_seen']) + + }) + + if servers_mx: + r.append({'type': ['domain'], + 'values': servers_mx, + 'Category': ['Network Activity'], + 'comment': ' List mx of %s first seen %s' % + (domain, + results['current_dns']['mx']['first_seen']) + + }) + if soa_hostnames: + r.append({'type': ['domain'], + 'values': soa_hostnames, + 'Category': ['Network Activity'], + 'comment': ' List soa of %s first seen %s' % + (domain, + results['current_dns']['soa']['first_seen']) + }) + + + return r, status_ok From 09c52788b8f2479f1b0571be7514c66220d42baf Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 16:11:24 +0200 Subject: [PATCH 04/78] add methods --- misp_modules/modules/expansion/dnstrails.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 3655269..d357ed8 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -163,3 +163,11 @@ def expand_domain_info(api, misperror,domain): return r, status_ok + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo \ No newline at end of file From f3962d2d0599ef689d2d52b458c36c9cf123fea5 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 16:17:32 +0200 Subject: [PATCH 05/78] add status ! --- misp_modules/modules/expansion/dnstrails.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index d357ed8..5d5e9d4 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -70,7 +70,7 @@ def handle_domain(api, domain, misperrors): if status_ok: result_filtered['results'].extend(r) else: - misperrors['error'] = 'Error pastries result' + misperrors['error'] = 'Error dns result' return misperrors return result_filtered @@ -91,6 +91,7 @@ def expand_domain_info(api, misperror,domain): results = api.domain(domain) if results: + status_ok = True if 'current_dns' in results: if 'values' in results['current_dns']['ns']: ns_servers = [ns_entry['nameserver'] for ns_entry in From 0275e3ecd8b9e976a12f2de2bfa1ac146f4b3fbe Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 16:20:35 +0200 Subject: [PATCH 06/78] changes keys --- misp_modules/modules/expansion/dnstrails.py | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 5d5e9d4..41151e6 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -19,7 +19,7 @@ mispattributes = { moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', 'description': 'Query on securitytrails.com', - 'module-type': ['expansion', 'hover']} + 'module-types': ['expansion', 'hover']} # config fields that your code expects from the site admin moduleconfig = ['apikey'] @@ -56,7 +56,7 @@ def handler(q=False): hostname = request['hostname'] return handle_domain(api, hostname, misperrors) else: - misperrors['error'] = "Unsupported attributes type" + misperrors['error'] = "Unsupported attributes types" return misperrors else: return False @@ -117,27 +117,27 @@ def expand_domain_info(api, misperror,domain): 'email' in soa_entry] if ns_servers: - r.append({'type': ['domain'], + r.append({'types': ['domain'], 'values': ns_servers, - 'Category': ['Network Activity'], + 'categories': ['Network Activity'], 'comment': 'List of name servers of %s first seen %s ' % (domain, results['current_dns']['ns']['first_seen']) }) if list_ipv4: - r.append({'type': ['domain|ip'], + r.append({'types': ['domain|ip'], 'values': ['%s|%s' % (domain, ipv4) for ipv4 in list_ipv4], - 'Category': ['Network Activity'], + 'categories': ['Network Activity'], 'comment': ' List ipv4 of %s first seen %s' % (domain, results['current_dns']['a']['first_seen']) }) if list_ipv6: - r.append({'type': ['domain|ip'], + r.append({'types': ['domain|ip'], 'values': ['%s|%s' % (domain, ipv6) for ipv6 in list_ipv6], - 'Category': ['Network Activity'], + 'categories': ['Network Activity'], 'comment': ' List ipv6 of %s first seen %s' % (domain, results['current_dns']['aaaa']['first_seen']) @@ -145,18 +145,18 @@ def expand_domain_info(api, misperror,domain): }) if servers_mx: - r.append({'type': ['domain'], + r.append({'types': ['domain'], 'values': servers_mx, - 'Category': ['Network Activity'], + 'categories': ['Network Activity'], 'comment': ' List mx of %s first seen %s' % (domain, results['current_dns']['mx']['first_seen']) }) if soa_hostnames: - r.append({'type': ['domain'], + r.append({'types': ['domain'], 'values': soa_hostnames, - 'Category': ['Network Activity'], + 'categories': ['Network Activity'], 'comment': ' List soa of %s first seen %s' % (domain, results['current_dns']['soa']['first_seen']) From 2d1adf4aa959aa9874ef4dc7b0ec79a133f3fbc2 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 16:30:47 +0200 Subject: [PATCH 07/78] change categories --- misp_modules/modules/expansion/dnstrails.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 41151e6..d2fd5e1 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -119,7 +119,7 @@ def expand_domain_info(api, misperror,domain): if ns_servers: r.append({'types': ['domain'], 'values': ns_servers, - 'categories': ['Network Activity'], + 'categories': ['Network activity'], 'comment': 'List of name servers of %s first seen %s ' % (domain, results['current_dns']['ns']['first_seen']) }) @@ -127,7 +127,8 @@ def expand_domain_info(api, misperror,domain): if list_ipv4: r.append({'types': ['domain|ip'], 'values': ['%s|%s' % (domain, ipv4) for ipv4 in list_ipv4], - 'categories': ['Network Activity'], + 'categories': ['Network activity'], + 'comment': ' List ipv4 of %s first seen %s' % (domain, results['current_dns']['a']['first_seen']) @@ -137,7 +138,7 @@ def expand_domain_info(api, misperror,domain): r.append({'types': ['domain|ip'], 'values': ['%s|%s' % (domain, ipv6) for ipv6 in list_ipv6], - 'categories': ['Network Activity'], + 'categories': ['Network activity'], 'comment': ' List ipv6 of %s first seen %s' % (domain, results['current_dns']['aaaa']['first_seen']) @@ -147,7 +148,7 @@ def expand_domain_info(api, misperror,domain): if servers_mx: r.append({'types': ['domain'], 'values': servers_mx, - 'categories': ['Network Activity'], + 'categories': ['Network activity'], 'comment': ' List mx of %s first seen %s' % (domain, results['current_dns']['mx']['first_seen']) @@ -156,7 +157,7 @@ def expand_domain_info(api, misperror,domain): if soa_hostnames: r.append({'types': ['domain'], 'values': soa_hostnames, - 'categories': ['Network Activity'], + 'categories': ['Network activity'], 'comment': ' List soa of %s first seen %s' % (domain, results['current_dns']['soa']['first_seen']) From 64847a8a04253f68b016e0283dd06cf5972a98ca Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 17:19:21 +0200 Subject: [PATCH 08/78] add expand subdomains --- misp_modules/modules/expansion/dnstrails.py | 25 ++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index d2fd5e1..ffb4055 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -2,6 +2,7 @@ import json import logging import sys from dnstrails import DnsTrails +from dnstrails import APIError log = logging.getLogger('dnstrails') log.setLevel(logging.DEBUG) @@ -163,9 +164,31 @@ def expand_domain_info(api, misperror,domain): results['current_dns']['soa']['first_seen']) }) - return r, status_ok + +def expand_subdomains(api, domain): + + r = [] + status_ok = False + + try: + results = api.subdomains(domain) + + if results: + status_ok = True + if 'subdomains' in results: + r.append({ + 'type': ['domain'], + 'values': ['%s.%s' % (sub,domain) for sub in results['subdomains']], + } + + ) + except APIError as e: + misperrors['error'] = e + return r, status_ok + + def introspection(): return mispattributes From 0965def6bfd9d4e6def23d8b442e7898238b534c Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 17:22:19 +0200 Subject: [PATCH 09/78] add expand subdomains --- misp_modules/modules/expansion/dnstrails.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index ffb4055..abef1e5 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -1,8 +1,9 @@ import json import logging import sys -from dnstrails import DnsTrails + from dnstrails import APIError +from dnstrails import DnsTrails log = logging.getLogger('dnstrails') log.setLevel(logging.DEBUG) @@ -68,6 +69,14 @@ def handle_domain(api, domain, misperrors): r, status_ok = expand_domain_info(api, misperrors, domain) + if status_ok: + result_filtered['results'].extend(r) + else: + misperrors['error'] = 'Error dns result' + return misperrors + + r, status_ok = expand_subdomains(api, domain) + if status_ok: result_filtered['results'].extend(r) else: From 78d6de9b7a47d4a997cfa996a9ae7f7b8014ab95 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 17:25:37 +0200 Subject: [PATCH 10/78] add categories and comments --- misp_modules/modules/expansion/dnstrails.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index abef1e5..1fd7500 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -190,6 +190,8 @@ def expand_subdomains(api, domain): r.append({ 'type': ['domain'], 'values': ['%s.%s' % (sub,domain) for sub in results['subdomains']], + 'categories': ['Network activity'], + 'comment': 'subdomains of %s' % domain } ) From f1c6095914381aa41feed5a2043989665e04eabc Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 17:26:56 +0200 Subject: [PATCH 11/78] typo --- misp_modules/modules/expansion/dnstrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 1fd7500..79afeb3 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -188,7 +188,7 @@ def expand_subdomains(api, domain): status_ok = True if 'subdomains' in results: r.append({ - 'type': ['domain'], + 'types': ['domain'], 'values': ['%s.%s' % (sub,domain) for sub in results['subdomains']], 'categories': ['Network activity'], 'comment': 'subdomains of %s' % domain From 34da5cdb767232bdf03c087f2368eb104f0b88d2 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Fri, 29 Jun 2018 17:57:11 +0200 Subject: [PATCH 12/78] add expand whois --- misp_modules/modules/expansion/dnstrails.py | 53 ++++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 79afeb3..d1c276a 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -16,7 +16,10 @@ log.addHandler(ch) misperrors = {'error': 'Error'} mispattributes = { 'input': ['hostname', 'domain', 'ip-src', 'ip-dst'], - 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'dns-soa-email'] + 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'dns-soa-email', + 'whois-registrant-email', 'whois-registrant-phone', + 'whois-registrant-name', + 'whois-registrar', 'whois-creation-date', 'domain'] } moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', @@ -77,6 +80,14 @@ def handle_domain(api, domain, misperrors): r, status_ok = expand_subdomains(api, domain) + if status_ok: + result_filtered['results'].extend(r) + else: + misperrors['error'] = 'Error dns result' + return misperrors + + r, status_ok = expand_whois(api, domain) + if status_ok: result_filtered['results'].extend(r) else: @@ -181,6 +192,7 @@ def expand_subdomains(api, domain): r = [] status_ok = False + try: results = api.subdomains(domain) @@ -200,10 +212,47 @@ def expand_subdomains(api, domain): return r, status_ok +def expand_whois(api, domain): + r = [] + status_ok = False + + try: + results = api.whois(domain) + + if results: + status_ok = True + item_registrant = __select_registrant_item(results) + + r.append({ + 'types': ['whois-registrant-email', 'whois-registrant-phone', + 'whois-registrant-name', 'whois-registrar', + 'whois-creation-date'], + 'values': [item_registrant['email'], + item_registrant['telephone'], + item_registrant['name'], results['registrarName'], + results['creationDate']], + 'categories': ['attribution'], + 'comment': 'whois information of %s by securitytrails' % domain + } + + ) + + except APIError as e: + misperrors['error'] = e + + return r, status_ok + def introspection(): return mispattributes def version(): moduleinfo['config'] = moduleconfig - return moduleinfo \ No newline at end of file + return moduleinfo + + +def __select_registrant_item(entry): + if 'contacts' in entry: + for c in entry['contacts']: + if c['type'] == 'registrant': + return entry From 2a8fb76e8483ee4df174b94275b0b3c883583aaa Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 14:56:20 +0200 Subject: [PATCH 13/78] add logs --- misp_modules/modules/expansion/dnstrails.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index d1c276a..cb00262 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -236,6 +236,9 @@ def expand_whois(api, domain): } ) + # TODO File "modules/expansion/dnstrails.py", line 230, in expand_whois + # 'values': [item_registrant['email'], + # TypeError: 'NoneType' object is not subscriptable except APIError as e: misperrors['error'] = e @@ -254,5 +257,6 @@ def version(): def __select_registrant_item(entry): if 'contacts' in entry: for c in entry['contacts']: + print(c) if c['type'] == 'registrant': return entry From f710162beda5cee2005cff6df50d30a09c1af25d Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 14:59:39 +0200 Subject: [PATCH 14/78] change errors --- misp_modules/modules/expansion/dnstrails.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index cb00262..bf16601 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -83,7 +83,7 @@ def handle_domain(api, domain, misperrors): if status_ok: result_filtered['results'].extend(r) else: - misperrors['error'] = 'Error dns result' + misperrors['error'] = 'Error subdomains result' return misperrors r, status_ok = expand_whois(api, domain) @@ -91,7 +91,7 @@ def handle_domain(api, domain, misperrors): if status_ok: result_filtered['results'].extend(r) else: - misperrors['error'] = 'Error dns result' + misperrors['error'] = 'Error whois result' return misperrors return result_filtered From e1a1648f14c1834892af9b036be494bf251b2b18 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 15:01:04 +0200 Subject: [PATCH 15/78] add logs --- misp_modules/modules/expansion/dnstrails.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index bf16601..9787a15 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -242,6 +242,7 @@ def expand_whois(api, domain): except APIError as e: misperrors['error'] = e + print(e) return r, status_ok From 714c15f079c91b82c796239238859501c84f3ea8 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 15:05:10 +0200 Subject: [PATCH 16/78] change return value --- misp_modules/modules/expansion/dnstrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 9787a15..5f2cc63 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -260,4 +260,4 @@ def __select_registrant_item(entry): for c in entry['contacts']: print(c) if c['type'] == 'registrant': - return entry + return c From 1223d93d52faea1105e8bd289691e33bc12e31a8 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 15:07:54 +0200 Subject: [PATCH 17/78] change name keys --- misp_modules/modules/expansion/dnstrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 5f2cc63..5a0cfcc 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -230,7 +230,7 @@ def expand_whois(api, domain): 'values': [item_registrant['email'], item_registrant['telephone'], item_registrant['name'], results['registrarName'], - results['creationDate']], + results['createdDate']], 'categories': ['attribution'], 'comment': 'whois information of %s by securitytrails' % domain } From 1d100833a4cade1d942e91c9f176ef0aea6aded7 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 15:12:27 +0200 Subject: [PATCH 18/78] concat results --- misp_modules/modules/expansion/dnstrails.py | 29 +++++++++++---------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 5a0cfcc..6f52dff 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -222,27 +222,28 @@ def expand_whois(api, domain): if results: status_ok = True item_registrant = __select_registrant_item(results) - - r.append({ - 'types': ['whois-registrant-email', 'whois-registrant-phone', + types = ['whois-registrant-email', 'whois-registrant-phone', 'whois-registrant-name', 'whois-registrar', - 'whois-creation-date'], - 'values': [item_registrant['email'], - item_registrant['telephone'], - item_registrant['name'], results['registrarName'], - results['createdDate']], - 'categories': ['attribution'], - 'comment': 'whois information of %s by securitytrails' % domain - } + 'whois-creation-date'] + values = [item_registrant['email'], + item_registrant['telephone'], + item_registrant['name'], results['registrarName'], + results['createdDate']] + for t, v in zip(types, values): + r.append({ + 'types': t, + 'values': v, + 'categories': ['attribution'], + 'comment': 'whois information of %s by securitytrails' % domain + } - ) + ) # TODO File "modules/expansion/dnstrails.py", line 230, in expand_whois # 'values': [item_registrant['email'], # TypeError: 'NoneType' object is not subscriptable except APIError as e: misperrors['error'] = e - print(e) return r, status_ok @@ -258,6 +259,6 @@ def version(): def __select_registrant_item(entry): if 'contacts' in entry: for c in entry['contacts']: - print(c) + if c['type'] == 'registrant': return c From b677cd5fc7ef1a69b938ff8020e15c42142b5dfd Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 15:16:02 +0200 Subject: [PATCH 19/78] change categories --- misp_modules/modules/expansion/dnstrails.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 6f52dff..8d17fa5 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -229,15 +229,14 @@ def expand_whois(api, domain): item_registrant['telephone'], item_registrant['name'], results['registrarName'], results['createdDate']] - for t, v in zip(types, values): - r.append({ + + r = [{ 'types': t, 'values': v, - 'categories': ['attribution'], + 'categories': ['Attribution'], 'comment': 'whois information of %s by securitytrails' % domain - } + } for t, v in zip(types, values)] - ) # TODO File "modules/expansion/dnstrails.py", line 230, in expand_whois # 'values': [item_registrant['email'], # TypeError: 'NoneType' object is not subscriptable From 21794249d0339b5910e6db09292119b51066b488 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 15:17:37 +0200 Subject: [PATCH 20/78] add logs --- misp_modules/modules/expansion/dnstrails.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 8d17fa5..8ebecda 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -209,6 +209,7 @@ def expand_subdomains(api, domain): ) except APIError as e: misperrors['error'] = e + return r, status_ok @@ -243,6 +244,7 @@ def expand_whois(api, domain): except APIError as e: misperrors['error'] = e + print(e) return r, status_ok From 495c720d0fecb6552757d026dc175598814c5e4b Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 16:31:39 +0200 Subject: [PATCH 21/78] add history ipv4 --- misp_modules/modules/expansion/dnstrails.py | 75 ++++++++++++++++----- 1 file changed, 58 insertions(+), 17 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 8ebecda..e0b78fa 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -9,7 +9,8 @@ log = logging.getLogger('dnstrails') log.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) log.addHandler(ch) @@ -94,13 +95,22 @@ def handle_domain(api, domain, misperrors): misperrors['error'] = 'Error whois result' return misperrors + r, status_ok = expand_history_ipv4(api, domain) + + if status_ok: + result_filtered['results'].extend(r) + else: + misperrors['error'] = 'Error history ipv4' + return misperrors + return result_filtered + def handle_ip(api, ip, misperrors): pass -def expand_domain_info(api, misperror,domain): +def expand_domain_info(api, misperror, domain): r = [] status_ok = False ns_servers = [] @@ -130,31 +140,33 @@ def expand_domain_info(api, misperror,domain): if 'values' in results['current_dns']['mx']: servers_mx = [mx_entry['hostname'] for mx_entry in - results['current_dns']['mx']['values'] if - 'hostname' in mx_entry] + results['current_dns']['mx']['values'] if + 'hostname' in mx_entry] if 'values' in results['current_dns']['soa']: soa_hostnames = [soa_entry['email'] for soa_entry in - results['current_dns']['soa']['values'] if - 'email' in soa_entry] + results['current_dns']['soa']['values'] if + 'email' in soa_entry] if ns_servers: r.append({'types': ['domain'], 'values': ns_servers, 'categories': ['Network activity'], 'comment': 'List of name servers of %s first seen %s ' % - (domain, results['current_dns']['ns']['first_seen']) - }) + (domain, + results['current_dns']['ns']['first_seen']) + }) if list_ipv4: r.append({'types': ['domain|ip'], - 'values': ['%s|%s' % (domain, ipv4) for ipv4 in list_ipv4], + 'values': ['%s|%s' % (domain, ipv4) for ipv4 in + list_ipv4], 'categories': ['Network activity'], 'comment': ' List ipv4 of %s first seen %s' % (domain, results['current_dns']['a']['first_seen']) - }) + }) if list_ipv6: r.append({'types': ['domain|ip'], 'values': ['%s|%s' % (domain, ipv6) for ipv6 in @@ -188,11 +200,9 @@ def expand_domain_info(api, misperror,domain): def expand_subdomains(api, domain): - r = [] status_ok = False - try: results = api.subdomains(domain) @@ -201,7 +211,8 @@ def expand_subdomains(api, domain): if 'subdomains' in results: r.append({ 'types': ['domain'], - 'values': ['%s.%s' % (sub,domain) for sub in results['subdomains']], + 'values': ['%s.%s' % (sub, domain) + for sub in results['subdomains']], 'categories': ['Network activity'], 'comment': 'subdomains of %s' % domain } @@ -224,7 +235,7 @@ def expand_whois(api, domain): status_ok = True item_registrant = __select_registrant_item(results) types = ['whois-registrant-email', 'whois-registrant-phone', - 'whois-registrant-name', 'whois-registrar', + 'whois-registrant-name', 'whois-registrar', 'whois-creation-date'] values = [item_registrant['email'], item_registrant['telephone'], @@ -232,10 +243,10 @@ def expand_whois(api, domain): results['createdDate']] r = [{ - 'types': t, - 'values': v, + 'types': t, + 'values': v, 'categories': ['Attribution'], - 'comment': 'whois information of %s by securitytrails' % domain + 'comment': 'whois information of %s by securitytrails' % domain } for t, v in zip(types, values)] # TODO File "modules/expansion/dnstrails.py", line 230, in expand_whois @@ -248,6 +259,36 @@ def expand_whois(api, domain): return r, status_ok + +def expand_history_ipv4(api, domain): + r = [] + status_ok = False + + try: + results = api.history_dns_ipv4(domain) + + if results: + status_ok = True + if 'records' in results: + for record in results['records']: + if 'values' in record: + r.append( + {'type': ['domain|ip'], + 'values': ['%s|%s' % (domain, record['ip'])], + 'categories': ['Newtwork activity'], + 'comment': 'last seen: %s first seen: %s' % + (record['last_seen'], + record['first_seen']) + } + ) + + except APIError as e: + misperrors['error'] = e + print(e) + + return r, status_ok + + def introspection(): return mispattributes From 602da3d1a3baa52005678025b2a320d87ada5286 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 16:35:01 +0200 Subject: [PATCH 22/78] control return of records --- misp_modules/modules/expansion/dnstrails.py | 27 +++++++++++---------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index e0b78fa..d8d9a53 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -234,20 +234,21 @@ def expand_whois(api, domain): if results: status_ok = True item_registrant = __select_registrant_item(results) - types = ['whois-registrant-email', 'whois-registrant-phone', - 'whois-registrant-name', 'whois-registrar', - 'whois-creation-date'] - values = [item_registrant['email'], - item_registrant['telephone'], - item_registrant['name'], results['registrarName'], - results['createdDate']] + if item_registrant: + types = ['whois-registrant-email', 'whois-registrant-phone', + 'whois-registrant-name', 'whois-registrar', + 'whois-creation-date'] + values = [item_registrant['email'], + item_registrant['telephone'], + item_registrant['name'], results['registrarName'], + results['createdDate']] - r = [{ - 'types': t, - 'values': v, - 'categories': ['Attribution'], - 'comment': 'whois information of %s by securitytrails' % domain - } for t, v in zip(types, values)] + r = [{ + 'types': t, + 'values': v, + 'categories': ['Attribution'], + 'comment': 'whois information of %s by securitytrails' % domain + } for t, v in zip(types, values)] # TODO File "modules/expansion/dnstrails.py", line 230, in expand_whois # 'values': [item_registrant['email'], From e9747a3379dbaaf225eec6411b8d2ca9fa7ad936 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 16:41:44 +0200 Subject: [PATCH 23/78] add time sleep in each request --- misp_modules/modules/expansion/dnstrails.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index d8d9a53..b352bd7 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -1,6 +1,7 @@ import json import logging import sys +import time from dnstrails import APIError from dnstrails import DnsTrails @@ -79,6 +80,7 @@ def handle_domain(api, domain, misperrors): misperrors['error'] = 'Error dns result' return misperrors + time.sleep(1) r, status_ok = expand_subdomains(api, domain) if status_ok: @@ -87,6 +89,7 @@ def handle_domain(api, domain, misperrors): misperrors['error'] = 'Error subdomains result' return misperrors + time.sleep(1) r, status_ok = expand_whois(api, domain) if status_ok: @@ -95,6 +98,7 @@ def handle_domain(api, domain, misperrors): misperrors['error'] = 'Error whois result' return misperrors + time.sleep(1) r, status_ok = expand_history_ipv4(api, domain) if status_ok: From 26950ea7de576906ae4ab5ae38b3f1cf4874d23e Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 16:51:31 +0200 Subject: [PATCH 24/78] change loop --- misp_modules/modules/expansion/dnstrails.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index b352bd7..f950500 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -277,15 +277,16 @@ def expand_history_ipv4(api, domain): if 'records' in results: for record in results['records']: if 'values' in record: - r.append( - {'type': ['domain|ip'], - 'values': ['%s|%s' % (domain, record['ip'])], - 'categories': ['Newtwork activity'], - 'comment': 'last seen: %s first seen: %s' % - (record['last_seen'], - record['first_seen']) - } - ) + for item in record['values']: + r.append( + {'type': ['domain|ip'], + 'values': ['%s|%s' % (domain, item['ip'])], + 'categories': ['Newtwork activity'], + 'comment': 'last seen: %s first seen: %s' % + (record['last_seen'], + record['first_seen']) + } + ) except APIError as e: misperrors['error'] = e From 9e6162a4349a14eb66cc83ad57eadb5f6ee456d9 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 16:53:06 +0200 Subject: [PATCH 25/78] change type --- misp_modules/modules/expansion/dnstrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index f950500..c30c747 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -281,7 +281,7 @@ def expand_history_ipv4(api, domain): r.append( {'type': ['domain|ip'], 'values': ['%s|%s' % (domain, item['ip'])], - 'categories': ['Newtwork activity'], + 'categories': ['Network activity'], 'comment': 'last seen: %s first seen: %s' % (record['last_seen'], record['first_seen']) From f2333a4978e016e2a870e3cf1c7b1e8fcf0e25e0 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 10 Jul 2018 16:55:13 +0200 Subject: [PATCH 26/78] change type --- misp_modules/modules/expansion/dnstrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index c30c747..a176489 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -279,7 +279,7 @@ def expand_history_ipv4(api, domain): if 'values' in record: for item in record['values']: r.append( - {'type': ['domain|ip'], + {'types': ['domain|ip'], 'values': ['%s|%s' % (domain, item['ip'])], 'categories': ['Network activity'], 'comment': 'last seen: %s first seen: %s' % From 3a96e189ed5f45780056226566da0fcdef0ab096 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 08:43:23 +0200 Subject: [PATCH 27/78] add ipv6 and ipv4 --- misp_modules/modules/expansion/dnstrails.py | 41 ++++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index a176489..02e1346 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -265,7 +265,7 @@ def expand_whois(api, domain): return r, status_ok -def expand_history_ipv4(api, domain): +def expand_history_ipv4_ipv6(api, domain): r = [] status_ok = False @@ -274,19 +274,14 @@ def expand_history_ipv4(api, domain): if results: status_ok = True - if 'records' in results: - for record in results['records']: - if 'values' in record: - for item in record['values']: - r.append( - {'types': ['domain|ip'], - 'values': ['%s|%s' % (domain, item['ip'])], - 'categories': ['Network activity'], - 'comment': 'last seen: %s first seen: %s' % - (record['last_seen'], - record['first_seen']) - } - ) + r.extend(__history_ip(results, domain)) + + time.sleep(1) + results = api.history_dns_aaaa(domain) + + if results: + status_ok = True + r.extend(__history_ip(results, domain)) except APIError as e: misperrors['error'] = e @@ -295,6 +290,24 @@ def expand_history_ipv4(api, domain): return r, status_ok +def __history_ip(results, domain): + r = [] + if 'records' in results: + for record in results['records']: + if 'values' in record: + for item in record['values']: + r.append( + {'types': ['domain|ip'], + 'values': ['%s|%s' % (domain, item['ip'])], + 'categories': ['Network activity'], + 'comment': 'last seen: %s first seen: %s' % + (record['last_seen'], + record['first_seen']) + } + ) + + return r + def introspection(): return mispattributes From 41635d43c772f15bb693260504b67df2eb88727a Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 08:49:59 +0200 Subject: [PATCH 28/78] correct typo --- misp_modules/modules/expansion/dnstrails.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 02e1346..c601481 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -99,7 +99,7 @@ def handle_domain(api, domain, misperrors): return misperrors time.sleep(1) - r, status_ok = expand_history_ipv4(api, domain) + r, status_ok = expand_history_ipv4_ipv6(api, domain) if status_ok: result_filtered['results'].extend(r) @@ -139,7 +139,7 @@ def expand_domain_info(api, misperror, domain): if 'values' in results['current_dns']['aaaa']: list_ipv6 = [ipv6_entry['ipv6'] for ipv6_entry in - results['current_ns']['aaaa']['values'] if + results['current_dns']['aaaa']['values'] if 'ipv6' in ipv6_entry] if 'values' in results['current_dns']['mx']: From 42c362d2fd6a629184b8887ff7876b4f7a8f52de Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 09:00:23 +0200 Subject: [PATCH 29/78] refactoring expand_whois --- misp_modules/modules/expansion/dnstrails.py | 68 +++++++++++++++++---- 1 file changed, 55 insertions(+), 13 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index c601481..7d18042 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -239,20 +239,62 @@ def expand_whois(api, domain): status_ok = True item_registrant = __select_registrant_item(results) if item_registrant: - types = ['whois-registrant-email', 'whois-registrant-phone', - 'whois-registrant-name', 'whois-registrar', - 'whois-creation-date'] - values = [item_registrant['email'], - item_registrant['telephone'], - item_registrant['name'], results['registrarName'], - results['createdDate']] - r = [{ - 'types': t, - 'values': v, - 'categories': ['Attribution'], - 'comment': 'whois information of %s by securitytrails' % domain - } for t, v in zip(types, values)] + if 'email' in item_registrant: + r.append( + { + 'types': ['whois-registrant-email'], + 'values': [item_registrant['email']], + 'categories': ['Attribution'], + 'comment': 'Whois information of %s by securitytrails' + % domain + } + ) + + if 'telephone' in item_registrant: + r.append( + { + 'types': ['whois-registrant-phone'], + 'values': [item_registrant['telephone']], + 'categories': ['Attribution'], + 'comment': 'Whois information of %s by securitytrails' + % domain + } + ) + + if 'name' in item_registrant: + r.append( + { + 'types': ['whois-registrant-name'], + 'values': [item_registrant['name']], + 'categories': ['Attribution'], + 'comment': 'Whois information of %s by securitytrails' + % domain + } + ) + + if 'registrarName' in item_registrant: + r.append( + { + 'types': ['whois-registrar'], + 'values': [item_registrant['registrarName']], + 'categories': ['Attribution'], + 'comment': 'Whois information of %s by securitytrails' + % domain + } + ) + + if 'createdDate' in item_registrant: + r.append( + { + 'types': ['whois-creation-date'], + 'values': [item_registrant['createdDate']], + 'categories': ['Attribution'], + 'comment': 'Whois information of %s by securitytrails' + % domain + } + ) + # TODO File "modules/expansion/dnstrails.py", line 230, in expand_whois # 'values': [item_registrant['email'], From dcdb6e589556a09a562f78a9ceb6684078eef0b0 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 09:02:47 +0200 Subject: [PATCH 30/78] switch type ip --- misp_modules/modules/expansion/dnstrails.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 7d18042..ee9f1ad 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -323,7 +323,7 @@ def expand_history_ipv4_ipv6(api, domain): if results: status_ok = True - r.extend(__history_ip(results, domain)) + r.extend(__history_ip(results, domain, type_ip='ipv6')) except APIError as e: misperrors['error'] = e @@ -332,7 +332,7 @@ def expand_history_ipv4_ipv6(api, domain): return r, status_ok -def __history_ip(results, domain): +def __history_ip(results, domain, type_ip='ip'): r = [] if 'records' in results: for record in results['records']: @@ -340,7 +340,7 @@ def __history_ip(results, domain): for item in record['values']: r.append( {'types': ['domain|ip'], - 'values': ['%s|%s' % (domain, item['ip'])], + 'values': ['%s|%s' % (domain, item[type_ip])], 'categories': ['Network activity'], 'comment': 'last seen: %s first seen: %s' % (record['last_seen'], From 54d996cb00b47046558c5ad30c12bda5066fe9aa Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 09:39:09 +0200 Subject: [PATCH 31/78] add history dns --- misp_modules/modules/expansion/dnstrails.py | 42 +++++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index ee9f1ad..24e252b 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -295,11 +295,6 @@ def expand_whois(api, domain): } ) - - # TODO File "modules/expansion/dnstrails.py", line 230, in expand_whois - # 'values': [item_registrant['email'], - # TypeError: 'NoneType' object is not subscriptable - except APIError as e: misperrors['error'] = e print(e) @@ -332,6 +327,37 @@ def expand_history_ipv4_ipv6(api, domain): return r, status_ok +def expand_history_dns(api, domain): + r = [] + status_ok = False + + try: + + results = api.history_dns_ns(domain) + if results: + status_ok = True + + if 'records' in results: + for record in results['records']: + if 'values' in record: + for item in record['values']: + r.append( + {'types': ['domain|ip'], + 'values': [ + '%s|%s' % (domain, item['nameserver'])], + 'categories': ['Network activity'], + 'comment': 'history DNS of %s last seen: %s first seen: %s' % + (domain, record['last_seen'], + record['first_seen']) + } + ) + + except APIError as e: + misperrors['error'] = e + + return r, status_ok + + def __history_ip(results, domain, type_ip='ip'): r = [] if 'records' in results: @@ -342,14 +368,16 @@ def __history_ip(results, domain, type_ip='ip'): {'types': ['domain|ip'], 'values': ['%s|%s' % (domain, item[type_ip])], 'categories': ['Network activity'], - 'comment': 'last seen: %s first seen: %s' % - (record['last_seen'], + 'comment': 'History IP on securitytrails %s ' + 'last seen: %s first seen: %s' % + (domain, record['last_seen'], record['first_seen']) } ) return r + def introspection(): return mispattributes From 43a49dafc6d993b7a9142e7f16993eb209c5038a Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 09:48:14 +0200 Subject: [PATCH 32/78] add history dns and handler exception --- misp_modules/modules/expansion/dnstrails.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 24e252b..89e45ec 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -77,7 +77,7 @@ def handle_domain(api, domain, misperrors): if status_ok: result_filtered['results'].extend(r) else: - misperrors['error'] = 'Error dns result' + misperrors['error'] = misperrors['error'] + ' Error DNS result' return misperrors time.sleep(1) @@ -86,7 +86,7 @@ def handle_domain(api, domain, misperrors): if status_ok: result_filtered['results'].extend(r) else: - misperrors['error'] = 'Error subdomains result' + misperrors['error'] = misperrors['error'] + ' Error subdomains result' return misperrors time.sleep(1) @@ -95,7 +95,7 @@ def handle_domain(api, domain, misperrors): if status_ok: result_filtered['results'].extend(r) else: - misperrors['error'] = 'Error whois result' + misperrors['error'] = misperrors['error'] + ' Error whois result' return misperrors time.sleep(1) @@ -104,9 +104,18 @@ def handle_domain(api, domain, misperrors): if status_ok: result_filtered['results'].extend(r) else: - misperrors['error'] = 'Error history ipv4' + misperrors['error'] = misperrors['error'] + ' Error history ipv4' return misperrors + time.sleep(1) + r, status_ok = expand_history_dns(api, domain) + + if status_ok: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors[ + 'error'] + ' Error in expand History DNS' + return misperrors return result_filtered From f47a64b3647fde0cf70d582dac831beada066779 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 11:24:49 +0200 Subject: [PATCH 33/78] add history mx and soa --- misp_modules/modules/expansion/dnstrails.py | 49 +++++++++++++++------ 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 89e45ec..0e79ad0 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -346,20 +346,23 @@ def expand_history_dns(api, domain): if results: status_ok = True - if 'records' in results: - for record in results['records']: - if 'values' in record: - for item in record['values']: - r.append( - {'types': ['domain|ip'], - 'values': [ - '%s|%s' % (domain, item['nameserver'])], - 'categories': ['Network activity'], - 'comment': 'history DNS of %s last seen: %s first seen: %s' % - (domain, record['last_seen'], - record['first_seen']) - } - ) + r.extend(__history_dns(results, domain, 'nameserver', 'ns')) + + time.sleep(1) + + results = api.history_dns_soa(results, domain) + + if results: + status_ok = True + r.extend(__history_dns(results, domain, 'email', 'soa')) + + time.sleep(1) + + results = api.history_dns_mx(domain) + + if results: + status_ok = True + r.extend(__history_dns(results, domain, 'host', 'mx')) except APIError as e: misperrors['error'] = e @@ -387,6 +390,24 @@ def __history_ip(results, domain, type_ip='ip'): return r +def __history_dns(results, domain, type_serv, service): + r = [] + + if 'records' in results: + for record in results['records']: + if 'values' in record: + for item in record['values']: + r.append( + {'types': ['domain|ip'], + 'values': [item[type_serv]], + 'categories': ['Network activity'], + 'comment': 'history %s of %s last seen: %s first seen: %s' % + (service, domain, record['last_seen'], + record['first_seen']) + } + ) + return r + def introspection(): return mispattributes From 74c611d2fb38c084084c0aca2a59cfb5189b5379 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 11:37:07 +0200 Subject: [PATCH 34/78] correct call function --- misp_modules/modules/expansion/dnstrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 0e79ad0..86a01a7 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -350,7 +350,7 @@ def expand_history_dns(api, domain): time.sleep(1) - results = api.history_dns_soa(results, domain) + results = api.history_dns_soa(domain) if results: status_ok = True From 560dacbf7e7491ac5bd75d62f91aa1d3d1aacf3f Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 11:40:22 +0200 Subject: [PATCH 35/78] add logs to debug --- misp_modules/modules/expansion/dnstrails.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 86a01a7..1a3f720 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -397,6 +397,7 @@ def __history_dns(results, domain, type_serv, service): for record in results['records']: if 'values' in record: for item in record['values']: + print(item) r.append( {'types': ['domain|ip'], 'values': [item[type_serv]], From 64e7f9c8b628115339d848b1ed57918167dc7a4b Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 11:47:10 +0200 Subject: [PATCH 36/78] change history dns --- misp_modules/modules/expansion/dnstrails.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 1a3f720..3343fa4 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -396,11 +396,23 @@ def __history_dns(results, domain, type_serv, service): if 'records' in results: for record in results['records']: if 'values' in record: - for item in record['values']: - print(item) + values = record['values'] + if type(values) is list: + + for item in record['values']: + r.append( + {'types': ['domain|ip'], + 'values': [item[type_serv]], + 'categories': ['Network activity'], + 'comment': 'history %s of %s last seen: %s first seen: %s' % + (service, domain, record['last_seen'], + record['first_seen']) + } + ) + else: r.append( {'types': ['domain|ip'], - 'values': [item[type_serv]], + 'values': [values[type_serv]], 'categories': ['Network activity'], 'comment': 'history %s of %s last seen: %s first seen: %s' % (service, domain, record['last_seen'], From 45c473aef504562b6bea2b37d8f671f827338ddf Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 11:52:10 +0200 Subject: [PATCH 37/78] change status --- misp_modules/modules/expansion/dnstrails.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 3343fa4..cab4778 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -344,7 +344,7 @@ def expand_history_dns(api, domain): results = api.history_dns_ns(domain) if results: - status_ok = True + r.extend(__history_dns(results, domain, 'nameserver', 'ns')) @@ -353,7 +353,6 @@ def expand_history_dns(api, domain): results = api.history_dns_soa(domain) if results: - status_ok = True r.extend(__history_dns(results, domain, 'email', 'soa')) time.sleep(1) @@ -367,6 +366,8 @@ def expand_history_dns(api, domain): except APIError as e: misperrors['error'] = e + status_ok = True + return r, status_ok From 45decc728dcd0391f0b1dfad08684e9844757b7f Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 11:55:31 +0200 Subject: [PATCH 38/78] debug --- misp_modules/modules/expansion/dnstrails.py | 71 +++++++++++---------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index cab4778..5ff9e25 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -72,42 +72,43 @@ def handler(q=False): def handle_domain(api, domain, misperrors): result_filtered = {"results": []} - r, status_ok = expand_domain_info(api, misperrors, domain) + # r, status_ok = expand_domain_info(api, misperrors, domain) + # + # if status_ok: + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + ' Error DNS result' + # return misperrors + # + # time.sleep(1) + # r, status_ok = expand_subdomains(api, domain) + # + # if status_ok: + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + ' Error subdomains result' + # return misperrors + # + # time.sleep(1) + # r, status_ok = expand_whois(api, domain) + # + # if status_ok: + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + ' Error whois result' + # return misperrors + # + # time.sleep(1) + # r, status_ok = expand_history_ipv4_ipv6(api, domain) + # + # if status_ok: + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + ' Error history ipv4' + # return misperrors + # + # time.sleep(1) - if status_ok: - result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + ' Error DNS result' - return misperrors - - time.sleep(1) - r, status_ok = expand_subdomains(api, domain) - - if status_ok: - result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + ' Error subdomains result' - return misperrors - - time.sleep(1) - r, status_ok = expand_whois(api, domain) - - if status_ok: - result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + ' Error whois result' - return misperrors - - time.sleep(1) - r, status_ok = expand_history_ipv4_ipv6(api, domain) - - if status_ok: - result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + ' Error history ipv4' - return misperrors - - time.sleep(1) r, status_ok = expand_history_dns(api, domain) if status_ok: From 386d38c88f7bacda2f905a8ff6f4a60c861074d1 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 11:55:59 +0200 Subject: [PATCH 39/78] add debug --- misp_modules/modules/expansion/dnstrails.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 5ff9e25..b97997a 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -112,6 +112,7 @@ def handle_domain(api, domain, misperrors): r, status_ok = expand_history_dns(api, domain) if status_ok: + print(r) result_filtered['results'].extend(r) else: misperrors['error'] = misperrors[ From 80e71f582c901bd866cc321f2cbe3f6bcfdd0935 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 11:58:42 +0200 Subject: [PATCH 40/78] debug ipv4 or ipv6 --- misp_modules/modules/expansion/dnstrails.py | 26 +++++++++++---------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index b97997a..9e3a80f 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -99,25 +99,27 @@ def handle_domain(api, domain, misperrors): # return misperrors # # time.sleep(1) - # r, status_ok = expand_history_ipv4_ipv6(api, domain) + r, status_ok = expand_history_ipv4_ipv6(api, domain) # - # if status_ok: - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + ' Error history ipv4' - # return misperrors - # - # time.sleep(1) - - r, status_ok = expand_history_dns(api, domain) if status_ok: print(r) result_filtered['results'].extend(r) else: - misperrors['error'] = misperrors[ - 'error'] + ' Error in expand History DNS' + misperrors['error'] = misperrors['error'] + ' Error history ipv4' return misperrors + + # time.sleep(1) + + # r, status_ok = expand_history_dns(api, domain) + # + # if status_ok: + # print(r) + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors[ + # 'error'] + ' Error in expand History DNS' + # return misperrors return result_filtered From fb262b451fe970a6e1b86fd191c8db2d459dd450 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 12:00:59 +0200 Subject: [PATCH 41/78] debug whois --- misp_modules/modules/expansion/dnstrails.py | 26 ++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 9e3a80f..5ceb3e7 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -90,24 +90,24 @@ def handle_domain(api, domain, misperrors): # return misperrors # # time.sleep(1) - # r, status_ok = expand_whois(api, domain) - # - # if status_ok: - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + ' Error whois result' - # return misperrors - # - # time.sleep(1) - r, status_ok = expand_history_ipv4_ipv6(api, domain) - # + r, status_ok = expand_whois(api, domain) if status_ok: - print(r) result_filtered['results'].extend(r) else: - misperrors['error'] = misperrors['error'] + ' Error history ipv4' + misperrors['error'] = misperrors['error'] + ' Error whois result' return misperrors + # + # time.sleep(1) + # r, status_ok = expand_history_ipv4_ipv6(api, domain) + # # + # + # if status_ok: + # print(r) + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + ' Error history ipv4' + # return misperrors # time.sleep(1) From dbeec4682e24482d319097d647cc20cddca9f320 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 12:02:31 +0200 Subject: [PATCH 42/78] add logs --- misp_modules/modules/expansion/dnstrails.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index 5ceb3e7..d1e6eaa 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -93,6 +93,7 @@ def handle_domain(api, domain, misperrors): r, status_ok = expand_whois(api, domain) if status_ok: + print(r) result_filtered['results'].extend(r) else: misperrors['error'] = misperrors['error'] + ' Error whois result' From f0a4c7190889192ca5723443218a248a758ab9be Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 12:08:01 +0200 Subject: [PATCH 43/78] add a test to check if the list is not empty --- misp_modules/modules/expansion/dnstrails.py | 82 +++++++++++---------- 1 file changed, 42 insertions(+), 40 deletions(-) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index d1e6eaa..eab1ae7 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -72,55 +72,57 @@ def handler(q=False): def handle_domain(api, domain, misperrors): result_filtered = {"results": []} - # r, status_ok = expand_domain_info(api, misperrors, domain) + r, status_ok = expand_domain_info(api, misperrors, domain) # - # if status_ok: - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + ' Error DNS result' - # return misperrors - # - # time.sleep(1) - # r, status_ok = expand_subdomains(api, domain) - # - # if status_ok: - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + ' Error subdomains result' - # return misperrors - # - # time.sleep(1) + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error DNS result' + return misperrors + + time.sleep(1) + r, status_ok = expand_subdomains(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error subdomains result' + return misperrors + + time.sleep(1) r, status_ok = expand_whois(api, domain) if status_ok: - print(r) - result_filtered['results'].extend(r) + if r: + result_filtered['results'].extend(r) else: misperrors['error'] = misperrors['error'] + ' Error whois result' return misperrors - # - # time.sleep(1) - # r, status_ok = expand_history_ipv4_ipv6(api, domain) - # # - # - # if status_ok: - # print(r) - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + ' Error history ipv4' - # return misperrors - # time.sleep(1) - - # r, status_ok = expand_history_dns(api, domain) + time.sleep(1) + r, status_ok = expand_history_ipv4_ipv6(api, domain) # - # if status_ok: - # print(r) - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors[ - # 'error'] + ' Error in expand History DNS' - # return misperrors + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error history ipv4' + return misperrors + + time.sleep(1) + + r, status_ok = expand_history_dns(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors[ + 'error'] + ' Error in expand History DNS' + return misperrors return result_filtered From a8ae6e06e9d23b835c724fa6099290fb09415534 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 12:09:34 +0200 Subject: [PATCH 44/78] add a test to check if the list is not empty --- misp_modules/modules/expansion/dnstrails.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/dnstrails.py index eab1ae7..5e85db5 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/dnstrails.py @@ -123,6 +123,8 @@ def handle_domain(api, domain, misperrors): misperrors['error'] = misperrors[ 'error'] + ' Error in expand History DNS' return misperrors + print(result_filtered) + print(misperrors) return result_filtered From 3a2aab6d7150f2ece96aad73b4b929996bdd8fa7 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 12:41:54 +0200 Subject: [PATCH 45/78] rename misp modules --- .../modules/expansion/{dnstrails.py => securitytrails.py} | 2 -- 1 file changed, 2 deletions(-) rename misp_modules/modules/expansion/{dnstrails.py => securitytrails.py} (99%) diff --git a/misp_modules/modules/expansion/dnstrails.py b/misp_modules/modules/expansion/securitytrails.py similarity index 99% rename from misp_modules/modules/expansion/dnstrails.py rename to misp_modules/modules/expansion/securitytrails.py index 5e85db5..6940907 100644 --- a/misp_modules/modules/expansion/dnstrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -353,8 +353,6 @@ def expand_history_dns(api, domain): results = api.history_dns_ns(domain) if results: - - r.extend(__history_dns(results, domain, 'nameserver', 'ns')) time.sleep(1) From 51067039daf8255e2a874d1bae7bcda43107b369 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 11 Jul 2018 13:03:47 +0200 Subject: [PATCH 46/78] correct typo --- misp_modules/modules/expansion/securitytrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 6940907..e97b493 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -26,7 +26,7 @@ mispattributes = { moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', 'description': 'Query on securitytrails.com', - 'module-types': ['expansion', 'hover']} + 'module-type': ['expansion', 'hover']} # config fields that your code expects from the site admin moduleconfig = ['apikey'] From 576b3c9b9bdb2c8c66268c44b041646a014bcbac Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 13:40:51 +0200 Subject: [PATCH 47/78] history whois dns --- .../modules/expansion/securitytrails.py | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index e97b493..e23184d 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -340,7 +340,7 @@ def expand_history_ipv4_ipv6(api, domain): except APIError as e: misperrors['error'] = e - print(e) + return [], False return r, status_ok @@ -372,12 +372,71 @@ def expand_history_dns(api, domain): except APIError as e: misperrors['error'] = e + return [], False status_ok = True return r, status_ok +def expand_history_whois(api, domain): + r = [] + status_ok = False + try: + results = api.history_whois(domain) + + if results: + + if 'items' in results['results']: + for item in results['results']['items']: + item_registrant = __select_registrant_item(item) + + r.extend( + { + 'type': ['domain'], + 'values': item['nameServers'], + 'categories': ['Network activity'], + 'comment': 'Whois history Name Servers of %s ' + 'Status: %s ' % (domain, item['status']) + + } + ) + if 'email' in item_registrant: + r.append( + { + 'types': ['whois-registrant-email'], + 'values': [item_registrant['email']], + 'categories': ['Attribution'], + 'comment': 'Whois history registrant email of %s' + 'Status: %s' % ( + domain, item['status']) + } + ) + + if 'telephone' in item_registrant: + r.append( + { + 'types': ['whois-registrant-phone'], + 'values': [item_registrant['telephone']], + 'categories': ['Attribution'], + 'comment': 'Whois history registrant phone of %s' + 'Status: %s' % ( + domain, item['status']) + } + ) + + + + + except APIError as e: + misperrors['error'] = e + return [], False + + + + return r, status_ok + + def __history_ip(results, domain, type_ip='ip'): r = [] if 'records' in results: From 4b0daee6f1389467d5bbae26242e71e281d84d28 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:04:37 +0200 Subject: [PATCH 48/78] test whois history --- .../modules/expansion/securitytrails.py | 98 ++++++++++--------- 1 file changed, 51 insertions(+), 47 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index e23184d..eb93756 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -72,59 +72,63 @@ def handler(q=False): def handle_domain(api, domain, misperrors): result_filtered = {"results": []} - r, status_ok = expand_domain_info(api, misperrors, domain) + # r, status_ok = expand_domain_info(api, misperrors, domain) + # # + # if status_ok: + # if r: + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + ' Error DNS result' + # return misperrors # - if status_ok: - if r: - result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + ' Error DNS result' - return misperrors - - time.sleep(1) - r, status_ok = expand_subdomains(api, domain) - - if status_ok: - if r: - result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + ' Error subdomains result' - return misperrors - - time.sleep(1) - r, status_ok = expand_whois(api, domain) - - if status_ok: - if r: - result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + ' Error whois result' - return misperrors - - time.sleep(1) - r, status_ok = expand_history_ipv4_ipv6(api, domain) + # time.sleep(1) + # r, status_ok = expand_subdomains(api, domain) # + # if status_ok: + # if r: + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + ' Error subdomains result' + # return misperrors + # + # time.sleep(1) + # r, status_ok = expand_whois(api, domain) + # + # if status_ok: + # if r: + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + ' Error whois result' + # return misperrors + # + # time.sleep(1) + # r, status_ok = expand_history_ipv4_ipv6(api, domain) + # # + # + # if status_ok: + # if r: + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + ' Error history ipv4' + # return misperrors + # + # time.sleep(1) + # + # r, status_ok = expand_history_dns(api, domain) + # + # if status_ok: + # if r: + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors[ + # 'error'] + ' Error in expand History DNS' + # return misperrors + + r, status_ok = expand_history_whois(api, domain) if status_ok: if r: result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + ' Error history ipv4' - return misperrors - - time.sleep(1) - - r, status_ok = expand_history_dns(api, domain) - - if status_ok: - if r: - result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors[ - 'error'] + ' Error in expand History DNS' - return misperrors - print(result_filtered) - print(misperrors) return result_filtered From 41587bd56823c2799dce55c68f982acc2f567c6f Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:14:43 +0200 Subject: [PATCH 49/78] correct typo --- misp_modules/modules/expansion/securitytrails.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index eb93756..043ded3 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -391,8 +391,8 @@ def expand_history_whois(api, domain): if results: - if 'items' in results['results']: - for item in results['results']['items']: + if 'items' in results['result']: + for item in results['result']['items']: item_registrant = __select_registrant_item(item) r.extend( From fb595c08aac4d7297fc169183879324bee1890b7 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:16:19 +0200 Subject: [PATCH 50/78] add logs --- misp_modules/modules/expansion/securitytrails.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 043ded3..bf35c7b 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -506,4 +506,5 @@ def __select_registrant_item(entry): for c in entry['contacts']: if c['type'] == 'registrant': + print(c) return c From 731c06a939778b3beeb188375073bbe765503797 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:17:16 +0200 Subject: [PATCH 51/78] add logs --- misp_modules/modules/expansion/securitytrails.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index bf35c7b..8ae5dc1 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -502,9 +502,10 @@ def version(): def __select_registrant_item(entry): + print(entry) if 'contacts' in entry: for c in entry['contacts']: - + print(c) if c['type'] == 'registrant': print(c) return c From 9063da88cdc6fad8b1ab2e21a9522eac87189118 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:27:59 +0200 Subject: [PATCH 52/78] correct key and return of functions --- .../modules/expansion/securitytrails.py | 74 ++++++++++--------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 8ae5dc1..845646a 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -394,40 +394,41 @@ def expand_history_whois(api, domain): if 'items' in results['result']: for item in results['result']['items']: item_registrant = __select_registrant_item(item) - - r.extend( - { - 'type': ['domain'], - 'values': item['nameServers'], - 'categories': ['Network activity'], - 'comment': 'Whois history Name Servers of %s ' - 'Status: %s ' % (domain, item['status']) - - } - ) - if 'email' in item_registrant: - r.append( + if item_registrant: + r.extend( { - 'types': ['whois-registrant-email'], - 'values': [item_registrant['email']], - 'categories': ['Attribution'], - 'comment': 'Whois history registrant email of %s' - 'Status: %s' % ( - domain, item['status']) + 'type': ['domain'], + 'values': item['nameServers'], + 'categories': ['Network activity'], + 'comment': 'Whois history Name Servers of %s ' + 'Status: %s ' % ( + domain, item['status']) + } ) + if 'email' in item_registrant: + r.append( + { + 'types': ['whois-registrant-email'], + 'values': [item_registrant['email']], + 'categories': ['Attribution'], + 'comment': 'Whois history registrant email of %s' + 'Status: %s' % ( + domain, item['status']) + } + ) - if 'telephone' in item_registrant: - r.append( - { - 'types': ['whois-registrant-phone'], - 'values': [item_registrant['telephone']], - 'categories': ['Attribution'], - 'comment': 'Whois history registrant phone of %s' - 'Status: %s' % ( - domain, item['status']) - } - ) + if 'telephone' in item_registrant: + r.append( + { + 'types': ['whois-registrant-phone'], + 'values': [item_registrant['telephone']], + 'categories': ['Attribution'], + 'comment': 'Whois history registrant phone of %s' + 'Status: %s' % ( + domain, item['status']) + } + ) @@ -502,10 +503,11 @@ def version(): def __select_registrant_item(entry): - print(entry) + if 'contacts' in entry: - for c in entry['contacts']: - print(c) - if c['type'] == 'registrant': - print(c) - return c + return list(filter(lambda x: x['type'] == 'registrant', + entry['contacts']))[0] + + if 'contact' in entry: + return list(filter(lambda x: x['type'] == 'registrant', + entry['contact']))[0] From 844b25b4cdcaa9f293bb7bb8db9aea4082219631 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:32:56 +0200 Subject: [PATCH 53/78] correct out of bound returns --- .../modules/expansion/securitytrails.py | 44 +++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 845646a..87ea1cd 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -266,7 +266,7 @@ def expand_whois(api, domain): r.append( { 'types': ['whois-registrant-email'], - 'values': [item_registrant['email']], + 'values': [item_registrant[0]['email']], 'categories': ['Attribution'], 'comment': 'Whois information of %s by securitytrails' % domain @@ -277,7 +277,7 @@ def expand_whois(api, domain): r.append( { 'types': ['whois-registrant-phone'], - 'values': [item_registrant['telephone']], + 'values': [item_registrant[0]['telephone']], 'categories': ['Attribution'], 'comment': 'Whois information of %s by securitytrails' % domain @@ -288,7 +288,7 @@ def expand_whois(api, domain): r.append( { 'types': ['whois-registrant-name'], - 'values': [item_registrant['name']], + 'values': [item_registrant[0]['name']], 'categories': ['Attribution'], 'comment': 'Whois information of %s by securitytrails' % domain @@ -299,7 +299,7 @@ def expand_whois(api, domain): r.append( { 'types': ['whois-registrar'], - 'values': [item_registrant['registrarName']], + 'values': [item_registrant[0]['registrarName']], 'categories': ['Attribution'], 'comment': 'Whois information of %s by securitytrails' % domain @@ -310,7 +310,7 @@ def expand_whois(api, domain): r.append( { 'types': ['whois-creation-date'], - 'values': [item_registrant['createdDate']], + 'values': [item_registrant[0]['createdDate']], 'categories': ['Attribution'], 'comment': 'Whois information of %s by securitytrails' % domain @@ -394,23 +394,24 @@ def expand_history_whois(api, domain): if 'items' in results['result']: for item in results['result']['items']: item_registrant = __select_registrant_item(item) - if item_registrant: - r.extend( - { - 'type': ['domain'], - 'values': item['nameServers'], - 'categories': ['Network activity'], - 'comment': 'Whois history Name Servers of %s ' - 'Status: %s ' % ( + r.extend( + { + 'type': ['domain'], + 'values': item['nameServers'], + 'categories': ['Network activity'], + 'comment': 'Whois history Name Servers of %s ' + 'Status: %s ' % ( domain, item['status']) - } - ) - if 'email' in item_registrant: + } + ) + if item_registrant: + + if 'email' in item_registrant[0]: r.append( { 'types': ['whois-registrant-email'], - 'values': [item_registrant['email']], + 'values': [item_registrant[0]['email']], 'categories': ['Attribution'], 'comment': 'Whois history registrant email of %s' 'Status: %s' % ( @@ -422,7 +423,7 @@ def expand_history_whois(api, domain): r.append( { 'types': ['whois-registrant-phone'], - 'values': [item_registrant['telephone']], + 'values': [item_registrant[0]['telephone']], 'categories': ['Attribution'], 'comment': 'Whois history registrant phone of %s' 'Status: %s' % ( @@ -430,9 +431,6 @@ def expand_history_whois(api, domain): } ) - - - except APIError as e: misperrors['error'] = e return [], False @@ -506,8 +504,8 @@ def __select_registrant_item(entry): if 'contacts' in entry: return list(filter(lambda x: x['type'] == 'registrant', - entry['contacts']))[0] + entry['contacts'])) if 'contact' in entry: return list(filter(lambda x: x['type'] == 'registrant', - entry['contact']))[0] + entry['contact'])) From d56bf550389fc23b44980e6335cba63c30238a94 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:33:52 +0200 Subject: [PATCH 54/78] add logs --- misp_modules/modules/expansion/securitytrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 87ea1cd..bb1c1b3 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -501,7 +501,7 @@ def version(): def __select_registrant_item(entry): - + print(entry) if 'contacts' in entry: return list(filter(lambda x: x['type'] == 'registrant', entry['contacts'])) From 9de201375b8221015fabdada7c511cf3cf42d11d Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:37:09 +0200 Subject: [PATCH 55/78] add logs --- misp_modules/modules/expansion/securitytrails.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index bb1c1b3..b113f1f 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -501,11 +501,13 @@ def version(): def __select_registrant_item(entry): - print(entry) + if 'contacts' in entry: return list(filter(lambda x: x['type'] == 'registrant', entry['contacts'])) if 'contact' in entry: + print(entry) + print('\r\n') return list(filter(lambda x: x['type'] == 'registrant', entry['contact'])) From a0cf9de590c5e126219877959c51f04621f2bd0d Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:38:38 +0200 Subject: [PATCH 56/78] add logs --- misp_modules/modules/expansion/securitytrails.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index b113f1f..0cbd6f8 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -509,5 +509,7 @@ def __select_registrant_item(entry): if 'contact' in entry: print(entry) print('\r\n') - return list(filter(lambda x: x['type'] == 'registrant', - entry['contact'])) + res = list(filter(lambda x: x['type'] == 'registrant', + entry['contact'])) + print(res) + return res From 86d94278162f0397fe0b092661555d1629cd31b1 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:42:33 +0200 Subject: [PATCH 57/78] add logs --- misp_modules/modules/expansion/securitytrails.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 0cbd6f8..77bdd2c 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -394,6 +394,7 @@ def expand_history_whois(api, domain): if 'items' in results['result']: for item in results['result']['items']: item_registrant = __select_registrant_item(item) + print(item_registrant) r.extend( { 'type': ['domain'], @@ -435,8 +436,6 @@ def expand_history_whois(api, domain): misperrors['error'] = e return [], False - - return r, status_ok From aa89a7fc4d2b69ca9e0b738aa890402e72c20bae Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:44:19 +0200 Subject: [PATCH 58/78] add logs --- misp_modules/modules/expansion/securitytrails.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 77bdd2c..68a7206 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -128,6 +128,7 @@ def handle_domain(api, domain, misperrors): if status_ok: if r: + print(r) result_filtered['results'].extend(r) return result_filtered From 86d236f859c8d6065d078853d512eb243927fddc Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:47:34 +0200 Subject: [PATCH 59/78] add status_ok to true --- misp_modules/modules/expansion/securitytrails.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 68a7206..8395c13 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -128,8 +128,12 @@ def handle_domain(api, domain, misperrors): if status_ok: if r: - print(r) + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + \ + ' Error in expand History Whois' + return result_filtered @@ -395,7 +399,7 @@ def expand_history_whois(api, domain): if 'items' in results['result']: for item in results['result']['items']: item_registrant = __select_registrant_item(item) - print(item_registrant) + r.extend( { 'type': ['domain'], @@ -436,7 +440,7 @@ def expand_history_whois(api, domain): except APIError as e: misperrors['error'] = e return [], False - + status_ok = True return r, status_ok @@ -507,9 +511,7 @@ def __select_registrant_item(entry): entry['contacts'])) if 'contact' in entry: - print(entry) - print('\r\n') res = list(filter(lambda x: x['type'] == 'registrant', entry['contact'])) - print(res) + return res From 0b0137829a03629b8972e6fd3c3a6d00bfb5b756 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:48:15 +0200 Subject: [PATCH 60/78] add logs --- misp_modules/modules/expansion/securitytrails.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 8395c13..c3d82a2 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -441,6 +441,7 @@ def expand_history_whois(api, domain): misperrors['error'] = e return [], False status_ok = True + print(r) return r, status_ok From 2f5381d7b2f38b6488dfbac004ea364a6d74b724 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:49:51 +0200 Subject: [PATCH 61/78] add logs --- misp_modules/modules/expansion/securitytrails.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index c3d82a2..36703f5 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -130,9 +130,10 @@ def handle_domain(api, domain, misperrors): if r: result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + \ - ' Error in expand History Whois' + else: + misperrors['error'] = misperrors['error'] + \ + ' Error in expand History Whois' + return misperrors return result_filtered From 0341bdc398b978a031b60ac6c32a7550f93a3efd Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:52:01 +0200 Subject: [PATCH 62/78] error call functions --- misp_modules/modules/expansion/securitytrails.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 36703f5..4b2cac6 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -401,9 +401,9 @@ def expand_history_whois(api, domain): for item in results['result']['items']: item_registrant = __select_registrant_item(item) - r.extend( + r.append( { - 'type': ['domain'], + 'types': ['domain'], 'values': item['nameServers'], 'categories': ['Network activity'], 'comment': 'Whois history Name Servers of %s ' From db35c9b0917c2bae5491a3a927943c69bde944fd Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:55:56 +0200 Subject: [PATCH 63/78] correct index error --- misp_modules/modules/expansion/securitytrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 4b2cac6..14cb8b8 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -426,7 +426,7 @@ def expand_history_whois(api, domain): } ) - if 'telephone' in item_registrant: + if 'telephone' in item_registrant[0]: r.append( { 'types': ['whois-registrant-phone'], From 5a422c2e5b370a23f1fa9668837a3e4c6f59626f Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:57:37 +0200 Subject: [PATCH 64/78] add whois expand to test --- .../modules/expansion/securitytrails.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 14cb8b8..d83eeed 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -91,17 +91,17 @@ def handle_domain(api, domain, misperrors): # misperrors['error'] = misperrors['error'] + ' Error subdomains result' # return misperrors # - # time.sleep(1) - # r, status_ok = expand_whois(api, domain) - # - # if status_ok: - # if r: - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + ' Error whois result' - # return misperrors - # - # time.sleep(1) + time.sleep(1) + r, status_ok = expand_whois(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error whois result' + return misperrors + + time.sleep(1) # r, status_ok = expand_history_ipv4_ipv6(api, domain) # # # @@ -268,7 +268,7 @@ def expand_whois(api, domain): item_registrant = __select_registrant_item(results) if item_registrant: - if 'email' in item_registrant: + if 'email' in item_registrant[0]: r.append( { 'types': ['whois-registrant-email'], @@ -279,7 +279,7 @@ def expand_whois(api, domain): } ) - if 'telephone' in item_registrant: + if 'telephone' in item_registrant[0]: r.append( { 'types': ['whois-registrant-phone'], @@ -290,7 +290,7 @@ def expand_whois(api, domain): } ) - if 'name' in item_registrant: + if 'name' in item_registrant[0]: r.append( { 'types': ['whois-registrant-name'], @@ -301,7 +301,7 @@ def expand_whois(api, domain): } ) - if 'registrarName' in item_registrant: + if 'registrarName' in item_registrant[0]: r.append( { 'types': ['whois-registrar'], @@ -312,7 +312,7 @@ def expand_whois(api, domain): } ) - if 'createdDate' in item_registrant: + if 'createdDate' in item_registrant[0]: r.append( { 'types': ['whois-creation-date'], From 3eda71219365a6ae489cd39cf7c7d084f07effcd Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:58:48 +0200 Subject: [PATCH 65/78] add whois expand to test --- .../modules/expansion/securitytrails.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index d83eeed..ab8087c 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -124,16 +124,16 @@ def handle_domain(api, domain, misperrors): # 'error'] + ' Error in expand History DNS' # return misperrors - r, status_ok = expand_history_whois(api, domain) - - if status_ok: - if r: - - result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + \ - ' Error in expand History Whois' - return misperrors + # r, status_ok = expand_history_whois(api, domain) + # + # if status_ok: + # if r: + # + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + \ + # ' Error in expand History Whois' + # return misperrors return result_filtered From 7f52a15d16c3ead17734180b55a078b452ce9122 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 14:59:50 +0200 Subject: [PATCH 66/78] add logs --- misp_modules/modules/expansion/securitytrails.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index ab8087c..3332026 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -509,9 +509,9 @@ def version(): def __select_registrant_item(entry): if 'contacts' in entry: - return list(filter(lambda x: x['type'] == 'registrant', - entry['contacts'])) - + res = list(filter(lambda x: x['type'] == 'registrant', + entry['contacts'])) + print(res) if 'contact' in entry: res = list(filter(lambda x: x['type'] == 'registrant', entry['contact'])) From 966f9603a9ec20ffa6055bb12cbf2a46a4793347 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Thu, 12 Jul 2018 15:02:46 +0200 Subject: [PATCH 67/78] add return --- misp_modules/modules/expansion/securitytrails.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 3332026..4b276da 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -267,7 +267,7 @@ def expand_whois(api, domain): status_ok = True item_registrant = __select_registrant_item(results) if item_registrant: - + print(item_registrant) if 'email' in item_registrant[0]: r.append( { @@ -511,7 +511,7 @@ def __select_registrant_item(entry): if 'contacts' in entry: res = list(filter(lambda x: x['type'] == 'registrant', entry['contacts'])) - print(res) + return res if 'contact' in entry: res = list(filter(lambda x: x['type'] == 'registrant', entry['contact'])) From a41cf59e0c28141fda01efa6aee840cd5fade869 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 17 Jul 2018 15:05:15 +0200 Subject: [PATCH 68/78] add searching domains --- .../modules/expansion/securitytrails.py | 42 +++++++++++++++++-- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 4b276da..b4d9609 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -442,7 +442,7 @@ def expand_history_whois(api, domain): misperrors['error'] = e return [], False status_ok = True - print(r) + return r, status_ok @@ -497,6 +497,40 @@ def __history_dns(results, domain, type_serv, service): ) return r + +def expand_searching_domain(api, ip): + r = [] + status_ok = False + + try: + results = api.searching_domains(ip) + + if results: + if 'records' in results: + res = [(r['host_provider'], r['hostname'], r['whois']) + for r in results['records']] + + for host_provider, hostname, whois in res: + comment = 'domain for %s by %s' % (ip, host_provider[0]) + if whois['registrant']: + comment = comment + ' registrar %s' % whois['registrar'] + + r.append( + { + 'types': ['domain'], + 'category': ['Network activity'], + 'values': [hostname], + 'comment': comment + + } + ) + status_ok = True + except APIError as e: + misperrors['error'] = e + return [], False + + return r, status_ok + def introspection(): return mispattributes @@ -507,13 +541,13 @@ def version(): def __select_registrant_item(entry): - + res = None if 'contacts' in entry: res = list(filter(lambda x: x['type'] == 'registrant', entry['contacts'])) - return res + if 'contact' in entry: res = list(filter(lambda x: x['type'] == 'registrant', entry['contact'])) - return res + return res From 999ae1f6f0ec2264a38ee557f9a4a7e4a533d188 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 17 Jul 2018 17:09:01 +0200 Subject: [PATCH 69/78] add searching domains --- .../modules/expansion/securitytrails.py | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index b4d9609..4aaca61 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -91,17 +91,17 @@ def handle_domain(api, domain, misperrors): # misperrors['error'] = misperrors['error'] + ' Error subdomains result' # return misperrors # - time.sleep(1) - r, status_ok = expand_whois(api, domain) - - if status_ok: - if r: - result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + ' Error whois result' - return misperrors - - time.sleep(1) + # time.sleep(1) + # r, status_ok = expand_whois(api, domain) + # + # if status_ok: + # if r: + # result_filtered['results'].extend(r) + # else: + # misperrors['error'] = misperrors['error'] + ' Error whois result' + # return misperrors + # + # time.sleep(1) # r, status_ok = expand_history_ipv4_ipv6(api, domain) # # # @@ -135,11 +135,23 @@ def handle_domain(api, domain, misperrors): # ' Error in expand History Whois' # return misperrors + return result_filtered def handle_ip(api, ip, misperrors): - pass + result_filtered = {"results": []} + + r, status_ok = expand_searching_domain(api, ip) + + if status_ok: + if r: + result_filtered['result'].extend(r) + else: + misperrors['error'] += ' Error in expand searching domain' + return misperrors + + return result_filtered def expand_domain_info(api, misperror, domain): From 431c1511a37d700fd6cd1e70f23fb18493bd6b4e Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 17 Jul 2018 17:20:30 +0200 Subject: [PATCH 70/78] correct param --- misp_modules/modules/expansion/securitytrails.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 4aaca61..53a0111 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -150,7 +150,7 @@ def handle_ip(api, ip, misperrors): else: misperrors['error'] += ' Error in expand searching domain' return misperrors - + return result_filtered @@ -515,7 +515,7 @@ def expand_searching_domain(api, ip): status_ok = False try: - results = api.searching_domains(ip) + results = api.searching_domains(ipv4=ip) if results: if 'records' in results: From 2706c4a82a6294341d276eeb151428ad4c5fe047 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 17 Jul 2018 17:21:38 +0200 Subject: [PATCH 71/78] correct key --- misp_modules/modules/expansion/securitytrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 53a0111..b6b3e65 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -524,7 +524,7 @@ def expand_searching_domain(api, ip): for host_provider, hostname, whois in res: comment = 'domain for %s by %s' % (ip, host_provider[0]) - if whois['registrant']: + if whois['registrar']: comment = comment + ' registrar %s' % whois['registrar'] r.append( From c785cae89b409a174714e8d012b4b6cccd062567 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 17 Jul 2018 17:22:48 +0200 Subject: [PATCH 72/78] correct key --- misp_modules/modules/expansion/securitytrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index b6b3e65..b2790fa 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -146,7 +146,7 @@ def handle_ip(api, ip, misperrors): if status_ok: if r: - result_filtered['result'].extend(r) + result_filtered['results'].extend(r) else: misperrors['error'] += ' Error in expand searching domain' return misperrors From 9d603344c2cb0c0560995252f8a9d3bdd50c51f4 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 17 Jul 2018 18:32:50 +0200 Subject: [PATCH 73/78] add searching_stats --- .../modules/expansion/securitytrails.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index b2790fa..5294a62 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -543,6 +543,31 @@ def expand_searching_domain(api, ip): return r, status_ok + +def expand_search_stats(api, ip, misperror): + r = [] + status_ok = False + + try: + result = api.searching_stats(ipv4=ip) + if result and 'top_organizations' in result: + comment = '' + for reg in result['top_organizations']: + comment += 'Organization %s used %s count: %s' % (reg['key'], + ip, + reg['count']) + r.append({'types': ['comment'], + 'categories': ['Other'], + 'values': comment, + }) + status_ok = True + except APIError as e: + misperrors['error'] = e + return [], False + + return r, status_ok + + def introspection(): return mispattributes From 8cbeda40a5043eb2494d1d41c9431dabfce9d067 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 17 Jul 2018 18:42:01 +0200 Subject: [PATCH 74/78] add searching_stats --- misp_modules/modules/expansion/securitytrails.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 5294a62..f8f822e 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -151,6 +151,17 @@ def handle_ip(api, ip, misperrors): misperrors['error'] += ' Error in expand searching domain' return misperrors + time.sleep(1) + + r, status_ok = expand_search_stats(api, ip, misperrors) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] += ' Error in expand searching stats' + return misperrors + return result_filtered @@ -549,7 +560,7 @@ def expand_search_stats(api, ip, misperror): status_ok = False try: - result = api.searching_stats(ipv4=ip) + result = api.search_stats(ipv4=ip) if result and 'top_organizations' in result: comment = '' for reg in result['top_organizations']: From 88859a0ba7123bd9683c9bdededb1a769a0546b7 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Tue, 17 Jul 2018 18:43:52 +0200 Subject: [PATCH 75/78] add logs --- misp_modules/modules/expansion/securitytrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index f8f822e..63f08c9 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -573,7 +573,7 @@ def expand_search_stats(api, ip, misperror): }) status_ok = True except APIError as e: - misperrors['error'] = e + misperrors['error'] = e.value return [], False return r, status_ok From f2df6dc538244210715c4ae45202871d8f3987ee Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 18 Jul 2018 10:47:42 +0200 Subject: [PATCH 76/78] last commit for release --- .../modules/expansion/securitytrails.py | 167 +++++++----------- 1 file changed, 65 insertions(+), 102 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 63f08c9..8c96f76 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -72,69 +72,66 @@ def handler(q=False): def handle_domain(api, domain, misperrors): result_filtered = {"results": []} - # r, status_ok = expand_domain_info(api, misperrors, domain) - # # - # if status_ok: - # if r: - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + ' Error DNS result' - # return misperrors - # - # time.sleep(1) - # r, status_ok = expand_subdomains(api, domain) - # - # if status_ok: - # if r: - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + ' Error subdomains result' - # return misperrors - # - # time.sleep(1) - # r, status_ok = expand_whois(api, domain) - # - # if status_ok: - # if r: - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + ' Error whois result' - # return misperrors - # - # time.sleep(1) - # r, status_ok = expand_history_ipv4_ipv6(api, domain) - # # - # - # if status_ok: - # if r: - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + ' Error history ipv4' - # return misperrors - # - # time.sleep(1) - # - # r, status_ok = expand_history_dns(api, domain) - # - # if status_ok: - # if r: - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors[ - # 'error'] + ' Error in expand History DNS' - # return misperrors + r, status_ok = expand_domain_info(api, misperrors, domain) - # r, status_ok = expand_history_whois(api, domain) - # - # if status_ok: - # if r: - # - # result_filtered['results'].extend(r) - # else: - # misperrors['error'] = misperrors['error'] + \ - # ' Error in expand History Whois' - # return misperrors + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error DNS result' + return misperrors + time.sleep(1) + r, status_ok = expand_subdomains(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error subdomains result' + return misperrors + + time.sleep(1) + r, status_ok = expand_whois(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error whois result' + return misperrors + + time.sleep(1) + r, status_ok = expand_history_ipv4_ipv6(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error history ipv4' + return misperrors + + time.sleep(1) + + r, status_ok = expand_history_dns(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors[ + 'error'] + ' Error in expand History DNS' + return misperrors + + r, status_ok = expand_history_whois(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + \ + ' Error in expand History Whois' + return misperrors return result_filtered @@ -151,17 +148,6 @@ def handle_ip(api, ip, misperrors): misperrors['error'] += ' Error in expand searching domain' return misperrors - time.sleep(1) - - r, status_ok = expand_search_stats(api, ip, misperrors) - - if status_ok: - if r: - result_filtered['results'].extend(r) - else: - misperrors['error'] += ' Error in expand searching stats' - return misperrors - return result_filtered @@ -274,7 +260,8 @@ def expand_subdomains(api, domain): ) except APIError as e: - misperrors['error'] = e + misperrors['error'] = e.value + return [], False return r, status_ok @@ -347,8 +334,8 @@ def expand_whois(api, domain): ) except APIError as e: - misperrors['error'] = e - print(e) + misperrors['error'] = e.value + return [], False return r, status_ok @@ -372,7 +359,7 @@ def expand_history_ipv4_ipv6(api, domain): r.extend(__history_ip(results, domain, type_ip='ipv6')) except APIError as e: - misperrors['error'] = e + misperrors['error'] = e.value return [], False return r, status_ok @@ -404,7 +391,7 @@ def expand_history_dns(api, domain): r.extend(__history_dns(results, domain, 'host', 'mx')) except APIError as e: - misperrors['error'] = e + misperrors['error'] = e.value return [], False status_ok = True @@ -462,7 +449,7 @@ def expand_history_whois(api, domain): ) except APIError as e: - misperrors['error'] = e + misperrors['error'] = e.value return [], False status_ok = True @@ -548,30 +535,6 @@ def expand_searching_domain(api, ip): } ) status_ok = True - except APIError as e: - misperrors['error'] = e - return [], False - - return r, status_ok - - -def expand_search_stats(api, ip, misperror): - r = [] - status_ok = False - - try: - result = api.search_stats(ipv4=ip) - if result and 'top_organizations' in result: - comment = '' - for reg in result['top_organizations']: - comment += 'Organization %s used %s count: %s' % (reg['key'], - ip, - reg['count']) - r.append({'types': ['comment'], - 'categories': ['Other'], - 'values': comment, - }) - status_ok = True except APIError as e: misperrors['error'] = e.value return [], False From c8e20d90879b024eb068cc74a21c848a8324e024 Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 18 Jul 2018 10:51:47 +0200 Subject: [PATCH 77/78] remove print --- misp_modules/modules/expansion/securitytrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 8c96f76..bf1c5b1 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -277,7 +277,7 @@ def expand_whois(api, domain): status_ok = True item_registrant = __select_registrant_item(results) if item_registrant: - print(item_registrant) + if 'email' in item_registrant[0]: r.append( { From 804e59ed8d6c48b7383de4327d45fae352197cbe Mon Sep 17 00:00:00 2001 From: Sebdraven Date: Wed, 18 Jul 2018 10:58:51 +0200 Subject: [PATCH 78/78] change type of status --- misp_modules/modules/expansion/securitytrails.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index bf1c5b1..325fa13 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -418,7 +418,7 @@ def expand_history_whois(api, domain): 'categories': ['Network activity'], 'comment': 'Whois history Name Servers of %s ' 'Status: %s ' % ( - domain, item['status']) + domain, ' '.join(item['status'])) } ) @@ -432,7 +432,8 @@ def expand_history_whois(api, domain): 'categories': ['Attribution'], 'comment': 'Whois history registrant email of %s' 'Status: %s' % ( - domain, item['status']) + domain, + ' '.join(item['status'])) } ) @@ -444,7 +445,8 @@ def expand_history_whois(api, domain): 'categories': ['Attribution'], 'comment': 'Whois history registrant phone of %s' 'Status: %s' % ( - domain, item['status']) + domain, + ' '.join(item['status'])) } )