From b79636ccfa18a2334b55acacae52737b2052be7f Mon Sep 17 00:00:00 2001 From: Golbark Date: Fri, 3 Apr 2020 03:15:03 -0700 Subject: [PATCH 1/3] new: usr: Censys Expansion module --- README.md | 1 + REQUIREMENTS | 1 + doc/expansion/censys_enrich.py | 8 + misp_modules/modules/expansion/__init__.py | 2 +- .../modules/expansion/censys_enrich.py | 179 ++++++++++++++++++ 5 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 doc/expansion/censys_enrich.py create mode 100644 misp_modules/modules/expansion/censys_enrich.py diff --git a/README.md b/README.md index fe37cd5..db199a0 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [RansomcoinDB check](misp_modules/modules/expansion/ransomcoindb.py) - An expansion hover module to query the [ransomcoinDB](https://ransomcoindb.concinnity-risks.com): it contains mapping between BTC addresses and malware hashes. Enrich MISP by querying for BTC -> hash or hash -> BTC addresses. * [BTC scam check](misp_modules/modules/expansion/btc_scam_check.py) - An expansion hover module to instantly check if a BTC address has been abused. * [BTC transactions](misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP. +* [Censys-enrich](misp_modules/modules/expansion/censys_enrich.py) - An expansion and module to retrieve information from censys.io about a particular IP or certificate. * [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. * [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate(s) seen. * [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to. diff --git a/REQUIREMENTS b/REQUIREMENTS index f98ae8b..c69b383 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -18,6 +18,7 @@ attrs==19.3.0 backscatter==0.2.4 beautifulsoup4==4.8.2 blockchain==1.4.4 +censys==0.0.8 certifi==2019.11.28 cffi==1.14.0 chardet==3.0.4 diff --git a/doc/expansion/censys_enrich.py b/doc/expansion/censys_enrich.py new file mode 100644 index 0000000..83e6d5f --- /dev/null +++ b/doc/expansion/censys_enrich.py @@ -0,0 +1,8 @@ +{ + "description": "An expansion module to enrich attributes in MISP by quering the censys.io API", + "requirements": ["API credentials to censys.io"], + "input": "IP, domain or certificate fingerprint (md5, sha1 or sha256)", + "output": "MISP objects retrieved from censys, including open ports, ASN, Location of the IP, x509 details", + "references": ["https://www.censys.io"], + "features": "This module takes an IP, hostname or a certificate fingerprint and attempts to enrich it by querying the Censys API." +} diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 2a99050..82264fa 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -16,4 +16,4 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'ods_enrich', 'odt_enrich', 'joesandbox_submit', 'joesandbox_query', 'urlhaus', 'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid', 'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', - 'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion'] + 'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion', 'censys_enrich'] diff --git a/misp_modules/modules/expansion/censys_enrich.py b/misp_modules/modules/expansion/censys_enrich.py new file mode 100644 index 0000000..e3f2acd --- /dev/null +++ b/misp_modules/modules/expansion/censys_enrich.py @@ -0,0 +1,179 @@ +# encoding: utf-8 +import json +import base64 +import codecs +from dateutil.parser import isoparse +from pymisp import MISPAttribute, MISPEvent, MISPObject +try: + import censys.base + import censys.ipv4 + import censys.websites + import censys.certificates +except ImportError: + print("Censys module not installed. Try 'pip install censys'") + +misperrors = {'error': 'Error'} +moduleconfig = ['api_id', 'api_secret'] +mispattributes = {'input': ['ip-src', 'ip-dst', 'domain', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256'], 'format': 'misp_standard'} +moduleinfo = {'version': '0.1', 'author': 'Loïc Fortemps', + 'description': 'Censys.io expansion module', 'module-type': ['expansion', 'hover']} + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + + if request.get('config'): + if (request['config'].get('api_id') is None) or (request['config'].get('api_secret') is None): + misperrors['error'] = "Censys API credentials are missing" + return misperrors + else: + misperrors['error'] = "Please provide config options" + return misperrors + + api_id = request['config']['api_id'] + api_secret = request['config']['api_secret'] + + if not request.get('attribute'): + return {'error': 'Unsupported input.'} + attribute = request['attribute'] + if not any(input_type == attribute['type'] for input_type in mispattributes['input']): + return {'error': 'Unsupported attributes type'} + + attribute = MISPAttribute() + attribute.from_dict(**request['attribute']) + + if attribute.type == 'ip-dst' or attribute.type == 'ip-src': + conn = censys.ipv4.CensysIPv4(api_id=api_id, api_secret=api_secret) + elif attribute.type == 'domain': + conn = censys.websites.CensysWebsites(api_id=api_id, api_secret=api_secret) + elif 'x509-fingerprint' in attribute.type: + conn = censys.certificates.CensysCertificates(api_id=api_id, api_secret=api_secret) + else: + return False + + try: + result = conn.view(attribute.value) + except censys.base.CensysNotFoundException: + misperrors['error'] = "Nothing could be found on Censys" + return misperrors + except Exception: + misperrors['error'] = "Connection issue" + return misperrors + + r = {'results': parse_response(result, attribute)} + return r + + +def parse_response(censys_output, attribute): + misp_event = MISPEvent() + misp_event.add_attribute(**attribute) + # Generic fields (for IP/Websites) + if "autonomous_system" in censys_output: + cen_as = censys_output['autonomous_system'] + asn_object = MISPObject('asn') + asn_object.add_attribute('asn', value=cen_as["asn"]) + asn_object.add_attribute('description', value=cen_as['name']) + asn_object.add_attribute('subnet-announced', value=cen_as['routed_prefix']) + asn_object.add_attribute('country', value=cen_as['country_code']) + asn_object.add_reference(attribute.uuid, 'associated-to') + misp_event.add_object(**asn_object) + + if "ip" in censys_output and "ports" in censys_output: + ip_object = MISPObject('ip-port') + ip_object.add_attribute('ip', value=censys_output['ip']) + for p in censys_output['ports']: + ip_object.add_attribute('dst-port', value=p) + ip_object.add_reference(attribute.uuid, 'associated-to') + misp_event.add_object(**ip_object) + + # We explore all ports to find https or ssh services + for k in censys_output.keys(): + if not isinstance(censys_output[k], dict): + continue + if 'https' in censys_output[k]: + try: + cert = censys_output[k]['https']['tls']['certificate'] + cert_obj = get_certificate_object(cert, attribute) + misp_event.add_object(**cert_obj) + except KeyError: + print("Error !") + if 'ssh' in censys_output[k]: + try: + cert = censys_output[k]['ssh']['v2']['server_host_key'] + # To enable once the type is merged + # misp_event.add_attribute(type='hasshserver-sha256', value=cert['fingerprint_sha256']) + except KeyError: + pass + + # Info from certificate query + if "parsed" in censys_output: + cert_obj = get_certificate_object(censys_output, attribute) + misp_event.add_object(**cert_obj) + + # Location can be present for IP/Websites results + if "location" in censys_output: + loc_obj = MISPObject('geolocation') + loc = censys_output['location'] + loc_obj.add_attribute('latitude', value=loc['latitude']) + loc_obj.add_attribute('longitude', value=loc['longitude']) + loc_obj.add_attribute('city', value=loc['city']) + loc_obj.add_attribute('country', value=loc['country']) + loc_obj.add_attribute('zipcode', value=loc['postal_code']) + if 'province' in loc: + loc_obj.add_attribute('region', value=loc['province']) + loc_obj.add_reference(attribute.uuid, 'associated-to') + misp_event.add_object(**loc_obj) + + event = json.loads(misp_event.to_json()) + return {'Object': event['Object'], 'Attribute': event['Attribute']} + + +def get_certificate_object(cert, attribute): + parsed = cert['parsed'] + cert_object = MISPObject('x509') + cert_object.add_attribute('x509-fingerprint-sha256', value=parsed['fingerprint_sha256']) + cert_object.add_attribute('x509-fingerprint-sha1', value=parsed['fingerprint_sha1']) + cert_object.add_attribute('x509-fingerprint-md5', value=parsed['fingerprint_md5']) + cert_object.add_attribute('serial-number', value=parsed['serial_number']) + cert_object.add_attribute('version', value=parsed['version']) + cert_object.add_attribute('subject', value=parsed['subject_dn']) + cert_object.add_attribute('issuer', value=parsed['issuer_dn']) + cert_object.add_attribute('validity-not-before', value=isoparse(parsed['validity']['start'])) + cert_object.add_attribute('validity-not-after', value=isoparse(parsed['validity']['end'])) + cert_object.add_attribute('self_signed', value=parsed['signature']['self_signed']) + cert_object.add_attribute('signature_algorithm', value=parsed['signature']['signature_algorithm']['name']) + + cert_object.add_attribute('pubkey-info-algorithm', value=parsed['subject_key_info']['key_algorithm']['name']) + + if 'rsa_public_key' in parsed['subject_key_info']: + pub_key = parsed['subject_key_info']['rsa_public_key'] + cert_object.add_attribute('pubkey-info-size', value=pub_key['length']) + cert_object.add_attribute('pubkey-info-exponent', value=pub_key['exponent']) + hex_mod = codecs.encode(base64.b64decode(pub_key['modulus']), 'hex').decode() + cert_object.add_attribute('pubkey-info-modulus', value=hex_mod) + + if "extensions" in parsed and "subject_alt_name" in parsed["extensions"]: + san = parsed["extensions"]["subject_alt_name"] + if "dns_names" in san: + for dns in san['dns_names']: + cert_object.add_attribute('dns_names', value=dns) + if "ip_addresses" in san: + for ip in san['ip_addresses']: + cert_object.add_attribute('ip', value=ip) + + if "raw" in cert: + cert_object.add_attribute('raw-base64', value=cert['raw']) + + cert_object.add_reference(attribute.uuid, 'associated-to') + return cert_object + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 500f0301a97917482cb52409bdd36914d3fe6ce5 Mon Sep 17 00:00:00 2001 From: Golbark Date: Tue, 7 Apr 2020 06:53:42 -0700 Subject: [PATCH 2/3] Adding support for more input types, including multi-types --- .../modules/expansion/censys_enrich.py | 112 +++++++++++++++--- 1 file changed, 93 insertions(+), 19 deletions(-) diff --git a/misp_modules/modules/expansion/censys_enrich.py b/misp_modules/modules/expansion/censys_enrich.py index e3f2acd..b1b89b8 100644 --- a/misp_modules/modules/expansion/censys_enrich.py +++ b/misp_modules/modules/expansion/censys_enrich.py @@ -14,7 +14,8 @@ except ImportError: misperrors = {'error': 'Error'} moduleconfig = ['api_id', 'api_secret'] -mispattributes = {'input': ['ip-src', 'ip-dst', 'domain', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256'], 'format': 'misp_standard'} +mispattributes = {'input': ['ip-src', 'ip-dst', 'domain', 'hostname', 'hostname|port', 'domain|ip', 'ip-dst|port', 'ip-src|port', + 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256'], 'format': 'misp_standard'} moduleinfo = {'version': '0.1', 'author': 'Loïc Fortemps', 'description': 'Censys.io expansion module', 'module-type': ['expansion', 'hover']} @@ -43,27 +44,53 @@ def handler(q=False): attribute = MISPAttribute() attribute.from_dict(**request['attribute']) + # Lists to accomodate multi-types attribute + conn = list() + types = list() + values = list() + results = list() - if attribute.type == 'ip-dst' or attribute.type == 'ip-src': - conn = censys.ipv4.CensysIPv4(api_id=api_id, api_secret=api_secret) - elif attribute.type == 'domain': - conn = censys.websites.CensysWebsites(api_id=api_id, api_secret=api_secret) - elif 'x509-fingerprint' in attribute.type: - conn = censys.certificates.CensysCertificates(api_id=api_id, api_secret=api_secret) + if "|" in attribute.type: + t_1, t_2 = attribute.type.split('|') + v_1, v_2 = attribute.value.split('|') + # We cannot use the port information + if t_2 == "port": + types.append(t_1) + values.append(v_1) + else: + types = [t_1, t_2] + values = [v_1, v_2] else: - return False + types.append(attribute.type) + values.append(attribute.value) - try: - result = conn.view(attribute.value) - except censys.base.CensysNotFoundException: + for t in types: + # ip, ip-src or ip-dst + if t[:2] == "ip": + conn.append(censys.ipv4.CensysIPv4(api_id=api_id, api_secret=api_secret)) + elif t == 'domain' or t == "hostname": + conn.append(censys.websites.CensysWebsites(api_id=api_id, api_secret=api_secret)) + elif 'x509-fingerprint' in t: + conn.append(censys.certificates.CensysCertificates(api_id=api_id, api_secret=api_secret)) + + found = True + for c in conn: + val = values.pop(0) + try: + r = c.view(val) + results.append(parse_response(r, attribute)) + found = True + except censys.base.CensysNotFoundException: + found = False + except Exception: + misperrors['error'] = "Connection issue" + return misperrors + + if not found: misperrors['error'] = "Nothing could be found on Censys" return misperrors - except Exception: - misperrors['error'] = "Connection issue" - return misperrors - r = {'results': parse_response(result, attribute)} - return r + return {'results': remove_duplicates(results)} def parse_response(censys_output, attribute): @@ -102,7 +129,7 @@ def parse_response(censys_output, attribute): if 'ssh' in censys_output[k]: try: cert = censys_output[k]['ssh']['v2']['server_host_key'] - # To enable once the type is merged + # TODO enable once the type is merged # misp_event.add_attribute(type='hasshserver-sha256', value=cert['fingerprint_sha256']) except KeyError: pass @@ -118,9 +145,11 @@ def parse_response(censys_output, attribute): loc = censys_output['location'] loc_obj.add_attribute('latitude', value=loc['latitude']) loc_obj.add_attribute('longitude', value=loc['longitude']) - loc_obj.add_attribute('city', value=loc['city']) + if 'city' in loc: + loc_obj.add_attribute('city', value=loc['city']) loc_obj.add_attribute('country', value=loc['country']) - loc_obj.add_attribute('zipcode', value=loc['postal_code']) + if 'postal_code' in loc: + loc_obj.add_attribute('zipcode', value=loc['postal_code']) if 'province' in loc: loc_obj.add_attribute('region', value=loc['province']) loc_obj.add_reference(attribute.uuid, 'associated-to') @@ -130,6 +159,51 @@ def parse_response(censys_output, attribute): return {'Object': event['Object'], 'Attribute': event['Attribute']} +# In case of multiple enrichment (ip and domain), we need to filter out similar objects +# TODO: make it more granular +def remove_duplicates(results): + # Only one enrichment was performed so no duplicate + if len(results) == 1: + return results[0] + elif len(results) == 2: + final_result = results[0] + obj_l2 = results[1]['Object'] + for o2 in obj_l2: + if o2['name'] == "asn": + key = "asn" + elif o2['name'] == "ip-port": + key = "ip" + elif o2['name'] == "x509": + key = "x509-fingerprint-sha256" + elif o2['name'] == "geolocation": + key = "latitude" + if not check_if_present(o2, key, final_result['Object']): + final_result['Object'].append(o2) + + return final_result + else: + return [] + + +def check_if_present(object, attribute_name, list_objects): + """ + Assert if a given object is present in the list. + + This function check if object (json format) is present in list_objects + using attribute_name for the matching + """ + for o in list_objects: + if o['name'] == object['name']: + for attr in object['Attribute']: + if attr['type'] == attribute_name: + value = attr['value'] + for attr2 in o['Attribute']: + if attr['type'] == attribute_name and attr['value'] == value: + return True + + return False + + def get_certificate_object(cert, attribute): parsed = cert['parsed'] cert_object = MISPObject('x509') From fd3c62c460dc52c18881a983a13b73f56774a063 Mon Sep 17 00:00:00 2001 From: Golbark Date: Wed, 8 Apr 2020 01:07:46 -0700 Subject: [PATCH 3/3] Fix variable issue in the loop --- misp_modules/modules/expansion/censys_enrich.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/misp_modules/modules/expansion/censys_enrich.py b/misp_modules/modules/expansion/censys_enrich.py index b1b89b8..0fc61ae 100644 --- a/misp_modules/modules/expansion/censys_enrich.py +++ b/misp_modules/modules/expansion/censys_enrich.py @@ -193,13 +193,15 @@ def check_if_present(object, attribute_name, list_objects): using attribute_name for the matching """ for o in list_objects: + # We first look for a match on the name if o['name'] == object['name']: for attr in object['Attribute']: + # Within the attributes, we look for the one to compare if attr['type'] == attribute_name: - value = attr['value'] - for attr2 in o['Attribute']: - if attr['type'] == attribute_name and attr['value'] == value: - return True + # Then we check the attributes of the other object and look for a match + for attr2 in o['Attribute']: + if attr2['type'] == attribute_name and attr2['value'] == attr['value']: + return True return False