Merge pull request #388 from Golbark/censys_expansion

new: usr: Censys Expansion module

README.md

@@ -26,6 +26,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
 * [RansomcoinDB check](misp_modules/modules/expansion/ransomcoindb.py) - An expansion hover module to query the [ransomcoinDB](https://ransomcoindb.concinnity-risks.com): it contains mapping between BTC addresses and malware hashes. Enrich MISP by querying for BTC -> hash or hash -> BTC addresses.
 * [BTC scam check](misp_modules/modules/expansion/btc_scam_check.py) - An expansion hover module to instantly check if a BTC address has been abused.
 * [BTC transactions](misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP.
+* [Censys-enrich](misp_modules/modules/expansion/censys_enrich.py) - An expansion and module to retrieve information from censys.io about a particular IP or certificate.
 * [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
 * [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate(s) seen.
 * [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to.

REQUIREMENTS

@@ -18,6 +18,7 @@ attrs==19.3.0
 backscatter==0.2.4
 beautifulsoup4==4.8.2
 blockchain==1.4.4
+censys==0.0.8
 certifi==2019.11.28
 cffi==1.14.0
 chardet==3.0.4

doc/expansion/censys_enrich.py

@@ -0,0 +1,8 @@
+{
+    "description": "An expansion module to enrich attributes in MISP by quering the censys.io API",
+    "requirements": ["API credentials to censys.io"],
+    "input": "IP, domain or certificate fingerprint (md5, sha1 or sha256)",
+    "output": "MISP objects retrieved from censys, including open ports, ASN, Location of the IP, x509 details",
+    "references": ["https://www.censys.io"],
+    "features": "This module takes an IP, hostname or a certificate fingerprint and attempts to enrich it by querying the Censys API."
+}

misp_modules/modules/expansion/__init__.py

@@ -16,4 +16,4 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c
            'ods_enrich', 'odt_enrich', 'joesandbox_submit', 'joesandbox_query', 'urlhaus',
            'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid',
            'assemblyline_submit', 'assemblyline_query', 'ransomcoindb',
-           'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion']
+           'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion', 'censys_enrich']

misp_modules/modules/expansion/censys_enrich.py

@@ -0,0 +1,255 @@
+# encoding: utf-8
+import json
+import base64
+import codecs
+from dateutil.parser import isoparse
+from pymisp import MISPAttribute, MISPEvent, MISPObject
+try:
+    import censys.base
+    import censys.ipv4
+    import censys.websites
+    import censys.certificates
+except ImportError:
+    print("Censys module not installed. Try 'pip install censys'")
+
+misperrors = {'error': 'Error'}
+moduleconfig = ['api_id', 'api_secret']
+mispattributes = {'input': ['ip-src', 'ip-dst', 'domain', 'hostname', 'hostname|port', 'domain|ip', 'ip-dst|port', 'ip-src|port',
+                  'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256'], 'format': 'misp_standard'}
+moduleinfo = {'version': '0.1', 'author': 'Loïc Fortemps',
+              'description': 'Censys.io expansion module', 'module-type': ['expansion', 'hover']}
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+
+    if request.get('config'):
+        if (request['config'].get('api_id') is None) or (request['config'].get('api_secret') is None):
+            misperrors['error'] = "Censys API credentials are missing"
+            return misperrors
+    else:
+        misperrors['error'] = "Please provide config options"
+        return misperrors
+
+    api_id = request['config']['api_id']
+    api_secret = request['config']['api_secret']
+
+    if not request.get('attribute'):
+        return {'error': 'Unsupported input.'}
+    attribute = request['attribute']
+    if not any(input_type == attribute['type'] for input_type in mispattributes['input']):
+        return {'error': 'Unsupported attributes type'}
+
+    attribute = MISPAttribute()
+    attribute.from_dict(**request['attribute'])
+    # Lists to accomodate multi-types attribute
+    conn = list()
+    types = list()
+    values = list()
+    results = list()
+
+    if "|" in attribute.type:
+        t_1, t_2 = attribute.type.split('|')
+        v_1, v_2 = attribute.value.split('|')
+        # We cannot use the port information
+        if t_2 == "port":
+            types.append(t_1)
+            values.append(v_1)
+        else:
+            types = [t_1, t_2]
+            values = [v_1, v_2]
+    else:
+        types.append(attribute.type)
+        values.append(attribute.value)
+
+    for t in types:
+        # ip, ip-src or ip-dst
+        if t[:2] == "ip":
+            conn.append(censys.ipv4.CensysIPv4(api_id=api_id, api_secret=api_secret))
+        elif t == 'domain' or t == "hostname":
+            conn.append(censys.websites.CensysWebsites(api_id=api_id, api_secret=api_secret))
+        elif 'x509-fingerprint' in t:
+            conn.append(censys.certificates.CensysCertificates(api_id=api_id, api_secret=api_secret))
+
+    found = True
+    for c in conn:
+        val = values.pop(0)
+        try:
+            r = c.view(val)
+            results.append(parse_response(r, attribute))
+            found = True
+        except censys.base.CensysNotFoundException:
+            found = False
+        except Exception:
+            misperrors['error'] = "Connection issue"
+            return misperrors
+
+    if not found:
+        misperrors['error'] = "Nothing could be found on Censys"
+        return misperrors
+
+    return {'results': remove_duplicates(results)}
+
+
+def parse_response(censys_output, attribute):
+    misp_event = MISPEvent()
+    misp_event.add_attribute(**attribute)
+    # Generic fields (for IP/Websites)
+    if "autonomous_system" in censys_output:
+        cen_as = censys_output['autonomous_system']
+        asn_object = MISPObject('asn')
+        asn_object.add_attribute('asn', value=cen_as["asn"])
+        asn_object.add_attribute('description', value=cen_as['name'])
+        asn_object.add_attribute('subnet-announced', value=cen_as['routed_prefix'])
+        asn_object.add_attribute('country', value=cen_as['country_code'])
+        asn_object.add_reference(attribute.uuid, 'associated-to')
+        misp_event.add_object(**asn_object)
+
+    if "ip" in censys_output and "ports" in censys_output:
+        ip_object = MISPObject('ip-port')
+        ip_object.add_attribute('ip', value=censys_output['ip'])
+        for p in censys_output['ports']:
+            ip_object.add_attribute('dst-port', value=p)
+        ip_object.add_reference(attribute.uuid, 'associated-to')
+        misp_event.add_object(**ip_object)
+
+    # We explore all ports to find https or ssh services
+    for k in censys_output.keys():
+        if not isinstance(censys_output[k], dict):
+            continue
+        if 'https' in censys_output[k]:
+            try:
+                cert = censys_output[k]['https']['tls']['certificate']
+                cert_obj = get_certificate_object(cert, attribute)
+                misp_event.add_object(**cert_obj)
+            except KeyError:
+                print("Error !")
+        if 'ssh' in censys_output[k]:
+            try:
+                cert = censys_output[k]['ssh']['v2']['server_host_key']
+                # TODO enable once the type is merged
+                # misp_event.add_attribute(type='hasshserver-sha256', value=cert['fingerprint_sha256'])
+            except KeyError:
+                pass
+
+    # Info from certificate query
+    if "parsed" in censys_output:
+        cert_obj = get_certificate_object(censys_output, attribute)
+        misp_event.add_object(**cert_obj)
+
+    # Location can be present for IP/Websites results
+    if "location" in censys_output:
+        loc_obj = MISPObject('geolocation')
+        loc = censys_output['location']
+        loc_obj.add_attribute('latitude', value=loc['latitude'])
+        loc_obj.add_attribute('longitude', value=loc['longitude'])
+        if 'city' in loc:
+            loc_obj.add_attribute('city', value=loc['city'])
+        loc_obj.add_attribute('country', value=loc['country'])
+        if 'postal_code' in loc:
+            loc_obj.add_attribute('zipcode', value=loc['postal_code'])
+        if 'province' in loc:
+            loc_obj.add_attribute('region', value=loc['province'])
+        loc_obj.add_reference(attribute.uuid, 'associated-to')
+        misp_event.add_object(**loc_obj)
+
+    event = json.loads(misp_event.to_json())
+    return {'Object': event['Object'], 'Attribute': event['Attribute']}
+
+
+# In case of multiple enrichment (ip and domain), we need to filter out similar objects
+# TODO: make it more granular
+def remove_duplicates(results):
+    # Only one enrichment was performed so no duplicate
+    if len(results) == 1:
+        return results[0]
+    elif len(results) == 2:
+        final_result = results[0]
+        obj_l2 = results[1]['Object']
+        for o2 in obj_l2:
+            if o2['name'] == "asn":
+                key = "asn"
+            elif o2['name'] == "ip-port":
+                key = "ip"
+            elif o2['name'] == "x509":
+                key = "x509-fingerprint-sha256"
+            elif o2['name'] == "geolocation":
+                key = "latitude"
+            if not check_if_present(o2, key, final_result['Object']):
+                final_result['Object'].append(o2)
+
+        return final_result
+    else:
+        return []
+
+
+def check_if_present(object, attribute_name, list_objects):
+    """
+    Assert if a given object is present in the list.
+
+    This function check if object (json format) is present in list_objects
+    using attribute_name for the matching
+    """
+    for o in list_objects:
+        # We first look for a match on the name
+        if o['name'] == object['name']:
+            for attr in object['Attribute']:
+                # Within the attributes, we look for the one to compare
+                if attr['type'] == attribute_name:
+                    # Then we check the attributes of the other object and look for a match
+                    for attr2 in o['Attribute']:
+                        if attr2['type'] == attribute_name and attr2['value'] == attr['value']:
+                            return True
+
+    return False
+
+
+def get_certificate_object(cert, attribute):
+    parsed = cert['parsed']
+    cert_object = MISPObject('x509')
+    cert_object.add_attribute('x509-fingerprint-sha256', value=parsed['fingerprint_sha256'])
+    cert_object.add_attribute('x509-fingerprint-sha1', value=parsed['fingerprint_sha1'])
+    cert_object.add_attribute('x509-fingerprint-md5', value=parsed['fingerprint_md5'])
+    cert_object.add_attribute('serial-number', value=parsed['serial_number'])
+    cert_object.add_attribute('version', value=parsed['version'])
+    cert_object.add_attribute('subject', value=parsed['subject_dn'])
+    cert_object.add_attribute('issuer', value=parsed['issuer_dn'])
+    cert_object.add_attribute('validity-not-before', value=isoparse(parsed['validity']['start']))
+    cert_object.add_attribute('validity-not-after', value=isoparse(parsed['validity']['end']))
+    cert_object.add_attribute('self_signed', value=parsed['signature']['self_signed'])
+    cert_object.add_attribute('signature_algorithm', value=parsed['signature']['signature_algorithm']['name'])
+
+    cert_object.add_attribute('pubkey-info-algorithm', value=parsed['subject_key_info']['key_algorithm']['name'])
+
+    if 'rsa_public_key' in parsed['subject_key_info']:
+        pub_key = parsed['subject_key_info']['rsa_public_key']
+        cert_object.add_attribute('pubkey-info-size', value=pub_key['length'])
+        cert_object.add_attribute('pubkey-info-exponent', value=pub_key['exponent'])
+        hex_mod = codecs.encode(base64.b64decode(pub_key['modulus']), 'hex').decode()
+        cert_object.add_attribute('pubkey-info-modulus', value=hex_mod)
+
+    if "extensions" in parsed and "subject_alt_name" in parsed["extensions"]:
+        san = parsed["extensions"]["subject_alt_name"]
+        if "dns_names" in san:
+            for dns in san['dns_names']:
+                cert_object.add_attribute('dns_names', value=dns)
+        if "ip_addresses" in san:
+            for ip in san['ip_addresses']:
+                cert_object.add_attribute('ip', value=ip)
+
+    if "raw" in cert:
+        cert_object.add_attribute('raw-base64', value=cert['raw'])
+
+    cert_object.add_reference(attribute.uuid, 'associated-to')
+    return cert_object
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo