diff --git a/.gitignore b/.gitignore index 323f87a..4c3db86 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,7 @@ site* .idea/* #venv -venv* \ No newline at end of file +venv* + +#vscode +.vscode* \ No newline at end of file diff --git a/documentation/website/import_mod/cof2misp.json b/documentation/website/import_mod/cof2misp.json new file mode 100644 index 0000000..cbbb0cc --- /dev/null +++ b/documentation/website/import_mod/cof2misp.json @@ -0,0 +1,12 @@ +{ + "description": "Passive DNS Common Output Format (COF) MISP importer", + "requirements": [ + "PyMISP" + ], + "features": "Takes as input a valid COF file or the output of the dnsdbflex utility and creates MISP objects for the input.", + "references": [ + "https://tools.ietf.org/id/draft-dulaunoy-dnsop-passive-dns-cof-08.html" + ], + "input": "Passive DNS output in Common Output Format (COF)", + "output": "MISP objects" +} diff --git a/misp_modules/lib/cof2misp/__init__.py b/misp_modules/lib/cof2misp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/misp_modules/modules/expansion/farsight_passivedns.py b/misp_modules/modules/expansion/farsight_passivedns.py index de18735..7cf6f66 100755 --- a/misp_modules/modules/expansion/farsight_passivedns.py +++ b/misp_modules/modules/expansion/farsight_passivedns.py @@ -2,7 +2,7 @@ import dnsdb2 import json from . import check_input_attribute, standard_error_message from datetime import datetime -from pymisp import MISPEvent, MISPObject +from pymisp import MISPEvent, MISPObject, Distribution misperrors = {'error': 'Error'} standard_query_input = [ @@ -43,7 +43,7 @@ moduleconfig = ['apikey', 'server', 'limit', 'flex_queries'] DEFAULT_DNSDB_SERVER = 'https://api.dnsdb.info' DEFAULT_LIMIT = 10 - +DEFAULT_DISTRIBUTION_SETTING = Distribution.your_organisation_only.value TYPE_TO_FEATURE = { "btc": "Bitcoin address", "dkim": "domainkeys identified mail", @@ -85,7 +85,7 @@ class FarsightDnsdbParser(): self.misp_event = MISPEvent() self.misp_event.add_attribute(**attribute) self.passivedns_mapping = { - 'bailiwick': {'type': 'text', 'object_relation': 'bailiwick'}, + 'bailiwick': {'type': 'domain', 'object_relation': 'bailiwick'}, 'count': {'type': 'counter', 'object_relation': 'count'}, 'raw_rdata': {'type': 'text', 'object_relation': 'raw_rdata'}, 'rdata': {'type': 'text', 'object_relation': 'rdata'}, @@ -103,6 +103,7 @@ class FarsightDnsdbParser(): comment = self.comment % (query_type, TYPE_TO_FEATURE[self.attribute['type']], self.attribute['value']) for result in results: passivedns_object = MISPObject('passive-dns') + passivedns_object.distribution = DEFAULT_DISTRIBUTION_SETTING if result.get('rdata') and isinstance(result['rdata'], list): for rdata in result.pop('rdata'): passivedns_object.add_attribute(**self._parse_attribute(comment, 'rdata', rdata)) @@ -121,7 +122,7 @@ class FarsightDnsdbParser(): return {'results': results} def _parse_attribute(self, comment, feature, value): - attribute = {'value': value, 'comment': comment} + attribute = {'value': value, 'comment': comment, 'distribution': DEFAULT_DISTRIBUTION_SETTING} attribute.update(self.passivedns_mapping[feature]) return attribute @@ -148,6 +149,8 @@ def handler(q=False): response = to_query(client, *args) except dnsdb2.DnsdbException as e: return {'error': e.__str__()} + except dnsdb2.exceptions.QueryError: + return {'error': 'Communication error occurs while executing a query, or the server reports an error due to invalid arguments.'} if not response: return {'error': f"Empty results on Farsight DNSDB for the {TYPE_TO_FEATURE[attribute['type']]}: {attribute['value']}."} parser = FarsightDnsdbParser(attribute) diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index d8db477..c777707 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- import json + +from pymisp import MISPEvent, MISPObject + try: from onyphe import Onyphe except ImportError: @@ -9,9 +12,10 @@ except ImportError: misperrors = {'error': 'Error'} mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'], - 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url']} + 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url'], + 'format': 'misp_standard'} # possible module-types: 'expansion', 'hover' or both -moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', +moduleinfo = {'version': '2', 'author': 'Sebastien Larinier @sebdraven', 'description': 'Query on Onyphe', 'module-type': ['expansion', 'hover']} @@ -19,84 +23,205 @@ moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', moduleconfig = ['apikey'] +class OnypheClient: + + def __init__(self, api_key, attribute): + self.onyphe_client = Onyphe(api_key=api_key) + self.attribute = attribute + self.misp_event = MISPEvent() + self.misp_event.add_attribute(**attribute) + + def get_results(self): + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] + for key in ('Attribute', 'Object') if key in event} + return results + + def get_query_onyphe(self): + if self.attribute['type'] == 'ip-src' or self.attribute['type'] == 'ip-dst': + self.__summary_ip() + if self.attribute['type'] == 'domain': + self.__summary_domain() + if self.attribute['type'] == 'hostname': + self.__summary_hostname() + + def __summary_ip(self): + results = self.onyphe_client.summary_ip(self.attribute['value']) + if 'results' in results: + for r in results['results']: + if 'domain' in r: + domain = r['domain'] + if type(domain) == list: + for d in domain: + self.__get_object_domain_ip(d, 'domain') + elif type(domain) == str: + self.__get_object_domain_ip(domain, 'domain') + + if 'hostname' in r: + hostname = r['hostname'] + if type(hostname) == list: + for d in hostname: + self.__get_object_domain_ip(d, 'domain') + elif type(hostname) == str: + self.__get_object_domain_ip(hostname, 'domain') + + if 'issuer' in r: + self.__get_object_certificate(r) + + def __summary_domain(self): + results = self.onyphe_client.summary_domain(self.attribute['value']) + if 'results' in results: + for r in results['results']: + + for domain in r.get('domain'): + self.misp_event.add_attribute('domain', domain) + for hostname in r.get('hostname'): + self.misp_event.add_attribute('hostname', hostname) + if 'ip' in r: + if type(r['ip']) is str: + self.__get_object_domain_ip(r['ip'], 'ip') + if type(r['ip']) is list: + for ip in r['ip']: + self.__get_object_domain_ip(ip, 'ip') + if 'issuer' in r: + self.__get_object_certificate(r) + + def __summary_hostname(self): + results = self.onyphe_client.summary_hostname(self.attribute['value']) + if 'results' in results: + + for r in results['results']: + if 'domain' in r: + if type(r['domain']) is str: + self.misp_event.add_attribute( + 'domain', r['domain']) + if type(r['domain']) is list: + for domain in r['domain']: + self.misp_event.add_attribute('domain', domain) + + if 'hostname' in r: + if type(r['hostname']) is str: + self.misp_event.add_attribute( + 'hostname', r['hostname']) + if type(r['hostname']) is list: + for hostname in r['hostname']: + self.misp_event.add_attribute( + 'hostname', hostname) + + if 'ip' in r: + if type(r['ip']) is str: + self.__get_object_domain_ip(r['ip'], 'ip') + if type(r['ip']) is list: + for ip in r['ip']: + self.__get_object_domain_ip(ip, 'ip') + + if 'issuer' in r: + self.__get_object_certificate(r) + + if 'cve' in r: + if type(r['cve']) is list: + for cve in r['cve']: + self.__get_object_cve(r, cve) + + def __get_object_certificate(self, r): + object_certificate = MISPObject('x509') + object_certificate.add_attribute('ip', self.attribute['value']) + object_certificate.add_attribute('serial-number', r['serial']) + object_certificate.add_attribute( + 'x509-fingerprint-sha256', r['fingerprint']['sha256']) + object_certificate.add_attribute( + 'x509-fingerprint-sha1', r['fingerprint']['sha1']) + object_certificate.add_attribute( + 'x509-fingerprint-md5', r['fingerprint']['md5']) + + signature = r['signature']['algorithm'] + value = '' + if 'sha256' in signature and 'RSA' in signature: + value = 'SHA256_WITH_RSA_ENCRYPTION' + elif 'sha1' in signature and 'RSA' in signature: + value = 'SHA1_WITH_RSA_ENCRYPTION' + if value: + object_certificate.add_attribute('signature_algorithm', value) + + object_certificate.add_attribute( + 'pubkey-info-algorithm', r['publickey']['algorithm']) + + if 'exponent' in r['publickey']: + object_certificate.add_attribute( + 'pubkey-info-exponent', r['publickey']['exponent']) + if 'length' in r['publickey']: + object_certificate.add_attribute( + 'pubkey-info-size', r['publickey']['length']) + + object_certificate.add_attribute('issuer', r['issuer']['commonname']) + object_certificate.add_attribute( + 'validity-not-before', r['validity']['notbefore']) + object_certificate.add_attribute( + 'validity-not-after', r['validity']['notbefore']) + object_certificate.add_reference(self.attribute['uuid'], 'related-to') + self.misp_event.add_object(object_certificate) + + def __get_object_domain_ip(self, obs, relation): + objet_domain_ip = MISPObject('domain-ip') + objet_domain_ip.add_attribute(relation, obs) + relation_attr = self.__get_relation_attribute() + if relation_attr: + objet_domain_ip.add_attribute( + relation_attr, self.attribute['value']) + objet_domain_ip.add_reference(self.attribute['uuid'], 'related-to') + self.misp_event.add_object(objet_domain_ip) + + def __get_relation_attribute(self): + + if self.attribute['type'] == 'ip-src': + return 'ip' + elif self.attribute['type'] == 'ip-dst': + return 'ip' + elif self.attribute['type'] == 'domain': + return 'domain' + elif self.attribute['type'] == 'hostname': + return 'domain' + + def __get_object_cve(self, item, cve): + attributes = [] + object_cve = MISPObject('vulnerability') + object_cve.add_attribute('id', cve) + object_cve.add_attribute('state', 'Published') + + if type(item['ip']) is list: + for ip in item['ip']: + attributes.extend( + list(filter(lambda x: x['value'] == ip, self.misp_event['Attribute']))) + for obj in self.misp_event['Object']: + attributes.extend( + list(filter(lambda x: x['value'] == ip, obj['Attribute']))) + if type(item['ip']) is str: + + for obj in self.misp_event['Object']: + for att in obj['Attribute']: + if att['value'] == item['ip']: + object_cve.add_reference(obj['uuid'], 'cve') + + self.misp_event.add_object(object_cve) + + def handler(q=False): if q: request = json.loads(q) + attribute = request['attribute'] if not request.get('config') or not request['config'].get('apikey'): misperrors['error'] = 'Onyphe authentication is missing' return misperrors - api = Onyphe(request['config'].get('apikey')) + api_key = request['config'].get('apikey') - if not api: - misperrors['error'] = 'Onyphe Error instance api' + onyphe_client = OnypheClient(api_key, attribute) + onyphe_client.get_query_onyphe() + results = onyphe_client.get_results() - ip = '' - if request.get('ip-src'): - ip = request['ip-src'] - elif request.get('ip-dst'): - ip = request['ip-dst'] - else: - misperrors['error'] = "Unsupported attributes type" - return misperrors - - return handle_expansion(api, ip, misperrors) - else: - return False - - -def handle_expansion(api, ip, misperrors): - result = api.ip(ip) - - if result['status'] == 'nok': - misperrors['error'] = result['message'] - return misperrors - - # categories = list(set([item['@category'] for item in result['results']])) - - result_filtered = {"results": []} - urls_pasties = [] - asn_list = [] - os_list = [] - domains_resolver = [] - domains_forward = [] - - for r in result['results']: - if r['@category'] == 'pastries': - if r['source'] == 'pastebin': - urls_pasties.append('https://pastebin.com/raw/%s' % r['key']) - elif r['@category'] == 'synscan': - asn_list.append(r['asn']) - os_target = r['os'] - if os_target != 'Unknown': - os_list.append(r['os']) - elif r['@category'] == 'resolver' and r['type'] == 'reverse': - domains_resolver.append(r['reverse']) - elif r['@category'] == 'resolver' and r['type'] == 'forward': - domains_forward.append(r['forward']) - - result_filtered['results'].append({'types': ['url'], 'values': urls_pasties, - 'categories': ['External analysis']}) - - result_filtered['results'].append({'types': ['AS'], 'values': list(set(asn_list)), - 'categories': ['Network activity']}) - - result_filtered['results'].append({'types': ['target-machine'], - 'values': list(set(os_list)), - 'categories': ['Targeting data']}) - - result_filtered['results'].append({'types': ['domain'], - 'values': list(set(domains_resolver)), - 'categories': ['Network activity'], - 'comment': 'resolver to %s' % ip}) - - result_filtered['results'].append({'types': ['domain'], - 'values': list(set(domains_forward)), - 'categories': ['Network activity'], - 'comment': 'forward to %s' % ip}) - return result_filtered + return {'results': results} def introspection(): diff --git a/misp_modules/modules/import_mod/cof2misp.py b/misp_modules/modules/import_mod/cof2misp.py index 96fd4c4..90998ec 100755 --- a/misp_modules/modules/import_mod/cof2misp.py +++ b/misp_modules/modules/import_mod/cof2misp.py @@ -81,7 +81,8 @@ def parse_and_insert_cof(data: str) -> dict: o = MISPObject(name='passive-dns', standalone=False, comment='created by cof2misp') # o.add_tag('tlp:amber') # FIXME: we'll want to add a tlp: tag to the object - o.add_attribute('bailiwick', value=entry['bailiwick'].rstrip('.')) + if 'bailiwick' in entry: + o.add_attribute('bailiwick', value=entry['bailiwick'].rstrip('.')) # # handle the combinations of rrtype (domain, ip) on both left and right side @@ -217,9 +218,9 @@ def handler(q=False): # Validate it # transform into MISP object # push to MISP - event_id = request['event_id'] + # event_id = request['event_id'] # event = misp.get_event(event_id) - print("event_id = %s" % event_id, file=sys.stderr) + # print("event_id = %s" % event_id, file=sys.stderr) try: data = base64.b64decode(request["data"]).decode('utf-8') if not data: diff --git a/tests/test_expansions.py b/tests/test_expansions.py index 84958ac..1bcc93e 100644 --- a/tests/test_expansions.py +++ b/tests/test_expansions.py @@ -310,6 +310,8 @@ class TestExpansions(unittest.TestCase): def test_onyphe(self): module_name = "onyphe" + if LiveCI: + return True query = {"module": module_name, "ip-src": "8.8.8.8"} if module_name in self.configs: query["config"] = self.configs[module_name] @@ -324,6 +326,8 @@ class TestExpansions(unittest.TestCase): def test_onyphe_full(self): module_name = "onyphe_full" + if LiveCI: + return True query = {"module": module_name, "ip-src": "8.8.8.8"} if module_name in self.configs: query["config"] = self.configs[module_name]