From e023f0b4700122d18f94a7e5663b2a712babfcb5 Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Tue, 10 Mar 2020 18:25:30 +0100 Subject: [PATCH 1/5] Cytomic Orion MISP Module An expansion module to enrich attributes in MISP and share indicators of compromise with Cytomic Orion --- README.md | 1 + misp_modules/modules/expansion/__init__.py | 2 +- .../modules/expansion/cytomic_orion.py | 183 ++++++++++++++++++ 3 files changed, 185 insertions(+), 1 deletion(-) create mode 100755 misp_modules/modules/expansion/cytomic_orion.py diff --git a/README.md b/README.md index 996f2d0..fe37cd5 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE). * [CVE advanced](misp_modules/modules/expansion/cve_advanced.py) - An expansion module to query the CIRCL CVE search API for more information about a vulnerability (CVE). * [Cuckoo submit](misp_modules/modules/expansion/cuckoo_submit.py) - A hover module to submit malware sample, url, attachment, domain to Cuckoo Sandbox. +* [Cytomic Orion](misp_modules/modules/expansion/cytomic_orion.py) - An expansion module to enrich attributes in MISP and share indicators of compromise with Cytomic Orion. * [DBL Spamhaus](misp_modules/modules/expansion/dbl_spamhaus.py) - a hover module to check Spamhaus DBL for a domain name. * [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes. * [docx-enrich](misp_modules/modules/expansion/docx_enrich.py) - an enrichment module to get text out of Word document into MISP (using free-text parser). diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 8eb0a92..53c8ad8 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -16,4 +16,4 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'ods_enrich', 'odt_enrich', 'joesandbox_submit', 'joesandbox_query', 'urlhaus', 'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid', 'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', - 'lastline_query', 'lastline_submit', 'sophoslabs_intelix'] + 'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion.py'] diff --git a/misp_modules/modules/expansion/cytomic_orion.py b/misp_modules/modules/expansion/cytomic_orion.py new file mode 100755 index 0000000..897840b --- /dev/null +++ b/misp_modules/modules/expansion/cytomic_orion.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 + +''' +Cytomic Orion MISP Module +An expansion module to enrich attributes in MISP and share indicators of compromise with Cytomic Orion + + +''' + +from pymisp import MISPAttribute, MISPEvent, MISPObject, MISPTag +import json +import requests +import re + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['md5'], 'format': 'misp_standard'} +moduleinfo = {'version': '0.3', 'author': 'Koen Van Impe', + 'description': 'an expansion module to enrich attributes in MISP and share indicators of compromise with Cytomic Orion', + 'module-type': ['expansion']} +moduleconfig = ['api_url', 'token_url', 'clientid', 'clientsecret', 'clientsecret', 'username', 'password', 'upload_timeframe', 'upload_tag', 'delete_tag', 'upload_ttlDays', 'upload_threat_level_id', 'limit_upload_events', 'limit_upload_attributes'] +# There are more config settings in this module than used by the enrichment +# There is also a PyMISP module which reuses the module config, and requires additional configuration, for example used for pushing indicators to the API + + +class CytomicParser(): + def __init__(self, attribute, config_object): + self.misp_event = MISPEvent() + self.attribute = MISPAttribute() + self.attribute.from_dict(**attribute) + self.misp_event.add_attribute(**self.attribute) + + self.config_object = config_object + + if self.config_object: + self.token = self.get_token() + else: + return {'error': 'Missing configuration'} + + def get_token(self): + try: + scope = self.config_object['scope'] + grant_type = self.config_object['grant_type'] + username = self.config_object['username'] + password = self.config_object['password'] + token_url = self.config_object['token_url'] + clientid = self.config_object['clientid'] + clientsecret = self.config_object['clientsecret'] + + if scope and grant_type and username and password: + data = {'scope': scope, 'grant_type': grant_type, 'username': username, 'password': password} + + if token_url and clientid and clientsecret: + access_token_response = requests.post(token_url, data=data, verify=False, allow_redirects=False, auth=(clientid, clientsecret)) + tokens = json.loads(access_token_response.text) + if 'access_token' in tokens: + return tokens['access_token'] + else: + self.result = {'error': 'No token received.'} + return + else: + self.result = {'error': 'No token_url, clientid or clientsecret supplied.'} + return + else: + self.result = {'error': 'No scope, grant_type, username or password supplied.'} + return + except Exception: + self.result = {'error': 'Unable to connect to token_url.'} + return + + def get_results(self): + if hasattr(self, 'result'): + return self.result + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object')} + return {'results': results} + + def parse(self, searchkey): + + if self.token: + + endpoint_fileinformation = self.config_object['endpoint_fileinformation'] + endpoint_machines = self.config_object['endpoint_machines'] + endpoint_machines_client = self.config_object['endpoint_machines_client'] + query_machines = self.config_object['query_machines'] + query_machine_info = self.config_object['query_machine_info'] + + # Update endpoint URLs + query_endpoint_fileinformation = endpoint_fileinformation.format(md5=searchkey) + query_endpoint_machines = endpoint_machines.format(md5=searchkey) + + # API calls + api_call_headers = {'Authorization': 'Bearer ' + self.token} + result_query_endpoint_fileinformation = requests.get(query_endpoint_fileinformation, headers=api_call_headers, verify=False) + json_result_query_endpoint_fileinformation = json.loads(result_query_endpoint_fileinformation.text) + + if json_result_query_endpoint_fileinformation: + + cytomic_object = MISPObject('cytomic-orion-file') + + cytomic_object.add_attribute('fileName', type='text', value=json_result_query_endpoint_fileinformation['fileName']) + cytomic_object.add_attribute('fileSize', type='text', value=json_result_query_endpoint_fileinformation['fileSize']) + cytomic_object.add_attribute('last-seen', type='datetime', value=json_result_query_endpoint_fileinformation['lastSeen']) + cytomic_object.add_attribute('first-seen', type='datetime', value=json_result_query_endpoint_fileinformation['firstSeen']) + cytomic_object.add_attribute('classification', type='text', value=json_result_query_endpoint_fileinformation['classification']) + cytomic_object.add_attribute('classificationName', type='text', value=json_result_query_endpoint_fileinformation['classificationName']) + self.misp_event.add_object(**cytomic_object) + + result_query_endpoint_machines = requests.get(query_endpoint_machines, headers=api_call_headers, verify=False) + json_result_query_endpoint_machines = json.loads(result_query_endpoint_machines.text) + + if json_result_query_endpoint_machines and len(json_result_query_endpoint_machines) > 0: + for machine in json_result_query_endpoint_machines: + + if machine['muid'] and query_machine_info: + query_endpoint_machines_client = endpoint_machines_client.format(muid=machine['muid']) + result_endpoint_machines_client = requests.get(query_endpoint_machines_client, headers=api_call_headers, verify=False) + json_result_endpoint_machines_client = json.loads(result_endpoint_machines_client.text) + + if json_result_endpoint_machines_client: + + cytomic_machine_object = MISPObject('cytomic-orion-machine') + + clienttag = [{'name': json_result_endpoint_machines_client['clientName']}] + + cytomic_machine_object.add_attribute('machineName', type='target-machine', value=json_result_endpoint_machines_client['machineName'], Tag=clienttag) + cytomic_machine_object.add_attribute('machineMuid', type='text', value=machine['muid']) + cytomic_machine_object.add_attribute('clientName', type='target-org', value=json_result_endpoint_machines_client['clientName'], Tag=clienttag) + cytomic_machine_object.add_attribute('clientId', type='text', value=machine['clientId']) + cytomic_machine_object.add_attribute('machinePath', type='text', value=machine['lastPath']) + cytomic_machine_object.add_attribute('first-seen', type='datetime', value=machine['firstSeen']) + cytomic_machine_object.add_attribute('last-seen', type='datetime', value=machine['lastSeen']) + cytomic_machine_object.add_attribute('creationDate', type='datetime', value=json_result_endpoint_machines_client['creationDate']) + cytomic_machine_object.add_attribute('clientCreationDateUTC', type='datetime', value=json_result_endpoint_machines_client['clientCreationDateUTC']) + cytomic_machine_object.add_attribute('lastSeenUtc', type='datetime', value=json_result_endpoint_machines_client['lastSeenUtc']) + self.misp_event.add_object(**cytomic_machine_object) + else: + self.result = {'error': 'No (valid) token.'} + return + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + + if not request.get('attribute'): + return {'error': 'Unsupported input.'} + + attribute = request['attribute'] + if not any(input_type == attribute['type'] for input_type in mispattributes['input']): + return {'error': 'Unsupported attributes type'} + + if not request.get('config'): + return {'error': 'Missing configuration'} + + config_object = { + 'clientid': request["config"].get("clientid"), + 'clientsecret': request["config"].get("clientsecret"), + 'scope': 'orion.api', + 'password': request["config"].get("password"), + 'username': request["config"].get("username"), + 'grant_type': 'password', + 'token_url': request["config"].get("token_url"), + 'endpoint_fileinformation': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/md5/{md5}/info'), + 'endpoint_machines': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/md5/{md5}/muids'), + 'endpoint_machines_client': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/muid/{muid}/info'), + 'query_machines': True, + 'query_machine_info': True + } + + cytomic_parser = CytomicParser(attribute, config_object) + cytomic_parser.parse(attribute['value']) + + return cytomic_parser.get_results() + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From c86f4a418053718eea07e1baa64d7d5db096a3d4 Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Tue, 10 Mar 2020 18:48:25 +0100 Subject: [PATCH 2/5] Make Travis (a little bit) happy --- misp_modules/modules/expansion/cytomic_orion.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/misp_modules/modules/expansion/cytomic_orion.py b/misp_modules/modules/expansion/cytomic_orion.py index 897840b..c7830e8 100755 --- a/misp_modules/modules/expansion/cytomic_orion.py +++ b/misp_modules/modules/expansion/cytomic_orion.py @@ -7,10 +7,10 @@ An expansion module to enrich attributes in MISP and share indicators of comprom ''' -from pymisp import MISPAttribute, MISPEvent, MISPObject, MISPTag +from pymisp import MISPAttribute, MISPEvent, MISPObject import json import requests -import re +import sys misperrors = {'error': 'Error'} mispattributes = {'input': ['md5'], 'format': 'misp_standard'} @@ -34,7 +34,7 @@ class CytomicParser(): if self.config_object: self.token = self.get_token() else: - return {'error': 'Missing configuration'} + sys.exit('Missing configuration') def get_token(self): try: @@ -108,10 +108,10 @@ class CytomicParser(): result_query_endpoint_machines = requests.get(query_endpoint_machines, headers=api_call_headers, verify=False) json_result_query_endpoint_machines = json.loads(result_query_endpoint_machines.text) - if json_result_query_endpoint_machines and len(json_result_query_endpoint_machines) > 0: + if query_machines and json_result_query_endpoint_machines and len(json_result_query_endpoint_machines) > 0: for machine in json_result_query_endpoint_machines: - if machine['muid'] and query_machine_info: + if query_machine_info and machine['muid']: query_endpoint_machines_client = endpoint_machines_client.format(muid=machine['muid']) result_endpoint_machines_client = requests.get(query_endpoint_machines_client, headers=api_call_headers, verify=False) json_result_endpoint_machines_client = json.loads(result_endpoint_machines_client.text) From 2713d3c6555f3e7f36a8aa0364a74a71ec1bd85a Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Tue, 10 Mar 2020 19:50:00 +0100 Subject: [PATCH 3/5] Update __init__ --- misp_modules/modules/expansion/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 53c8ad8..2a99050 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -16,4 +16,4 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'ods_enrich', 'odt_enrich', 'joesandbox_submit', 'joesandbox_query', 'urlhaus', 'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid', 'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', - 'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion.py'] + 'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion'] From d2f0d8027bd3f380198a22bbfb7ca300fd39b1fb Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Wed, 11 Mar 2020 11:56:12 +0100 Subject: [PATCH 4/5] Documentation for Cytomic Orion --- doc/expansion/cytomic_orion.py | 9 +++++++++ doc/logos/cytomic_orion.png | Bin 0 -> 898 bytes 2 files changed, 9 insertions(+) create mode 100644 doc/expansion/cytomic_orion.py create mode 100644 doc/logos/cytomic_orion.png diff --git a/doc/expansion/cytomic_orion.py b/doc/expansion/cytomic_orion.py new file mode 100644 index 0000000..6f87657 --- /dev/null +++ b/doc/expansion/cytomic_orion.py @@ -0,0 +1,9 @@ +{ + "description": "An expansion module to enrich attributes in MISP by quering the Cytomic Orion API", + "logo": "logos/cytomic_orion.png", + "requirements": ["Access (license) to Cytomic Orion"], + "input": "MD5, hash of the sample / malware to search for.", + "output": "MISP objects with sightings of the hash in Cytomic Orion. Includes files and machines.", + "references": ["https://www.vanimpe.eu/2020/03/10/integrating-misp-and-cytomic-orion/", "https://www.cytomicmodel.com/solutions/"], + "features": "This module takes an MD5 hash and searches for occurrences of this hash in the Cytomic Orion database. Returns observed files and machines." +} diff --git a/doc/logos/cytomic_orion.png b/doc/logos/cytomic_orion.png new file mode 100644 index 0000000000000000000000000000000000000000..45704e9278088bb5305b3b0b20fcb4158b8fe2ee GIT binary patch literal 898 zcmV-|1AY97P)fVtcD? zYC!oEge*lRQ+<$8WgRLQmX~&DAQ85>=1)@bO%R&r)SEd4S*k8mCidD-B(Y&t=*w~O z;XcTeR4$)D$+J2)=BFSvN6ND|M32|YY?V2PiyQjV5Nq}ZH~WGw9F^OSNMlH3F?>0Q z@M}P>toG{WoDSjU4Yc_bWRt1fD#q4Gya7Tas1l?LVoPKTTb+PjmogaJD4YdTYTlZb zTOf0LGTGWRr^DO;Td?kexJ+sgQ75me@*y$5@)V?-^gAGHOjLcSJqB?jZSHq& z5la3U2oGU;3Tv==+&u(wsWhjeo5JiIWIy!~8udwIkjq+iE&2|K@VjBS|D1z#7yI@0 zYjnUR`DKuB6AdGC4#L_|y;Lfc_s_WDs(-pHR6l<%iXj?D}4NAl(nE7D=@~eG1aa z4GaqJ=UtHQr*8GJt9r@%35d(3xB*iBD53dfSZL718|U7yLCk6B&+H43fx!tRE3aao z464#g`}l7Be7mDA2n;&MpK{{z2c6ip^SU>^{zP366bgkxp-?Ck3WY+UP$(1%h5AeS Y2jOxQJ{Y2W1^@s607*qoM6N<$g6^}Z-~a#s literal 0 HcmV?d00001 From fe34023866da011b175709a862d6efec579b56b3 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Thu, 12 Mar 2020 11:02:43 +0100 Subject: [PATCH 5/5] csvimport: Return error if input is not valid UTF-8 --- misp_modules/modules/import_mod/csvimport.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py index 8bfbbe9..38e5f96 100644 --- a/misp_modules/modules/import_mod/csvimport.py +++ b/misp_modules/modules/import_mod/csvimport.py @@ -256,7 +256,11 @@ def handler(q=False): return False request = json.loads(q) if request.get('data'): - data = base64.b64decode(request['data']).decode('utf-8') + try: + data = base64.b64decode(request['data']).decode('utf-8') + except UnicodeDecodeError: + misperrors['error'] = "Input is not valid UTF-8" + return misperrors else: misperrors['error'] = "Unsupported attributes type" return misperrors