#!/usr/bin/env python # -*- coding: utf-8 -*- ''' Koen Van Impe Cytomic Automation Put this script in crontab to run every /15 or /60 */15 * * * * mispuser /usr/bin/python3 /home/mispuser/PyMISP/examples/cytomic_orion.py Fetches the configuration set in the Cytomic Orion enrichment module - events : upload events tagged with the 'upload' tag, all the attributes supported by Cytomic Orion - upload : upload attributes flagged with the 'upload' tag (only attributes supported by Cytomic Orion) - delete : delete attributes flagged with the 'upload' tag (only attributes supported by Cytomic Orion) ''' from pymisp import ExpandedPyMISP from keys import misp_url, misp_key, misp_verifycert import argparse import os import re import sys import requests import json import urllib3 def get_token(token_url, clientid, clientsecret, scope, grant_type, username, password): ''' Get oAuth2 token Configuration settings are fetched first from the MISP module configu ''' try: if scope and grant_type and username and password: data = {'scope': scope, 'grant_type': grant_type, 'username': username, 'password': password} if token_url and clientid and clientsecret: access_token_response = requests.post(token_url, data=data, verify=False, allow_redirects=False, auth=(clientid, clientsecret)) tokens = json.loads(access_token_response.text) if 'access_token' in tokens: access_token = tokens['access_token'] return access_token else: sys.exit('No token received') else: sys.exit('No token_url, clientid or clientsecret supplied') else: sys.exit('No scope, grant_type, username or password supplied') except Exception: sys.exit('Unable to connect to token_url') def get_config(url, key, misp_verifycert): ''' Get the module config and the settings needed to access the API Also contains the settings to do the query ''' try: misp_headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': key} req = requests.get(url + 'servers/serverSettings.json', verify=misp_verifycert, headers=misp_headers) if req.status_code == 200: req_json = req.json() if 'finalSettings' in req_json: finalSettings = req_json['finalSettings'] clientid = clientsecret = scope = username = password = grant_type = api_url = token_url = '' module_enabled = False scope = 'orion.api' grant_type = 'password' limit_upload_events = 50 limit_upload_attributes = 50 ttlDays = "1" last_attributes = '5d' post_threat_level_id = 2 for el in finalSettings: # Is the module enabled? if el['setting'] == 'Plugin.Enrichment_cytomic_orion_enabled': module_enabled = el['value'] if module_enabled is False: break elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_clientid': clientid = el['value'] elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_clientsecret': clientsecret = el['value'] elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_username': username = el['value'] elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_password': password = el['value'] elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_api_url': api_url = el['value'].replace('\\/', '/') elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_token_url': token_url = el['value'].replace('\\/', '/') elif el['setting'] == 'MISP.baseurl': misp_baseurl = el['value'] elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_threat_level_id': if el['value']: try: post_threat_level_id = int(el['value']) except: continue elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_ttlDays': if el['value']: try: ttlDays = "{last_days}".format(last_days=int(el['value'])) except: continue elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_timeframe': if el['value']: try: last_attributes = "{last_days}d".format(last_days=int(el['value'])) except: continue elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_tag': upload_tag = el['value'] elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_delete_tag': delete_tag = el['value'] elif el['setting'] == 'Plugin.Enrichment_limit_upload_events': if el['value']: try: limit_upload_events = "{limit_upload_events}".format(limit_upload_events=int(el['value'])) except: continue elif el['setting'] == 'Plugin.Enrichment_limit_upload_attributes': if el['value']: try: limit_upload_attributes = "{limit_upload_attributes}".format(limit_upload_attributes=int(el['value'])) except: continue else: sys.exit('Did not receive a 200 code from MISP') if module_enabled and api_url and token_url and clientid and clientsecret and username and password and grant_type: return {'cytomic_policy': 'Detect', 'upload_timeframe': last_attributes, 'upload_tag': upload_tag, 'delete_tag': delete_tag, 'upload_ttlDays': ttlDays, 'post_threat_level_id': post_threat_level_id, 'clientid': clientid, 'clientsecret': clientsecret, 'scope': scope, 'username': username, 'password': password, 'grant_type': grant_type, 'api_url': api_url, 'token_url': token_url, 'misp_baseurl': misp_baseurl, 'limit_upload_events': limit_upload_events, 'limit_upload_attributes': limit_upload_attributes} else: sys.exit('Did not receive all the necessary configuration information from MISP') except Exception as e: sys.exit('Unable to get module config from MISP') class cytomicobject: misp = None lst_evtid = None lst_attuuid = None lst_attuuid_error = None endpoint_ioc = None api_call_headers = None post_data = None args = None tag = None limit_events = None limit_attributes = None atttype_misp = None atttype_cytomic = None attlabel_cytomic = None att_types = { "ip-dst": {"ip": "ipioc"}, "ip-src": {"ip": "ipioc"}, "url": {"url": "urlioc"}, "md5": {"hash": "filehashioc"}, "domain": {"domain": "domainioc"}, "hostname": {"domain": "domainioc"}, "domain|ip": {"domain": "domainioc"}, "hostname|port": {"domain": "domainioc"} } debug = True error = False res = False res_msg = None def collect_events_ids(cytomicobj, moduleconfig): # Get events that contain Cytomic tag. try: evt_result = cytomicobj.misp.search(controller='events', limit=cytomicobj.limit_events, tags=cytomicobj.tag, last=moduleconfig['upload_timeframe'], published=True, deleted=False, pythonify=True) cytomicobj.lst_evtid = ['x', 'y'] for evt in evt_result: evt = cytomicobj.misp.get_event(event=evt['id'], pythonify=True) if len(evt.tags) > 0: for tg in evt.tags: if tg.name == cytomicobj.tag: if not cytomicobj.lst_evtid: cytomicobj.lst_evtid = str(evt['id']) else: if not evt['id'] in cytomicobj.lst_evtid: cytomicobj.lst_evtid.append(str(evt['id'])) break cytomicobj.lst_evtid.remove('x') cytomicobj.lst_evtid.remove('y') except Exception: cytomicobj.error = True if cytomicobj.debug: sys.exit('Unable to collect events ids') def find_eventid(cytomicobj, evtid): # Get events that contain Cytomic tag. try: cytomicobj.res = False for id in cytomicobj.lst_evtid: if id == evtid: cytomicobj.res = True break except Exception: cytomicobj.error = True if cytomicobj.debug: sys.exit('Unable to collect events ids') def print_result_events(cytomicobj): try: if cytomicobj.res_msg is not None: for key, msg in cytomicobj.res_msg.items(): if msg is not None: print(key, msg) except Exception: cytomicobj.error = True if cytomicobj.debug: sys.exit('Unable to print result') def set_postdata(cytomicobj, moduleconfig, attribute): # Set JSON to send to the API. try: if cytomicobj.args.upload or cytomicobj.args.events: event = attribute['Event'] event_title = event['info'] event_id = event['id'] threat_level_id = int(event['threat_level_id']) if moduleconfig['post_threat_level_id'] <= threat_level_id: if cytomicobj.atttype_misp == 'domain|ip' or cytomicobj.atttype_misp == 'hostname|port': post_value = attribute['value'].split('|')[0] else: post_value = attribute['value'] if cytomicobj.atttype_misp == 'url' and 'http' not in post_value: pass else: if cytomicobj.post_data is None: cytomicobj.post_data = [{cytomicobj.attlabel_cytomic: post_value, 'AdditionalData': '{} {}'.format(cytomicobj.atttype_misp, attribute['comment']).strip(), 'Source': 'Uploaded from MISP', 'Policy': moduleconfig['cytomic_policy'], 'Description': '{} - {}'.format(event_id, event_title).strip()}] else: if post_value not in str(cytomicobj.post_data): cytomicobj.post_data.append({cytomicobj.attlabel_cytomic: post_value, 'AdditionalData': '{} {}'.format(cytomicobj.atttype_misp, attribute['comment']).strip(), 'Source': 'Uploaded from MISP', 'Policy': moduleconfig['cytomic_policy'], 'Description': '{} - {}'.format(event_id, event_title).strip()}) else: if cytomicobject.debug: print('Event %s skipped because of lower threat level' % event_id) else: event = attribute['Event'] threat_level_id = int(event['threat_level_id']) if moduleconfig['post_threat_level_id'] <= threat_level_id: if cytomicobj.atttype_misp == 'domain|ip' or cytomicobj.atttype_misp == 'hostname|port': post_value = attribute['value'].split('|')[0] else: post_value = attribute['value'] if cytomicobj.atttype_misp == 'url' and 'http' not in post_value: pass else: if cytomicobj.post_data is None: cytomicobj.post_data = [{cytomicobj.attlabel_cytomic: post_value}] else: cytomicobj.post_data.append({cytomicobj.attlabel_cytomic: post_value}) else: if cytomicobject.debug: print('Event %s skipped because of lower threat level' % event_id) except Exception: cytomicobj.error = True if cytomicobj.debug: sys.exit('Unable to process post-data') def send_postdata(cytomicobj, evtid=None): # Batch post to upload event attributes. try: if cytomicobj.post_data is not None: if cytomicobj.debug: print('POST: {} {}'.format(cytomicobj.endpoint_ioc, cytomicobj.post_data)) result_post_endpoint_ioc = requests.post(cytomicobj.endpoint_ioc, headers=cytomicobj.api_call_headers, json=cytomicobj.post_data, verify=False) json_result_post_endpoint_ioc = json.loads(result_post_endpoint_ioc.text) print(result_post_endpoint_ioc) if 'true' not in (result_post_endpoint_ioc.text): cytomicobj.error = True if evtid is not None: if cytomicobj.res_msg['Event: ' + str(evtid)] is None: cytomicobj.res_msg['Event: ' + str(evtid)] = '(Send POST data: errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.' else: cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (Send POST data -else: errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.' if cytomicobj.debug: print('RESULT: {}'.format(json_result_post_endpoint_ioc)) else: if evtid is None: cytomicobj.error = True except Exception: cytomicobj.error = True if cytomicobj.debug: sys.exit('Unable to post attributes') def process_attributes(cytomicobj, moduleconfig, evtid=None): # Get attributes to process. try: for misptype, cytomictypes in cytomicobject.att_types.items(): cytomicobj.atttype_misp = misptype for cytomiclabel, cytomictype in cytomictypes.items(): cytomicobj.attlabel_cytomic = cytomiclabel cytomicobj.atttype_cytomic = cytomictype cytomicobj.post_data = None icont = 0 if cytomicobj.args.upload or cytomicobj.args.events: cytomicobj.endpoint_ioc = moduleconfig['api_url'] + '/iocs/' + cytomicobj.atttype_cytomic + '?ttlDays=' + str(moduleconfig['upload_ttlDays']) else: cytomicobj.endpoint_ioc = moduleconfig['api_url'] + '/iocs/eraser/' + cytomicobj.atttype_cytomic # Get attributes to upload/delete and prepare JSON # If evtid is set; we're called from --events if cytomicobject.debug: print("\nSearching for attributes of type %s" % cytomicobj.atttype_misp) if evtid is None: cytomicobj.error = False attr_result = cytomicobj.misp.search(controller='attributes', last=moduleconfig['upload_timeframe'], limit=cytomicobj.limit_attributes, type_attribute=cytomicobj.atttype_misp, tag=cytomicobj.tag, published=True, deleted=False, includeProposals=False, include_context=True, to_ids=True) else: if cytomicobj.error: break # We don't search with tags; we have an event for which we want to upload all events attr_result = cytomicobj.misp.search(controller='attributes', eventid=evtid, last=moduleconfig['upload_timeframe'], limit=cytomicobj.limit_attributes, type_attribute=cytomicobj.atttype_misp, published=True, deleted=False, includeProposals=False, include_context=True, to_ids=True) cytomicobj.lst_attuuid = ['x', 'y'] if len(attr_result['Attribute']) > 0: for attribute in attr_result['Attribute']: if evtid is not None: if cytomicobj.error: cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.' break if icont >= cytomicobj.limit_attributes: if not cytomicobj.error and cytomicobj.post_data is not None: # Send data to Cytomic send_postdata(cytomicobj, evtid) if not cytomicobj.error: if 'Event: ' + str(evtid) in cytomicobj.res_msg: if cytomicobj.res_msg['Event: ' + str(evtid)] is None: cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.attlabel_cytomic + 's: ' + str(icont) else: cytomicobj.res_msg['Event: ' + str(evtid)] += ' | ' + cytomicobj.attlabel_cytomic + 's: ' + str(icont) else: if cytomicobject.debug: print('Data sent (' + cytomicobj.attlabel_cytomic + '): ' + str(icont)) cytomicobj.post_data = None if cytomicobj.error: if evtid is not None: cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.' break icont = 0 if evtid is None: event = attribute['Event'] event_id = event['id'] find_eventid(cytomicobj, str(event_id)) if not cytomicobj.res: if not cytomicobj.lst_attuuid: cytomicobj.lst_attuuid = attribute['uuid'] else: if not attribute['uuid'] in cytomicobj.lst_attuuid: cytomicobj.lst_attuuid.append(attribute['uuid']) icont += 1 # Prepare data to send set_postdata(cytomicobj, moduleconfig, attribute) else: icont += 1 # Prepare data to send set_postdata(cytomicobj, moduleconfig, attribute) if not cytomicobj.error: # Send data to Cytomic send_postdata(cytomicobj, evtid) if not cytomicobj.error and cytomicobj.post_data is not None and icont > 0: # Data sent; process response if cytomicobj.res_msg is not None and 'Event: ' + str(evtid) in cytomicobj.res_msg: if cytomicobj.res_msg['Event: ' + str(evtid)] is None: cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.attlabel_cytomic + 's: ' + str(icont) else: cytomicobj.res_msg['Event: ' + str(evtid)] += ' | ' + cytomicobj.attlabel_cytomic + 's: ' + str(icont) else: if cytomicobject.debug: print('Data sent (' + cytomicobj.attlabel_cytomic + '): ' + str(icont)) if not cytomicobj.error: cytomicobj.lst_attuuid.remove('x') cytomicobj.lst_attuuid.remove('y') # Untag attributes untag_attributes(cytomicobj) except Exception: cytomicobj.error = True if cytomicobj.debug: sys.exit('Unable to get attributes') def untag_event(evtid): # Remove tag of the event being processed. try: cytomicobj.records = 0 evt = cytomicobj.misp.get_event(event=evtid, pythonify=True) if len(evt.tags) > 0: for tg in evt.tags: if tg.name == cytomicobj.tag: cytomicobj.misp.untag(evt['uuid'], cytomicobj.tag) cytomicobj.records += 1 cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (event untagged)' break except Exception: cytomicobj.error = True if cytomicobj.debug: sys.exit('Unable to untag events') def process_events(cytomicobj, moduleconfig): # Get events that contain Cytomic tag. try: collect_events_ids(cytomicobj, moduleconfig) total_attributes_sent = 0 for evtid in cytomicobj.lst_evtid: cytomicobj.error = False if cytomicobj.res_msg is None: cytomicobj.res_msg = {'Event: ' + str(evtid): None} else: cytomicobj.res_msg['Event: ' + str(evtid)] = None if cytomicobject.debug: print('Event id: ' + str(evtid)) # get attributes of each known type of the event / prepare data to send / send data to Cytomic process_attributes(cytomicobj, moduleconfig, evtid) if not cytomicobj.error: untag_event(evtid) except Exception: cytomicobj.error = True if cytomicobj.debug: sys.exit('Unable to process events ids') def untag_attributes(cytomicobj): # Remove tag of attributes sent. try: icont = 0 if len(cytomicobj.lst_attuuid) > 0: for uuid in cytomicobj.lst_attuuid: attr = cytomicobj.misp.get_attribute(attribute=uuid, pythonify=True) if len(attr.tags) > 0: for tg in attr.tags: if tg.name == cytomicobj.tag: cytomicobj.misp.untag(uuid, cytomicobj.tag) icont += 1 break print('Attributes untagged (' + str(icont) + ')') except Exception: cytomicobj.error = True if cytomicobj.debug: sys.exit('Unable to untag attributes') def process_attributes_upload(cytomicobj, moduleconfig): # get attributes of each known type / prepare data to send / send data to Cytomic try: collect_events_ids(cytomicobj, moduleconfig) process_attributes(cytomicobj, moduleconfig) except Exception: cytomicobj.error = True if cytomicobj.debug: sys.exit('Unable to upload attributes to Cytomic') def process_attributes_delete(cytomicobj, moduleconfig): # get attributes of each known type / prepare data to send / send data to Cytomic try: collect_events_ids(cytomicobj, moduleconfig) process_attributes(cytomicobj, moduleconfig) except Exception: cytomicobj.error = True if cytomicobj.debug: sys.exit('Unable to delete attributes in Cytomic') if __name__ == '__main__': parser = argparse.ArgumentParser(description='Upload or delete indicators to Cytomic API') group = parser.add_mutually_exclusive_group() group.add_argument('--events', action='store_true', help='Upload events indicators') group.add_argument('--upload', action='store_true', help='Upload indicators') group.add_argument('--delete', action='store_true', help='Delete indicators') args = parser.parse_args() if not args.upload and not args.delete and not args.events: sys.exit("No valid action for the API") if misp_verifycert is False: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) module_config = get_config(misp_url, misp_key, misp_verifycert) cytomicobj = cytomicobject misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, debug=cytomicobject.debug) cytomicobj.misp = misp cytomicobj.args = args access_token = get_token(module_config['token_url'], module_config['clientid'], module_config['clientsecret'], module_config['scope'], module_config['grant_type'], module_config['username'], module_config['password']) cytomicobj.api_call_headers = {'Authorization': 'Bearer ' + access_token} if cytomicobj.debug: print('Received access token') if cytomicobj.args.events: cytomicobj.tag = module_config['upload_tag'] cytomicobj.limit_events = module_config['limit_upload_events'] cytomicobj.limit_attributes = module_config['limit_upload_attributes'] process_events(cytomicobj, module_config) print_result_events(cytomicobj) elif cytomicobj.args.upload: cytomicobj.tag = module_config['upload_tag'] cytomicobj.limit_events = 0 cytomicobj.limit_attributes = module_config['limit_upload_attributes'] process_attributes_upload(cytomicobj, module_config) else: cytomicobj.tag = module_config['delete_tag'] cytomicobj.limit_events = 0 cytomicobj.limit_attributes = module_config['limit_upload_attributes'] process_attributes_delete(cytomicobj, module_config)