From 6616561e9609052854dcd770c9efee4ccbf6142e Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Wed, 11 Mar 2020 14:00:34 +0100 Subject: [PATCH 1/4] VMRay Automation with ExpandedPyMISP --- examples/vmray_automation.py | 213 +++++++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 examples/vmray_automation.py diff --git a/examples/vmray_automation.py b/examples/vmray_automation.py new file mode 100644 index 0000000..569d28e --- /dev/null +++ b/examples/vmray_automation.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +''' +Koen Van Impe + +VMRay automatic import +Put this script in crontab to run every /15 or /60 + */5 * * * * mispuser /usr/bin/python3 /home/mispuser/PyMISP/examples/vmray_automation.py + +Calls "vmray_import" for all events that have an 'incomplete' VMray analysis + +Do inline config in "main" + +''' + +from pymisp import ExpandedPyMISP, MISPAttribute +from keys import misp_url, misp_key, misp_verifycert +import argparse +import os +import json +import datetime +import time + +import requests +import sys + +# Suppress those "Unverified HTTPS request is being made" +import urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + +def get_vmray_config(url, key, misp_verifycert, default_wait_period): + ''' + Fetch configuration settings from MISP + Includes VMRay API and modules URL + ''' + + try: + misp_headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': key} + req = requests.get(url + 'servers/serverSettings.json', verify=misp_verifycert, headers=misp_headers) + + if req.status_code == 200: + req_json = req.json() + if 'finalSettings' in req_json: + finalSettings = req_json['finalSettings'] + vmray_api = '' + vmray_url = '' + vmray_wait_period = 0 + + for el in finalSettings: + # Is the vmray import module enabled? + if el['setting'] == 'Plugin.Import_vmray_import_enabled': + vmray_import_enabled = el['value'] + if vmray_import_enabled is False: + break + # Get the VMRay API key from the MISP settings + elif el['setting'] == 'Plugin.Import_vmray_import_apikey': + vmray_api = el['value'] + # The VMRay URL to query + elif el['setting'] == 'Plugin.Import_vmray_import_url': + vmray_url = el['value'].replace('/', '\\/') + # MISP modules - Port? + elif el['setting'] == 'Plugin.Import_services_port': + module_import_port = el['value'] + if module_import_port: + module_import_port = str(module_import_port) + else: + module_import_port = "6666" + # MISP modules - URL + elif el['setting'] == 'Plugin.Import_services_url': + module_import_url = el['value'].replace('\/\/', '//') + # Wait period + elif el['setting'] == 'Plugin.Import_vmray_import_wait_period': + vmray_wait_period = abs(int(el['value'])) + + if vmray_wait_period < 1: + vmray_wait_period = default_wait_period + else: + sys.exit('Did not receive a 200 code from MISP') + + if vmray_import_enabled and vmray_api and vmray_url and module_import_port and module_import_url: + return {'vmray_wait_period': vmray_wait_period, 'vmray_api': vmray_api, 'vmray_url': vmray_url, 'module_import_port': module_import_port, 'module_import_url': module_import_url} + else: + sys.exit('Did not receive all the necessary configuration information from MISP') + + except Exception as e: + sys.exit('Unable to get VMRay config from MISP') + + +def search_vmray_incomplete(m, url, wait_period, module_import_url, module_import_port, vmray_url, vmray_api, vmray_attribute_category, vmray_include_analysisid, vmray_include_imphash_ssdeep, vmray_include_extracted_files, vmray_include_analysisdetails, vmray_include_vtidetails, custom_tags_incomplete, custom_tags_complete): + ''' + Search for the events with VMRay samples that are marked incomplete + and then update these events + ''' + + controller = 'attributes' + vmray_value = 'VMRay Sample ID:' # How sample IDs are stored in MISP + req = None + + # Search for the events + try: + result = m.search(controller, tags=custom_tags_incomplete) + + attribute = result['Attribute'] + + if len(attribute) == 0: + sys.exit("No VMRay attributes found that match %s" % custom_tags_incomplete) + + timestamp = int(attribute[0]["timestamp"]) + # Not enough time has gone by to lookup the analysis jobs + if int((time.time() - timestamp) / 60) < int(wait_period): + if module_DEBUG: + r_timestamp = datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S') + print("Attribute to recent for wait_period (%s minutes) - timestamp attribute: %s (%s minutes old)" % (wait_period, r_timestamp, round((int(time.time() - timestamp) / 60), 2))) + return False + + if module_DEBUG: + print("All attributes older than %s" % int(wait_period)) + + for att in attribute: + value = att['value'] + + if vmray_value in value: # We found a sample ID + att_id = att['id'] + att_uuid = att['uuid'] + + # VMRay Sample IDs are stored as VMRay Sample ID: 2796577 + vmray_sample_id = value.split(vmray_value)[1].strip() + if vmray_sample_id.isdigit(): + event_id = att['event_id'] + + if module_DEBUG: + print("Found event %s with matching tags %s for sample id %s " % (event_id, custom_tags_incomplete, vmray_sample_id)) + + # Prepare request to send to vmray_import via misp modules + misp_modules_url = module_import_url + ':' + module_import_port + '/query' + misp_modules_headers = {'Content-Type': 'application/json'} + misp_modules_body = '{ "sample_id":"' + vmray_sample_id + '","module":"vmray_import","event_id":"' + event_id + '","config":{"apikey":"' + vmray_api + '","url":"' + vmray_url + '","include_analysisid":"' + vmray_include_analysisid + '","include_analysisdetails":"' + vmray_include_analysisdetails + '","include_extracted_files":"' + vmray_include_extracted_files + '","include_imphash_ssdeep":"' + vmray_include_imphash_ssdeep + '","include_vtidetails":"' + vmray_include_vtidetails + '","sample_id":"' + vmray_sample_id + '"},"data":""}' + req = requests.post(misp_modules_url, data=misp_modules_body, headers=misp_modules_headers) + if module_DEBUG and req is not None: + print("Response code from submitting to MISP modules %s" % (req.status_code)) + + # Succesful response from the misp modules? + if req.status_code == 200: + req_json = req.json() + if "error" in req_json: + print("Error code in reply %s " % req_json["error"]) + continue + else: + results = req_json["results"] + + # Walk through all results in the misp-module reply + for el in results: + to_ids = True + values = el['values'] + types = el['types'] + if "to_ids" in el: + to_ids = el['to_ids'] + if "text" in types: + to_ids = False + comment = el['comment'] + if len(comment) < 1: + comment = "Enriched via the vmray_import module" + + # Attribute can belong in different types + for attr_type in types: + try: + new_attribute = MISPAttribute() + new_attribute.type = attr_type + new_attribute.category = vmray_attribute_category + new_attribute.value = values + new_attribute.to_ids = to_ids + new_attribute.comment = comment + r = m.add_attribute(event_id, new_attribute) + if module_DEBUG: + print("Add event %s: %s as %s (%s) (toids: %s)" % (event_id, values, attr_type, comment, to_ids)) + except Exception as e: + if module_DEBUG: + print("Unable to add attribute %s as type %s for event %s" % (values, attr_type, event_id)) + continue + + # Remove 'incomplete' state tags + m.untag(att_uuid, custom_tags_incomplete) + # Update tags to 'complete' state + m.tag(att_uuid, custom_tags_complete) + if module_DEBUG: + print("Updated event %s" % event_id) + + else: + sys.exit('MISP modules did not return HTTP 200 code (event %s ; sampleid %s)' % (event_id, vmray_sample_id)) + + except Exception as e: + sys.exit("Invalid response received from MISP : %s", e) + + +if __name__ == '__main__': + + module_DEBUG = True + + # Set some defaults to be used in this module + vmray_attribute_category = 'External analysis' + vmray_include_analysisid = '0' + vmray_include_imphash_ssdeep = '0' + vmray_include_extracted_files = '0' + vmray_include_analysisdetails = '0' + vmray_include_vtidetails = '0' + custom_tags_incomplete = 'workflow:state="incomplete"' + custom_tags_complete = 'workflow:state="complete"' + default_wait_period = 30 + + misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, debug=module_DEBUG) + vmray_config = get_vmray_config(misp_url, misp_key, misp_verifycert, default_wait_period) + search_vmray_incomplete(misp, misp_url, vmray_config['vmray_wait_period'], vmray_config['module_import_url'], vmray_config['module_import_port'], vmray_config['vmray_url'], vmray_config['vmray_api'], vmray_attribute_category, vmray_include_analysisid, vmray_include_imphash_ssdeep, vmray_include_extracted_files, vmray_include_analysisdetails, vmray_include_vtidetails, custom_tags_incomplete, custom_tags_complete) From 65e4e3b4ec7414c33f92e60beb352951d449e8ed Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Wed, 11 Mar 2020 14:07:44 +0100 Subject: [PATCH 2/4] Minor updates to vmray_automation for travis --- examples/vmray_automation.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/examples/vmray_automation.py b/examples/vmray_automation.py index 569d28e..670b3e0 100644 --- a/examples/vmray_automation.py +++ b/examples/vmray_automation.py @@ -30,11 +30,6 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) def get_vmray_config(url, key, misp_verifycert, default_wait_period): - ''' - Fetch configuration settings from MISP - Includes VMRay API and modules URL - ''' - try: misp_headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': key} req = requests.get(url + 'servers/serverSettings.json', verify=misp_verifycert, headers=misp_headers) @@ -80,19 +75,13 @@ def get_vmray_config(url, key, misp_verifycert, default_wait_period): if vmray_import_enabled and vmray_api and vmray_url and module_import_port and module_import_url: return {'vmray_wait_period': vmray_wait_period, 'vmray_api': vmray_api, 'vmray_url': vmray_url, 'module_import_port': module_import_port, 'module_import_url': module_import_url} - else: - sys.exit('Did not receive all the necessary configuration information from MISP') + sys.exit('Did not receive all the necessary configuration information from MISP') except Exception as e: sys.exit('Unable to get VMRay config from MISP') def search_vmray_incomplete(m, url, wait_period, module_import_url, module_import_port, vmray_url, vmray_api, vmray_attribute_category, vmray_include_analysisid, vmray_include_imphash_ssdeep, vmray_include_extracted_files, vmray_include_analysisdetails, vmray_include_vtidetails, custom_tags_incomplete, custom_tags_complete): - ''' - Search for the events with VMRay samples that are marked incomplete - and then update these events - ''' - controller = 'attributes' vmray_value = 'VMRay Sample ID:' # How sample IDs are stored in MISP req = None @@ -148,7 +137,7 @@ def search_vmray_incomplete(m, url, wait_period, module_import_url, module_impor continue else: results = req_json["results"] - + # Walk through all results in the misp-module reply for el in results: to_ids = True From 3b38de345527fc9b25d0d7e553dcb02f40f92890 Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Wed, 11 Mar 2020 14:17:05 +0100 Subject: [PATCH 3/4] Add organisations from CSV --- examples/add_organisations.py | 57 +++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 examples/add_organisations.py diff --git a/examples/add_organisations.py b/examples/add_organisations.py new file mode 100644 index 0000000..e8bc57a --- /dev/null +++ b/examples/add_organisations.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import ExpandedPyMISP, MISPOrganisation, MISPSharingGroup +from keys import misp_url, misp_key, misp_verifycert +import argparse +import csv + + +# Suppress those "Unverified HTTPS request is being made" +import urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Add organizations from a CSV file') + parser.add_argument("-c", "--csv-import", required=True, help="The CSV file containing the organizations. Format 'orgname,nationality,sector,type,contacts,uuid,local,sharingroup_uuid'") + args = parser.parse_args() + + misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert) + + # CSV format + # orgname,nationality,sector,type,contacts,uuid,local,sharingroup + with open(args.csv_import) as csv_file: + count_orgs = 0 + csv_reader = csv.reader(csv_file, delimiter=',') + for row in csv_reader: + + org = MISPOrganisation() + org.name = row[0] + print("Process {}".format(org.name)) + org.nationality = row[1] + org.sector = row[2] + org.type = row[3] + org.contacts = row[4] + org.uuid = row[5] + org.local = row[6] + + add_org = misp.add_organisation(org, pythonify=True) + + if 'errors' in add_org: + print(add_org['errors']) + else: + count_orgs = count_orgs + 1 + org_uuid = add_org.uuid + + if org_uuid: + sharinggroup = MISPSharingGroup() + sharinggroup_uuid = row[7] + + if sharinggroup_uuid: + sharinggroup.uuid = sharinggroup_uuid + add_sharing = misp.add_org_to_sharing_group(sharinggroup, org) + else: + print("Organisation {} not added to sharing group, missing sharing group uuid".format(org.name)) + + print("Import finished, {} organisations added".format(count_orgs)) From b4e17a8d0277fb7b32e36bb86d937db8871683f7 Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Wed, 11 Mar 2020 14:34:13 +0100 Subject: [PATCH 4/4] Cytomic Orion API access --- examples/cytomic_orion.py | 549 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 549 insertions(+) create mode 100755 examples/cytomic_orion.py diff --git a/examples/cytomic_orion.py b/examples/cytomic_orion.py new file mode 100755 index 0000000..4b9b3df --- /dev/null +++ b/examples/cytomic_orion.py @@ -0,0 +1,549 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +''' +Koen Van Impe + +Cytomic Automation +Put this script in crontab to run every /15 or /60 + */15 * * * * mispuser /usr/bin/python3 /home/mispuser/PyMISP/examples/cytomic_orion.py + + +Fetches the configuration set in the Cytomic Orion enrichment module +- events : upload events tagged with the 'upload' tag, all the attributes supported by Cytomic Orion +- upload : upload attributes flagged with the 'upload' tag (only attributes supported by Cytomic Orion) +- delete : delete attributes flagged with the 'upload' tag (only attributes supported by Cytomic Orion) + +''' + +from pymisp import ExpandedPyMISP +from keys import misp_url, misp_key, misp_verifycert +import argparse +import os +import re +import sys +import requests +import json +import urllib3 + + +def get_token(token_url, clientid, clientsecret, scope, grant_type, username, password): + ''' + Get oAuth2 token + Configuration settings are fetched first from the MISP module configu + ''' + + try: + if scope and grant_type and username and password: + data = {'scope': scope, 'grant_type': grant_type, 'username': username, 'password': password} + + if token_url and clientid and clientsecret: + access_token_response = requests.post(token_url, data=data, verify=False, allow_redirects=False, auth=(clientid, clientsecret)) + tokens = json.loads(access_token_response.text) + if 'access_token' in tokens: + access_token = tokens['access_token'] + return access_token + else: + sys.exit('No token received') + else: + sys.exit('No token_url, clientid or clientsecret supplied') + else: + sys.exit('No scope, grant_type, username or password supplied') + except Exception: + sys.exit('Unable to connect to token_url') + + +def get_config(url, key, misp_verifycert): + ''' + Get the module config and the settings needed to access the API + Also contains the settings to do the query + ''' + try: + misp_headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': key} + req = requests.get(url + 'servers/serverSettings.json', verify=misp_verifycert, headers=misp_headers) + if req.status_code == 200: + req_json = req.json() + if 'finalSettings' in req_json: + finalSettings = req_json['finalSettings'] + + clientid = clientsecret = scope = username = password = grant_type = api_url = token_url = '' + module_enabled = False + scope = 'orion.api' + grant_type = 'password' + limit_upload_events = 50 + limit_upload_attributes = 50 + ttlDays = "1" + last_attributes = '5d' + post_threat_level_id = 2 + for el in finalSettings: + # Is the module enabled? + if el['setting'] == 'Plugin.Enrichment_cytomic_orion_enabled': + module_enabled = el['value'] + if module_enabled is False: + break + elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_clientid': + clientid = el['value'] + elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_clientsecret': + clientsecret = el['value'] + elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_username': + username = el['value'] + elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_password': + password = el['value'] + elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_api_url': + api_url = el['value'].replace('\\/', '/') + elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_token_url': + token_url = el['value'].replace('\\/', '/') + elif el['setting'] == 'MISP.baseurl': + misp_baseurl = el['value'] + elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_threat_level_id': + if el['value']: + try: + post_threat_level_id = int(el['value']) + except: + continue + elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_ttlDays': + if el['value']: + try: + ttlDays = "{last_days}".format(last_days=int(el['value'])) + except: + continue + elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_timeframe': + if el['value']: + try: + last_attributes = "{last_days}d".format(last_days=int(el['value'])) + except: + continue + elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_tag': + upload_tag = el['value'] + elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_delete_tag': + delete_tag = el['value'] + elif el['setting'] == 'Plugin.Enrichment_limit_upload_events': + if el['value']: + try: + limit_upload_events = "{limit_upload_events}".format(limit_upload_events=int(el['value'])) + except: + continue + elif el['setting'] == 'Plugin.Enrichment_limit_upload_attributes': + if el['value']: + try: + limit_upload_attributes = "{limit_upload_attributes}".format(limit_upload_attributes=int(el['value'])) + except: + continue + else: + sys.exit('Did not receive a 200 code from MISP') + + if module_enabled and api_url and token_url and clientid and clientsecret and username and password and grant_type: + + return {'cytomic_policy': 'Detect', + 'upload_timeframe': last_attributes, + 'upload_tag': upload_tag, + 'delete_tag': delete_tag, + 'upload_ttlDays': ttlDays, + 'post_threat_level_id': post_threat_level_id, + 'clientid': clientid, + 'clientsecret': clientsecret, + 'scope': scope, + 'username': username, + 'password': password, + 'grant_type': grant_type, + 'api_url': api_url, + 'token_url': token_url, + 'misp_baseurl': misp_baseurl, + 'limit_upload_events': limit_upload_events, + 'limit_upload_attributes': limit_upload_attributes} + else: + sys.exit('Did not receive all the necessary configuration information from MISP') + + except Exception as e: + sys.exit('Unable to get module config from MISP') + + +class cytomicobject: + misp = None + lst_evtid = None + lst_attuuid = None + lst_attuuid_error = None + endpoint_ioc = None + api_call_headers = None + post_data = None + args = None + tag = None + limit_events = None + limit_attributes = None + atttype_misp = None + atttype_cytomic = None + attlabel_cytomic = None + att_types = { + "ip-dst": {"ip": "ipioc"}, + "ip-src": {"ip": "ipioc"}, + "url": {"url": "urlioc"}, + "md5": {"hash": "filehashioc"}, + "domain": {"domain": "domainioc"}, + "hostname": {"domain": "domainioc"}, + "domain|ip": {"domain": "domainioc"}, + "hostname|port": {"domain": "domainioc"} + } + debug = True + error = False + res = False + res_msg = None + + +def collect_events_ids(cytomicobj, moduleconfig): + # Get events that contain Cytomic tag. + try: + evt_result = cytomicobj.misp.search(controller='events', limit=cytomicobj.limit_events, tags=cytomicobj.tag, last=moduleconfig['upload_timeframe'], published=True, deleted=False, pythonify=True) + cytomicobj.lst_evtid = ['x', 'y'] + for evt in evt_result: + evt = cytomicobj.misp.get_event(event=evt['id'], pythonify=True) + if len(evt.tags) > 0: + for tg in evt.tags: + if tg.name == cytomicobj.tag: + if not cytomicobj.lst_evtid: + cytomicobj.lst_evtid = str(evt['id']) + else: + if not evt['id'] in cytomicobj.lst_evtid: + cytomicobj.lst_evtid.append(str(evt['id'])) + break + cytomicobj.lst_evtid.remove('x') + cytomicobj.lst_evtid.remove('y') + except Exception: + cytomicobj.error = True + if cytomicobj.debug: + sys.exit('Unable to collect events ids') + + +def find_eventid(cytomicobj, evtid): + # Get events that contain Cytomic tag. + try: + cytomicobj.res = False + for id in cytomicobj.lst_evtid: + if id == evtid: + cytomicobj.res = True + break + except Exception: + cytomicobj.error = True + if cytomicobj.debug: + sys.exit('Unable to collect events ids') + + +def print_result_events(cytomicobj): + try: + if cytomicobj.res_msg is not None: + for key, msg in cytomicobj.res_msg.items(): + if msg is not None: + print(key, msg) + except Exception: + cytomicobj.error = True + if cytomicobj.debug: + sys.exit('Unable to print result') + + +def set_postdata(cytomicobj, moduleconfig, attribute): + # Set JSON to send to the API. + try: + + if cytomicobj.args.upload or cytomicobj.args.events: + event = attribute['Event'] + event_title = event['info'] + event_id = event['id'] + threat_level_id = int(event['threat_level_id']) + if moduleconfig['post_threat_level_id'] <= threat_level_id: + + if cytomicobj.atttype_misp == 'domain|ip' or cytomicobj.atttype_misp == 'hostname|port': + post_value = attribute['value'].split('|')[0] + else: + post_value = attribute['value'] + + if cytomicobj.atttype_misp == 'url' and 'http' not in post_value: + pass + else: + if cytomicobj.post_data is None: + cytomicobj.post_data = [{cytomicobj.attlabel_cytomic: post_value, 'AdditionalData': '{} {}'.format(cytomicobj.atttype_misp, attribute['comment']).strip(), 'Source': 'Uploaded from MISP', 'Policy': moduleconfig['cytomic_policy'], 'Description': '{} - {}'.format(event_id, event_title).strip()}] + else: + if post_value not in str(cytomicobj.post_data): + cytomicobj.post_data.append({cytomicobj.attlabel_cytomic: post_value, 'AdditionalData': '{} {}'.format(cytomicobj.atttype_misp, attribute['comment']).strip(), 'Source': 'Uploaded from MISP', 'Policy': moduleconfig['cytomic_policy'], 'Description': '{} - {}'.format(event_id, event_title).strip()}) + else: + if cytomicobject.debug: + print('Event %s skipped because of lower threat level' % event_id) + else: + event = attribute['Event'] + threat_level_id = int(event['threat_level_id']) + if moduleconfig['post_threat_level_id'] <= threat_level_id: + if cytomicobj.atttype_misp == 'domain|ip' or cytomicobj.atttype_misp == 'hostname|port': + post_value = attribute['value'].split('|')[0] + else: + post_value = attribute['value'] + + if cytomicobj.atttype_misp == 'url' and 'http' not in post_value: + pass + else: + if cytomicobj.post_data is None: + cytomicobj.post_data = [{cytomicobj.attlabel_cytomic: post_value}] + else: + cytomicobj.post_data.append({cytomicobj.attlabel_cytomic: post_value}) + else: + if cytomicobject.debug: + print('Event %s skipped because of lower threat level' % event_id) + except Exception: + cytomicobj.error = True + if cytomicobj.debug: + sys.exit('Unable to process post-data') + + +def send_postdata(cytomicobj, evtid=None): + # Batch post to upload event attributes. + try: + if cytomicobj.post_data is not None: + if cytomicobj.debug: + print('POST: {} {}'.format(cytomicobj.endpoint_ioc, cytomicobj.post_data)) + result_post_endpoint_ioc = requests.post(cytomicobj.endpoint_ioc, headers=cytomicobj.api_call_headers, json=cytomicobj.post_data, verify=False) + json_result_post_endpoint_ioc = json.loads(result_post_endpoint_ioc.text) + print(result_post_endpoint_ioc) + if 'true' not in (result_post_endpoint_ioc.text): + cytomicobj.error = True + if evtid is not None: + if cytomicobj.res_msg['Event: ' + str(evtid)] is None: + cytomicobj.res_msg['Event: ' + str(evtid)] = '(Send POST data: errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.' + else: + cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (Send POST data -else: errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.' + if cytomicobj.debug: + print('RESULT: {}'.format(json_result_post_endpoint_ioc)) + else: + if evtid is None: + cytomicobj.error = True + except Exception: + cytomicobj.error = True + if cytomicobj.debug: + sys.exit('Unable to post attributes') + + +def process_attributes(cytomicobj, moduleconfig, evtid=None): + # Get attributes to process. + try: + for misptype, cytomictypes in cytomicobject.att_types.items(): + cytomicobj.atttype_misp = misptype + for cytomiclabel, cytomictype in cytomictypes.items(): + cytomicobj.attlabel_cytomic = cytomiclabel + cytomicobj.atttype_cytomic = cytomictype + cytomicobj.post_data = None + icont = 0 + if cytomicobj.args.upload or cytomicobj.args.events: + cytomicobj.endpoint_ioc = moduleconfig['api_url'] + '/iocs/' + cytomicobj.atttype_cytomic + '?ttlDays=' + str(moduleconfig['upload_ttlDays']) + else: + cytomicobj.endpoint_ioc = moduleconfig['api_url'] + '/iocs/eraser/' + cytomicobj.atttype_cytomic + + # Get attributes to upload/delete and prepare JSON + # If evtid is set; we're called from --events + if cytomicobject.debug: + print("\nSearching for attributes of type %s" % cytomicobj.atttype_misp) + + if evtid is None: + cytomicobj.error = False + attr_result = cytomicobj.misp.search(controller='attributes', last=moduleconfig['upload_timeframe'], limit=cytomicobj.limit_attributes, type_attribute=cytomicobj.atttype_misp, tag=cytomicobj.tag, published=True, deleted=False, includeProposals=False, include_context=True, to_ids=True) + else: + if cytomicobj.error: + break + # We don't search with tags; we have an event for which we want to upload all events + attr_result = cytomicobj.misp.search(controller='attributes', eventid=evtid, last=moduleconfig['upload_timeframe'], limit=cytomicobj.limit_attributes, type_attribute=cytomicobj.atttype_misp, published=True, deleted=False, includeProposals=False, include_context=True, to_ids=True) + + cytomicobj.lst_attuuid = ['x', 'y'] + + if len(attr_result['Attribute']) > 0: + for attribute in attr_result['Attribute']: + if evtid is not None: + if cytomicobj.error: + cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.' + break + if icont >= cytomicobj.limit_attributes: + if not cytomicobj.error and cytomicobj.post_data is not None: + # Send data to Cytomic + send_postdata(cytomicobj, evtid) + if not cytomicobj.error: + if 'Event: ' + str(evtid) in cytomicobj.res_msg: + if cytomicobj.res_msg['Event: ' + str(evtid)] is None: + cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.attlabel_cytomic + 's: ' + str(icont) + else: + cytomicobj.res_msg['Event: ' + str(evtid)] += ' | ' + cytomicobj.attlabel_cytomic + 's: ' + str(icont) + else: + if cytomicobject.debug: + print('Data sent (' + cytomicobj.attlabel_cytomic + '): ' + str(icont)) + + cytomicobj.post_data = None + if cytomicobj.error: + if evtid is not None: + cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.' + break + icont = 0 + + if evtid is None: + event = attribute['Event'] + event_id = event['id'] + find_eventid(cytomicobj, str(event_id)) + if not cytomicobj.res: + if not cytomicobj.lst_attuuid: + cytomicobj.lst_attuuid = attribute['uuid'] + else: + if not attribute['uuid'] in cytomicobj.lst_attuuid: + cytomicobj.lst_attuuid.append(attribute['uuid']) + icont += 1 + # Prepare data to send + set_postdata(cytomicobj, moduleconfig, attribute) + else: + icont += 1 + # Prepare data to send + set_postdata(cytomicobj, moduleconfig, attribute) + + if not cytomicobj.error: + # Send data to Cytomic + send_postdata(cytomicobj, evtid) + + if not cytomicobj.error and cytomicobj.post_data is not None and icont > 0: + # Data sent; process response + if cytomicobj.res_msg is not None and 'Event: ' + str(evtid) in cytomicobj.res_msg: + if cytomicobj.res_msg['Event: ' + str(evtid)] is None: + cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.attlabel_cytomic + 's: ' + str(icont) + else: + cytomicobj.res_msg['Event: ' + str(evtid)] += ' | ' + cytomicobj.attlabel_cytomic + 's: ' + str(icont) + else: + if cytomicobject.debug: + print('Data sent (' + cytomicobj.attlabel_cytomic + '): ' + str(icont)) + + if not cytomicobj.error: + cytomicobj.lst_attuuid.remove('x') + cytomicobj.lst_attuuid.remove('y') + # Untag attributes + untag_attributes(cytomicobj) + except Exception: + cytomicobj.error = True + if cytomicobj.debug: + sys.exit('Unable to get attributes') + + +def untag_event(evtid): + # Remove tag of the event being processed. + try: + cytomicobj.records = 0 + evt = cytomicobj.misp.get_event(event=evtid, pythonify=True) + if len(evt.tags) > 0: + for tg in evt.tags: + if tg.name == cytomicobj.tag: + cytomicobj.misp.untag(evt['uuid'], cytomicobj.tag) + cytomicobj.records += 1 + cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (event untagged)' + break + except Exception: + cytomicobj.error = True + if cytomicobj.debug: + sys.exit('Unable to untag events') + + +def process_events(cytomicobj, moduleconfig): + # Get events that contain Cytomic tag. + try: + collect_events_ids(cytomicobj, moduleconfig) + total_attributes_sent = 0 + for evtid in cytomicobj.lst_evtid: + cytomicobj.error = False + if cytomicobj.res_msg is None: + cytomicobj.res_msg = {'Event: ' + str(evtid): None} + else: + cytomicobj.res_msg['Event: ' + str(evtid)] = None + if cytomicobject.debug: + print('Event id: ' + str(evtid)) + + # get attributes of each known type of the event / prepare data to send / send data to Cytomic + process_attributes(cytomicobj, moduleconfig, evtid) + if not cytomicobj.error: + untag_event(evtid) + except Exception: + cytomicobj.error = True + if cytomicobj.debug: + sys.exit('Unable to process events ids') + + +def untag_attributes(cytomicobj): + # Remove tag of attributes sent. + try: + icont = 0 + if len(cytomicobj.lst_attuuid) > 0: + for uuid in cytomicobj.lst_attuuid: + attr = cytomicobj.misp.get_attribute(attribute=uuid, pythonify=True) + if len(attr.tags) > 0: + for tg in attr.tags: + if tg.name == cytomicobj.tag: + cytomicobj.misp.untag(uuid, cytomicobj.tag) + icont += 1 + break + print('Attributes untagged (' + str(icont) + ')') + except Exception: + cytomicobj.error = True + if cytomicobj.debug: + sys.exit('Unable to untag attributes') + + +def process_attributes_upload(cytomicobj, moduleconfig): + # get attributes of each known type / prepare data to send / send data to Cytomic + try: + collect_events_ids(cytomicobj, moduleconfig) + process_attributes(cytomicobj, moduleconfig) + except Exception: + cytomicobj.error = True + if cytomicobj.debug: + sys.exit('Unable to upload attributes to Cytomic') + + +def process_attributes_delete(cytomicobj, moduleconfig): + # get attributes of each known type / prepare data to send / send data to Cytomic + try: + collect_events_ids(cytomicobj, moduleconfig) + process_attributes(cytomicobj, moduleconfig) + except Exception: + cytomicobj.error = True + if cytomicobj.debug: + sys.exit('Unable to delete attributes in Cytomic') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Upload or delete indicators to Cytomic API') + group = parser.add_mutually_exclusive_group() + group.add_argument('--events', action='store_true', help='Upload events indicators') + group.add_argument('--upload', action='store_true', help='Upload indicators') + group.add_argument('--delete', action='store_true', help='Delete indicators') + args = parser.parse_args() + if not args.upload and not args.delete and not args.events: + sys.exit("No valid action for the API") + + if misp_verifycert is False: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + module_config = get_config(misp_url, misp_key, misp_verifycert) + cytomicobj = cytomicobject + misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, debug=cytomicobject.debug) + + cytomicobj.misp = misp + cytomicobj.args = args + + access_token = get_token(module_config['token_url'], module_config['clientid'], module_config['clientsecret'], module_config['scope'], module_config['grant_type'], module_config['username'], module_config['password']) + cytomicobj.api_call_headers = {'Authorization': 'Bearer ' + access_token} + if cytomicobj.debug: + print('Received access token') + + if cytomicobj.args.events: + cytomicobj.tag = module_config['upload_tag'] + cytomicobj.limit_events = module_config['limit_upload_events'] + cytomicobj.limit_attributes = module_config['limit_upload_attributes'] + process_events(cytomicobj, module_config) + print_result_events(cytomicobj) + + elif cytomicobj.args.upload: + cytomicobj.tag = module_config['upload_tag'] + cytomicobj.limit_events = 0 + cytomicobj.limit_attributes = module_config['limit_upload_attributes'] + process_attributes_upload(cytomicobj, module_config) + + else: + cytomicobj.tag = module_config['delete_tag'] + cytomicobj.limit_events = 0 + cytomicobj.limit_attributes = module_config['limit_upload_attributes'] + process_attributes_delete(cytomicobj, module_config)