From 6616561e9609052854dcd770c9efee4ccbf6142e Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Wed, 11 Mar 2020 14:00:34 +0100 Subject: [PATCH 1/3] VMRay Automation with ExpandedPyMISP --- examples/vmray_automation.py | 213 +++++++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 examples/vmray_automation.py diff --git a/examples/vmray_automation.py b/examples/vmray_automation.py new file mode 100644 index 0000000..569d28e --- /dev/null +++ b/examples/vmray_automation.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +''' +Koen Van Impe + +VMRay automatic import +Put this script in crontab to run every /15 or /60 + */5 * * * * mispuser /usr/bin/python3 /home/mispuser/PyMISP/examples/vmray_automation.py + +Calls "vmray_import" for all events that have an 'incomplete' VMray analysis + +Do inline config in "main" + +''' + +from pymisp import ExpandedPyMISP, MISPAttribute +from keys import misp_url, misp_key, misp_verifycert +import argparse +import os +import json +import datetime +import time + +import requests +import sys + +# Suppress those "Unverified HTTPS request is being made" +import urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + +def get_vmray_config(url, key, misp_verifycert, default_wait_period): + ''' + Fetch configuration settings from MISP + Includes VMRay API and modules URL + ''' + + try: + misp_headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': key} + req = requests.get(url + 'servers/serverSettings.json', verify=misp_verifycert, headers=misp_headers) + + if req.status_code == 200: + req_json = req.json() + if 'finalSettings' in req_json: + finalSettings = req_json['finalSettings'] + vmray_api = '' + vmray_url = '' + vmray_wait_period = 0 + + for el in finalSettings: + # Is the vmray import module enabled? + if el['setting'] == 'Plugin.Import_vmray_import_enabled': + vmray_import_enabled = el['value'] + if vmray_import_enabled is False: + break + # Get the VMRay API key from the MISP settings + elif el['setting'] == 'Plugin.Import_vmray_import_apikey': + vmray_api = el['value'] + # The VMRay URL to query + elif el['setting'] == 'Plugin.Import_vmray_import_url': + vmray_url = el['value'].replace('/', '\\/') + # MISP modules - Port? + elif el['setting'] == 'Plugin.Import_services_port': + module_import_port = el['value'] + if module_import_port: + module_import_port = str(module_import_port) + else: + module_import_port = "6666" + # MISP modules - URL + elif el['setting'] == 'Plugin.Import_services_url': + module_import_url = el['value'].replace('\/\/', '//') + # Wait period + elif el['setting'] == 'Plugin.Import_vmray_import_wait_period': + vmray_wait_period = abs(int(el['value'])) + + if vmray_wait_period < 1: + vmray_wait_period = default_wait_period + else: + sys.exit('Did not receive a 200 code from MISP') + + if vmray_import_enabled and vmray_api and vmray_url and module_import_port and module_import_url: + return {'vmray_wait_period': vmray_wait_period, 'vmray_api': vmray_api, 'vmray_url': vmray_url, 'module_import_port': module_import_port, 'module_import_url': module_import_url} + else: + sys.exit('Did not receive all the necessary configuration information from MISP') + + except Exception as e: + sys.exit('Unable to get VMRay config from MISP') + + +def search_vmray_incomplete(m, url, wait_period, module_import_url, module_import_port, vmray_url, vmray_api, vmray_attribute_category, vmray_include_analysisid, vmray_include_imphash_ssdeep, vmray_include_extracted_files, vmray_include_analysisdetails, vmray_include_vtidetails, custom_tags_incomplete, custom_tags_complete): + ''' + Search for the events with VMRay samples that are marked incomplete + and then update these events + ''' + + controller = 'attributes' + vmray_value = 'VMRay Sample ID:' # How sample IDs are stored in MISP + req = None + + # Search for the events + try: + result = m.search(controller, tags=custom_tags_incomplete) + + attribute = result['Attribute'] + + if len(attribute) == 0: + sys.exit("No VMRay attributes found that match %s" % custom_tags_incomplete) + + timestamp = int(attribute[0]["timestamp"]) + # Not enough time has gone by to lookup the analysis jobs + if int((time.time() - timestamp) / 60) < int(wait_period): + if module_DEBUG: + r_timestamp = datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S') + print("Attribute to recent for wait_period (%s minutes) - timestamp attribute: %s (%s minutes old)" % (wait_period, r_timestamp, round((int(time.time() - timestamp) / 60), 2))) + return False + + if module_DEBUG: + print("All attributes older than %s" % int(wait_period)) + + for att in attribute: + value = att['value'] + + if vmray_value in value: # We found a sample ID + att_id = att['id'] + att_uuid = att['uuid'] + + # VMRay Sample IDs are stored as VMRay Sample ID: 2796577 + vmray_sample_id = value.split(vmray_value)[1].strip() + if vmray_sample_id.isdigit(): + event_id = att['event_id'] + + if module_DEBUG: + print("Found event %s with matching tags %s for sample id %s " % (event_id, custom_tags_incomplete, vmray_sample_id)) + + # Prepare request to send to vmray_import via misp modules + misp_modules_url = module_import_url + ':' + module_import_port + '/query' + misp_modules_headers = {'Content-Type': 'application/json'} + misp_modules_body = '{ "sample_id":"' + vmray_sample_id + '","module":"vmray_import","event_id":"' + event_id + '","config":{"apikey":"' + vmray_api + '","url":"' + vmray_url + '","include_analysisid":"' + vmray_include_analysisid + '","include_analysisdetails":"' + vmray_include_analysisdetails + '","include_extracted_files":"' + vmray_include_extracted_files + '","include_imphash_ssdeep":"' + vmray_include_imphash_ssdeep + '","include_vtidetails":"' + vmray_include_vtidetails + '","sample_id":"' + vmray_sample_id + '"},"data":""}' + req = requests.post(misp_modules_url, data=misp_modules_body, headers=misp_modules_headers) + if module_DEBUG and req is not None: + print("Response code from submitting to MISP modules %s" % (req.status_code)) + + # Succesful response from the misp modules? + if req.status_code == 200: + req_json = req.json() + if "error" in req_json: + print("Error code in reply %s " % req_json["error"]) + continue + else: + results = req_json["results"] + + # Walk through all results in the misp-module reply + for el in results: + to_ids = True + values = el['values'] + types = el['types'] + if "to_ids" in el: + to_ids = el['to_ids'] + if "text" in types: + to_ids = False + comment = el['comment'] + if len(comment) < 1: + comment = "Enriched via the vmray_import module" + + # Attribute can belong in different types + for attr_type in types: + try: + new_attribute = MISPAttribute() + new_attribute.type = attr_type + new_attribute.category = vmray_attribute_category + new_attribute.value = values + new_attribute.to_ids = to_ids + new_attribute.comment = comment + r = m.add_attribute(event_id, new_attribute) + if module_DEBUG: + print("Add event %s: %s as %s (%s) (toids: %s)" % (event_id, values, attr_type, comment, to_ids)) + except Exception as e: + if module_DEBUG: + print("Unable to add attribute %s as type %s for event %s" % (values, attr_type, event_id)) + continue + + # Remove 'incomplete' state tags + m.untag(att_uuid, custom_tags_incomplete) + # Update tags to 'complete' state + m.tag(att_uuid, custom_tags_complete) + if module_DEBUG: + print("Updated event %s" % event_id) + + else: + sys.exit('MISP modules did not return HTTP 200 code (event %s ; sampleid %s)' % (event_id, vmray_sample_id)) + + except Exception as e: + sys.exit("Invalid response received from MISP : %s", e) + + +if __name__ == '__main__': + + module_DEBUG = True + + # Set some defaults to be used in this module + vmray_attribute_category = 'External analysis' + vmray_include_analysisid = '0' + vmray_include_imphash_ssdeep = '0' + vmray_include_extracted_files = '0' + vmray_include_analysisdetails = '0' + vmray_include_vtidetails = '0' + custom_tags_incomplete = 'workflow:state="incomplete"' + custom_tags_complete = 'workflow:state="complete"' + default_wait_period = 30 + + misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, debug=module_DEBUG) + vmray_config = get_vmray_config(misp_url, misp_key, misp_verifycert, default_wait_period) + search_vmray_incomplete(misp, misp_url, vmray_config['vmray_wait_period'], vmray_config['module_import_url'], vmray_config['module_import_port'], vmray_config['vmray_url'], vmray_config['vmray_api'], vmray_attribute_category, vmray_include_analysisid, vmray_include_imphash_ssdeep, vmray_include_extracted_files, vmray_include_analysisdetails, vmray_include_vtidetails, custom_tags_incomplete, custom_tags_complete) From 65e4e3b4ec7414c33f92e60beb352951d449e8ed Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Wed, 11 Mar 2020 14:07:44 +0100 Subject: [PATCH 2/3] Minor updates to vmray_automation for travis --- examples/vmray_automation.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/examples/vmray_automation.py b/examples/vmray_automation.py index 569d28e..670b3e0 100644 --- a/examples/vmray_automation.py +++ b/examples/vmray_automation.py @@ -30,11 +30,6 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) def get_vmray_config(url, key, misp_verifycert, default_wait_period): - ''' - Fetch configuration settings from MISP - Includes VMRay API and modules URL - ''' - try: misp_headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': key} req = requests.get(url + 'servers/serverSettings.json', verify=misp_verifycert, headers=misp_headers) @@ -80,19 +75,13 @@ def get_vmray_config(url, key, misp_verifycert, default_wait_period): if vmray_import_enabled and vmray_api and vmray_url and module_import_port and module_import_url: return {'vmray_wait_period': vmray_wait_period, 'vmray_api': vmray_api, 'vmray_url': vmray_url, 'module_import_port': module_import_port, 'module_import_url': module_import_url} - else: - sys.exit('Did not receive all the necessary configuration information from MISP') + sys.exit('Did not receive all the necessary configuration information from MISP') except Exception as e: sys.exit('Unable to get VMRay config from MISP') def search_vmray_incomplete(m, url, wait_period, module_import_url, module_import_port, vmray_url, vmray_api, vmray_attribute_category, vmray_include_analysisid, vmray_include_imphash_ssdeep, vmray_include_extracted_files, vmray_include_analysisdetails, vmray_include_vtidetails, custom_tags_incomplete, custom_tags_complete): - ''' - Search for the events with VMRay samples that are marked incomplete - and then update these events - ''' - controller = 'attributes' vmray_value = 'VMRay Sample ID:' # How sample IDs are stored in MISP req = None @@ -148,7 +137,7 @@ def search_vmray_incomplete(m, url, wait_period, module_import_url, module_impor continue else: results = req_json["results"] - + # Walk through all results in the misp-module reply for el in results: to_ids = True From 3b38de345527fc9b25d0d7e553dcb02f40f92890 Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Wed, 11 Mar 2020 14:17:05 +0100 Subject: [PATCH 3/3] Add organisations from CSV --- examples/add_organisations.py | 57 +++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 examples/add_organisations.py diff --git a/examples/add_organisations.py b/examples/add_organisations.py new file mode 100644 index 0000000..e8bc57a --- /dev/null +++ b/examples/add_organisations.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pymisp import ExpandedPyMISP, MISPOrganisation, MISPSharingGroup +from keys import misp_url, misp_key, misp_verifycert +import argparse +import csv + + +# Suppress those "Unverified HTTPS request is being made" +import urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Add organizations from a CSV file') + parser.add_argument("-c", "--csv-import", required=True, help="The CSV file containing the organizations. Format 'orgname,nationality,sector,type,contacts,uuid,local,sharingroup_uuid'") + args = parser.parse_args() + + misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert) + + # CSV format + # orgname,nationality,sector,type,contacts,uuid,local,sharingroup + with open(args.csv_import) as csv_file: + count_orgs = 0 + csv_reader = csv.reader(csv_file, delimiter=',') + for row in csv_reader: + + org = MISPOrganisation() + org.name = row[0] + print("Process {}".format(org.name)) + org.nationality = row[1] + org.sector = row[2] + org.type = row[3] + org.contacts = row[4] + org.uuid = row[5] + org.local = row[6] + + add_org = misp.add_organisation(org, pythonify=True) + + if 'errors' in add_org: + print(add_org['errors']) + else: + count_orgs = count_orgs + 1 + org_uuid = add_org.uuid + + if org_uuid: + sharinggroup = MISPSharingGroup() + sharinggroup_uuid = row[7] + + if sharinggroup_uuid: + sharinggroup.uuid = sharinggroup_uuid + add_sharing = misp.add_org_to_sharing_group(sharinggroup, org) + else: + print("Organisation {} not added to sharing group, missing sharing group uuid".format(org.name)) + + print("Import finished, {} organisations added".format(count_orgs))