From 0f49b27794e14ff3369ba4c0da85d59583ecacca Mon Sep 17 00:00:00 2001 From: Koen Van Impe Date: Wed, 1 May 2019 22:48:07 +0200 Subject: [PATCH] Automation script that links vmray_submit and vmray_import Import finished VMRay tasks ; add attributes to event Makes use of the 'incomplete' workflow taxonomy Needs to be put in a cronjob to run in the background --- examples/vmray_automation.py | 211 +++++++++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 examples/vmray_automation.py diff --git a/examples/vmray_automation.py b/examples/vmray_automation.py new file mode 100644 index 0000000..cdf73e3 --- /dev/null +++ b/examples/vmray_automation.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +''' +Koen Van Impe + +VMRay automatic import +Put this script in crontab to run every /15 or /60 + */5 * * * * mispuser /usr/bin/python3 /home/mispuser/PyMISP/examples/vmray_automation.py + +Calls "vmray_import" for all events that have an 'incomplete' VMray analysis + +Do inline config in "main" + +''' + +from pymisp import PyMISP +from keys import misp_url, misp_key,misp_verifycert +import argparse +import os +import json +import datetime +import time + +import requests +import sys + +# Suppress those "Unverified HTTPS request is being made" +import urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + +def init(url, key): + ''' + Template to get MISP module started + ''' + return PyMISP(url, key, misp_verifycert, 'json') + + +def get_vmray_config(url, key, default_wait_period): + ''' + Fetch configuration settings from MISP + Includes VMRay API and modules URL + ''' + + try: + misp_headers = {'Content-Type': 'application/json','Accept': 'application/json', 'Authorization': key } + req = requests.get(url + 'servers/serverSettings.json', verify=False, headers=misp_headers) + + if req.status_code == 200: + req_json=req.json() + if 'finalSettings' in req_json: + finalSettings = req_json['finalSettings'] + vmray_api = '' + vmray_url = '' + vmray_wait_period = 0 + + for el in finalSettings: + # Is the vmray import module enabled? + if el['setting'] == 'Plugin.Import_vmray_import_enabled': + vmray_import_enabled = el['value'] + if vmray_import_enabled == False: + break + # Get the VMRay API key from the MISP settings + elif el['setting'] == 'Plugin.Import_vmray_import_apikey': + vmray_api = el['value'] + # The VMRay URL to query + elif el['setting'] == 'Plugin.Import_vmray_import_url': + vmray_url = el['value'].replace('/','\\/') + # MISP modules - Port? + elif el['setting'] == 'Plugin.Import_services_port': + module_import_port = el['value'] + # MISP modules - URL + elif el['setting'] == 'Plugin.Import_services_url': + module_import_url = el['value'].replace('\/\/', '//') + # Wait period + elif el['setting'] == 'Plugin.Import_vmray_import_wait_period': + vmray_wait_period = abs(int(el['value'])) + + if vmray_wait_period < 1: + vmray_wait_period = default_wait_period + else: + sys.exit('Did not receive a 200 code from MISP') + + if vmray_import_enabled and vmray_api and vmray_url and module_import_port and module_import_url: + return {'vmray_wait_period': vmray_wait_period, 'vmray_api': vmray_api, 'vmray_url': vmray_url, 'module_import_port': module_import_port, 'module_import_url': module_import_url} + else: + sys.exit('Did not receive all the necessary configuration information from MISP') + + except Exception as e: + sys.exit('Unable to get VMRay config from MISP') + + + +def search_vmray_incomplete(m, url, wait_period, module_import_url, module_import_port, vmray_url, vmray_api, vmray_attribute_category, vmray_include_analysisid, vmray_include_imphash_ssdeep, vmray_include_extracted_files, vmray_include_analysisdetails, vmray_include_vtidetails, custom_tags_incomplete, custom_tags_complete): + ''' + Search for the events with VMRay samples that are marked incomplete + and then update these events + ''' + + controller = 'attributes' + vmray_value = 'VMRay Sample ID:' # How sample IDs are stored in MISP + req = None + + # Search for the events + try: + result = m.search(controller, tags=custom_tags_incomplete) + response = result['response'] + + if len(response) == 0: + sys.exit("No VMRay attributes found that match %s" % custom_tags_incomplete) + + attribute = response['Attribute'] + + if len(attribute) == 0: + sys.exit("No VMRay attributes found that match %s" % custom_tags_incomplete) + + timestamp = int(attribute[0]["timestamp"]) + # Not enough time has gone by to lookup the analysis jobs + if int((time.time() - timestamp) / 60) < int(wait_period): + if module_DEBUG: + print("Attribute to recent - %s " % (int(time.time() - timestamp) / 60) ) + return False + + if module_DEBUG: + print("All attributes older than %s" % int(wait_period)) + + for att in attribute: + value = att['value'] + + if vmray_value in value: # We found a sample ID + att_id = att['id'] + att_uuid = att['uuid'] + + # VMRay Sample IDs are stored as VMRay Sample ID: 2796577 + vmray_sample_id = value.split(vmray_value)[1].strip() + if vmray_sample_id.isdigit(): + event_id = att['event_id'] + if module_DEBUG: + print("Found event %s with matching tags %s for sample id %s " % (event_id,custom_tags_incomplete,vmray_sample_id) ) + + # Prepare request to send to vmray_import via misp modules + misp_modules_url = module_import_url + ':' + module_import_port + '/query' + misp_modules_headers = {'Content-Type': 'application/json'} + misp_modules_body = '{ "sample_id":"' + vmray_sample_id + '","module":"vmray_import","event_id":"' + event_id + '","config":{"apikey":"' + vmray_api + '","url":"' + vmray_url + '","include_analysisid":"' + vmray_include_analysisid + '","include_analysisdetails":"' + vmray_include_analysisdetails + '","include_extracted_files":"' + vmray_include_extracted_files + '","include_imphash_ssdeep":"' + vmray_include_imphash_ssdeep + '","include_vtidetails":"' + vmray_include_vtidetails + '","sample_id":"' + vmray_sample_id + '"},"data":""}' + req = requests.post(misp_modules_url, data=misp_modules_body, headers=misp_modules_headers) + if module_DEBUG and req is not None: + print("Response code from submitting to MISP modules %s" % (req.status_code) ) + + # Succesful response from the misp modules? + if req.status_code == 200: + req_json=req.json() + if "error" in req_json: + print("Error code in reply %s " % req_json["error"]) + continue + else: + results = req_json["results"] + + # Walk through all results in the misp-module reply + for el in results: + to_ids = True + values = el['values'] + types = el['types'] + if "text" in types: + to_ids = False + comment = el['comment'] + if len(comment) < 1: + comment = "Enriched via the vmray_import module" + + # Attribute can belong in different types + for type in types: + try: + r = m.add_named_attribute( event_id, type, values, vmray_attribute_category, to_ids, comment) + if module_DEBUG: + print("Add event %s: %s as %s (%s)" % (event_id, values, type, comment)) + except Exception as e: + continue + if module_DEBUG: + print("Unable to add attribute %s as type %s for event %s" % (values, type, event_id)) + + # Remove 'incomplete' state tags + m.untag(att_uuid, custom_tags_incomplete) + # Update tags to 'complete' state + m.tag(att_uuid, custom_tags_complete) + if module_DEBUG: + print("Updated event %s" % event_id) + + else: + sys.exit('MISP modules did not return HTTP 200 code (event %s ; sampleid %s)' % (event_id,vmray_sample_id) ) + + except Exception as e: + sys.exit("Invalid response received from MISP : %s", e) + + +if __name__ == '__main__': + + module_DEBUG = True + + # Set some defaults to be used in this module + vmray_attribute_category = 'External analysis' + vmray_include_analysisid = '0' + vmray_include_imphash_ssdeep = '0' + vmray_include_extracted_files = '0' + vmray_include_analysisdetails = '0' + vmray_include_vtidetails = '0' + custom_tags_incomplete = 'workflow:state="incomplete"' + custom_tags_complete = 'workflow:state="complete"' + default_wait_period = 30 + + misp = init(misp_url, misp_key) + vmray_config = get_vmray_config(misp_url, misp_key, default_wait_period) + search_vmray_incomplete(misp, misp_url, vmray_config['vmray_wait_period'], vmray_config['module_import_url'], vmray_config['module_import_port'], vmray_config['vmray_url'], vmray_config['vmray_api'], vmray_attribute_category, vmray_include_analysisid, vmray_include_imphash_ssdeep, vmray_include_extracted_files, vmray_include_analysisdetails, vmray_include_vtidetails, custom_tags_incomplete, custom_tags_complete)