mirror of https://github.com/MISP/PyMISP
Merge branch 'master' of github.com:MISP/PyMISP
Raphaël Vinot
commit
8c63ad5d1c
|
|
@ -0,0 +1,57 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from pymisp import ExpandedPyMISP, MISPOrganisation, MISPSharingGroup
|
||||
from keys import misp_url, misp_key, misp_verifycert
|
||||
import argparse
|
||||
import csv
|
||||
|
||||
|
||||
# Suppress those "Unverified HTTPS request is being made"
|
||||
import urllib3
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='Add organizations from a CSV file')
|
||||
parser.add_argument("-c", "--csv-import", required=True, help="The CSV file containing the organizations. Format 'orgname,nationality,sector,type,contacts,uuid,local,sharingroup_uuid'")
|
||||
args = parser.parse_args()
|
||||
|
||||
misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
|
||||
|
||||
# CSV format
|
||||
# orgname,nationality,sector,type,contacts,uuid,local,sharingroup
|
||||
with open(args.csv_import) as csv_file:
|
||||
count_orgs = 0
|
||||
csv_reader = csv.reader(csv_file, delimiter=',')
|
||||
for row in csv_reader:
|
||||
|
||||
org = MISPOrganisation()
|
||||
org.name = row[0]
|
||||
print("Process {}".format(org.name))
|
||||
org.nationality = row[1]
|
||||
org.sector = row[2]
|
||||
org.type = row[3]
|
||||
org.contacts = row[4]
|
||||
org.uuid = row[5]
|
||||
org.local = row[6]
|
||||
|
||||
add_org = misp.add_organisation(org, pythonify=True)
|
||||
|
||||
if 'errors' in add_org:
|
||||
print(add_org['errors'])
|
||||
else:
|
||||
count_orgs = count_orgs + 1
|
||||
org_uuid = add_org.uuid
|
||||
|
||||
if org_uuid:
|
||||
sharinggroup = MISPSharingGroup()
|
||||
sharinggroup_uuid = row[7]
|
||||
|
||||
if sharinggroup_uuid:
|
||||
sharinggroup.uuid = sharinggroup_uuid
|
||||
add_sharing = misp.add_org_to_sharing_group(sharinggroup, org)
|
||||
else:
|
||||
print("Organisation {} not added to sharing group, missing sharing group uuid".format(org.name))
|
||||
|
||||
print("Import finished, {} organisations added".format(count_orgs))
|
||||
|
|
@ -0,0 +1,549 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
'''
|
||||
Koen Van Impe
|
||||
|
||||
Cytomic Automation
|
||||
Put this script in crontab to run every /15 or /60
|
||||
*/15 * * * * mispuser /usr/bin/python3 /home/mispuser/PyMISP/examples/cytomic_orion.py
|
||||
|
||||
|
||||
Fetches the configuration set in the Cytomic Orion enrichment module
|
||||
- events : upload events tagged with the 'upload' tag, all the attributes supported by Cytomic Orion
|
||||
- upload : upload attributes flagged with the 'upload' tag (only attributes supported by Cytomic Orion)
|
||||
- delete : delete attributes flagged with the 'upload' tag (only attributes supported by Cytomic Orion)
|
||||
|
||||
'''
|
||||
|
||||
from pymisp import ExpandedPyMISP
|
||||
from keys import misp_url, misp_key, misp_verifycert
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import requests
|
||||
import json
|
||||
import urllib3
|
||||
|
||||
|
||||
def get_token(token_url, clientid, clientsecret, scope, grant_type, username, password):
|
||||
'''
|
||||
Get oAuth2 token
|
||||
Configuration settings are fetched first from the MISP module configu
|
||||
'''
|
||||
|
||||
try:
|
||||
if scope and grant_type and username and password:
|
||||
data = {'scope': scope, 'grant_type': grant_type, 'username': username, 'password': password}
|
||||
|
||||
if token_url and clientid and clientsecret:
|
||||
access_token_response = requests.post(token_url, data=data, verify=False, allow_redirects=False, auth=(clientid, clientsecret))
|
||||
tokens = json.loads(access_token_response.text)
|
||||
if 'access_token' in tokens:
|
||||
access_token = tokens['access_token']
|
||||
return access_token
|
||||
else:
|
||||
sys.exit('No token received')
|
||||
else:
|
||||
sys.exit('No token_url, clientid or clientsecret supplied')
|
||||
else:
|
||||
sys.exit('No scope, grant_type, username or password supplied')
|
||||
except Exception:
|
||||
sys.exit('Unable to connect to token_url')
|
||||
|
||||
|
||||
def get_config(url, key, misp_verifycert):
|
||||
'''
|
||||
Get the module config and the settings needed to access the API
|
||||
Also contains the settings to do the query
|
||||
'''
|
||||
try:
|
||||
misp_headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': key}
|
||||
req = requests.get(url + 'servers/serverSettings.json', verify=misp_verifycert, headers=misp_headers)
|
||||
if req.status_code == 200:
|
||||
req_json = req.json()
|
||||
if 'finalSettings' in req_json:
|
||||
finalSettings = req_json['finalSettings']
|
||||
|
||||
clientid = clientsecret = scope = username = password = grant_type = api_url = token_url = ''
|
||||
module_enabled = False
|
||||
scope = 'orion.api'
|
||||
grant_type = 'password'
|
||||
limit_upload_events = 50
|
||||
limit_upload_attributes = 50
|
||||
ttlDays = "1"
|
||||
last_attributes = '5d'
|
||||
post_threat_level_id = 2
|
||||
for el in finalSettings:
|
||||
# Is the module enabled?
|
||||
if el['setting'] == 'Plugin.Enrichment_cytomic_orion_enabled':
|
||||
module_enabled = el['value']
|
||||
if module_enabled is False:
|
||||
break
|
||||
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_clientid':
|
||||
clientid = el['value']
|
||||
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_clientsecret':
|
||||
clientsecret = el['value']
|
||||
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_username':
|
||||
username = el['value']
|
||||
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_password':
|
||||
password = el['value']
|
||||
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_api_url':
|
||||
api_url = el['value'].replace('\\/', '/')
|
||||
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_token_url':
|
||||
token_url = el['value'].replace('\\/', '/')
|
||||
elif el['setting'] == 'MISP.baseurl':
|
||||
misp_baseurl = el['value']
|
||||
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_threat_level_id':
|
||||
if el['value']:
|
||||
try:
|
||||
post_threat_level_id = int(el['value'])
|
||||
except:
|
||||
continue
|
||||
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_ttlDays':
|
||||
if el['value']:
|
||||
try:
|
||||
ttlDays = "{last_days}".format(last_days=int(el['value']))
|
||||
except:
|
||||
continue
|
||||
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_timeframe':
|
||||
if el['value']:
|
||||
try:
|
||||
last_attributes = "{last_days}d".format(last_days=int(el['value']))
|
||||
except:
|
||||
continue
|
||||
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_upload_tag':
|
||||
upload_tag = el['value']
|
||||
elif el['setting'] == 'Plugin.Enrichment_cytomic_orion_delete_tag':
|
||||
delete_tag = el['value']
|
||||
elif el['setting'] == 'Plugin.Enrichment_limit_upload_events':
|
||||
if el['value']:
|
||||
try:
|
||||
limit_upload_events = "{limit_upload_events}".format(limit_upload_events=int(el['value']))
|
||||
except:
|
||||
continue
|
||||
elif el['setting'] == 'Plugin.Enrichment_limit_upload_attributes':
|
||||
if el['value']:
|
||||
try:
|
||||
limit_upload_attributes = "{limit_upload_attributes}".format(limit_upload_attributes=int(el['value']))
|
||||
except:
|
||||
continue
|
||||
else:
|
||||
sys.exit('Did not receive a 200 code from MISP')
|
||||
|
||||
if module_enabled and api_url and token_url and clientid and clientsecret and username and password and grant_type:
|
||||
|
||||
return {'cytomic_policy': 'Detect',
|
||||
'upload_timeframe': last_attributes,
|
||||
'upload_tag': upload_tag,
|
||||
'delete_tag': delete_tag,
|
||||
'upload_ttlDays': ttlDays,
|
||||
'post_threat_level_id': post_threat_level_id,
|
||||
'clientid': clientid,
|
||||
'clientsecret': clientsecret,
|
||||
'scope': scope,
|
||||
'username': username,
|
||||
'password': password,
|
||||
'grant_type': grant_type,
|
||||
'api_url': api_url,
|
||||
'token_url': token_url,
|
||||
'misp_baseurl': misp_baseurl,
|
||||
'limit_upload_events': limit_upload_events,
|
||||
'limit_upload_attributes': limit_upload_attributes}
|
||||
else:
|
||||
sys.exit('Did not receive all the necessary configuration information from MISP')
|
||||
|
||||
except Exception as e:
|
||||
sys.exit('Unable to get module config from MISP')
|
||||
|
||||
|
||||
class cytomicobject:
|
||||
misp = None
|
||||
lst_evtid = None
|
||||
lst_attuuid = None
|
||||
lst_attuuid_error = None
|
||||
endpoint_ioc = None
|
||||
api_call_headers = None
|
||||
post_data = None
|
||||
args = None
|
||||
tag = None
|
||||
limit_events = None
|
||||
limit_attributes = None
|
||||
atttype_misp = None
|
||||
atttype_cytomic = None
|
||||
attlabel_cytomic = None
|
||||
att_types = {
|
||||
"ip-dst": {"ip": "ipioc"},
|
||||
"ip-src": {"ip": "ipioc"},
|
||||
"url": {"url": "urlioc"},
|
||||
"md5": {"hash": "filehashioc"},
|
||||
"domain": {"domain": "domainioc"},
|
||||
"hostname": {"domain": "domainioc"},
|
||||
"domain|ip": {"domain": "domainioc"},
|
||||
"hostname|port": {"domain": "domainioc"}
|
||||
}
|
||||
debug = True
|
||||
error = False
|
||||
res = False
|
||||
res_msg = None
|
||||
|
||||
|
||||
def collect_events_ids(cytomicobj, moduleconfig):
|
||||
# Get events that contain Cytomic tag.
|
||||
try:
|
||||
evt_result = cytomicobj.misp.search(controller='events', limit=cytomicobj.limit_events, tags=cytomicobj.tag, last=moduleconfig['upload_timeframe'], published=True, deleted=False, pythonify=True)
|
||||
cytomicobj.lst_evtid = ['x', 'y']
|
||||
for evt in evt_result:
|
||||
evt = cytomicobj.misp.get_event(event=evt['id'], pythonify=True)
|
||||
if len(evt.tags) > 0:
|
||||
for tg in evt.tags:
|
||||
if tg.name == cytomicobj.tag:
|
||||
if not cytomicobj.lst_evtid:
|
||||
cytomicobj.lst_evtid = str(evt['id'])
|
||||
else:
|
||||
if not evt['id'] in cytomicobj.lst_evtid:
|
||||
cytomicobj.lst_evtid.append(str(evt['id']))
|
||||
break
|
||||
cytomicobj.lst_evtid.remove('x')
|
||||
cytomicobj.lst_evtid.remove('y')
|
||||
except Exception:
|
||||
cytomicobj.error = True
|
||||
if cytomicobj.debug:
|
||||
sys.exit('Unable to collect events ids')
|
||||
|
||||
|
||||
def find_eventid(cytomicobj, evtid):
|
||||
# Get events that contain Cytomic tag.
|
||||
try:
|
||||
cytomicobj.res = False
|
||||
for id in cytomicobj.lst_evtid:
|
||||
if id == evtid:
|
||||
cytomicobj.res = True
|
||||
break
|
||||
except Exception:
|
||||
cytomicobj.error = True
|
||||
if cytomicobj.debug:
|
||||
sys.exit('Unable to collect events ids')
|
||||
|
||||
|
||||
def print_result_events(cytomicobj):
|
||||
try:
|
||||
if cytomicobj.res_msg is not None:
|
||||
for key, msg in cytomicobj.res_msg.items():
|
||||
if msg is not None:
|
||||
print(key, msg)
|
||||
except Exception:
|
||||
cytomicobj.error = True
|
||||
if cytomicobj.debug:
|
||||
sys.exit('Unable to print result')
|
||||
|
||||
|
||||
def set_postdata(cytomicobj, moduleconfig, attribute):
|
||||
# Set JSON to send to the API.
|
||||
try:
|
||||
|
||||
if cytomicobj.args.upload or cytomicobj.args.events:
|
||||
event = attribute['Event']
|
||||
event_title = event['info']
|
||||
event_id = event['id']
|
||||
threat_level_id = int(event['threat_level_id'])
|
||||
if moduleconfig['post_threat_level_id'] <= threat_level_id:
|
||||
|
||||
if cytomicobj.atttype_misp == 'domain|ip' or cytomicobj.atttype_misp == 'hostname|port':
|
||||
post_value = attribute['value'].split('|')[0]
|
||||
else:
|
||||
post_value = attribute['value']
|
||||
|
||||
if cytomicobj.atttype_misp == 'url' and 'http' not in post_value:
|
||||
pass
|
||||
else:
|
||||
if cytomicobj.post_data is None:
|
||||
cytomicobj.post_data = [{cytomicobj.attlabel_cytomic: post_value, 'AdditionalData': '{} {}'.format(cytomicobj.atttype_misp, attribute['comment']).strip(), 'Source': 'Uploaded from MISP', 'Policy': moduleconfig['cytomic_policy'], 'Description': '{} - {}'.format(event_id, event_title).strip()}]
|
||||
else:
|
||||
if post_value not in str(cytomicobj.post_data):
|
||||
cytomicobj.post_data.append({cytomicobj.attlabel_cytomic: post_value, 'AdditionalData': '{} {}'.format(cytomicobj.atttype_misp, attribute['comment']).strip(), 'Source': 'Uploaded from MISP', 'Policy': moduleconfig['cytomic_policy'], 'Description': '{} - {}'.format(event_id, event_title).strip()})
|
||||
else:
|
||||
if cytomicobject.debug:
|
||||
print('Event %s skipped because of lower threat level' % event_id)
|
||||
else:
|
||||
event = attribute['Event']
|
||||
threat_level_id = int(event['threat_level_id'])
|
||||
if moduleconfig['post_threat_level_id'] <= threat_level_id:
|
||||
if cytomicobj.atttype_misp == 'domain|ip' or cytomicobj.atttype_misp == 'hostname|port':
|
||||
post_value = attribute['value'].split('|')[0]
|
||||
else:
|
||||
post_value = attribute['value']
|
||||
|
||||
if cytomicobj.atttype_misp == 'url' and 'http' not in post_value:
|
||||
pass
|
||||
else:
|
||||
if cytomicobj.post_data is None:
|
||||
cytomicobj.post_data = [{cytomicobj.attlabel_cytomic: post_value}]
|
||||
else:
|
||||
cytomicobj.post_data.append({cytomicobj.attlabel_cytomic: post_value})
|
||||
else:
|
||||
if cytomicobject.debug:
|
||||
print('Event %s skipped because of lower threat level' % event_id)
|
||||
except Exception:
|
||||
cytomicobj.error = True
|
||||
if cytomicobj.debug:
|
||||
sys.exit('Unable to process post-data')
|
||||
|
||||
|
||||
def send_postdata(cytomicobj, evtid=None):
|
||||
# Batch post to upload event attributes.
|
||||
try:
|
||||
if cytomicobj.post_data is not None:
|
||||
if cytomicobj.debug:
|
||||
print('POST: {} {}'.format(cytomicobj.endpoint_ioc, cytomicobj.post_data))
|
||||
result_post_endpoint_ioc = requests.post(cytomicobj.endpoint_ioc, headers=cytomicobj.api_call_headers, json=cytomicobj.post_data, verify=False)
|
||||
json_result_post_endpoint_ioc = json.loads(result_post_endpoint_ioc.text)
|
||||
print(result_post_endpoint_ioc)
|
||||
if 'true' not in (result_post_endpoint_ioc.text):
|
||||
cytomicobj.error = True
|
||||
if evtid is not None:
|
||||
if cytomicobj.res_msg['Event: ' + str(evtid)] is None:
|
||||
cytomicobj.res_msg['Event: ' + str(evtid)] = '(Send POST data: errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.'
|
||||
else:
|
||||
cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (Send POST data -else: errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.'
|
||||
if cytomicobj.debug:
|
||||
print('RESULT: {}'.format(json_result_post_endpoint_ioc))
|
||||
else:
|
||||
if evtid is None:
|
||||
cytomicobj.error = True
|
||||
except Exception:
|
||||
cytomicobj.error = True
|
||||
if cytomicobj.debug:
|
||||
sys.exit('Unable to post attributes')
|
||||
|
||||
|
||||
def process_attributes(cytomicobj, moduleconfig, evtid=None):
|
||||
# Get attributes to process.
|
||||
try:
|
||||
for misptype, cytomictypes in cytomicobject.att_types.items():
|
||||
cytomicobj.atttype_misp = misptype
|
||||
for cytomiclabel, cytomictype in cytomictypes.items():
|
||||
cytomicobj.attlabel_cytomic = cytomiclabel
|
||||
cytomicobj.atttype_cytomic = cytomictype
|
||||
cytomicobj.post_data = None
|
||||
icont = 0
|
||||
if cytomicobj.args.upload or cytomicobj.args.events:
|
||||
cytomicobj.endpoint_ioc = moduleconfig['api_url'] + '/iocs/' + cytomicobj.atttype_cytomic + '?ttlDays=' + str(moduleconfig['upload_ttlDays'])
|
||||
else:
|
||||
cytomicobj.endpoint_ioc = moduleconfig['api_url'] + '/iocs/eraser/' + cytomicobj.atttype_cytomic
|
||||
|
||||
# Get attributes to upload/delete and prepare JSON
|
||||
# If evtid is set; we're called from --events
|
||||
if cytomicobject.debug:
|
||||
print("\nSearching for attributes of type %s" % cytomicobj.atttype_misp)
|
||||
|
||||
if evtid is None:
|
||||
cytomicobj.error = False
|
||||
attr_result = cytomicobj.misp.search(controller='attributes', last=moduleconfig['upload_timeframe'], limit=cytomicobj.limit_attributes, type_attribute=cytomicobj.atttype_misp, tag=cytomicobj.tag, published=True, deleted=False, includeProposals=False, include_context=True, to_ids=True)
|
||||
else:
|
||||
if cytomicobj.error:
|
||||
break
|
||||
# We don't search with tags; we have an event for which we want to upload all events
|
||||
attr_result = cytomicobj.misp.search(controller='attributes', eventid=evtid, last=moduleconfig['upload_timeframe'], limit=cytomicobj.limit_attributes, type_attribute=cytomicobj.atttype_misp, published=True, deleted=False, includeProposals=False, include_context=True, to_ids=True)
|
||||
|
||||
cytomicobj.lst_attuuid = ['x', 'y']
|
||||
|
||||
if len(attr_result['Attribute']) > 0:
|
||||
for attribute in attr_result['Attribute']:
|
||||
if evtid is not None:
|
||||
if cytomicobj.error:
|
||||
cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.'
|
||||
break
|
||||
if icont >= cytomicobj.limit_attributes:
|
||||
if not cytomicobj.error and cytomicobj.post_data is not None:
|
||||
# Send data to Cytomic
|
||||
send_postdata(cytomicobj, evtid)
|
||||
if not cytomicobj.error:
|
||||
if 'Event: ' + str(evtid) in cytomicobj.res_msg:
|
||||
if cytomicobj.res_msg['Event: ' + str(evtid)] is None:
|
||||
cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.attlabel_cytomic + 's: ' + str(icont)
|
||||
else:
|
||||
cytomicobj.res_msg['Event: ' + str(evtid)] += ' | ' + cytomicobj.attlabel_cytomic + 's: ' + str(icont)
|
||||
else:
|
||||
if cytomicobject.debug:
|
||||
print('Data sent (' + cytomicobj.attlabel_cytomic + '): ' + str(icont))
|
||||
|
||||
cytomicobj.post_data = None
|
||||
if cytomicobj.error:
|
||||
if evtid is not None:
|
||||
cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (errors uploading attributes, event NOT untagged). If the problem persists, please review the format of the value of the attributes is correct.'
|
||||
break
|
||||
icont = 0
|
||||
|
||||
if evtid is None:
|
||||
event = attribute['Event']
|
||||
event_id = event['id']
|
||||
find_eventid(cytomicobj, str(event_id))
|
||||
if not cytomicobj.res:
|
||||
if not cytomicobj.lst_attuuid:
|
||||
cytomicobj.lst_attuuid = attribute['uuid']
|
||||
else:
|
||||
if not attribute['uuid'] in cytomicobj.lst_attuuid:
|
||||
cytomicobj.lst_attuuid.append(attribute['uuid'])
|
||||
icont += 1
|
||||
# Prepare data to send
|
||||
set_postdata(cytomicobj, moduleconfig, attribute)
|
||||
else:
|
||||
icont += 1
|
||||
# Prepare data to send
|
||||
set_postdata(cytomicobj, moduleconfig, attribute)
|
||||
|
||||
if not cytomicobj.error:
|
||||
# Send data to Cytomic
|
||||
send_postdata(cytomicobj, evtid)
|
||||
|
||||
if not cytomicobj.error and cytomicobj.post_data is not None and icont > 0:
|
||||
# Data sent; process response
|
||||
if cytomicobj.res_msg is not None and 'Event: ' + str(evtid) in cytomicobj.res_msg:
|
||||
if cytomicobj.res_msg['Event: ' + str(evtid)] is None:
|
||||
cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.attlabel_cytomic + 's: ' + str(icont)
|
||||
else:
|
||||
cytomicobj.res_msg['Event: ' + str(evtid)] += ' | ' + cytomicobj.attlabel_cytomic + 's: ' + str(icont)
|
||||
else:
|
||||
if cytomicobject.debug:
|
||||
print('Data sent (' + cytomicobj.attlabel_cytomic + '): ' + str(icont))
|
||||
|
||||
if not cytomicobj.error:
|
||||
cytomicobj.lst_attuuid.remove('x')
|
||||
cytomicobj.lst_attuuid.remove('y')
|
||||
# Untag attributes
|
||||
untag_attributes(cytomicobj)
|
||||
except Exception:
|
||||
cytomicobj.error = True
|
||||
if cytomicobj.debug:
|
||||
sys.exit('Unable to get attributes')
|
||||
|
||||
|
||||
def untag_event(evtid):
|
||||
# Remove tag of the event being processed.
|
||||
try:
|
||||
cytomicobj.records = 0
|
||||
evt = cytomicobj.misp.get_event(event=evtid, pythonify=True)
|
||||
if len(evt.tags) > 0:
|
||||
for tg in evt.tags:
|
||||
if tg.name == cytomicobj.tag:
|
||||
cytomicobj.misp.untag(evt['uuid'], cytomicobj.tag)
|
||||
cytomicobj.records += 1
|
||||
cytomicobj.res_msg['Event: ' + str(evtid)] = cytomicobj.res_msg['Event: ' + str(evtid)] + ' (event untagged)'
|
||||
break
|
||||
except Exception:
|
||||
cytomicobj.error = True
|
||||
if cytomicobj.debug:
|
||||
sys.exit('Unable to untag events')
|
||||
|
||||
|
||||
def process_events(cytomicobj, moduleconfig):
|
||||
# Get events that contain Cytomic tag.
|
||||
try:
|
||||
collect_events_ids(cytomicobj, moduleconfig)
|
||||
total_attributes_sent = 0
|
||||
for evtid in cytomicobj.lst_evtid:
|
||||
cytomicobj.error = False
|
||||
if cytomicobj.res_msg is None:
|
||||
cytomicobj.res_msg = {'Event: ' + str(evtid): None}
|
||||
else:
|
||||
cytomicobj.res_msg['Event: ' + str(evtid)] = None
|
||||
if cytomicobject.debug:
|
||||
print('Event id: ' + str(evtid))
|
||||
|
||||
# get attributes of each known type of the event / prepare data to send / send data to Cytomic
|
||||
process_attributes(cytomicobj, moduleconfig, evtid)
|
||||
if not cytomicobj.error:
|
||||
untag_event(evtid)
|
||||
except Exception:
|
||||
cytomicobj.error = True
|
||||
if cytomicobj.debug:
|
||||
sys.exit('Unable to process events ids')
|
||||
|
||||
|
||||
def untag_attributes(cytomicobj):
|
||||
# Remove tag of attributes sent.
|
||||
try:
|
||||
icont = 0
|
||||
if len(cytomicobj.lst_attuuid) > 0:
|
||||
for uuid in cytomicobj.lst_attuuid:
|
||||
attr = cytomicobj.misp.get_attribute(attribute=uuid, pythonify=True)
|
||||
if len(attr.tags) > 0:
|
||||
for tg in attr.tags:
|
||||
if tg.name == cytomicobj.tag:
|
||||
cytomicobj.misp.untag(uuid, cytomicobj.tag)
|
||||
icont += 1
|
||||
break
|
||||
print('Attributes untagged (' + str(icont) + ')')
|
||||
except Exception:
|
||||
cytomicobj.error = True
|
||||
if cytomicobj.debug:
|
||||
sys.exit('Unable to untag attributes')
|
||||
|
||||
|
||||
def process_attributes_upload(cytomicobj, moduleconfig):
|
||||
# get attributes of each known type / prepare data to send / send data to Cytomic
|
||||
try:
|
||||
collect_events_ids(cytomicobj, moduleconfig)
|
||||
process_attributes(cytomicobj, moduleconfig)
|
||||
except Exception:
|
||||
cytomicobj.error = True
|
||||
if cytomicobj.debug:
|
||||
sys.exit('Unable to upload attributes to Cytomic')
|
||||
|
||||
|
||||
def process_attributes_delete(cytomicobj, moduleconfig):
|
||||
# get attributes of each known type / prepare data to send / send data to Cytomic
|
||||
try:
|
||||
collect_events_ids(cytomicobj, moduleconfig)
|
||||
process_attributes(cytomicobj, moduleconfig)
|
||||
except Exception:
|
||||
cytomicobj.error = True
|
||||
if cytomicobj.debug:
|
||||
sys.exit('Unable to delete attributes in Cytomic')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='Upload or delete indicators to Cytomic API')
|
||||
group = parser.add_mutually_exclusive_group()
|
||||
group.add_argument('--events', action='store_true', help='Upload events indicators')
|
||||
group.add_argument('--upload', action='store_true', help='Upload indicators')
|
||||
group.add_argument('--delete', action='store_true', help='Delete indicators')
|
||||
args = parser.parse_args()
|
||||
if not args.upload and not args.delete and not args.events:
|
||||
sys.exit("No valid action for the API")
|
||||
|
||||
if misp_verifycert is False:
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
module_config = get_config(misp_url, misp_key, misp_verifycert)
|
||||
cytomicobj = cytomicobject
|
||||
misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, debug=cytomicobject.debug)
|
||||
|
||||
cytomicobj.misp = misp
|
||||
cytomicobj.args = args
|
||||
|
||||
access_token = get_token(module_config['token_url'], module_config['clientid'], module_config['clientsecret'], module_config['scope'], module_config['grant_type'], module_config['username'], module_config['password'])
|
||||
cytomicobj.api_call_headers = {'Authorization': 'Bearer ' + access_token}
|
||||
if cytomicobj.debug:
|
||||
print('Received access token')
|
||||
|
||||
if cytomicobj.args.events:
|
||||
cytomicobj.tag = module_config['upload_tag']
|
||||
cytomicobj.limit_events = module_config['limit_upload_events']
|
||||
cytomicobj.limit_attributes = module_config['limit_upload_attributes']
|
||||
process_events(cytomicobj, module_config)
|
||||
print_result_events(cytomicobj)
|
||||
|
||||
elif cytomicobj.args.upload:
|
||||
cytomicobj.tag = module_config['upload_tag']
|
||||
cytomicobj.limit_events = 0
|
||||
cytomicobj.limit_attributes = module_config['limit_upload_attributes']
|
||||
process_attributes_upload(cytomicobj, module_config)
|
||||
|
||||
else:
|
||||
cytomicobj.tag = module_config['delete_tag']
|
||||
cytomicobj.limit_events = 0
|
||||
cytomicobj.limit_attributes = module_config['limit_upload_attributes']
|
||||
process_attributes_delete(cytomicobj, module_config)
|
||||
|
|
@ -0,0 +1,202 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
'''
|
||||
Koen Van Impe
|
||||
|
||||
VMRay automatic import
|
||||
Put this script in crontab to run every /15 or /60
|
||||
*/5 * * * * mispuser /usr/bin/python3 /home/mispuser/PyMISP/examples/vmray_automation.py
|
||||
|
||||
Calls "vmray_import" for all events that have an 'incomplete' VMray analysis
|
||||
|
||||
Do inline config in "main"
|
||||
|
||||
'''
|
||||
|
||||
from pymisp import ExpandedPyMISP, MISPAttribute
|
||||
from keys import misp_url, misp_key, misp_verifycert
|
||||
import argparse
|
||||
import os
|
||||
import json
|
||||
import datetime
|
||||
import time
|
||||
|
||||
import requests
|
||||
import sys
|
||||
|
||||
# Suppress those "Unverified HTTPS request is being made"
|
||||
import urllib3
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
|
||||
def get_vmray_config(url, key, misp_verifycert, default_wait_period):
|
||||
try:
|
||||
misp_headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': key}
|
||||
req = requests.get(url + 'servers/serverSettings.json', verify=misp_verifycert, headers=misp_headers)
|
||||
|
||||
if req.status_code == 200:
|
||||
req_json = req.json()
|
||||
if 'finalSettings' in req_json:
|
||||
finalSettings = req_json['finalSettings']
|
||||
vmray_api = ''
|
||||
vmray_url = ''
|
||||
vmray_wait_period = 0
|
||||
|
||||
for el in finalSettings:
|
||||
# Is the vmray import module enabled?
|
||||
if el['setting'] == 'Plugin.Import_vmray_import_enabled':
|
||||
vmray_import_enabled = el['value']
|
||||
if vmray_import_enabled is False:
|
||||
break
|
||||
# Get the VMRay API key from the MISP settings
|
||||
elif el['setting'] == 'Plugin.Import_vmray_import_apikey':
|
||||
vmray_api = el['value']
|
||||
# The VMRay URL to query
|
||||
elif el['setting'] == 'Plugin.Import_vmray_import_url':
|
||||
vmray_url = el['value'].replace('/', '\\/')
|
||||
# MISP modules - Port?
|
||||
elif el['setting'] == 'Plugin.Import_services_port':
|
||||
module_import_port = el['value']
|
||||
if module_import_port:
|
||||
module_import_port = str(module_import_port)
|
||||
else:
|
||||
module_import_port = "6666"
|
||||
# MISP modules - URL
|
||||
elif el['setting'] == 'Plugin.Import_services_url':
|
||||
module_import_url = el['value'].replace('\/\/', '//')
|
||||
# Wait period
|
||||
elif el['setting'] == 'Plugin.Import_vmray_import_wait_period':
|
||||
vmray_wait_period = abs(int(el['value']))
|
||||
|
||||
if vmray_wait_period < 1:
|
||||
vmray_wait_period = default_wait_period
|
||||
else:
|
||||
sys.exit('Did not receive a 200 code from MISP')
|
||||
|
||||
if vmray_import_enabled and vmray_api and vmray_url and module_import_port and module_import_url:
|
||||
return {'vmray_wait_period': vmray_wait_period, 'vmray_api': vmray_api, 'vmray_url': vmray_url, 'module_import_port': module_import_port, 'module_import_url': module_import_url}
|
||||
sys.exit('Did not receive all the necessary configuration information from MISP')
|
||||
|
||||
except Exception as e:
|
||||
sys.exit('Unable to get VMRay config from MISP')
|
||||
|
||||
|
||||
def search_vmray_incomplete(m, url, wait_period, module_import_url, module_import_port, vmray_url, vmray_api, vmray_attribute_category, vmray_include_analysisid, vmray_include_imphash_ssdeep, vmray_include_extracted_files, vmray_include_analysisdetails, vmray_include_vtidetails, custom_tags_incomplete, custom_tags_complete):
|
||||
controller = 'attributes'
|
||||
vmray_value = 'VMRay Sample ID:' # How sample IDs are stored in MISP
|
||||
req = None
|
||||
|
||||
# Search for the events
|
||||
try:
|
||||
result = m.search(controller, tags=custom_tags_incomplete)
|
||||
|
||||
attribute = result['Attribute']
|
||||
|
||||
if len(attribute) == 0:
|
||||
sys.exit("No VMRay attributes found that match %s" % custom_tags_incomplete)
|
||||
|
||||
timestamp = int(attribute[0]["timestamp"])
|
||||
# Not enough time has gone by to lookup the analysis jobs
|
||||
if int((time.time() - timestamp) / 60) < int(wait_period):
|
||||
if module_DEBUG:
|
||||
r_timestamp = datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
|
||||
print("Attribute to recent for wait_period (%s minutes) - timestamp attribute: %s (%s minutes old)" % (wait_period, r_timestamp, round((int(time.time() - timestamp) / 60), 2)))
|
||||
return False
|
||||
|
||||
if module_DEBUG:
|
||||
print("All attributes older than %s" % int(wait_period))
|
||||
|
||||
for att in attribute:
|
||||
value = att['value']
|
||||
|
||||
if vmray_value in value: # We found a sample ID
|
||||
att_id = att['id']
|
||||
att_uuid = att['uuid']
|
||||
|
||||
# VMRay Sample IDs are stored as VMRay Sample ID: 2796577
|
||||
vmray_sample_id = value.split(vmray_value)[1].strip()
|
||||
if vmray_sample_id.isdigit():
|
||||
event_id = att['event_id']
|
||||
|
||||
if module_DEBUG:
|
||||
print("Found event %s with matching tags %s for sample id %s " % (event_id, custom_tags_incomplete, vmray_sample_id))
|
||||
|
||||
# Prepare request to send to vmray_import via misp modules
|
||||
misp_modules_url = module_import_url + ':' + module_import_port + '/query'
|
||||
misp_modules_headers = {'Content-Type': 'application/json'}
|
||||
misp_modules_body = '{ "sample_id":"' + vmray_sample_id + '","module":"vmray_import","event_id":"' + event_id + '","config":{"apikey":"' + vmray_api + '","url":"' + vmray_url + '","include_analysisid":"' + vmray_include_analysisid + '","include_analysisdetails":"' + vmray_include_analysisdetails + '","include_extracted_files":"' + vmray_include_extracted_files + '","include_imphash_ssdeep":"' + vmray_include_imphash_ssdeep + '","include_vtidetails":"' + vmray_include_vtidetails + '","sample_id":"' + vmray_sample_id + '"},"data":""}'
|
||||
req = requests.post(misp_modules_url, data=misp_modules_body, headers=misp_modules_headers)
|
||||
if module_DEBUG and req is not None:
|
||||
print("Response code from submitting to MISP modules %s" % (req.status_code))
|
||||
|
||||
# Succesful response from the misp modules?
|
||||
if req.status_code == 200:
|
||||
req_json = req.json()
|
||||
if "error" in req_json:
|
||||
print("Error code in reply %s " % req_json["error"])
|
||||
continue
|
||||
else:
|
||||
results = req_json["results"]
|
||||
|
||||
# Walk through all results in the misp-module reply
|
||||
for el in results:
|
||||
to_ids = True
|
||||
values = el['values']
|
||||
types = el['types']
|
||||
if "to_ids" in el:
|
||||
to_ids = el['to_ids']
|
||||
if "text" in types:
|
||||
to_ids = False
|
||||
comment = el['comment']
|
||||
if len(comment) < 1:
|
||||
comment = "Enriched via the vmray_import module"
|
||||
|
||||
# Attribute can belong in different types
|
||||
for attr_type in types:
|
||||
try:
|
||||
new_attribute = MISPAttribute()
|
||||
new_attribute.type = attr_type
|
||||
new_attribute.category = vmray_attribute_category
|
||||
new_attribute.value = values
|
||||
new_attribute.to_ids = to_ids
|
||||
new_attribute.comment = comment
|
||||
r = m.add_attribute(event_id, new_attribute)
|
||||
if module_DEBUG:
|
||||
print("Add event %s: %s as %s (%s) (toids: %s)" % (event_id, values, attr_type, comment, to_ids))
|
||||
except Exception as e:
|
||||
if module_DEBUG:
|
||||
print("Unable to add attribute %s as type %s for event %s" % (values, attr_type, event_id))
|
||||
continue
|
||||
|
||||
# Remove 'incomplete' state tags
|
||||
m.untag(att_uuid, custom_tags_incomplete)
|
||||
# Update tags to 'complete' state
|
||||
m.tag(att_uuid, custom_tags_complete)
|
||||
if module_DEBUG:
|
||||
print("Updated event %s" % event_id)
|
||||
|
||||
else:
|
||||
sys.exit('MISP modules did not return HTTP 200 code (event %s ; sampleid %s)' % (event_id, vmray_sample_id))
|
||||
|
||||
except Exception as e:
|
||||
sys.exit("Invalid response received from MISP : %s", e)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
module_DEBUG = True
|
||||
|
||||
# Set some defaults to be used in this module
|
||||
vmray_attribute_category = 'External analysis'
|
||||
vmray_include_analysisid = '0'
|
||||
vmray_include_imphash_ssdeep = '0'
|
||||
vmray_include_extracted_files = '0'
|
||||
vmray_include_analysisdetails = '0'
|
||||
vmray_include_vtidetails = '0'
|
||||
custom_tags_incomplete = 'workflow:state="incomplete"'
|
||||
custom_tags_complete = 'workflow:state="complete"'
|
||||
default_wait_period = 30
|
||||
|
||||
misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, debug=module_DEBUG)
|
||||
vmray_config = get_vmray_config(misp_url, misp_key, misp_verifycert, default_wait_period)
|
||||
search_vmray_incomplete(misp, misp_url, vmray_config['vmray_wait_period'], vmray_config['module_import_url'], vmray_config['module_import_port'], vmray_config['vmray_url'], vmray_config['vmray_api'], vmray_attribute_category, vmray_include_analysisid, vmray_include_imphash_ssdeep, vmray_include_extracted_files, vmray_include_analysisdetails, vmray_include_vtidetails, custom_tags_incomplete, custom_tags_complete)
|
||||
Loading…
Reference in New Issue