diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 4a1bde9..94c8c06 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -4,7 +4,7 @@ import sys sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3]))) __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl', - 'countrycode', 'cve', 'cve_advanced', 'dns', 'btc_steroids', 'domaintools', 'eupi', + 'countrycode', 'cve', 'cve_advanced', 'dns', 'btc_steroids', 'domaintools', 'eupi', 'eql', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', diff --git a/misp_modules/modules/expansion/eql.py b/misp_modules/modules/expansion/eql.py new file mode 100644 index 0000000..46cc05e --- /dev/null +++ b/misp_modules/modules/expansion/eql.py @@ -0,0 +1,84 @@ +""" +Export module for converting MISP events into Endgame EQL queries +""" +import json +import logging + +misperrors = {"error": "Error"} + +moduleinfo = { + "version": "0.1", + "author": "92 COS DOM", + "description": "Generates EQL queries from events", + "module-type": ["expansion"] +} + +# Map of MISP fields => Endgame fields +fieldmap = { + "ip-src": "source_address", + "ip-dst": "destination_address", + "filename": "file_name" +} + +# Describe what events have what fields +event_types = { + "source_address": "network", + "destination_address": "network", + "file_name": "file" +} + +# combine all the MISP fields from fieldmap into one big list +mispattributes = { + "input": list(fieldmap.keys()) +} + + +def handler(q=False): + """ + Convert a MISP query into a CSV file matching the ThreatConnect Structured Import file format. + Input + q: Query dictionary + """ + if q is False or not q: + return False + + # Check if we were given a configuration + request = json.loads(q) + config = request.get("config", {"Default_Source": ""}) + logging.info("Setting config to: %s", config) + + for supportedType in fieldmap.keys(): + if request.get(supportedType): + attrType = supportedType + + if attrType: + eqlType = fieldmap[attrType] + event_type = event_types[eqlType] + fullEql = "{} where {} == \"{}\"".format(event_type, eqlType, request[attrType]) + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + + response = [] + response.append({'types': ['comment'], 'categories': ['External analysis'], 'values': fullEql, 'comment': "Event EQL queries"}) + return {'results': response} + + +def introspection(): + """ + Relay the supported attributes to MISP. + No Input + Output + Dictionary of supported MISP attributes + """ + return mispattributes + + +def version(): + """ + Relay module version and associated metadata to MISP. + No Input + Output + moduleinfo: metadata output containing all potential configuration values + """ + return moduleinfo diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py index 21ff089..a88437b 100644 --- a/misp_modules/modules/expansion/securitytrails.py +++ b/misp_modules/modules/expansion/securitytrails.py @@ -37,14 +37,15 @@ def handler(q=False): request = json.loads(q) - if not request.get('config') and not (request['config'].get('apikey')): - misperrors['error'] = 'DNS authentication is missing' + if not request.get('config') or not (request['config'].get('apikey')): + misperrors['error'] = 'SecurityTrails authentication is missing' return misperrors api = DnsTrails(request['config'].get('apikey')) if not api: - misperrors['error'] = 'Onyphe Error instance api' + misperrors['error'] = 'SecurityTrails Error instance api' + return misperrors if request.get('ip-src'): ip = request['ip-src'] return handle_ip(api, ip, misperrors) @@ -92,9 +93,6 @@ def handle_domain(api, domain, misperrors): if status_ok: if r: result_filtered['results'].extend(r) - else: - misperrors['error'] = misperrors['error'] + ' Error whois result' - return misperrors time.sleep(1) r, status_ok = expand_history_ipv4_ipv6(api, domain) diff --git a/misp_modules/modules/expansion/shodan.py b/misp_modules/modules/expansion/shodan.py index fbdf5cd..5a4b792 100755 --- a/misp_modules/modules/expansion/shodan.py +++ b/misp_modules/modules/expansion/shodan.py @@ -27,8 +27,8 @@ def handler(q=False): misperrors['error'] = "Unsupported attributes type" return misperrors - if not request.get('config') and not (request['config'].get('apikey')): - misperrors['error'] = 'shodan authentication is missing' + if not request.get('config') or not request['config'].get('apikey'): + misperrors['error'] = 'Shodan authentication is missing' return misperrors api = shodan.Shodan(request['config'].get('apikey')) diff --git a/misp_modules/modules/expansion/urlhaus.py b/misp_modules/modules/expansion/urlhaus.py index 21a3718..30b78ee 100644 --- a/misp_modules/modules/expansion/urlhaus.py +++ b/misp_modules/modules/expansion/urlhaus.py @@ -60,7 +60,7 @@ class PayloadQuery(URLhaus): def query_api(self): hash_type = self.attribute.type file_object = MISPObject('file') - if self.attribute.event_id != '0': + if hasattr(self.attribute, 'object_id') and hasattr(self.attribute, 'event_id') and self.attribute.event_id != '0': file_object.id = self.attribute.object_id response = requests.post(self.url, data={'{}_hash'.format(hash_type): self.attribute.value}).json() other_hash_type = 'md5' if hash_type == 'sha256' else 'sha256' diff --git a/misp_modules/modules/expansion/urlscan.py b/misp_modules/modules/expansion/urlscan.py index f8dccbb..e6af7f6 100644 --- a/misp_modules/modules/expansion/urlscan.py +++ b/misp_modules/modules/expansion/urlscan.py @@ -22,7 +22,7 @@ moduleinfo = { moduleconfig = ['apikey'] misperrors = {'error': 'Error'} mispattributes = { - 'input': ['hostname', 'domain', 'url'], + 'input': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url'], 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url', 'text', 'link', 'hash'] } @@ -31,10 +31,9 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if (request.get('config')): - if (request['config'].get('apikey') is None): - misperrors['error'] = 'urlscan apikey is missing' - return misperrors + if not request.get('config') or not request['config'].get('apikey'): + misperrors['error'] = 'Urlscan apikey is missing' + return misperrors client = urlscanAPI(request['config']['apikey']) r = {'results': []} diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index c6263fc..cd0e738 100644 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -172,12 +172,13 @@ class VirusTotalParser(object): return attribute.uuid def parse_vt_object(self, query_result): - vt_object = MISPObject('virustotal-report') - vt_object.add_attribute('permalink', type='link', value=query_result['permalink']) - detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total']) - vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio) - self.misp_event.add_object(**vt_object) - return vt_object.uuid + if query_result['response_code'] == 1: + vt_object = MISPObject('virustotal-report') + vt_object.add_attribute('permalink', type='link', value=query_result['permalink']) + detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total']) + vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio) + self.misp_event.add_object(**vt_object) + return vt_object.uuid def parse_error(status_code): diff --git a/misp_modules/modules/expansion/virustotal_public.py b/misp_modules/modules/expansion/virustotal_public.py index 7074826..69c2c85 100644 --- a/misp_modules/modules/expansion/virustotal_public.py +++ b/misp_modules/modules/expansion/virustotal_public.py @@ -56,11 +56,12 @@ class VirusTotalParser(): self.misp_event.add_object(**domain_ip_object) def parse_vt_object(self, query_result): - vt_object = MISPObject('virustotal-report') - vt_object.add_attribute('permalink', type='link', value=query_result['permalink']) - detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total']) - vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio) - self.misp_event.add_object(**vt_object) + if query_result['response_code'] == 1: + vt_object = MISPObject('virustotal-report') + vt_object.add_attribute('permalink', type='link', value=query_result['permalink']) + detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total']) + vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio) + self.misp_event.add_object(**vt_object) def get_query_result(self, query_type): params = {query_type: self.attribute.value, 'apikey': self.apikey} diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py index 1affbd2..77dec0d 100644 --- a/misp_modules/modules/export_mod/__init__.py +++ b/misp_modules/modules/export_mod/__init__.py @@ -1,2 +1,2 @@ -__all__ = ['cef_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport', +__all__ = ['cef_export', 'mass_eql_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport', 'threatStream_misp_export', 'osqueryexport', 'nexthinkexport'] diff --git a/misp_modules/modules/export_mod/mass_eql_export.py b/misp_modules/modules/export_mod/mass_eql_export.py new file mode 100644 index 0000000..f42874d --- /dev/null +++ b/misp_modules/modules/export_mod/mass_eql_export.py @@ -0,0 +1,99 @@ +""" +Export module for converting MISP events into Endgame EQL queries +""" +import base64 +import io +import json +import logging + +misperrors = {"error": "Error"} + +moduleinfo = { + "version": "0.1", + "author": "92 COS DOM", + "description": "Export MISP event in Event Query Language", + "module-type": ["export"] +} + +# Map of MISP fields => Endgame fields +fieldmap = { + "ip-src": "source_address", + "ip-dst": "destination_address", + "filename": "file_name" +} + +# Describe what events have what fields +event_types = { + "source_address": "network", + "destination_address": "network", + "file_name": "file" +} + +# combine all the MISP fields from fieldmap into one big list +mispattributes = { + "input": list(fieldmap.keys()) +} + + +def handler(q=False): + """ + Convert a MISP query into a CSV file matching the ThreatConnect Structured Import file format. + Input + q: Query dictionary + """ + if q is False or not q: + return False + + # Check if we were given a configuration + request = json.loads(q) + config = request.get("config", {"Default_Source": ""}) + logging.info("Setting config to: %s", config) + + response = io.StringIO() + + # start parsing MISP data + queryDict = {} + for event in request["data"]: + for attribute in event["Attribute"]: + if attribute["type"] in mispattributes["input"]: + logging.debug("Adding %s to EQL query", attribute["value"]) + event_type = event_types[fieldmap[attribute["type"]]] + if event_type not in queryDict.keys(): + queryDict[event_type] = {} + queryDict[event_type][attribute["value"]] = fieldmap[attribute["type"]] + i = 0 + for query in queryDict.keys(): + response.write("{} where\n".format(query)) + for value in queryDict[query].keys(): + if i != 0: + response.write(" or\n") + response.write("\t{} == \"{}\"".format(queryDict[query][value], value)) + i += 1 + + return {"response": [], "data": str(base64.b64encode(bytes(response.getvalue(), 'utf-8')), 'utf-8')} + + +def introspection(): + """ + Relay the supported attributes to MISP. + No Input + Output + Dictionary of supported MISP attributes + """ + modulesetup = { + "responseType": "application/txt", + "outputFileExtension": "txt", + "userConfig": {}, + "inputSource": [] + } + return modulesetup + + +def version(): + """ + Relay module version and associated metadata to MISP. + No Input + Output + moduleinfo: metadata output containing all potential configuration values + """ + return moduleinfo