Merge branch 'master' of github.com:MISP/misp-modules into tests

misp_modules/modules/expansion/__init__.py

@@ -4,7 +4,7 @@ import sys
 sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3])))
 
 __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl',
-           'countrycode', 'cve', 'cve_advanced', 'dns', 'btc_steroids', 'domaintools', 'eupi',
+           'countrycode', 'cve', 'cve_advanced', 'dns', 'btc_steroids', 'domaintools', 'eupi', 'eql',
            'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal',
            'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep',
            'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon',

misp_modules/modules/expansion/eql.py

@@ -0,0 +1,84 @@
+"""
+Export module for converting MISP events into Endgame EQL queries
+"""
+import json
+import logging
+
+misperrors = {"error": "Error"}
+
+moduleinfo = {
+    "version": "0.1",
+    "author": "92 COS DOM",
+    "description": "Generates EQL queries from events",
+    "module-type": ["expansion"]
+}
+
+# Map of MISP fields => Endgame fields
+fieldmap = {
+    "ip-src": "source_address",
+    "ip-dst": "destination_address",
+    "filename": "file_name"
+}
+
+# Describe what events have what fields
+event_types = {
+    "source_address": "network",
+    "destination_address": "network",
+    "file_name": "file"
+}
+
+# combine all the MISP fields from fieldmap into one big list
+mispattributes = {
+    "input": list(fieldmap.keys())
+}
+
+
+def handler(q=False):
+    """
+    Convert a MISP query into a CSV file matching the ThreatConnect Structured Import file format.
+    Input
+        q: Query dictionary
+    """
+    if q is False or not q:
+        return False
+
+    # Check if we were given a configuration
+    request = json.loads(q)
+    config = request.get("config", {"Default_Source": ""})
+    logging.info("Setting config to: %s", config)
+
+    for supportedType in fieldmap.keys():
+        if request.get(supportedType):
+            attrType = supportedType
+
+    if attrType:
+        eqlType = fieldmap[attrType]
+        event_type = event_types[eqlType]
+        fullEql = "{} where {} == \"{}\"".format(event_type, eqlType, request[attrType])
+    else:
+        misperrors['error'] = "Unsupported attributes type"
+        return misperrors
+
+    response = []
+    response.append({'types': ['comment'], 'categories': ['External analysis'], 'values': fullEql, 'comment': "Event EQL queries"})
+    return {'results': response}
+
+
+def introspection():
+    """
+    Relay the supported attributes to MISP.
+    No Input
+    Output
+        Dictionary of supported MISP attributes
+    """
+    return mispattributes
+
+
+def version():
+    """
+    Relay module version and associated metadata to MISP.
+    No Input
+    Output
+        moduleinfo: metadata output containing all potential configuration values
+    """
+    return moduleinfo

misp_modules/modules/expansion/securitytrails.py

@@ -37,14 +37,15 @@ def handler(q=False):
 
         request = json.loads(q)
 
-        if not request.get('config') and not (request['config'].get('apikey')):
+        if not request.get('config') or not (request['config'].get('apikey')):
-            misperrors['error'] = 'DNS authentication is missing'
+            misperrors['error'] = 'SecurityTrails authentication is missing'
             return misperrors
 
         api = DnsTrails(request['config'].get('apikey'))
 
         if not api:
-            misperrors['error'] = 'Onyphe Error instance api'
+            misperrors['error'] = 'SecurityTrails Error instance api'
+            return misperrors
         if request.get('ip-src'):
             ip = request['ip-src']
             return handle_ip(api, ip, misperrors)
@@ -92,9 +93,6 @@ def handle_domain(api, domain, misperrors):
     if status_ok:
         if r:
             result_filtered['results'].extend(r)
-    else:
-        misperrors['error'] = misperrors['error'] + ' Error whois result'
-        return misperrors
 
     time.sleep(1)
     r, status_ok = expand_history_ipv4_ipv6(api, domain)

misp_modules/modules/expansion/shodan.py

@@ -27,8 +27,8 @@ def handler(q=False):
         misperrors['error'] = "Unsupported attributes type"
         return misperrors
 
-    if not request.get('config') and not (request['config'].get('apikey')):
+    if not request.get('config') or not request['config'].get('apikey'):
-        misperrors['error'] = 'shodan authentication is missing'
+        misperrors['error'] = 'Shodan authentication is missing'
         return misperrors
     api = shodan.Shodan(request['config'].get('apikey'))
 

misp_modules/modules/expansion/urlhaus.py

@@ -60,7 +60,7 @@ class PayloadQuery(URLhaus):
     def query_api(self):
         hash_type = self.attribute.type
         file_object = MISPObject('file')
-        if self.attribute.event_id != '0':
+        if hasattr(self.attribute, 'object_id') and hasattr(self.attribute, 'event_id') and self.attribute.event_id != '0':
             file_object.id = self.attribute.object_id
         response = requests.post(self.url, data={'{}_hash'.format(hash_type): self.attribute.value}).json()
         other_hash_type = 'md5' if hash_type == 'sha256' else 'sha256'

misp_modules/modules/expansion/urlscan.py

@@ -22,7 +22,7 @@ moduleinfo = {
 moduleconfig = ['apikey']
 misperrors = {'error': 'Error'}
 mispattributes = {
-    'input': ['hostname', 'domain', 'url'],
+    'input': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url'],
     'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url', 'text', 'link', 'hash']
 }
 
@@ -31,10 +31,9 @@ def handler(q=False):
     if q is False:
         return False
     request = json.loads(q)
-    if (request.get('config')):
+    if not request.get('config') or not request['config'].get('apikey'):
-        if (request['config'].get('apikey') is None):
+        misperrors['error'] = 'Urlscan apikey is missing'
-            misperrors['error'] = 'urlscan apikey is missing'
+        return misperrors
-            return misperrors
     client = urlscanAPI(request['config']['apikey'])
 
     r = {'results': []}

misp_modules/modules/expansion/virustotal.py

@@ -172,12 +172,13 @@ class VirusTotalParser(object):
         return attribute.uuid
 
     def parse_vt_object(self, query_result):
-        vt_object = MISPObject('virustotal-report')
+        if query_result['response_code'] == 1:
-        vt_object.add_attribute('permalink', type='link', value=query_result['permalink'])
+            vt_object = MISPObject('virustotal-report')
-        detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total'])
+            vt_object.add_attribute('permalink', type='link', value=query_result['permalink'])
-        vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio)
+            detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total'])
-        self.misp_event.add_object(**vt_object)
+            vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio)
-        return vt_object.uuid
+            self.misp_event.add_object(**vt_object)
+            return vt_object.uuid
 
 
 def parse_error(status_code):

misp_modules/modules/expansion/virustotal_public.py

@@ -56,11 +56,12 @@ class VirusTotalParser():
         self.misp_event.add_object(**domain_ip_object)
 
     def parse_vt_object(self, query_result):
-        vt_object = MISPObject('virustotal-report')
+        if query_result['response_code'] == 1:
-        vt_object.add_attribute('permalink', type='link', value=query_result['permalink'])
+            vt_object = MISPObject('virustotal-report')
-        detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total'])
+            vt_object.add_attribute('permalink', type='link', value=query_result['permalink'])
-        vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio)
+            detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total'])
-        self.misp_event.add_object(**vt_object)
+            vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio)
+            self.misp_event.add_object(**vt_object)
 
     def get_query_result(self, query_type):
         params = {query_type: self.attribute.value, 'apikey': self.apikey}

misp_modules/modules/export_mod/__init__.py

@@ -1,2 +1,2 @@
-__all__ = ['cef_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport',
+__all__ = ['cef_export', 'mass_eql_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport',
            'threatStream_misp_export', 'osqueryexport', 'nexthinkexport']

misp_modules/modules/export_mod/mass_eql_export.py

@@ -0,0 +1,99 @@
+"""
+Export module for converting MISP events into Endgame EQL queries
+"""
+import base64
+import io
+import json
+import logging
+
+misperrors = {"error": "Error"}
+
+moduleinfo = {
+    "version": "0.1",
+    "author": "92 COS DOM",
+    "description": "Export MISP event in Event Query Language",
+    "module-type": ["export"]
+}
+
+# Map of MISP fields => Endgame fields
+fieldmap = {
+    "ip-src": "source_address",
+    "ip-dst": "destination_address",
+    "filename": "file_name"
+}
+
+# Describe what events have what fields
+event_types = {
+    "source_address": "network",
+    "destination_address": "network",
+    "file_name": "file"
+}
+
+# combine all the MISP fields from fieldmap into one big list
+mispattributes = {
+    "input": list(fieldmap.keys())
+}
+
+
+def handler(q=False):
+    """
+    Convert a MISP query into a CSV file matching the ThreatConnect Structured Import file format.
+    Input
+        q: Query dictionary
+    """
+    if q is False or not q:
+        return False
+
+    # Check if we were given a configuration
+    request = json.loads(q)
+    config = request.get("config", {"Default_Source": ""})
+    logging.info("Setting config to: %s", config)
+
+    response = io.StringIO()
+
+    # start parsing MISP data
+    queryDict = {}
+    for event in request["data"]:
+        for attribute in event["Attribute"]:
+            if attribute["type"] in mispattributes["input"]:
+                logging.debug("Adding %s to EQL query", attribute["value"])
+                event_type = event_types[fieldmap[attribute["type"]]]
+                if event_type not in queryDict.keys():
+                    queryDict[event_type] = {}
+                queryDict[event_type][attribute["value"]] = fieldmap[attribute["type"]]
+    i = 0
+    for query in queryDict.keys():
+        response.write("{} where\n".format(query))
+        for value in queryDict[query].keys():
+            if i != 0:
+                response.write(" or\n")
+            response.write("\t{} == \"{}\"".format(queryDict[query][value], value))
+            i += 1
+
+    return {"response": [], "data": str(base64.b64encode(bytes(response.getvalue(), 'utf-8')), 'utf-8')}
+
+
+def introspection():
+    """
+    Relay the supported attributes to MISP.
+    No Input
+    Output
+        Dictionary of supported MISP attributes
+    """
+    modulesetup = {
+        "responseType": "application/txt",
+        "outputFileExtension": "txt",
+        "userConfig": {},
+        "inputSource": []
+    }
+    return modulesetup
+
+
+def version():
+    """
+    Relay module version and associated metadata to MISP.
+    No Input
+    Output
+        moduleinfo: metadata output containing all potential configuration values
+    """
+    return moduleinfo