Merge branch 'main' into feat/EN-5047/MISP-manual-update

Pipfile

@@ -18,7 +18,7 @@ pypdns = "*"
 pypssl = "*"
 pyeupi = "*"
 uwhois = {editable = true,git = "https://github.com/Rafiot/uwhoisd.git",ref = "testing",subdirectory = "client"}
-pymisp = {editable = true,extras = ["fileobjects,openioc,virustotal,pdfexport"],git = "https://github.com/MISP/PyMISP.git"}
+pymisp = {editable = true,extras = ["fileobjects,openioc,pdfexport"],git = "https://github.com/MISP/PyMISP.git"}
 pyonyphe = {editable = true,git = "https://github.com/sebdraven/pyonyphe"}
 pydnstrails = {editable = true,git = "https://github.com/sebdraven/pydnstrails"}
 pytesseract = "*"
@@ -60,6 +60,7 @@ geoip2 = "*"
 apiosintDS = "*"
 assemblyline_client = "*"
 vt-graph-api = "*"
+trustar = "*"
 
 [requires]
 python_version = "3"

README.md

@@ -57,6 +57,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
 * [Lastline query](misp_modules/modules/expansion/lastline_query.py) - Query Lastline with the link to an analysis and parse the report.
 * [macaddress.io](misp_modules/modules/expansion/macaddress_io.py) - a hover module to retrieve vendor details and other information regarding a given MAC address or an OUI from [MAC address Vendor Lookup](https://macaddress.io). See [integration tutorial here](https://macaddress.io/integrations/MISP-module).
 * [macvendors](misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information.
+* [MALWAREbazaar](misp_modules/modules/expansion/malwarebazaar.py) - an expansion module to query MALWAREbazaar with some payload.
 * [ocr-enrich](misp_modules/modules/expansion/ocr_enrich.py) - an enrichment module to get OCRized data from images into MISP.
 * [ods-enrich](misp_modules/modules/expansion/ods_enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser).
 * [odt-enrich](misp_modules/modules/expansion/odt_enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser).
@@ -68,6 +69,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
 * [pptx-enrich](misp_modules/modules/expansion/pptx_enrich.py) - an enrichment module to get text out of PowerPoint document into MISP (using free-text parser).
 * [qrcode](misp_modules/modules/expansion/qrcode.py) - a module decode QR code, barcode and similar codes from an image and enrich with the decoded values.
 * [rbl](misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute.
+* [recordedfuture](misp_modules/modules/expansion/recordedfuture.py) - a hover and expansion module for enriching MISP attributes with threat intelligence from Recorded Future.
 * [reversedns](misp_modules/modules/expansion/reversedns.py) - Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes.
 * [securitytrails](misp_modules/modules/expansion/securitytrails.py) - an expansion module for [securitytrails](https://securitytrails.com/).
 * [shodan](misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module.
@@ -78,6 +80,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
 * [STIX2 pattern syntax validator](misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - a module to check a STIX2 pattern syntax.
 * [ThreatCrowd](misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/).
 * [threatminer](misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/).
+* [TruSTAR Enrich](misp_modules/modules/expansion/trustar_enrich.py) - an expansion module to enrich MISP data with [TruSTAR](https://www.trustar.co/).
 * [urlhaus](misp_modules/modules/expansion/urlhaus.py) - Query urlhaus to get additional data about a domain, hash, hostname, ip or url.
 * [urlscan](misp_modules/modules/expansion/urlscan.py) - an expansion module to query [urlscan.io](https://urlscan.io).
 * [virustotal](misp_modules/modules/expansion/virustotal.py) - an expansion module to query the [VirusTotal](https://www.virustotal.com/gui/home) API with a high request rate limit required. (More details about the API: [here](https://developers.virustotal.com/reference))

doc/README.md

@@ -502,13 +502,15 @@ Module to query a local copy of Maxmind's Geolite database.
 
 Module to access GreyNoise.io API
 - **features**:
->The module takes an IP address as input and queries Greynoise for some additional information about it. The result is returned as text.
+>The module takes an IP address as input and queries Greynoise for some additional information about it: basically it checks whether a given IP address is “Internet background noise”, or has been observed scanning or attacking devices across the Internet. The result is returned as text.
 - **input**:
 >An IP address.
 - **output**:
 >Additional information about the IP fetched from Greynoise API.
 - **references**:
 >https://greynoise.io/, https://github.com/GreyNoise-Intelligence/api.greynoise.io
+- **requirements**:
+>A Greynoise API key.
 
 -----
 
@@ -715,6 +717,22 @@ Module to access Macvendors API.
 
 -----
 
+#### [malwarebazaar](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/malwarebazaar.py)
+
+Query the MALWAREbazaar API to get additional information about the input hash attribute.
+- **features**:
+>The module takes a hash attribute as input and queries MALWAREbazaar's API to fetch additional data about it. The result, if the payload is known on the databases, is at least one file object describing the file the input hash is related to.
+>
+>The module is using the new format of modules able to return object since the result is one or multiple MISP object(s).
+- **input**:
+>A hash attribute (md5, sha1 or sha256).
+- **output**:
+>File object(s) related to the input attribute found on MALWAREbazaar databases.
+- **references**:
+>https://bazaar.abuse.ch/
+
+-----
+
 #### [ocr-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ocr-enrich.py)
 
 Module to process some optical character recognition on pictures.
@@ -948,6 +966,24 @@ Module to check an IPv4 address against known RBLs.
 
 -----
 
+#### [recordedfuture](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/recordedfuture.py)
+
+<img src=logos/recordedfuture.png height=60>
+
+Module to enrich attributes with threat intelligence from Recorded Future.
+- **features**:
+>Enrich an attribute to add a custom enrichment object to the event. The object contains a copy of the enriched attribute with added tags presenting risk score and triggered risk rules from Recorded Future. Malware and Threat Actors related to the enriched indicator in Recorded Future is matched against MISP's galaxy clusters and applied as galaxy tags. The custom enrichment object also includes a list of related indicators from Recorded Future (IP's, domains, hashes, URL's and vulnerabilities) added as additional attributes.
+- **input**:
+>A MISP attribute of one of the following types: ip, ip-src, ip-dst, domain, hostname, md5, sha1, sha256, uri, url, vulnerability, weakness.
+- **output**:
+>A MISP object containing a copy of the enriched attribute with added tags from Recorded Future and a list of new attributes related to the enriched attribute.
+- **references**:
+>https://www.recordedfuture.com/
+- **requirements**:
+>A Recorded Future API token.
+
+-----
+
 #### [reversedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/reversedns.py)
 
 Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes.

doc/expansion/greynoise.json

@@ -1,9 +1,9 @@
 {
     "description": "Module to access GreyNoise.io API",
     "logo": "logos/greynoise.png",
-    "requirements": [],
+    "requirements": ["A Greynoise API key."],
     "input": "An IP address.",
     "output": "Additional information about the IP fetched from Greynoise API.",
     "references": ["https://greynoise.io/", "https://github.com/GreyNoise-Intelligence/api.greynoise.io"],
-    "features": "The module takes an IP address as input and queries Greynoise for some additional information about it. The result is returned as text."
+    "features": "The module takes an IP address as input and queries Greynoise for some additional information about it: basically it checks whether a given IP address is “Internet background noise”, or has been observed scanning or attacking devices across the Internet. The result is returned as text."
 }

doc/expansion/malwarebazaar.json

@@ -0,0 +1,8 @@
+{
+    "description": "Query the MALWAREbazaar API to get additional information about the input hash attribute.",
+    "requirements": [],
+    "input": "A hash attribute (md5, sha1 or sha256).",
+    "output": "File object(s) related to the input attribute found on MALWAREbazaar databases.",
+    "references": ["https://bazaar.abuse.ch/"],
+    "features": "The module takes a hash attribute as input and queries MALWAREbazaar's API to fetch additional data about it. The result, if the payload is known on the databases, is at least one file object describing the file the input hash is related to.\n\nThe module is using the new format of modules able to return object since the result is one or multiple MISP object(s)."
+}

doc/expansion/recordedfuture.json

@@ -0,0 +1,9 @@
+{
+    "description": "Module to enrich attributes with threat intelligence from Recorded Future.",
+    "logo": "logos/recordedfuture.png",
+    "requirements": ["A Recorded Future API token."],
+    "input": "A MISP attribute of one of the following types: ip, ip-src, ip-dst, domain, hostname, md5, sha1, sha256, uri, url, vulnerability, weakness.",
+    "output": "A MISP object containing a copy of the enriched attribute with added tags from Recorded Future and a list of new attributes related to the enriched attribute.",
+    "references": ["https://www.recordedfuture.com/"],
+    "features": "Enrich an attribute to add a custom enrichment object to the event. The object contains a copy of the enriched attribute with added tags presenting risk score and triggered risk rules from Recorded Future. Malware and Threat Actors related to the enriched indicator in Recorded Future is matched against MISP's galaxy clusters and applied as galaxy tags. The custom enrichment object also includes a list of related indicators from Recorded Future (IP's, domains, hashes, URL's and vulnerabilities) added as additional attributes."
+}

misp_modules/modules/expansion/__init__.py

@@ -16,6 +16,16 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c
            'qrcode', 'ocr_enrich', 'pdf_enrich', 'docx_enrich', 'xlsx_enrich', 'pptx_enrich',
            'ods_enrich', 'odt_enrich', 'joesandbox_submit', 'joesandbox_query', 'urlhaus',
            'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid',
-           'assemblyline_submit', 'assemblyline_query', 'ransomcoindb',
+           'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', 'malwarebazaar',
            'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion', 'censys_enrich',
-           'trustar_enrich']
+           'trustar_enrich', 'recordedfuture']
+
+
+minimum_required_fields = ('type', 'uuid', 'value')
+
+checking_error = 'containing at least a "type" field and a "value" field'
+standard_error_message = 'This module requires an "attribute" field as input'
+
+
+def check_input_attribute(attribute, requirements=minimum_required_fields):
+    return all(feature in attribute for feature in requirements)

misp_modules/modules/expansion/apivoid.py

@@ -1,5 +1,6 @@
 import json
 import requests
+from . import check_input_attribute, standard_error_message
 from pymisp import MISPAttribute, MISPEvent, MISPObject
 
 misperrors = {'error': 'Error'}
@@ -74,7 +75,11 @@ def handler(q=False):
     request = json.loads(q)
     if not request.get('config', {}).get('apikey'):
         return {'error': 'An API key for APIVoid is required.'}
-    attribute = request.get('attribute')
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
+    attribute = request['attribute']
+    if attribute['type'] not in mispattributes['input']:
+        return {'error': 'Unsupported attribute type.'}
     apikey = request['config']['apikey']
     apivoid_parser = APIVoidParser(attribute)
     apivoid_parser.parse_domain(apikey)

misp_modules/modules/expansion/assemblyline_query.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 import json
+from . import check_input_attribute, standard_error_message
 from assemblyline_client import Client, ClientError
 from collections import defaultdict
 from pymisp import MISPAttribute, MISPEvent, MISPObject
@@ -139,6 +140,10 @@ def handler(q=False):
     if q is False:
         return False
     request = json.loads(q)
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
+    if request['attribute']['type'] not in mispattributes['input']:
+        return {'error': 'Unsupported attribute type.'}
     if not request.get('config'):
         return {"error": "Missing configuration."}
     if not request['config'].get('apiurl'):

misp_modules/modules/expansion/censys_enrich.py

@@ -3,6 +3,7 @@ import json
 import base64
 import codecs
 from dateutil.parser import isoparse
+from . import check_input_attribute, standard_error_message
 from pymisp import MISPAttribute, MISPEvent, MISPObject
 try:
     import censys.base
@@ -36,11 +37,11 @@ def handler(q=False):
     api_id = request['config']['api_id']
     api_secret = request['config']['api_secret']
 
-    if not request.get('attribute'):
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
-        return {'error': 'Unsupported input.'}
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
     attribute = request['attribute']
     if not any(input_type == attribute['type'] for input_type in mispattributes['input']):
-        return {'error': 'Unsupported attributes type'}
+        return {'error': 'Unsupported attribute type.'}
 
     attribute = MISPAttribute()
     attribute.from_dict(**request['attribute'])

misp_modules/modules/expansion/circl_passivedns.py

@@ -1,8 +1,9 @@
 import json
 import pypdns
+from . import check_input_attribute, standard_error_message
 from pymisp import MISPAttribute, MISPEvent, MISPObject
 
-mispattributes = {'input': ['hostname', 'domain', 'ip-src', 'ip-dst'], 'format': 'misp_standard'}
+mispattributes = {'input': ['hostname', 'domain', 'ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port'], 'format': 'misp_standard'}
 moduleinfo = {'version': '0.2', 'author': 'Alexandre Dulaunoy',
               'description': 'Module to access CIRCL Passive DNS',
               'module-type': ['expansion', 'hover']}
@@ -24,12 +25,19 @@ class PassiveDNSParser():
         results = {key: event[key] for key in ('Attribute', 'Object')}
         return {'results': results}
 
-    def parse(self, value):
+    def parse(self):
+        value = self.attribute.value.split('|')[0] if '|' in self.attribute.type else self.attribute.value
+
         try:
-            results = self.pdns.query(self.attribute.value)
+            results = self.pdns.query(value)
         except Exception:
             self.result = {'error': 'There is an authentication error, please make sure you supply correct credentials.'}
             return
+
+        if not results:
+            self.result = {'error': 'Not found'}
+            return
+
         mapping = {'count': 'counter', 'origin': 'text',
                    'time_first': 'datetime', 'rrtype': 'text',
                    'rrname': 'text', 'rdata': 'text',
@@ -51,13 +59,13 @@ def handler(q=False):
     if not request['config'].get('username') or not request['config'].get('password'):
         return {'error': 'CIRCL Passive DNS authentication is incomplete, please provide your username and password.'}
     authentication = (request['config']['username'], request['config']['password'])
-    if not request.get('attribute'):
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
-        return {'error': 'Unsupported input.'}
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
     attribute = request['attribute']
     if not any(input_type == attribute['type'] for input_type in mispattributes['input']):
-        return {'error': 'Unsupported attributes type'}
+        return {'error': 'Unsupported attribute type.'}
     pdns_parser = PassiveDNSParser(attribute, authentication)
-    pdns_parser.parse(attribute['value'])
+    pdns_parser.parse()
     return pdns_parser.get_results()
 
 

misp_modules/modules/expansion/circl_passivessl.py

@@ -1,8 +1,9 @@
 import json
 import pypssl
+from . import check_input_attribute, standard_error_message
 from pymisp import MISPAttribute, MISPEvent, MISPObject
 
-mispattributes = {'input': ['ip-src', 'ip-dst'], 'format': 'misp_standard'}
+mispattributes = {'input': ['ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port'], 'format': 'misp_standard'}
 moduleinfo = {'version': '0.2', 'author': 'Raphaël Vinot',
               'description': 'Module to access CIRCL Passive SSL',
               'module-type': ['expansion', 'hover']}
@@ -31,12 +32,23 @@ class PassiveSSLParser():
         results = {key: event[key] for key in ('Attribute', 'Object')}
         return {'results': results}
 
-    def parse(self, value):
+    def parse(self):
+        value = self.attribute.value.split('|')[0] if '|' in self.attribute.type else self.attribute.value
+
         try:
-            results = self.pssl.query(self.attribute.value)
+            results = self.pssl.query(value)
         except Exception:
             self.result = {'error': 'There is an authentication error, please make sure you supply correct credentials.'}
             return
+
+        if not results:
+            self.result = {'error': 'Not found'}
+            return
+
+        if 'error' in results:
+            self.result = {'error': results['error']}
+            return
+
         for ip_address, certificates in results.items():
             ip_uuid = self._handle_ip_attribute(ip_address)
             for certificate in certificates['certificates']:
@@ -72,13 +84,13 @@ def handler(q=False):
     if not request['config'].get('username') or not request['config'].get('password'):
         return {'error': 'CIRCL Passive SSL authentication is incomplete, please provide your username and password.'}
     authentication = (request['config']['username'], request['config']['password'])
-    if not request.get('attribute'):
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
-        return {'error': 'Unsupported input.'}
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
     attribute = request['attribute']
     if not any(input_type == attribute['type'] for input_type in mispattributes['input']):
-        return {'error': 'Unsupported attributes type'}
+        return {'error': 'Unsupported attribute type.'}
     pssl_parser = PassiveSSLParser(attribute, authentication)
-    pssl_parser.parse(attribute['value'])
+    pssl_parser.parse()
     return pssl_parser.get_results()
 
 

misp_modules/modules/expansion/cve_advanced.py

@@ -1,7 +1,8 @@
-from collections import defaultdict
-from pymisp import MISPEvent, MISPObject
 import json
 import requests
+from . import check_input_attribute, standard_error_message
+from collections import defaultdict
+from pymisp import MISPEvent, MISPObject
 
 misperrors = {'error': 'Error'}
 mispattributes = {'input': ['vulnerability'], 'format': 'misp_standard'}
@@ -108,7 +109,9 @@ def handler(q=False):
     if q is False:
         return False
     request = json.loads(q)
-    attribute = request.get('attribute')
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
+    attribute = request['attribute']
     if attribute.get('type') != 'vulnerability':
         misperrors['error'] = 'Vulnerability id missing.'
         return misperrors

misp_modules/modules/expansion/cytomic_orion.py

@@ -7,6 +7,7 @@ An expansion module to enrich attributes in MISP and share indicators of comprom
 
 '''
 
+from . import check_input_attribute, standard_error_message
 from pymisp import MISPAttribute, MISPEvent, MISPObject
 import json
 import requests
@@ -146,9 +147,11 @@ def handler(q=False):
     if not request.get('attribute'):
         return {'error': 'Unsupported input.'}
 
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
     attribute = request['attribute']
     if not any(input_type == attribute['type'] for input_type in mispattributes['input']):
-        return {'error': 'Unsupported attributes type'}
+        return {'error': 'Unsupported attribute type.'}
 
     if not request.get('config'):
         return {'error': 'Missing configuration'}

misp_modules/modules/expansion/greynoise.py

@@ -3,35 +3,59 @@ import json
 
 misperrors = {'error': 'Error'}
 mispattributes = {'input': ['ip-dst', 'ip-src'], 'output': ['text']}
-moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab <aurelien.schwab+dev@gmail.com>', 'description': 'Module to access GreyNoise.io API.', 'module-type': ['hover']}
+moduleinfo = {
-moduleconfig = ['user-agent']  # TODO take this into account in the code
+    'version': '0.2',
+    'author': 'Aurélien Schwab <aurelien.schwab+dev@gmail.com>',
+    'description': 'Module to access GreyNoise.io API.',
+    'module-type': ['hover']
+}
+moduleconfig = ['api_key']
 
-greynoise_api_url = 'http://api.greynoise.io:8888/v1/query/ip'
+greynoise_api_url = 'https://api.greynoise.io/v2/noise/quick/'
-default_user_agent = 'MISP-Module'
+codes_mapping = {
+    '0x00': 'The IP has never been observed scanning the Internet',
+    '0x01': 'The IP has been observed by the GreyNoise sensor network',
+    '0x02': 'The IP has been observed scanning the GreyNoise sensor network, but has not completed a full connection, meaning this can be spoofed',
+    '0x03': 'The IP is adjacent to another host that has been directly observed by the GreyNoise sensor network',
+    '0x04': 'Reserved',
+    '0x05': 'This IP is commonly spoofed in Internet-scan activity',
+    '0x06': 'This IP has been observed as noise, but this host belongs to a cloud provider where IPs can be cycled frequently',
+    '0x07': 'This IP is invalid',
+    '0x08': 'This IP was classified as noise, but has not been observed engaging in Internet-wide scans or attacks in over 60 days'
+}
 
 
 def handler(q=False):
     if q is False:
         return False
     request = json.loads(q)
+    if not request.get('config') or not request['config'].get('api_key'):
+        return {'error': 'Missing Greynoise API key.'}
+    headers = {
+        'Accept': 'application/json',
+        'key': request['config']['api_key']
+    }
     for input_type in mispattributes['input']:
         if input_type in request:
             ip = request[input_type]
             break
     else:
-        misperrors['error'] = "Unsupported attributes type"
+        misperrors['error'] = "Unsupported attributes type."
         return misperrors
-    data = {'ip': ip}
+    response = requests.get(f'{greynoise_api_url}{ip}', headers=headers)  # Real request
-    r = requests.post(greynoise_api_url, data=data, headers={'user-agent': default_user_agent})  # Real request
+    if response.status_code == 200:  # OK (record found)
-    if r.status_code == 200:  # OK (record found)
+        return {'results': [{'types': mispattributes['output'], 'values': codes_mapping[response.json()['code']]}]}
-        response = r.text
+    # There is an error
-        if response:
+    errors = {
-            return {'results': [{'types': mispattributes['output'], 'values': response}]}
+        400: "Bad request.",
-    elif r.status_code == 404:  # Not found (not an error)
+        401: "Unauthorized. Please check your API key.",
-        return {'results': [{'types': mispattributes['output'], 'values': 'No data'}]}
+        429: "Too many requests. You've hit the rate-limit."
-    else:  # Real error
+    }
-        misperrors['error'] = 'GreyNoise API not accessible (HTTP ' + str(r.status_code) + ')'
+    try:
-        return misperrors['error']
+        misperrors['error'] = errors[response.status_code]
+    except KeyError:
+        misperrors['error'] = f'GreyNoise API not accessible (HTTP {response.status_code})'
+    return misperrors['error']
 
 
 def introspection():

misp_modules/modules/expansion/ipasn.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 
 import json
+from . import check_input_attribute, standard_error_message
 from pyipasnhistory import IPASNHistory
 from pymisp import MISPAttribute, MISPEvent, MISPObject
 
@@ -34,11 +35,11 @@ def handler(q=False):
     if q is False:
         return False
     request = json.loads(q)
-    if request.get('attribute') and request['attribute'].get('type') in mispattributes['input']:
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
-        toquery = request['attribute']['value']
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
-    else:
+    if request['attribute']['type'] not in mispattributes['input']:
-        misperrors['error'] = "Unsupported attributes type"
+        return {'error': 'Unsupported attribute type.'}
-        return misperrors
+    toquery = request['attribute']['value']
 
     ipasn = IPASNHistory()
     values = ipasn.query(toquery)

misp_modules/modules/expansion/joesandbox_query.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import jbxapi
 import json
+from . import check_input_attribute, checking_error, standard_error_message
 from joe_parser import JoeParser
 
 misperrors = {'error': 'Error'}
@@ -27,6 +28,10 @@ def handler(q=False):
     if not apikey:
         return {'error': 'No API key provided'}
 
+    if not request.get('attribute') or not check_input_attribute(request['attribute'], requirements=('type', 'value')):
+        return {'error': f'{standard_error_message}, {checking_error} that is the link to the Joe Sandbox report.'}
+    if request['attribute']['type'] != 'link':
+        return {'error': 'Unsupported attribute type.'}
     url = request['attribute']['value']
     if "/submissions/" not in url:
         return {'error': "The URL does not point to a Joe Sandbox analysis."}

misp_modules/modules/expansion/lastline_query.py

@@ -3,8 +3,8 @@
 Module (type "expansion") to query a Lastline report from an analysis link.
 """
 import json
-
 import lastline_api
+from . import check_input_attribute, checking_error, standard_error_message
 
 
 misperrors = {
@@ -52,6 +52,8 @@ def handler(q=False):
     try:
         config = request["config"]
         auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config)
+        if not request.get('attribute') or not check_input_attribute(request['attribute'], requirements=('type', 'value')):
+            return {'error': f'{standard_error_message}, {checking_error} that is the link to a Lastline analysis.'}
         analysis_link = request['attribute']['value']
         # The API url changes based on the analysis link host name
         api_url = lastline_api.get_portal_url_from_task_link(analysis_link)

misp_modules/modules/expansion/malwarebazaar.py

@@ -0,0 +1,57 @@
+import json
+import requests
+from . import check_input_attribute, checking_error, standard_error_message
+from pymisp import MISPEvent, MISPObject
+
+mispattributes = {'input': ['md5', 'sha1', 'sha256'],
+                  'format': 'misp_standard'}
+moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
+              'description': 'Query Malware Bazaar to get additional information about the input hash.',
+              'module-type': ['expansion', 'hover']}
+moduleconfig = []
+
+
+def parse_response(response):
+    mapping = {'file_name': {'type': 'filename', 'object_relation': 'filename'},
+               'file_size': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'},
+               'file_type_mime': {'type': 'mime-type', 'object_relation': 'mimetype'},
+               'md5_hash': {'type': 'md5', 'object_relation': 'md5'},
+               'sha1_hash': {'type': 'sha1', 'object_relation': 'sha1'},
+               'sha256_hash': {'type': 'sha256', 'object_relation': 'sha256'},
+               'ssdeep': {'type': 'ssdeep', 'object_relation': 'ssdeep'}}
+    misp_event = MISPEvent()
+    for data in response:
+        misp_object = MISPObject('file')
+        for feature, attribute in mapping.items():
+            if feature in data:
+                misp_attribute = {'value': data[feature]}
+                misp_attribute.update(attribute)
+                misp_object.add_attribute(**misp_attribute)
+        misp_event.add_object(**misp_object)
+    return {'results': {'Object': [json.loads(misp_object.to_json()) for misp_object in misp_event.objects]}}
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    if not request.get('attribute') or not check_input_attribute(request['attribute'], requirements=('type', 'value')):
+        return {'error': f'{standard_error_message}, {checking_error} that is the hash to submit to Malware Bazaar.'}
+    attribute = request['attribute']
+    if attribute['type'] not in mispattributes['input']:
+        return {'error': 'Unsupported attribute type.'}
+    url = 'https://mb-api.abuse.ch/api/v1/'
+    response = requests.post(url, data={'query': 'get_info', 'hash': attribute['value']}).json()
+    query_status = response['query_status']
+    if query_status == 'ok':
+        return parse_response(response['data'])
+    return {'error': 'Hash not found on MALWAREbazzar' if query_status == 'hash_not_found' else f'Problem encountered during the query: {query_status}'}
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/ransomcoindb.py

@@ -1,4 +1,5 @@
 import json
+from . import check_input_attribute, checking_error, standard_error_message
 from ._ransomcoindb import ransomcoindb
 from pymisp import MISPObject
 
@@ -28,6 +29,10 @@ def handler(q=False):
     q = json.loads(q)
     if "config" not in q or "api-key" not in q["config"]:
         return {"error": "Ransomcoindb API key is missing"}
+    if not q.get('attribute') or not check_input_attribute(q['attribute'], requirements=('type', 'value')):
+        return {'error': f'{standard_error_message}, {checking_error}.'}
+    if q['attribute']['type'] not in mispattributes['input']:
+        return {'error': 'Unsupported attribute type.'}
     api_key = q["config"]["api-key"]
     r = {"results": []}
 

misp_modules/modules/expansion/rbl.py

@@ -88,18 +88,18 @@ def handler(q=False):
     else:
         misperrors['error'] = "Unsupported attributes type"
         return misperrors
-    listed = []
+    listeds = []
-    info = []
+    infos = []
     ipRev = '.'.join(ip.split('.')[::-1])
     for rbl in rbls:
         query = '{}.{}'.format(ipRev, rbl)
         try:
             txt = resolver.query(query, 'TXT')
-            listed.append(query)
+            listeds.append(query)
-            info.append([str(t) for t in txt])
+            infos.append([str(t) for t in txt])
         except Exception:
             continue
-    result = "\n".join(["{}: {}".format(l, "  -  ".join(i)) for l, i in zip(listed, info)])
+    result = "\n".join([f"{listed}: {'  -  '.join(info)}" for listed, info in zip(listeds, infos)])
     if not result:
         return {'error': 'No data found by querying known RBLs'}
     return {'results': [{'types': mispattributes.get('output'), 'values': result}]}

misp_modules/modules/expansion/recordedfuture.py

@@ -0,0 +1,285 @@
+import json
+import logging
+import requests
+from . import check_input_attribute, checking_error, standard_error_message
+from urllib.parse import quote
+from pymisp import MISPAttribute, MISPEvent, MISPTag, MISPObject
+
+moduleinfo = {'version': '1.0', 'author': 'Recorded Future',
+              'description': 'Module to retrieve data from Recorded Future',
+              'module-type': ['expansion', 'hover']}
+
+moduleconfig = ['token']
+
+misperrors = {'error': 'Error'}
+
+mispattributes = {'input': ['ip', 'ip-src', 'ip-dst', 'domain', 'hostname', 'md5', 'sha1', 'sha256',
+                            'uri', 'url', 'vulnerability', 'weakness'],
+                  'output': ['ip', 'ip-src', 'ip-dst', 'domain', 'hostname', 'md5', 'sha1', 'sha256',
+                             'uri', 'url', 'vulnerability', 'weakness', 'email-src', 'text'],
+                  'format': 'misp_standard'}
+
+LOGGER = logging.getLogger('recorded_future')
+LOGGER.setLevel(logging.INFO)
+
+
+def rf_lookup(api_token: str, category: str, ioc: str) -> requests.Response:
+    """Do a lookup call using Recorded Future's ConnectAPI."""
+    auth_header = {"X-RFToken": api_token}
+    parsed_ioc = quote(ioc, safe='')
+    url = f'https://api.recordedfuture.com/v2/{category}/{parsed_ioc}?fields=risk%2CrelatedEntities'
+    response = requests.get(url, headers=auth_header)
+    response.raise_for_status()
+    return response
+
+
+class GalaxyFinder:
+    """A class for finding MISP galaxy matches to Recorded Future data."""
+    def __init__(self):
+        self.session = requests.Session()
+        self.sources = {
+            'RelatedThreatActor': ['https://raw.githubusercontent.com/MISP/misp-galaxy/'
+                                   'main/clusters/threat-actor.json'],
+            'RelatedMalware': ['https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/banker.json',
+                               'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/botnet.json',
+                               'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/exploit-kit.json',
+                               'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/rat.json',
+                               'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/ransomware.json',
+                               'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/malpedia.json']
+        }
+        self.galaxy_clusters = {}
+
+    def pull_galaxy_cluster(self, related_type: str):
+        """Fetches galaxy clusters for the related_type from the remote json files specified as self.sources."""
+        # Only fetch clusters if not fetched previously
+        if not self.galaxy_clusters.get(related_type):
+            for source in self.sources.get(related_type):
+                response = self.session.get(source)
+                if response.ok:
+                    name = source.split('/')[-1].split('.')[0]
+                    self.galaxy_clusters[related_type] = {name: response.json()}
+                else:
+                    LOGGER.info(f'pull_galaxy_cluster failed for source: {source},'
+                                f' got response: {response}, {response.reason}.')
+
+    def find_galaxy_match(self, indicator: str, related_type: str) -> str:
+        """Searches the clusters of the related_type for a match with the indicator.
+           :returns the first matching galaxy string or an empty string if no galaxy match is found.
+        """
+        self.pull_galaxy_cluster(related_type)
+        try:
+            for cluster_name, cluster in self.galaxy_clusters[related_type].items():
+                for value in cluster['values']:
+                    try:
+                        if indicator in value['meta']['synonyms'] or indicator in value['value']:
+                            value = value['value']
+                            return f'misp-galaxy:{cluster_name}="{value}"'
+                    except KeyError:
+                        pass
+        except KeyError:
+            pass
+        return ''
+
+
+class RFColors:
+    """Class for setting signature RF-colors."""
+    def __init__(self):
+        self.rf_white = '#CCCCCC'
+        self.rf_yellow = '#FFCE00'
+        self.rf_red = '#CF0A2C'
+
+    def riskscore_color(self, risk_score: int) -> str:
+        """Returns appropriate hex-colors according to risk score."""
+        risk_score = int(risk_score)
+        if risk_score < 25:
+            return self.rf_white
+        elif risk_score < 65:
+            return self.rf_yellow
+        else:
+            return self.rf_red
+
+    def riskrule_color(self, risk_rule_criticality: int) -> str:
+        """Returns appropriate hex-colors according to risk rule criticality."""
+        risk_rule_criticality = int(risk_rule_criticality)
+        if risk_rule_criticality == 1:
+            return self.rf_white
+        elif risk_rule_criticality == 2:
+            return self.rf_yellow
+        else:  # risk_rule_criticality == 3 or 4
+            return self.rf_red
+
+
+class RFEnricher:
+    """Class for enriching an attribute with data from Recorded Future.
+       The enrichment data is returned as a custom MISP object.
+    """
+    def __init__(self, api_token: str, attribute_props: dict):
+        self.api_token = api_token
+        self.event = MISPEvent()
+        self.enrichment_object = MISPObject('Recorded Future Enrichment')
+        self.enrichment_object.from_dict(**{'meta-category': 'misc',
+                                            'description': 'An object containing the enriched attribute and related '
+                                                           'entities from Recorded Future.',
+                                            'distribution': 0})
+
+        # Create a copy of enriched attribute to add tags to
+        temp_attr = MISPAttribute()
+        temp_attr.from_dict(**attribute_props)
+        self.enriched_attribute = MISPAttribute()
+        self.enriched_attribute.from_dict(**{'value': temp_attr.value, 'type': temp_attr.type, 'distribution': 0})
+
+        self.related_attributes = []
+        self.color_picker = RFColors()
+        self.galaxy_finder = GalaxyFinder()
+
+        # Mapping from MISP-type to RF-type
+        self.type_to_rf_category = {'ip': 'ip', 'ip-src': 'ip', 'ip-dst': 'ip',
+                                    'domain': 'domain', 'hostname': 'domain',
+                                    'md5': 'hash', 'sha1': 'hash', 'sha256': 'hash',
+                                    'uri': 'url', 'url': 'url',
+                                    'vulnerability': 'vulnerability', 'weakness': 'vulnerability'}
+
+        # Related entities from RF portrayed as related attributes in MISP
+        self.related_attribute_types = ['RelatedIpAddress', 'RelatedInternetDomainName', 'RelatedHash',
+                                        'RelatedEmailAddress', 'RelatedCyberVulnerability']
+        # Related entities from RF portrayed as tags in MISP
+        self.galaxy_tag_types = ['RelatedMalware', 'RelatedThreatActor']
+
+    def enrich(self):
+        """Run the enrichment."""
+        category = self.type_to_rf_category.get(self.enriched_attribute.type)
+
+        try:
+            response = rf_lookup(self.api_token, category, self.enriched_attribute.value)
+            json_response = json.loads(response.content)
+        except requests.HTTPError as error:
+            misperrors['error'] = f'Error when requesting data from Recorded Future. ' \
+                                  f'{error.response} : {error.response.reason}'
+            raise error
+
+        try:
+            # Add risk score and risk rules as tags to the enriched attribute
+            risk_score = json_response['data']['risk']['score']
+            hex_color = self.color_picker.riskscore_color(risk_score)
+            tag_name = f'recorded-future:risk-score="{risk_score}"'
+            self.add_tag(tag_name, hex_color)
+            for evidence in json_response['data']['risk']['evidenceDetails']:
+                risk_rule = evidence['rule']
+                criticality = evidence['criticality']
+                hex_color = self.color_picker.riskrule_color(criticality)
+                tag_name = f'recorded-future:risk-rule="{risk_rule}"'
+                self.add_tag(tag_name, hex_color)
+
+            # Retrieve related entities
+            for related_entity in json_response['data']['relatedEntities']:
+                related_type = related_entity['type']
+                if related_type in self.related_attribute_types:
+                    # Related entities returned as additional attributes
+                    for related in related_entity['entities']:
+                        if int(related["count"]) > 4:
+                            indicator = related['entity']['name']
+                            self.add_related_attribute(indicator, related_type)
+                elif related_type in self.galaxy_tag_types:
+                    # Related entities added as galaxy-tags to the enriched attribute
+                    galaxy_tags = []
+                    for related in related_entity['entities']:
+                        if int(related["count"]) > 4:
+                            indicator = related['entity']['name']
+                            galaxy = self.galaxy_finder.find_galaxy_match(indicator, related_type)
+                            # Handle deduplication of galaxy tags
+                            if galaxy and galaxy not in galaxy_tags:
+                                galaxy_tags.append(galaxy)
+                    for galaxy in galaxy_tags:
+                        self.add_tag(galaxy)
+        except KeyError as error:
+            misperrors['error'] = 'Unexpected format in Recorded Future api response.'
+            raise error
+
+    def add_related_attribute(self, indicator: str, related_type: str) -> None:
+        """Helper method for adding an indicator to the related attribute list."""
+        out_type = self.get_output_type(related_type, indicator)
+        attribute = MISPAttribute()
+        attribute.from_dict(**{'value': indicator, 'type': out_type, 'distribution': 0})
+        self.related_attributes.append((related_type, attribute))
+
+    def add_tag(self, tag_name: str, hex_color: str = None) -> None:
+        """Helper method for adding a tag to the enriched attribute."""
+        tag = MISPTag()
+        tag_properties = {'name': tag_name}
+        if hex_color:
+            tag_properties['colour'] = hex_color
+        tag.from_dict(**tag_properties)
+        self.enriched_attribute.add_tag(tag)
+
+    def get_output_type(self, related_type: str, indicator: str) -> str:
+        """Helper method for translating a Recorded Future related type to a MISP output type."""
+        output_type = 'text'
+        if related_type == 'RelatedIpAddress':
+            output_type = 'ip-dst'
+        elif related_type == 'RelatedInternetDomainName':
+            output_type = 'domain'
+        elif related_type == 'RelatedHash':
+            hash_len = len(indicator)
+            if hash_len == 64:
+                output_type = 'sha256'
+            elif hash_len == 40:
+                output_type = 'sha1'
+            elif hash_len == 32:
+                output_type = 'md5'
+        elif related_type == 'RelatedEmailAddress':
+            output_type = 'email-src'
+        elif related_type == 'RelatedCyberVulnerability':
+            signature = indicator.split('-')[0]
+            if signature == 'CVE':
+                output_type = 'vulnerability'
+            elif signature == 'CWE':
+                output_type = 'weakness'
+        return output_type
+
+    def get_results(self) -> dict:
+        """Build and return the enrichment results."""
+        self.enrichment_object.add_attribute('Enriched attribute', **self.enriched_attribute)
+        for related_type, attribute in self.related_attributes:
+            self.enrichment_object.add_attribute(related_type, **attribute)
+        self.event.add_object(**self.enrichment_object)
+        event = json.loads(self.event.to_json())
+        result = {key: event[key] for key in ['Object'] if key in event}
+        return {'results': result}
+
+
+def handler(q=False):
+    """Handle enrichment."""
+    if q is False:
+        return False
+    request = json.loads(q)
+
+    if request.get('config') and request['config'].get('token'):
+        token = request['config'].get('token')
+    else:
+        misperrors['error'] = 'Missing Recorded Future token.'
+        return misperrors
+    if not request.get('attribute') or not check_input_attribute(request['attribute'], requirements=('type', 'value')):
+        return {'error': f'{standard_error_message}, {checking_error}.'}
+    if request['attribute']['type'] not in mispattributes['input']:
+        return {'error': 'Unsupported attribute type.'}
+
+    input_attribute = request.get('attribute')
+    rf_enricher = RFEnricher(token, input_attribute)
+    try:
+        rf_enricher.enrich()
+    except (requests.HTTPError, KeyError):
+        return misperrors
+
+    return rf_enricher.get_results()
+
+
+def introspection():
+    """Returns a dict of the supported attributes."""
+    return mispattributes
+
+
+def version():
+    """Returns a dict with the version and the associated meta-data
+    including potential configurations required of the module."""
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/sigma_queries.py

@@ -12,7 +12,7 @@ mispattributes = {'input': ['sigma'], 'output': ['text']}
 moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'module-type': ['expansion', 'hover'],
               'description': 'An expansion hover module to display the result of sigma queries.'}
 moduleconfig = []
-sigma_targets = ('es-dsl', 'es-qs', 'graylog', 'kibana', 'xpack-watcher', 'logpoint', 'splunk', 'grep', 'wdatp', 'splunkxml', 'arcsight', 'qualys')
+sigma_targets = ('es-dsl', 'es-qs', 'graylog', 'kibana', 'xpack-watcher', 'logpoint', 'splunk', 'grep', 'mdatp', 'splunkxml', 'arcsight', 'qualys')
 
 
 def handler(q=False):

misp_modules/modules/expansion/sophoslabs_intelix.py

@@ -1,7 +1,8 @@
-from pymisp import MISPEvent, MISPObject
 import json
 import requests
 import base64
+from . import check_input_attribute, checking_error, standard_error_message
+from pymisp import MISPEvent, MISPObject
 from urllib.parse import quote
 
 moduleinfo = {'version': '1.0',
@@ -105,13 +106,25 @@ def handler(q=False):
         misperrors['error'] = "Missing client_id or client_secret value for SOPHOSLabs Intelix. \
             It's free to sign up here https://aws.amazon.com/marketplace/pp/B07SLZPMCS."
         return misperrors
+    to_check = (('type', 'value'), ('type', 'value1'))
+    if not j.get('attribute') or not any(check_input_attribute(j['attribute'], requirements=check) for check in to_check):
+        return {'error': f'{standard_error_message}, {checking_error}.'}
+    attribute = j['attribute']
+    if attribute['type'] not in misp_types_in:
+        return {'error': 'Unsupported attribute type.'}
     client = SophosLabsApi(j['config']['client_id'], j['config']['client_secret'])
-    if j['attribute']['type'] == "sha256":
+    mapping = {
-        client.hash_lookup(j['attribute']['value1'])
+        'sha256': 'hash_lookup',
-    if j['attribute']['type'] in ['ip-dst', 'ip-src', 'ip']:
+        'ip-dst': 'ip_lookup',
-        client.ip_lookup(j["attribute"]["value1"])
+        'ip-src': 'ip_lookup',
-    if j['attribute']['type'] in ['uri', 'url', 'domain', 'hostname']:
+        'ip': 'ip_lookup',
-        client.url_lookup(j["attribute"]["value1"])
+        'uri': 'url_lookup',
+        'url': 'url_lookup',
+        'domain': 'url_lookup',
+        'hostname': 'url_lookup'
+    }
+    attribute_value = attribute['value'] if 'value' in attribute else attribute['value1']
+    getattr(client, mapping[attribute['type']])(attribute_value)
     return client.get_result()
 
 

misp_modules/modules/expansion/trustar_enrich.py

@@ -2,6 +2,7 @@ import json
 import pymisp
 from base64 import b64encode
 from collections import OrderedDict
+from . import check_input_attribute, checking_error, standard_error_message
 from pymisp import MISPAttribute, MISPEvent, MISPObject
 from trustar import TruStar, Indicator
 from urllib.parse import quote
@@ -183,7 +184,11 @@ def handler(q=False):
         misperrors['error'] = "Your TruSTAR API key and secret are required for indicator enrichment."
         return misperrors
 
+    if not request.get('attribute') or not check_input_attribute(request['attribute'], requirements=('type', 'value')):
+        return {'error': f'{standard_error_message}, {checking_error}.'}
     attribute = request['attribute']
+    if attribute['type'] not in mispattributes['input']:
+        return {'error': 'Unsupported attribute type.'}
     trustar_parser = TruSTARParser(attribute, config)
     metadata = None
     summary = None

misp_modules/modules/expansion/urlhaus.py

@@ -1,6 +1,8 @@
-from pymisp import MISPAttribute, MISPEvent, MISPObject
+# -*- coding: utf-8 -*-
 import json
 import requests
+from . import check_input_attribute, standard_error_message
+from pymisp import MISPAttribute, MISPEvent, MISPObject
 
 misperrors = {'error': 'Error'}
 mispattributes = {'input': ['domain', 'hostname', 'ip-src', 'ip-dst', 'md5', 'sha256', 'url'],
@@ -134,7 +136,11 @@ def handler(q=False):
     if q is False:
         return False
     request = json.loads(q)
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
     attribute = request['attribute']
+    if attribute['type'] not in mispattributes['input']:
+        return {'error': 'Unsupported attribute type.'}
     urlhaus_parser = _misp_type_mapping[attribute['type']](attribute)
     return urlhaus_parser.query_api()
 

misp_modules/modules/expansion/virustotal.py

@@ -1,6 +1,7 @@
-from pymisp import MISPAttribute, MISPEvent, MISPObject
 import json
 import requests
+from . import check_input_attribute, standard_error_message
+from pymisp import MISPAttribute, MISPEvent, MISPObject
 
 misperrors = {'error': 'Error'}
 mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "url"],
@@ -52,7 +53,7 @@ class VirusTotalParser(object):
                          'downloaded': 'downloaded-from',
                          'referrer': 'referring'}
         siblings = (self.parse_siblings(domain) for domain in req['domain_siblings'])
-        uuid = self.parse_resolutions(req['resolutions'], req['subdomains'], siblings)
+        uuid = self.parse_resolutions(req['resolutions'], req['subdomains'] if 'subdomains' in req else None, siblings)
         for feature_type, relationship in feature_types.items():
             for feature in ('undetected_{}_samples', 'detected_{}_samples'):
                 for sample in req.get(feature.format(feature_type), [])[:self.limit]:
@@ -195,6 +196,11 @@ def handler(q=False):
     if not request.get('config') or not request['config'].get('apikey'):
         misperrors['error'] = "A VirusTotal api key is required for this module."
         return misperrors
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
+    if request['attribute']['type'] not in mispattributes['input']:
+        return {'error': 'Unsupported attribute type.'}
+
     event_limit = request['config'].get('event_limit')
     if not isinstance(event_limit, int):
         event_limit = 5

misp_modules/modules/expansion/virustotal_public.py

@@ -1,6 +1,7 @@
-from pymisp import MISPAttribute, MISPEvent, MISPObject
 import json
 import requests
+from . import check_input_attribute, standard_error_message
+from pymisp import MISPAttribute, MISPEvent, MISPObject
 
 misperrors = {'error': 'Error'}
 mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "url"],
@@ -174,7 +175,11 @@ def handler(q=False):
     if not request.get('config') or not request['config'].get('apikey'):
         misperrors['error'] = "A VirusTotal api key is required for this module."
         return misperrors
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
     attribute = request['attribute']
+    if attribute['type'] not in mispattributes['input']:
+        return {'error': 'Unsupported attribute type.'}
     query_type, to_call = misp_type_mapping[attribute['type']]
     parser = to_call(request['config']['apikey'], attribute)
     query_result = parser.get_query_result(query_type)

misp_modules/modules/expansion/xforceexchange.py

@@ -1,6 +1,7 @@
 import requests
 import json
 import sys
+from . import check_input_attribute, standard_error_message
 from collections import defaultdict
 from pymisp import MISPAttribute, MISPEvent, MISPObject
 from requests.auth import HTTPBasicAuth
@@ -160,6 +161,10 @@ def handler(q=False):
         return misperrors
     key = request["config"]["apikey"]
     password = request['config']['apipassword']
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
+        return {'error': f'{standard_error_message} which should contain at least a type, a value and an uuid.'}
+    if request['attribute']['type'] not in mispattributes['input']:
+        return {'error': 'Unsupported attribute type.'}
     parser = XforceExchange(request['attribute'], key, password)
     parser.parse()
     return parser.get_result()

misp_modules/modules/import_mod/__init__.py

@@ -15,5 +15,4 @@ __all__ = [
     'threatanalyzer_import',
     'csvimport',
     'joe_import',
-    'trustar_import',
 ]

misp_modules/modules/import_mod/csvimport.py

@@ -244,11 +244,11 @@ def __any_mandatory_misp_field(header):
 
 
 def __special_parsing(data, delimiter):
-    return list(tuple(l.strip() for l in line[0].split(delimiter)) for line in csv.reader(io.TextIOWrapper(io.BytesIO(data.encode()), encoding='utf-8')) if line and not line[0].startswith('#'))
+    return list(tuple(part.strip() for part in line[0].split(delimiter)) for line in csv.reader(io.TextIOWrapper(io.BytesIO(data.encode()), encoding='utf-8')) if line and not line[0].startswith('#'))
 
 
 def __standard_parsing(data):
-    return list(tuple(l.strip() for l in line) for line in csv.reader(io.TextIOWrapper(io.BytesIO(data.encode()), encoding='utf-8')) if line and not line[0].startswith('#'))
+    return list(tuple(part.strip() for part in line) for line in csv.reader(io.TextIOWrapper(io.BytesIO(data.encode()), encoding='utf-8')) if line and not line[0].startswith('#'))
 
 
 def handler(q=False):

misp_modules/modules/import_mod/threatanalyzer_import.py

@@ -99,7 +99,7 @@ def handler(q=False):
             results = process_analysis_json(json.loads(data.decode('utf-8')))
         except ValueError:
             log.warning('MISP modules {0} failed: uploaded file is not a zip or json file.'.format(request['module']))
-            return {'error': 'Uploaded file is not a zip or json file.'.format(request['module'])}
+            return {'error': 'Uploaded file is not a zip or json file.'}
             pass
     # keep only unique entries based on the value field
     results = list({v['values']: v for v in results}.values())

misp_modules/modules/import_mod/trustar_import.py

@@ -1,7 +0,0 @@
-import base64
-import json
-
-from trustar import TruStar
-
-misp_errors = {'error': "Error"}
-

tests/test_expansions.py

@@ -229,11 +229,24 @@ class TestExpansions(unittest.TestCase):
         self.assertEqual(to_check, 'OK (Not Found)', response)
 
     def test_greynoise(self):
-        query = {"module": "greynoise", "ip-dst": "1.1.1.1"}
+        module_name = 'greynoise'
-        response = self.misp_modules_post(query)
+        query = {"module": module_name, "ip-dst": "1.1.1.1"}
-        value = self.get_values(response)
+        if module_name in self.configs:
-        if value != 'GreyNoise API not accessible (HTTP 429)':
+            query['config'] = self.configs[module_name]
-            self.assertTrue(value.startswith('{"ip":"1.1.1.1","status":"ok"'))
+            response = self.misp_modules_post(query)
+            try:
+                self.assertEqual(self.get_values(response), 'This IP is commonly spoofed in Internet-scan activity')
+            except Exception:
+                self.assertIn(
+                    self.get_errors(reponse),
+                    (
+                        "Unauthorized. Please check your API key.",
+                        "Too many requests. You've hit the rate-limit."
+                    )
+                )
+        else:
+            response = self.misp_modules_post(query)
+            self.assertEqual(self.get_errors(response), 'Missing Greynoise API key.')
 
     def test_ipasn(self):
         query = {"module": "ipasn",