Merge branch 'main' of github.com:MISP/misp-modules into main

README.md

@@ -58,6 +58,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
 * [macaddress.io](misp_modules/modules/expansion/macaddress_io.py) - a hover module to retrieve vendor details and other information regarding a given MAC address or an OUI from [MAC address Vendor Lookup](https://macaddress.io). See [integration tutorial here](https://macaddress.io/integrations/MISP-module).
 * [macvendors](misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information.
 * [MALWAREbazaar](misp_modules/modules/expansion/malwarebazaar.py) - an expansion module to query MALWAREbazaar with some payload.
+* [McAfee MVISION Insights](misp_modules/modules/expansion/mcafee_insights_enrich.py) - an expansion module enrich IOCs with McAfee MVISION Insights.
 * [ocr-enrich](misp_modules/modules/expansion/ocr_enrich.py) - an enrichment module to get OCRized data from images into MISP.
 * [ods-enrich](misp_modules/modules/expansion/ods_enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser).
 * [odt-enrich](misp_modules/modules/expansion/odt_enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser).

REQUIREMENTS

@@ -5,141 +5,146 @@
 #    pipenv lock --requirements
 #
 
+-i https://pypi.org/simple
 -e .
--e git+https://github.com/D4-project/BGP-Ranking.git/@fd9c0e03af9b61d4bf0b67ac73c7208a55178a54#egg=pybgpranking&subdirectory=client
+-e git+https://github.com/D4-project/BGP-Ranking.git/@68de39f6c5196f796055c1ac34504054d688aa59#egg=pybgpranking&subdirectory=client
--e git+https://github.com/D4-project/IPASN-History.git/@fc5e48608afc113e101ca6421bf693b7b9753f9e#egg=pyipasnhistory&subdirectory=client
+-e git+https://github.com/D4-project/IPASN-History.git/@a2853c39265cecdd0c0d16850bd34621c0551b87#egg=pyipasnhistory&subdirectory=client
--e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471
+-e git+https://github.com/MISP/PyIntel471.git@917272fafa8e12102329faca52173e90c5256968#egg=pyintel471
--e git+https://github.com/Rafiot/uwhoisd.git@783bba09b5a6964f25566089826a1be4b13f2a22#egg=uwhois&subdirectory=client
 -e git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader
 -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
 -e git+https://github.com/sebdraven/pyonyphe@1ce15581beebb13e841193a08a2eb6f967855fcb#egg=pyonyphe
-aiohttp==3.7.3; python_version >= '3.6'
+aiohttp==3.7.4.post0
 antlr4-python3-runtime==4.8; python_version >= '3'
 apiosintds==1.8.3
 argparse==1.4.0
-assemblyline-client==4.0.1
+assemblyline-client==4.1.0
 async-timeout==3.0.1; python_full_version >= '3.5.3'
-attrs==20.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+attrs==21.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+backports.zoneinfo==0.2.1; python_version < '3.9'
 backscatter==0.2.4
 beautifulsoup4==4.9.3
 bidict==0.21.2; python_version >= '3.6'
 blockchain==1.4.4
-censys==1.1.1
+certifi==2021.5.30
-certifi==2020.12.5
+cffi==1.14.6
-cffi==1.14.4
+chardet==4.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-chardet==3.0.4
+charset-normalizer==2.0.4; python_version >= '3'
 clamd==1.0.2
 click-plugins==1.1.1
-click==7.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+click==8.0.1; python_version >= '3.6'
 colorama==0.4.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 colorclass==2.2.0
 compressed-rtf==1.0.6
-configparser==5.0.1; python_version >= '3.6'
+configparser==5.0.2; python_version >= '3.6'
-cryptography==3.3.1
+cryptography==3.4.7; python_version >= '3.6'
-decorator==4.4.2
+decorator==5.0.9; python_version >= '3.5'
-deprecated==1.2.11; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+deprecated==1.2.12; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-dnsdb2==1.1.2
+dnsdb2==1.1.3
-dnspython3==1.15.0
+dnspython==2.1.0
-domaintools-api==0.5.2
+domaintools-api==0.5.4
-easygui==0.98.1
+easygui==0.98.2
 ebcdic==1.1.1
 enum-compat==0.0.3
-extract-msg==0.28.1
+extract-msg==0.28.7
 ez-setup==0.9
 ezodf==0.3.2
+filelock==3.0.12
 future==0.18.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
 futures==3.1.1
-geoip2==4.1.0
+geoip2==4.2.0
-httplib2==0.18.1
+httplib2==0.19.1
 idna-ssl==1.1.0; python_version < '3.7'
-idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+idna==3.2; python_version >= '3'
 imapclient==2.1.0
 isodate==0.6.0
-jbxapi==3.14.0
+itsdangerous==2.0.1; python_version >= '3.6'
-json-log-formatter==0.3.0
+jbxapi==3.17.2
+json-log-formatter==0.4.0
 jsonschema==3.2.0
-ndjson==0.3.1
+lark-parser==0.11.3
-lark-parser==0.11.1
+lief==0.11.5
-lief==0.11.0
+lxml==4.6.3
-lxml==4.6.2
 maclookup==1.0.3
 markdownify==0.5.3
 maxminddb==2.0.3; python_version >= '3.6'
-msoffcrypto-tool==4.11.0
+more-itertools==8.8.0; python_version >= '3.5'
+msoffcrypto-tool==4.12.0; python_version >= '3' and platform_python_implementation != 'PyPy' or (platform_system != 'Windows' and platform_system != 'Darwin')
 multidict==5.1.0; python_version >= '3.6'
 np==1.0.2
-numpy==1.19.5; python_version >= '3.6'
+numpy==1.21.2; python_version < '3.11' and python_version >= '3.7'
 oauth2==1.9.0.post1
 olefile==0.46; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-oletools==0.56
+oletools==0.56.2
-opencv-python==4.5.1.48
+opencv-python==4.5.3.56
-openpyxl
+pandas-ods-reader==0.1.2
-pandas-ods-reader==0.0.7
+pandas==1.3.2
-pandas==1.1.5
+passivetotal==2.5.4
-passivetotal==1.0.31
 pcodedmp==1.2.6
-pdftotext==2.1.5
+pdftotext==2.2.0
-pillow==8.1.0
+pillow==8.3.1
 progressbar2==3.53.1
 psutil==5.8.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
 pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pycryptodome==3.9.9; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pycryptodome==3.10.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-pycryptodomex==3.9.9; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pycryptodomex==3.10.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 pydeep==0.4
 pyeupi==1.1
-pyfaup==1.2
 pygeoip==0.3.2
-pymisp[email,fileobjects,openioc,pdfexport]==2.4.137.1
+pymisp[email,fileobjects,openioc,pdfexport]==2.4.148
-pyopenssl==20.0.1
 pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pypdns==1.5.1
+pypdns==1.5.2
-pypssl==2.1
+pypssl==2.2
-pyrsistent==0.17.3; python_version >= '3.5'
+pyrsistent==0.18.0; python_version >= '3.6'
-pytesseract==0.3.7
+pytesseract==0.3.8
 python-baseconv==1.2.2
-python-dateutil==2.8.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-python-docx==0.8.10
+python-docx==0.8.11
-python-engineio==4.0.0
+python-engineio==4.2.1; python_version >= '3.6'
-python-magic==0.4.18
+python-magic==0.4.24
-python-pptx==0.6.18
+python-pptx==0.6.19
-python-socketio[client]==5.0.4
+python-socketio[client]==5.4.0; python_version >= '3.6'
-python-utils==2.5.2
+python-utils==2.5.6
 pytz==2019.3
 pyyaml==5.4.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
-pyeti-python3==1.0.0
 pyzbar==0.1.8
-pyzipper==0.3.4; python_version >= '3.5'
+pyzipper==0.3.5; python_version >= '3.5'
-rdflib==5.0.0
+rdflib==6.0.0; python_version >= '3.7'
 redis==3.5.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-reportlab==3.5.59
+reportlab==3.6.1
-requests-cache==0.5.2
+requests-cache==0.6.4; python_version >= '3.6'
-requests[security]==2.25.1
+requests-file==1.5.1
+requests[security]==2.26.0
 rtfde==0.0.2
-shodan==1.24.0
+ruamel.yaml.clib==0.2.6; python_version < '3.10' and platform_python_implementation == 'CPython'
-sigmatools==0.18.1
+ruamel.yaml==0.17.13; python_version >= '3'
-six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+shodan==1.25.0
-socialscan==1.4.1
+sigmatools==0.20
+six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+socialscan==1.4.2
 socketio-client==0.5.7.4
-soupsieve==2.1; python_version >= '3'
+soupsieve==2.2.1; python_version >= '3'
 sparqlwrapper==1.8.5
 stix2-patterns==1.3.2
-tabulate==0.8.7
+tabulate==0.8.9
+tau-clients==0.1.3
+tldextract==3.1.0; python_version >= '3.5'
 tornado==6.1; python_version >= '3.5'
-tqdm==4.56.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+tqdm==4.62.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-trustar==0.3.34
+trustar==0.3.35
-typing-extensions==3.7.4.3; python_version < '3.8'
+typing-extensions==3.10.0.0
-tzlocal==2.1
+tzlocal==3.0; python_version >= '3.6'
 unicodecsv==0.14.1
 url-normalize==1.4.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
 urlarchiver==0.2
-urllib3==1.26.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'
+urllib3==1.26.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
 validators==0.14.0
-vt-graph-api==1.0.1
+vt-graph-api==1.1.2
-vulners==1.5.9
+vt-py==0.7.2
-wand==0.6.5
+vulners==1.5.12
-websocket-client==0.57.0
+wand==0.6.7
+websocket-client==1.2.1
 wrapt==1.12.1
 xlrd==2.0.1
-xlsxwriter==1.3.7
+xlsxwriter==3.0.1; python_version >= '3.4'
 yara-python==3.8.1
 yarl==1.6.3; python_version >= '3.6'

misp_modules/modules/expansion/mcafee_insights_enrich.py

@@ -0,0 +1,239 @@
+# Written by mohlcyber 13.08.2021
+# MISP Module for McAfee MVISION Insights to query campaign details
+
+import json
+import logging
+import requests
+import sys
+
+from . import check_input_attribute, standard_error_message
+from pymisp import MISPAttribute, MISPEvent, MISPObject
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ["md5", "sha1", "sha256"],
+                  'format': 'misp_standard'}
+
+# possible module-types: 'expansion', 'hover' or both
+moduleinfo = {'version': '1', 'author': 'Martin Ohl',
+              'description': 'Lookup McAfee MVISION Insights Details',
+              'module-type': ['hover']}
+
+# config fields that your code expects from the site admin
+moduleconfig = ['api_key', 'client_id', 'client_secret']
+
+
+class MVAPI():
+    def __init__(self, attribute, api_key, client_id, client_secret):
+        self.misp_event = MISPEvent()
+        self.attribute = MISPAttribute()
+        self.attribute.from_dict(**attribute)
+        self.misp_event.add_attribute(**self.attribute)
+
+        self.base_url = 'https://api.mvision.mcafee.com'
+        self.session = requests.Session()
+
+        self.api_key = api_key
+        auth = (client_id, client_secret)
+
+        self.logging()
+        self.auth(auth)
+
+    def logging(self):
+        self.logger = logging.getLogger('logs')
+        self.logger.setLevel('INFO')
+        handler = logging.StreamHandler()
+        formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(message)s")
+        handler.setFormatter(formatter)
+        self.logger.addHandler(handler)
+
+    def auth(self, auth):
+        iam_url = "https://iam.mcafee-cloud.com/iam/v1.1/token"
+
+        headers = {
+            'x-api-key': self.api_key,
+            'Content-Type': 'application/vnd.api+json'
+        }
+
+        payload = {
+            "grant_type": "client_credentials",
+            "scope": "ins.user ins.suser ins.ms.r"
+        }
+
+        res = self.session.post(iam_url, headers=headers, auth=auth, data=payload)
+
+        if res.status_code != 200:
+            self.logger.error('Could not authenticate to get the IAM token: {0} - {1}'.format(res.status_code, res.text))
+            sys.exit()
+        else:
+            self.logger.info('Successful authenticated.')
+            access_token = res.json()['access_token']
+            headers['Authorization'] = 'Bearer ' + access_token
+            self.session.headers = headers
+
+    def search_ioc(self):
+        filters = {
+            'filter[type][eq]': self.attribute.type,
+            'filter[value]': self.attribute.value,
+            'fields': 'id, type, value, coverage, uid, is_coat, is_sdb_dirty, category, comment, campaigns, threat, prevalence'
+        }
+        res = self.session.get(self.base_url + '/insights/v2/iocs', params=filters)
+
+        if res.ok:
+            if len(res.json()['data']) == 0:
+                self.logger.info('No Hash details in MVISION Insights found.')
+            else:
+                self.logger.info('Successfully retrieved MVISION Insights details.')
+                self.logger.debug(res.text)
+                return res.json()
+        else:
+            self.logger.error('Error in search_ioc. HTTP {0} - {1}'.format(str(res.status_code), res.text))
+            sys.exit()
+
+    def prep_result(self, ioc):
+        res = ioc['data'][0]
+        results = []
+
+        # Parse out Attribute Category
+        category_attr = {
+            'type': 'text',
+            'object_relation': 'text',
+            'value': 'Attribute Category: {0}'.format(res['attributes']['category'])
+        }
+        results.append(category_attr)
+
+        # Parse out Attribute Comment
+        comment_attr = {
+            'type': 'text',
+            'object_relation': 'text',
+            'value': 'Attribute Comment: {0}'.format(res['attributes']['comment'])
+        }
+        results.append(comment_attr)
+
+        # Parse out Attribute Dat Coverage
+        cover_attr = {
+            'type': 'text',
+            'object_relation': 'text',
+            'value': 'Dat Version Coverage: {0}'.format(res['attributes']['coverage']['dat_version']['min'])
+        }
+        results.append(cover_attr)
+
+        # Parse out if Dirty
+        cover_attr = {
+            'type': 'text',
+            'object_relation': 'text',
+            'value': 'Is Dirty: {0}'.format(res['attributes']['is-sdb-dirty'])
+        }
+        results.append(cover_attr)
+
+        # Parse our targeted countries
+        countries_dict = []
+        countries = res['attributes']['prevalence']['countries']
+
+        for country in countries:
+            countries_dict.append(country['iso_code'])
+
+        country_attr = {
+            'type': 'text',
+            'object_relation': 'text',
+            'value': 'Targeted Countries: {0}'.format(countries_dict)
+        }
+        results.append(country_attr)
+
+        # Parse out targeted sectors
+        sectors_dict = []
+        sectors = res['attributes']['prevalence']['sectors']
+
+        for sector in sectors:
+            sectors_dict.append(sector['sector'])
+
+        sector_attr = {
+            'type': 'text',
+            'object_relation': 'text',
+            'value': 'Targeted Sectors: {0}'.format(sectors_dict)
+        }
+        results.append(sector_attr)
+
+        # Parse out Threat Classification
+        threat_class_attr = {
+            'type': 'text',
+            'object_relation': 'text',
+            'value': 'Threat Classification: {0}'.format(res['attributes']['threat']['classification'])
+        }
+        results.append(threat_class_attr)
+
+        # Parse out Threat Name
+        threat_name_attr = {
+            'type': 'text',
+            'object_relation': 'text',
+            'value': 'Threat Name: {0}'.format(res['attributes']['threat']['name'])
+        }
+        results.append(threat_name_attr)
+
+        # Parse out Threat Severity
+        threat_sev_attr = {
+            'type': 'text',
+            'object_relation': 'text',
+            'value': 'Threat Severity: {0}'.format(res['attributes']['threat']['severity'])
+        }
+        results.append(threat_sev_attr)
+
+        # Parse out Attribute ID
+        attr_id = {
+            'type': 'text',
+            'object_relation': 'text',
+            'value': 'Attribute ID: {0}'.format(res['id'])
+        }
+        results.append(attr_id)
+
+        # Parse out Campaign Relationships
+        campaigns = ioc['included']
+
+        for campaign in campaigns:
+            campaign_attr = {
+                'type': 'campaign-name',
+                'object_relation': 'campaign-name',
+                'value': campaign['attributes']['name']
+            }
+            results.append(campaign_attr)
+
+        mv_insights_obj = MISPObject(name='MVISION Insights Details')
+        for mvi_res in results:
+            mv_insights_obj.add_attribute(**mvi_res)
+        mv_insights_obj.add_reference(self.attribute.uuid, 'mvision-insights-details')
+
+        self.misp_event.add_object(mv_insights_obj)
+
+        event = json.loads(self.misp_event.to_json())
+        results_mvi = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])}
+
+        return {'results': results_mvi}
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+
+    if not request.get('config') or not request['config'].get('api_key') or not request['config'].get('client_id') or not request['config'].get('client_secret'):
+        misperrors['error'] = "Please provide MVISION API Key, Client ID and Client Secret."
+        return misperrors
+    if request['attribute']['type'] not in mispattributes['input']:
+        return {'error': 'Unsupported attribute type. Please use {0}'.format(mispattributes['input'])}
+
+    api_key = request['config']['api_key']
+    client_id = request['config']['client_id']
+    client_secret = request['config']['client_secret']
+    attribute = request['attribute']
+
+    mvi = MVAPI(attribute, api_key, client_id, client_secret)
+    res = mvi.search_ioc()
+    return mvi.prep_result(res)
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta:__legacy__"