merge

2024-05-13 20:28:41 +02:00 · 2024-05-13 20:28:41 +02:00 · 636dc3cdfa
parent 6b7260229b 55b0914185
commit 636dc3cdfa
15 changed files with 298 additions and 14 deletions
--- a/1
+++ b/1
@ -76,6 +76,7 @@ ndjson = "0.3.1"
 Jinja2 = "3.1.2"
 mattermostdriver = "7.3.2"
 openpyxl = "*"
+slack-sdk = "3.27.1"

 [requires]
 python_version = "3.7"
--- a/README.md
+++ b/README.md
@ -3,10 +3,12 @@
 [![Python package](https://github.com/MISP/misp-modules/actions/workflows/python-package.yml/badge.svg)](https://github.com/MISP/misp-modules/actions/workflows/python-package.yml)[![Coverage Status](https://coveralls.io/repos/github/MISP/misp-modules/badge.svg?branch=main)](https://coveralls.io/github/MISP/misp-modules?branch=main)
 [![codecov](https://codecov.io/gh/MISP/misp-modules/branch/main/graph/badge.svg)](https://codecov.io/gh/MISP/misp-modules)

-MISP modules are autonomous modules that can be used to extend [MISP](https://github.com/MISP/MISP) for new services such as expansion, import and export.
+MISP modules are autonomous modules that can be used to extend [MISP](https://github.com/MISP/MISP) for new services such as expansion, import, export and workflow action.
+
+MISP modules can be also installed and used without MISP as a [standalone tool accessible via a convenient web interface](./website).

 The modules are written in Python 3 following a simple API interface. The objective is to ease the extensions of MISP functionalities
-without modifying core components. The API is available via a simple REST API which is independent from MISP installation or configuration.
+without modifying core components. The API is available via a simple REST API which is independent from MISP installation or configuration and can be used with other tools.

 For more information: [Extending MISP with Python modules](https://www.misp-project.org/misp-training/3.1-misp-modules.pdf) slides from [MISP training](https://github.com/MISP/misp-training).

@ -83,6 +85,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
 * [Socialscan](misp_modules/modules/expansion/socialscan.py) - a hover module to check if an email address or a username is used on different online platforms, using the [socialscan](https://github.com/iojw/socialscan) python library
 * [SophosLabs Intelix](misp_modules/modules/expansion/sophoslabs_intelix.py) - SophosLabs Intelix is an API for Threat Intelligence and Analysis (free tier available). [SophosLabs](https://aws.amazon.com/marketplace/pp/B07SLZPMCS)
 * [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
+* [stairwell](misp_modules/modules/expansion/stairwell.py) - an expansion module to enrich hash observables with the Stairwell API
 * [STIX2 pattern syntax validator](misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - a module to check a STIX2 pattern syntax.
 * [ThreatCrowd](misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/).
 * [threatminer](misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/).
@ -577,7 +580,7 @@ cd ../

 ## Documentation

-In order to provide documentation about some modules that require specific input / output / configuration, the [doc](doc) directory contains detailed information about the general purpose, requirements, features, input and ouput of each of these modules:
+In order to provide documentation about some modules that require specific input / output / configuration, the [index.md](docs/index.md) file within the [docs](docs) directory contains detailed information about the general purpose, requirements, features, input and ouput of each of these modules:

 - ***description** - quick description of the general purpose of the module, as the one given by the moduleinfo
 - **requirements** - special libraries needed to make the module work
--- a/1
+++ b/1
@ -152,6 +152,7 @@ sigmatools==0.19.1
 sigmf==1.1.1
 simplejson==3.19.1 ; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'
 six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+slack-sdk==3.27.1
 sniffio==1.3.0 ; python_version >= '3.7'
 socialscan==1.4
 socketio-client==0.5.7.4
--- a/docs/index.md
+++ b/docs/index.md
@ -66,6 +66,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
 * [Sigma queries](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/sigma_queries.py) - Experimental expansion module querying a sigma rule to convert it into all the available SIEM signatures.
 * [Sigma syntax validator](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/sigma_syntax_validator.py) - Sigma syntax validator.
 * [sourcecache](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
+* [stairwell](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/stairwell.py) - an expansion module to enrich hash observables with the Stairwell API
 * [STIX2 pattern syntax validator](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - a module to check a STIX2 pattern syntax.
 * [ThreatCrowd](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/).
 * [threatminer](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/).
--- a/documentation/README.md
+++ b/documentation/README.md
@ -1604,6 +1604,26 @@ Module to cache web pages of analysis reports, OSINT sources. The module returns

 -----

+#### [stairwell](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/stairwell.py)
+
+<img src=logos/stairwell.png height=60>
+
+An expansion module to enrich hash observables with the Stairwell API.
+- **features**:
+>This module takes a file hash as input and queries the Stairwell API. It will create a misp-object with the additional enrichment intel.
+- **input**:
+>MD5, SHA1, or SHA256
+- **output**:
+>A stairwell misp-object with additional enrichment intel.
+- **references**:
+>https://docs.stairwell.com
+- **requirements**:
+>- json
+>- pymisp
+>- requests
+
+-----
+
 #### [stix2_pattern_syntax_validator](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py)

 <img src=logos/stix.png height=60>
--- a/documentation/logos/stairwell.png
+++ b/documentation/logos/stairwell.png
--- a/documentation/website/expansion/stairwell.json
+++ b/documentation/website/expansion/stairwell.json
@ -0,0 +1,14 @@
+{
+  "description": "Module to query the Stairwell API to get additional information about the input hash attribute",
+  "logo": "stairwell.png",
+  "requirements": [
+    "Access to Stairwell platform (apikey)"
+  ],
+  "input": "A hash attribute (md5, sha1, sha256).",
+  "output": "File object related to the input attribute found on Stairwell platform.",
+  "references": [
+    "https://stairwell.com",
+    "https://docs.stairwell.com"
+  ],
+  "features": "The module takes a hash attribute as input and queries Stariwell's API to fetch additional data about it. The result, if the payload is observed in Stariwell, is a file object describing the file the input hash is related to."
+}
--- a/misp_modules/init.py
+++ b/misp_modules/init.py
@ -294,7 +294,8 @@ def main():

    application = tornado.web.Application(service)
    try:
-        application.listen(args.port, address=args.listen)
+        server = tornado.httpserver.HTTPServer(application, max_buffer_size=1073741824)  # buffer size increase when large MISP event are submitted - GH issue 662
+        server.listen(args.port, args.listen)
    except Exception as e:
        if e.errno == 98:
            pids = psutil.pids()
--- a/misp_modules/lib/misp-objects
+++ b/misp_modules/lib/misp-objects
@ -1 +1 @@
-Subproject commit 9c8b9504257c65459cfedf4f3231aee74f77dbe3
+Subproject commit a193e03ad200baddcdc0d5fad1cc1d8bd1276b7f
--- a/misp_modules/modules/action_mod/init.py
+++ b/misp_modules/modules/action_mod/init.py
@ -1 +1 @@
-__all__ = ['testaction', 'mattermost']
+__all__ = ['testaction', 'mattermost', 'slack']
--- a/misp_modules/modules/action_mod/slack.py
+++ b/misp_modules/modules/action_mod/slack.py
@ -0,0 +1,86 @@
+import json
+from slack_sdk import WebClient
+from slack_sdk.errors import SlackApiError
+from ._utils import utils
+
+misperrors = {'error': 'Error'}
+
+# config fields that your code expects from the site admin
+moduleconfig = {
+    'params': {
+        'slack_bot_token': {
+            'type': 'string',
+            'description': 'The Slack bot token generated when you created the bot account',
+        },
+        'channel_id': {
+            'type': 'string',
+            'description': 'The channel ID you want to post messages to',
+        },
+        'message_template': {
+            'type': 'large_string',
+            'description': 'The template to be used to generate the message to be posted',
+            'value': 'The **template** will be rendered using *Jinja2*!',
+            'jinja_supported': True,
+        },
+    },
+    # Blocking modules break the execution of the current action
+    'blocking': False,
+    # Indicates whether parts of the data passed to this module should be filtered.
+    'support_filters': True,
+    # Indicates whether the data passed to this module should be compliant with the MISP core format
+    'expect_misp_core_format': False,
+}
+
+# returns either "boolean" or "data"
+# Boolean is used to simply signal that the execution has finished.
+# For blocking modules, the actual boolean value determines whether we break execution
+returns = 'boolean'
+
+moduleinfo = {'version': '0.1', 'author': 'goodlandsecurity',
+              'description': 'Simplistic module to send messages to a Slack channel.',
+              'module-type': ['action']}
+
+
+def create_post(request):
+    params = request['params']
+    slack_token = params['slack_bot_token']
+    channel_id = params['channel_id']
+
+    client = WebClient(token=slack_token)
+
+    data = request.get('matchingData', request.get('data', {}))
+
+    if params['message_template']:
+        message = utils.renderTemplate(data, params['message_template'])
+    else:
+        message = '```\n{}\n```'.format(json.dumps(data))
+
+    try:
+        client.chat_postMessage(channel=channel_id, text=message)
+        return True
+    except SlackApiError as e:
+        error_message = e.response['error']
+        print(f"Error posting message: {error_message}")
+        return False
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    create_post(request)
+    return {"data": True}
+
+
+def introspection():
+    modulesetup = {}
+    try:
+        modulesetup['config'] = moduleconfig
+    except NameError:
+        pass
+    return modulesetup
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo
--- a/misp_modules/modules/expansion/init.py
+++ b/misp_modules/modules/expansion/init.py
@ -20,7 +20,7 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c
           'trustar_enrich', 'recordedfuture', 'html_to_markdown', 'socialscan', 'passive-ssh',
           'qintel_qsentry', 'mwdb', 'hashlookup', 'mmdb_lookup', 'ipqs_fraud_and_risk_scoring',
           'clamav', 'jinja_template_rendering','hyasinsight', 'variotdbs', 'crowdsec',
-           'extract_url_components', 'ipinfo', 'whoisfreaks', 'ip2locationio', 'vysion',
+           'extract_url_components', 'ipinfo', 'whoisfreaks', 'ip2locationio', 'vysion', 'stairwell',
           'google_threat_intelligence']


--- a/misp_modules/modules/expansion/dns.py
+++ b/misp_modules/modules/expansion/dns.py
@ -4,7 +4,7 @@ import dns.resolver
 misperrors = {'error': 'Error'}
 mispattributes = {'input': ['hostname', 'domain', 'domain|ip'], 'output': ['ip-src',
                                                                           'ip-dst']}
-moduleinfo = {'version': '0.2', 'author': 'Alexandre Dulaunoy',
+moduleinfo = {'version': '0.3', 'author': 'Alexandre Dulaunoy',
              'description': 'Simple DNS expansion service to resolve IP address from MISP attributes',
              'module-type': ['expansion', 'hover']}

@ -43,8 +43,8 @@ def handler(q=False):
    except dns.exception.Timeout:
        misperrors['error'] = "Timeout"
        return misperrors
-    except Exception:
-        misperrors['error'] = "DNS resolving error"
+    except Exception as e:
+        misperrors['error'] = f'DNS resolving error {e}'
        return misperrors

    r = {'results': [{'types': mispattributes['output'],
--- a/misp_modules/modules/expansion/stairwell.py
+++ b/misp_modules/modules/expansion/stairwell.py
@ -0,0 +1,149 @@
+import json
+import re
+import requests
+from pymisp import MISPEvent, MISPObject
+from . import check_input_attribute, checking_error, standard_error_message
+
+
+misperrors = {
+    'error': 'Error'
+}
+mispattributes = {
+    'input': [
+        'md5',
+        'sha1',
+        'sha256'
+    ],
+    'format': 'misp_standard'
+}
+moduleinfo = {
+    'version': '0.1',
+    'author': 'goodlandsecurity',
+    'description': 'Enrich hash observables with the Stairwell API',
+    'module-type': ['expansion']
+}
+moduleconfig = ["apikey"]
+
+
+def parse_response(response: dict):
+    attribute_mapping = {
+        'environments': {'type': 'comment', 'object_relation': 'environment', 'distribution': 5},
+        'imphash': {'type': 'imphash', 'object_relation': 'impash', 'distribution': 5},
+        'magic': {'type': 'comment', 'object_relation': 'magic', 'distribution': 5},
+        'malEval': {
+            'probabilityBucket': {'type': 'comment', 'object_relation': 'malEval-probability', 'distribution': 5},
+            'severity': {'type': 'comment', 'object_relation': 'malEval-severity', 'distribution': 5}
+        },
+        'md5': {'type': 'md5', 'object_relation': 'md5', 'distribution': 5},
+        'mimeType': {'type': 'mime-type', 'object_relation': 'mime-type', 'distribution': 5},
+        'sha1': {'type': 'sha1', 'object_relation': 'sha1', 'distribution': 5},
+        'sha256': {'type': 'sha256', 'object_relation': 'sha256', 'distribution': 5},
+        'shannonEntropy': {'type': 'float', 'object_relation': 'entropy', 'distribution': 5},
+        'size': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes', 'distribution': 5},
+        'stairwellFirstSeenTime': {'type': 'datetime', 'object_relation': 'stairwell-first-seen', 'distribution': 5},
+        'tlsh': {'type': 'tlsh', 'object_relation': 'tlsh', 'distribution': 5},
+        'yaraRuleMatches': {'type': 'text', 'object_relation': 'yara-rule-match', 'comment': 'matching Stairwell yara rule name', 'distribution': 5}
+    }
+    environments_mapping = {
+        "NCS2SM-YHB2KT-SAFUDX-JC7F6WYA": "Florian's Open Rules",
+        "VR9Z98-4KU7ZC-PCNFEG-FURQ66FW": "Jotti",
+        "D7W6M6-BA9BS4-BQ23Z4-NKCNWQ96": "Malshare",
+        "D4447Q-WJJL6P-W7ME89-WHXJK8TW": "Malware Bazaar",
+        "XAKLND-DKWP3Z-56RL88-6XJ5NH46": "Pro Rules",
+        "GMEELM-K226XF-F95XZL-7VEJFKZ6": "Public Samples",
+        "5HEG8N-9T7UPG-8SZJ7T-2J4XSDC6": "RH-ISAC",
+        "2NN2BJ-HDVQHS-49824H-2SEDBBLJ": "RH-ISAC Malware Sharing",
+        "VCZTNF-8S76AK-LUU53W-2SWFFZWJ": "Stairwell Experimental Rules",
+        "GEG6FU-MRARGF-TLTM6X-H6MGDT5E": "Stairwell Methodology Rules",
+        "EB3DXY-3ZYFVH-6HNKJQ-GAPKHESS": "Stairwell OSINT Rules",
+        "NQNJM6-5LSCAF-3MC5FJ-W8EKGW6N": "Stairwell Research Rules",
+        "TT9GM5-JUMD8H-9828FL-GAW5NNXE": "stairwell-public-verdicts",
+        "MKYSAR-3XN9MB-3VAK3R-888ZJUTJ": "Threat Report Feeds",
+        "6HP5R3-ZM7DAN-RB4732-X6QPCJ36": "Virusshare",
+        "TV6WCV-7Y79LE-BK79EY-C8GUEY46": "vxintel"
+    }
+
+    misp_event = MISPEvent()
+    misp_object = MISPObject('stairwell')
+    for feature, attribute in attribute_mapping.items():
+        if feature in response.keys() and response[feature]:
+            if feature == 'yaraRuleMatches':
+                for rule in response[feature]:
+                    env_pattern = r'\b[A-Z0-9]{6}-[A-Z0-9]{6}-[A-Z0-9]{6}-[A-Z0-9]{8}\b'
+                    env = re.findall(env_pattern, rule.split('yaraRules/')[0])[0]
+                    misp_attribute = {
+                        'value': rule.split('yaraRules/')[1],
+                        'comment': f'Rule from: {environments_mapping.get(env, "Unknown UUID!")}'
+                    }
+                    misp_attribute.update(attribute)
+                    misp_object.add_attribute(**misp_attribute)
+            elif feature == 'environments':
+                for env in response[feature]:
+                    misp_attribute = {
+                        'value': environments_mapping.get(env, f'Unknown Environment: {env}'),
+                        'comment': 'Hash observed in'
+                    }
+                    misp_attribute.update(attribute)
+                    misp_object.add_attribute(**misp_attribute)
+            elif feature == 'malEval':
+                for attr in attribute:
+                    misp_attribute = {'value': response[feature][attr]}
+                    misp_attribute.update(attribute[attr])
+                    misp_object.add_attribute(**misp_attribute)
+            else:
+                misp_attribute = {'value': response[feature]}
+                misp_attribute.update(attribute)
+                attr = misp_object.add_attribute(**misp_attribute)
+                if feature in ('md5', 'sha1', 'sha256'):
+                    for label in response['malEval']['labels']:
+                        attr.add_tag(label)
+    misp_event.add_object(**misp_object)
+
+    event = json.loads(misp_event.to_json())
+    results = {'Object': event['Object']}
+
+    return {'results': results}
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    if not request.get('config') or not request['config'].get('apikey'):
+        misperrors['error'] = 'A Stairwell api key is required for this module!'
+        return misperrors
+    if not request.get('attribute') or not check_input_attribute(request['attribute'], requirements=('type', 'value')):
+        misperrors['error'] = f'{standard_error_message}, {checking_error}.'
+        return misperrors
+    attribute = request['attribute']
+    if attribute['type'] not in mispattributes['input']:
+        misperrors['error'] = 'Unsupported attribute type!'
+        return misperrors
+
+    headers = {
+        "Accept": "application/json",
+        "Authorization": request['config']['apikey'],
+        "User-Agent": f"misp-module {__file__} {moduleinfo['version']}"
+    }
+    url = f"https://app.stairwell.com/v1/objects/{attribute['value']}/metadata"
+    response = requests.get(url=url, headers=headers).json()
+
+    if response.get('code') == 16:  # bad auth
+        return {'error': f"{response['message']} Is api key valid?"}
+    elif response.get('code') == 5:  # not found
+        return {'error': f"{attribute['type']}:{attribute['value']} {response['message']}"}
+    elif response.get('code') == 2:  # encoding/hex: invalid byte
+        return {'error': response['message']}
+    elif response.get('code'):  # catchall for potential unforeseen errors
+        return {'error': response['message'], 'code': response['code']}
+    else:
+        return parse_response(response)
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo
--- a/misp_modules/modules/expansion/virustotal.py
+++ b/misp_modules/modules/expansion/virustotal.py
@ -4,11 +4,11 @@ from . import check_input_attribute, standard_error_message
 from pymisp import MISPAttribute, MISPEvent, MISPObject

 misperrors = {'error': 'Error'}
-mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "url"],
+mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "url", "ip-src|port", "ip-dst|port"],
                  'format': 'misp_standard'}

 # possible module-types: 'expansion', 'hover' or both
-moduleinfo = {'version': '5', 'author': 'Hannah Ward',
+moduleinfo = {'version': '6', 'author': 'Hannah Ward',
              'description': 'Enrich observables with the VirusTotal v3 API',
              'module-type': ['expansion']}

@ -29,7 +29,8 @@ class VirusTotalParser:
        self.input_types_mapping = {'ip-src': self.parse_ip, 'ip-dst': self.parse_ip,
                                    'domain': self.parse_domain, 'hostname': self.parse_domain,
                                    'md5': self.parse_hash, 'sha1': self.parse_hash,
-                                    'sha256': self.parse_hash, 'url': self.parse_url}
+                                    'sha256': self.parse_hash, 'url': self.parse_url,
+                                    'ip-src|port': self.parse_ip_port, 'ip-dst|port': self.parse_ip_port}
        self.proxies = None

    @staticmethod
@ -51,7 +52,11 @@ class VirusTotalParser:
    def add_vt_report(self, report: vt.Object) -> str:
        analysis = report.get('last_analysis_stats')
        total = self.get_total_analysis(analysis, report.get('known_distributors'))
-        permalink = f'https://www.virustotal.com/gui/{report.type}/{report.id}'
+        if report.type == 'ip_address':
+            rtype = 'ip-address'
+        else:
+            rtype = report.type
+        permalink = f'https://www.virustotal.com/gui/{rtype}/{report.id}'

        vt_object = MISPObject('virustotal-report')
        vt_object.add_attribute('permalink', type='link', value=permalink)
@ -160,6 +165,9 @@ class VirusTotalParser:

        self.misp_event.add_object(**file_object)
        return file_object.uuid
+    def parse_ip_port(self, ipport: str) -> str:
+        ip = ipport.split('|')[0]
+        self.parse_ip(ip)

    def parse_ip(self, ip: str) -> str:
        ip_report = self.client.get_object(f'/ip_addresses/{ip}')