diff --git a/.gitignore b/.gitignore index 10c6e69..9bf7be8 100644 --- a/.gitignore +++ b/.gitignore @@ -113,3 +113,6 @@ storage/repl # Config files of running instance *config.py + +# python-o365 token file +o365_token.txt diff --git a/MUA/Microsoft/Office365/README.md b/MUA/Microsoft/Office365/README.md new file mode 100644 index 0000000..a127b6a --- /dev/null +++ b/MUA/Microsoft/Office365/README.md @@ -0,0 +1,113 @@ +# O365MISPClient + +A mail_to_misp client to connect your O365 mail infrastructure to [MISP](https://github.com/MISP/MISP) in order to create events based on the information contained within emails. + + +## Getting Started +### OAuth Setup (Pre Requisite) +You will need to register your application at [Microsoft Apps](https://apps.dev.microsoft.com/). Steps below: + +1. Login to https://apps.dev.microsoft.com/ +2. Create an app, note your app id (**client_id**) +3. Generate a new password (**client_secret**) under **Application Secrets** section +4. Under the **Platform** section, add a new Web platform and set "https://outlook.office365.com/owa/" as the redirect URL +5. Under **Microsoft Graph Permissions** section, Add the below delegated permission (or based on what scopes you plan to use) + 1. offline_access + 2. Mail.Read + 3. Mail.Read.Shared +6. Note the **client_id** and **client_secret** as they will be using for establishing the connection through the api + +Detailed documentation for getting [Oauth Authentication](https://github.com/O365/python-o365/blob/master/README.md#oauth-authentication) configured. + +## Token Storage +When authenticating you will retrieve oauth tokens. If you don't want a one time access you will have to store the token somewhere. O365 makes no assumptions on where to store the token and tries to abstract this from the library usage point of view. + +You can choose where and how to store tokens by using the proper Token Backend. + +**Take care: the access (and refresh) token must remain protected from unauthorized users.** + +To store the token you will have to provide a properly configured TokenBackend. + + +### FileSystemTokenBackend +(Default backend): Stores and retrieves tokens from the file system. Tokens are stored as files. You can explicitly initialize this as shown below. +```python +from O365.utils import FileSystemTokenBackend + + +# initialize Mail2MISP +m2m = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config) + +tb = FileSystemTokenBackend(token_path='/path/to/store/token', token_filename='o365_token.txt') + +# initialize O365MISPClient +o365 = m2m.O365MISPClient( + client_id=o365_client_id, + client_secret=o365_client_secret, + tenant_id=o365_tenant_id, + resource=o365_resource, + scopes=o365_scopes, + token_backend=tb +) +``` + +As this is the default backend, you do not need to explicitly initialize FileSystemTokenBackend. If ```token_backend``` is ```None``` the token file will be saved as ```o365_token.txt``` to the directory the script is running in. + +```python +# initialize Mail2MISP +m2m = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config) + +# initialize O365MISPClient +o365 = m2m.O365MISPClient( + client_id=o365_client_id, + client_secret=o365_client_secret, + tenant_id=o365_tenant_id, + resource=o365_resource, + scopes=o365_scopes, + token_backend=None +) +``` + +### AWSSecretsBackend +Stores and retrieves tokens from an AWS Secrets Management vault. +```python +from O365.utils import AWSSecretsBackend + + +# initialize Mail2MISP +m2m = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config) + +tb = AWSSecretsBackend(secret_name='o365_m2m_token', region_name='us-east-1') + +# initialize O365MISPClient +o365 = m2m.O365MISPClient( + client_id=o365_client_id, + client_secret=o365_client_secret, + tenant_id=o365_tenant_id, + resource=o365_resource, + scopes=o365_scopes, + token_backend=tb +) +``` + +### EnvTokenBackend +Stores and retrieves tokens from environment variables. +```python +from O365.utils import EnvTokenBackend + + +# initialize Mail2MISP +m2m = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config) + +tb = EnvTokenBackend('O365_M2M_TOKEN') + +# initialize O365MISPClient +o365 = m2m.O365MISPClient( + client_id=o365_client_id, + client_secret=o365_client_secret, + tenant_id=o365_tenant_id, + resource=o365_resource, + scopes=o365_scopes, + token_backend=tb +) +``` \ No newline at end of file diff --git a/README.md b/README.md index 659e7ec..f67ec1a 100644 --- a/README.md +++ b/README.md @@ -73,11 +73,15 @@ The implemented workflow is mainly for mail servers like Postfix. Client side im `Email -> mail_to_misp` -2. Apple Mail [unmaintained] +2. Office 365 + +`Email -> Outlook -> O365MISPClient -> mail_to_misp` + +3. Apple Mail [unmaintained] `Email -> Apple Mail -> Mail rule -> AppleScript -> mail_to_misp -> PyMISP -> MISP` -3. Mozilla Thunderbird [unmaintained] +4. Mozilla Thunderbird [unmaintained] `Email -> Thunderbird -> Mail rule -> filterscript -> thunderbird_wrapper -> mail_to_misp -> PyMISP -> MISP` @@ -120,6 +124,15 @@ $ sudo chmod 770 /etc/authbind/byport/25 `$ python3 fake_smtp.py` +### Office 365 +- Full [documentation](MUA/Microsoft/Office365/README.md) for getting started in MUA/Microsoft/Office365 +- Built-in O365MISPClient for Mail2MISP +- Uses new Mail2MISP methods: `.load_o365_email` and `.process_o365_email_body` + +Run mail_to_misp_o365.py to get the last 1 day of messages + +`$ python3 mail_to_misp_0365.py -nd 1` + ### Apple Mail [unmaintained] 1. Mail rule script @@ -193,6 +206,10 @@ poetry install -E fileobjects -E openioc -E virustotal -E email -E url - ftfy from https://github.com/LuminosoInsight/python-ftfy (to fix unicode text) - defang from https://github.com/Rafiot/defang.git (fork of: https://bitbucket.org/johannestaas/defang) +### Office 365 + +- O365 from https://github.com/O365/python-o365 + ### Thunderbird [unmaintained] - https://github.com/rommelfs/filterscript (modified fork from https://github.com/adamnew123456/filterscript) diff --git a/mail2misp/mail2misp.py b/mail2misp/mail2misp.py index 0a5ee14..327616b 100644 --- a/mail2misp/mail2misp.py +++ b/mail2misp/mail2misp.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- - +import base64 import re import syslog import html @@ -8,13 +8,19 @@ import os from io import BytesIO from ipaddress import ip_address from email import message_from_bytes, policy, message +from email.parser import BytesParser -from . import urlmarker -from . import hashmarker +from . import urlmarker, hashmarker from pyfaup.faup import Faup # type: ignore from pymisp import ExpandedPyMISP, MISPEvent, MISPObject, MISPSighting, InvalidMISPObject from pymisp.tools import EMailObject, make_binary_objects, VTReportObject from defang import refang # type: ignore + +from datetime import datetime +from O365 import Account +from O365.message import Message +from O365.utils import AWSS3Backend, AWSSecretsBackend, EnvTokenBackend, FileSystemTokenBackend, FirestoreBackend +from typing import Iterator, List, Optional, Union try: import dns.resolver HAS_DNS = True @@ -78,6 +84,41 @@ class Mail2MISP(): self.misp_event.analysis = self.config.default_analysis self.misp_event.add_tag(self.config.id_tag) + def load_o365_email(self, msg: Message): + self.msg = msg + + try: + self.sender = self.msg.sender.address + except Exception as ex: + self.sender = "" + if self.debug: + syslog.syslog(ex) + + try: + self.reply_to = self.msg.reply_to[0].address + except Exception as ex: + self.reply_to = None + if self.debug: + syslog.syslog(ex) + + try: + self.subject = self.msg.subject + # remove words from subject + for removeword in self.config.removelist: + self.subject = re.sub(removeword, "", self.subject).strip() + except Exception as ex: + self.subject = "" + if self.debug: + syslog.syslog(ex) + + # initialize the MISP event + self.misp_event = MISPEvent() + self.misp_event.info = self.subject + self.misp_event.distribution = self.config.default_distribution + self.misp_event.threat_level_id = self.config.default_threat_level + self.misp_event.analysis = self.config.default_analysis + self.misp_event.add_tag(self.config.id_tag) + def sighting(self, value, source): if self.offline: raise Exception('The script is running in offline mode, ') @@ -123,6 +164,19 @@ class Mail2MISP(): [self.misp_event.add_object(section) for section in sections] return forwarded_emails + def _find_o365_attached_forward(self, msg: Message): + forwarded_emails = [] + if msg.has_attachments: + if msg.attachments.download_attachments(): + for attachment in msg.attachments: + if '.eml' in attachment.name: + decoded_attachment = base64.b64decode(attachment.content) + pseudofile = BytesIO(decoded_attachment) + eml = BytesParser(policy=policy.default).parse(pseudofile) + if isinstance(eml, message.EmailMessage): + forwarded_emails.append(self.forwarded_email(pseudofile=pseudofile)) + return forwarded_emails + def email_from_spamtrap(self): '''The email comes from a spamtrap and should be attached as-is.''' raw_body = self.original_mail.get_body(preferencelist=('html', 'plain')) @@ -197,6 +251,31 @@ class Mail2MISP(): self.clean_email_body = '' self._find_attached_forward() + def process_o365_email_body(self): + if self.msg: + self.clean_email_body = html.unescape(self.msg.body) + if re.search(r"
You don't often get email from .*?
", self.clean_email_body): + self.clean_email_body = re.sub(r"
You don't often get email from .*?
", "", html.unescape(self.msg.body)) + # Check if there are config lines in the body & convert them to a python dictionary: + # :: => {: } + self.config_from_email_body = {k.strip(): v.strip() for k, v in re.findall(f'{self.config.body_config_prefix}:(.*):(.*)', self.clean_email_body)} + if self.config_from_email_body: + # ... remove the config lines from the body + self.clean_email_body = re.sub(rf'^{self.config.body_config_prefix}.*\n?', '', html.unescape(self.msg.body), flags=re.MULTILINE) + # Check if autopublish key is present and valid + if self.config_from_email_body.get('m2mkey') == self.config.m2m_key: + if self.config_from_email_body.get('distribution') is not None: + self.misp_event.distribution = self.config_from_email_body.get('distribution') + if self.config_from_email_body.get('threat_level') is not None: + self.misp_event.threat_level_id = self.config_from_email_body.get('threat_level') + if self.config_from_email_body.get('analysis') is not None: + self.misp_event.analysis = self.config_from_email_body.get('analysis') + if self.config_from_email_body.get('publish'): + self.misp_event.publish() + else: + self.clean_email_body = '' + self._find_o365_attached_forward(self.msg) + def process_body_iocs(self, email_object=None): if email_object: body = html.unescape(email_object.email.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode('utf8', 'surrogateescape')) @@ -416,7 +495,10 @@ class Mail2MISP(): for value, source in self.sightings_to_add: self.sighting(value, source) if self.config.freetext: - self.misp.freetext(event, string=self.original_mail.get_body(preferencelist=('html', 'plain')), adhereToWarninglists=self.config.enforcewarninglist) + if self.config.o365_freetext: + self.misp.freetext(event, string=self.clean_email_body, adhereToWarninglists=self.config.enforcewarninglist) + else: + self.misp.freetext(event, string=self.original_mail.get_body(preferencelist=('html', 'plain')), adhereToWarninglists=self.config.enforcewarninglist) return event def get_attached_emails(self, pseudofile): @@ -438,3 +520,76 @@ class Mail2MISP(): # all attachments are identified as message.EmailMessage so filtering on extension for now. forwarded_emails.append(BytesIO(attachment_content)) return forwarded_emails + + class O365MISPClient: + """ + A client (MUA) to allow mail_to_misp to interact with Microsoft Graph and Office 365 API to get email messages. + """ + def __init__( + self, + client_id: str, + client_secret: str, + tenant_id: str, + resource: str, + scopes: List[str], + token_backend: Optional[ + Union[AWSS3Backend, AWSSecretsBackend, EnvTokenBackend, FileSystemTokenBackend, FirestoreBackend] + ] = None, + ): + """ + Init O365MISPClient + :param client_id: OAuth Client ID + :param client_secret: OAuth Client Secret + :param tenant_id: Your Tenant ID + :param resource: The email address you want to access + :param scopes: The permission scopes for the resource + :param token_backend: The backend used for storing OAuth token + """ + self.scopes = scopes + self.resource = resource + self.o365_acct = Account( + credentials=(client_id, client_secret), + auth_flow_type='authorization', + tenant_id=tenant_id, + token_backend=token_backend + ) + if not self.o365_acct.is_authenticated: + self.o365_acct.authenticate(scopes=self.scopes) + self.mailbox = self.o365_acct.mailbox(resource=self.resource) + self.inbox = self.mailbox.inbox_folder() + self.query_properties = [ + 'internet_message_headers', + 'subject', + 'body', + 'unique_body', + 'from', + 'reply_to', + 'is_read', + 'is_draft', + 'received_date_time', + 'has_attachments', + 'attachments' + ] + + def get_email_messages(self, from_time: datetime, to_time: datetime, folder: Optional[str] = None) -> Iterator[Message]: + """ + Get messages for a certain timeframe. Defaults to looking for messages in the Inbox folder, however by + supplying a folder name as a parameter you can change where to get the messages from. + + :param from_time: start time to search for + :param to_time: end time to search for + :param folder: specific folder to get messages from (don't supply if getting from the inbox folder) + :return: an iterator of O365.messages.Message from the resource + """ + query = self.mailbox.new_query().select(*self.query_properties) + # https://learn.microsoft.com/en-us/graph/api/resources/message?view=graph-rest-1.0#properties + query = query.chain('and').on_attribute('received_date_time').greater(from_time) + query = query.chain('and').on_attribute('received_date_time').less(to_time) + + if folder: + messages = self.mailbox.get_folder(folder_name=folder).get_messages(query=query) + else: + messages = self.inbox.get_messages(query=query) + + return messages + diff --git a/mail_to_misp_config.py-example b/mail_to_misp_config.py-example index 823adc5..2509e5e 100644 --- a/mail_to_misp_config.py-example +++ b/mail_to_misp_config.py-example @@ -11,6 +11,18 @@ default_analysis = 1 id_tag = 'host:m2m:tld' freetext = False +# O365MISPClient config +o365_freetext = False # must be enabled in addition to the above freetext to use this on o365 email messages +o365_client_id = 'YOUR_O365_CLIENT_ID' +o365_client_secret = 'YOUR_O365_CLIENT_SECRET' +o365_tenant_id = 'YOUR_O365_TENANT_ID' +o365_resource = 'YOUR_O365_INBOX' # misp@yourdomain.com or whatever inbox you are reading mail from +o365_scopes = [ + 'offline_access', # Highly recommended to add this. If not you will have to re-authenticate every hour. + 'https://graph.microsoft.com/Mail.Read', # To read my mailbox + # 'https://graph.microsoft.com/Mail.Read.Shared' # To read another user/shared mailbox +] + body_config_prefix = 'm2m' # every line in the body starting with this value will be skipped from the IOCs m2m_key = 'YOUSETYOURKEYHERE' m2m_benign_attachment_keyword = 'benign' @@ -24,7 +36,8 @@ ignore_carrier_mail = False ignore_nullsize_attachments = False excludelist = ('google.com', 'microsoft.com') -externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net') +externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net', 'urlscan.io', + 'abuse.ch', 'tria.ge', 'bleepingcomputer.com', 'any.run', 'urlvoid.com', 'intezer.com') internallist = ('internal.system.local') noidsflaglist = ('myexternalip.com', 'ipinfo.io', 'icanhazip.com', 'wtfismyip.com', 'ipecho.net', 'api.ipify.org', 'checkip.amazonaws.com', 'whatismyipaddress.com', 'google.com', @@ -48,13 +61,16 @@ sighting_source = "YOUR_MAIL_TO_MISP_IDENTIFIER" # Remove "Re:", "Fwd:" and {Spam?} from subject # add: "[\(\[].*?[\)\]]" to remove everything between [] and (): i.e. [tag] -removelist = (r"Re:", r"Fwd:", r"\{Spam\?\} ") +removelist = (r"Re:", r"Fwd:", r"\{Spam\?\} ", r"RE:", r"FW:") # TLP tag setup # Tuples contain different variations of spelling -tlptags = {'tlp:amber': ['tlp:amber', 'tlp: amber', 'tlp amber'], - 'tlp:green': ['tlp:green', 'tlp: green', 'tlp green'], - 'tlp:white': ['tlp:white', 'tlp: white', 'tlp white'] +tlptags = {'tlp:amber': ['tlp:amber', 'tlp: amber', 'tlp amber', 'tlp :amber'], + 'tlp:amber+strict': ['tlp:amber+strict', 'tlp: amber+strict', 'tlp amber+strict', 'tlp :amber+strict'], + 'tlp:green': ['tlp:green', 'tlp: green', 'tlp green', 'tlp :green'], + 'tlp:white': ['tlp:white', 'tlp: white', 'tlp white', 'tlp :white'], + 'tlp:clear': ['tlp:clear', 'tlp: clear', 'tlp clear', 'tlp :clear'], + 'tlp:red': ['tlp:red', 'tlp: red', 'tlp red', 'tlp :red'] } tlptag_default = sorted(tlptags.keys())[0] @@ -65,11 +81,26 @@ malwaretags = {'locky': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:rans 'Pony': ['misp-galaxy:tool="Hancitor"'], 'ursnif': ['misp-galaxy:tool="Snifula"'], 'NanoCore': ['misp-galaxy:tool="NanoCoreRAT"'], - 'trickbot': ['misp-galaxy:tool="Trick Bot"'] + 'trickbot': ['misp-galaxy:tool="Trick Bot"'], + 'agenttesla': ['misp-galaxy:mitre-malware="Agent Tesla - S0331"'], + 'formbook': ['misp-galaxy:malpedia="Formbook"'], + 'remcos': ['misp-galaxy:mitre-tool="Remcos - S0332"'], + 'snake keylogger': ['misp-galaxy:malpedia="404 Keylogger"'], + 'icedid': ['misp-galaxy:malpedia="IcedID"'], + 'zloader': ['misp-galaxy:malpedia="Zloader"'], + 'lokibot': ['misp-galaxy:mitre-malware="Lokibot - S0447"'], + 'valyria': ['misp-galaxy:malpedia="POWERSTATS"'], + 'guloader': ['misp-galaxy:mitre-malware="GuLoader - S0561"'], + 'avemaria': ['misp-galaxy:mitre-malware="WarzoneRAT - S0670"'], + 'warzone': ['misp-galaxy:mitre-malware="WarzoneRAT - S0670"'], + 'hancitor': ['misp-galaxy:malpedia="Hancitor"'], + 'async': ['misp-galaxy:malpedia="AsyncRAT"'], + 'emotet': ['misp-galaxy:mitre-malware="Emotet - S0367"'] } # Tags to be set depending on the presence of other tags -dependingtags = {'tlp:white': ['circl:osint-feed'] +dependingtags = {'tlp:white': ['circl:osint-feed'], + 'tlp:clear': ['circl:osint-feed'] } # Known identifiers for forwarded messages diff --git a/mail_to_misp_o365.py b/mail_to_misp_o365.py new file mode 100644 index 0000000..b290a24 --- /dev/null +++ b/mail_to_misp_o365.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import argparse +import importlib +import os +import sys +import syslog +from datetime import datetime, timedelta, timezone +from itertools import tee +from pathlib import Path + +from mail2misp import Mail2MISP + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Push mail from O365 into a MISP instance') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('-nd', '--days', help='Number of days to search back in inbox') + group.add_argument('-nh', '--hours', help='Number of hours to search back in inbox') + parser.add_argument('-f', '--folder', help='Folder name that contains email messages to parse') + args = parser.parse_args() + + syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_USER) + syslog.syslog("[+] O365 job started.") + + # import config module + os.chdir(Path(__file__).parent) + + configmodule = Path(__file__).name.replace('o365.py', 'config') + if Path(f"{configmodule}.py").exists(): + config = importlib.import_module(configmodule) + try: + misp_url = config.misp_url + misp_key = config.misp_key + misp_verifycert = config.misp_verifycert + o365_client_id = config.o365_client_id + o365_client_secret = config.o365_client_secret + o365_tenant_id = config.o365_tenant_id + o365_resource = config.o365_resource + o365_scopes = config.o365_scopes + debug = config.debug + except Exception as ex: + print("There is a problem with the configuration. A mandatory configuration variable is not set.") + print("Did you just update? mail_to_misp might have new configuration variables.") + print("Please compare with the configuration example.") + print("\nTrace:") + print(ex) + sys.exit(-2) + else: + print(f"Couldn't locate config file {configmodule}.py") + sys.exit(-1) + + # set message search period to look for emails + to_time = datetime.now(timezone.utc) + + if args.days: + from_time = (to_time - timedelta(days=int(args.days))) + else: + from_time = (to_time - timedelta(hours=int(args.hours))) + + # initialize Mail2MISP + m2m = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config) + + # initialize O365MISPClient + o365 = m2m.O365MISPClient( + client_id=o365_client_id, + client_secret=o365_client_secret, + tenant_id=o365_tenant_id, + resource=o365_resource, + scopes=o365_scopes, + token_backend=None # if not supplied will default to using FileSystemTokenBackend, which stores the token in a + # txt file on disk in the directory the script is executed from + ) + + messages = o365.get_email_messages( + from_time=from_time, + to_time=to_time, + folder=args.folder if args.folder else None # defaults to searching the resource's inbox folder if None + ) + + messages1, messages2 = tee(messages, 2) + + syslog.syslog(f"[*] Found {len(list(messages1))} messages to process and send to MISP!") + + for msg in messages2: + m2m.load_o365_email(msg) + if debug: + syslog.syslog(f"[*] Processing email with subject: {m2m.subject}") + m2m.process_o365_email_body() + m2m.process_body_iocs() + m2m.add_event() + syslog.syslog("[-] O365 job finished.") diff --git a/pyproject.toml b/pyproject.toml index d4b044d..ef6c605 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "mail2misp" -version = "0.1.0" +version = "0.2.0" description = "Importer of EML files into a MISP instance" authors = ["Raphaël Vinot "] license = "AGPL-3.0" @@ -39,6 +39,7 @@ flask-bootstrap = "^3.3.7.1" gunicorn = "^20.1.0" chardet = "^5.2.0" aiosmtpd = "^1.4.4.post2" +O365 = {extras = ["o365"], version = "^2.0.31", optional = true} [tool.poetry.extras] fileobjects = ['python-magic', 'pydeep', 'lief'] @@ -49,6 +50,7 @@ pdfexport = ['reportlab'] url = ['pyfaup'] email = ['extract_msg', "RTFDE", "oletools"] brotli = ['urllib3'] +o365 = ['O365'] [tool.poetry.dev-dependencies] nose = "^1.3.7" diff --git a/setup.py b/setup.py index 7b6271c..418440a 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from setuptools import setup setup( name='mail2misp', - version='0.1', + version='0.2', author='Raphaël Vinot', author_email='raphael.vinot@circl.lu', maintainer='Raphaël Vinot',