mirror of https://github.com/MISP/mail_to_misp
Merge pull request #87 from goodlandsecurity/o365-client
Adding Mail2MISP client for Microsoft Graph/O365 APImain
commit
799041bdc1
|
@ -113,3 +113,6 @@ storage/repl
|
||||||
|
|
||||||
# Config files of running instance
|
# Config files of running instance
|
||||||
*config.py
|
*config.py
|
||||||
|
|
||||||
|
# python-o365 token file
|
||||||
|
o365_token.txt
|
||||||
|
|
|
@ -0,0 +1,113 @@
|
||||||
|
# O365MISPClient
|
||||||
|
|
||||||
|
A mail_to_misp client to connect your O365 mail infrastructure to [MISP](https://github.com/MISP/MISP) in order to create events based on the information contained within emails.
|
||||||
|
|
||||||
|
|
||||||
|
## Getting Started
|
||||||
|
### OAuth Setup (Pre Requisite)
|
||||||
|
You will need to register your application at [Microsoft Apps](https://apps.dev.microsoft.com/). Steps below:
|
||||||
|
|
||||||
|
1. Login to https://apps.dev.microsoft.com/
|
||||||
|
2. Create an app, note your app id (**client_id**)
|
||||||
|
3. Generate a new password (**client_secret**) under **Application Secrets** section
|
||||||
|
4. Under the **Platform** section, add a new Web platform and set "https://outlook.office365.com/owa/" as the redirect URL
|
||||||
|
5. Under **Microsoft Graph Permissions** section, Add the below delegated permission (or based on what scopes you plan to use)
|
||||||
|
1. offline_access
|
||||||
|
2. Mail.Read
|
||||||
|
3. Mail.Read.Shared
|
||||||
|
6. Note the **client_id** and **client_secret** as they will be using for establishing the connection through the api
|
||||||
|
|
||||||
|
Detailed documentation for getting [Oauth Authentication](https://github.com/O365/python-o365/blob/master/README.md#oauth-authentication) configured.
|
||||||
|
|
||||||
|
## Token Storage
|
||||||
|
When authenticating you will retrieve oauth tokens. If you don't want a one time access you will have to store the token somewhere. O365 makes no assumptions on where to store the token and tries to abstract this from the library usage point of view.
|
||||||
|
|
||||||
|
You can choose where and how to store tokens by using the proper Token Backend.
|
||||||
|
|
||||||
|
**Take care: the access (and refresh) token must remain protected from unauthorized users.**
|
||||||
|
|
||||||
|
To store the token you will have to provide a properly configured TokenBackend.
|
||||||
|
|
||||||
|
|
||||||
|
### FileSystemTokenBackend
|
||||||
|
(Default backend): Stores and retrieves tokens from the file system. Tokens are stored as files. You can explicitly initialize this as shown below.
|
||||||
|
```python
|
||||||
|
from O365.utils import FileSystemTokenBackend
|
||||||
|
|
||||||
|
|
||||||
|
# initialize Mail2MISP
|
||||||
|
m2m = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config)
|
||||||
|
|
||||||
|
tb = FileSystemTokenBackend(token_path='/path/to/store/token', token_filename='o365_token.txt')
|
||||||
|
|
||||||
|
# initialize O365MISPClient
|
||||||
|
o365 = m2m.O365MISPClient(
|
||||||
|
client_id=o365_client_id,
|
||||||
|
client_secret=o365_client_secret,
|
||||||
|
tenant_id=o365_tenant_id,
|
||||||
|
resource=o365_resource,
|
||||||
|
scopes=o365_scopes,
|
||||||
|
token_backend=tb
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
As this is the default backend, you do not need to explicitly initialize FileSystemTokenBackend. If ```token_backend``` is ```None``` the token file will be saved as ```o365_token.txt``` to the directory the script is running in.
|
||||||
|
|
||||||
|
```python
|
||||||
|
# initialize Mail2MISP
|
||||||
|
m2m = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config)
|
||||||
|
|
||||||
|
# initialize O365MISPClient
|
||||||
|
o365 = m2m.O365MISPClient(
|
||||||
|
client_id=o365_client_id,
|
||||||
|
client_secret=o365_client_secret,
|
||||||
|
tenant_id=o365_tenant_id,
|
||||||
|
resource=o365_resource,
|
||||||
|
scopes=o365_scopes,
|
||||||
|
token_backend=None
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
### AWSSecretsBackend
|
||||||
|
Stores and retrieves tokens from an AWS Secrets Management vault.
|
||||||
|
```python
|
||||||
|
from O365.utils import AWSSecretsBackend
|
||||||
|
|
||||||
|
|
||||||
|
# initialize Mail2MISP
|
||||||
|
m2m = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config)
|
||||||
|
|
||||||
|
tb = AWSSecretsBackend(secret_name='o365_m2m_token', region_name='us-east-1')
|
||||||
|
|
||||||
|
# initialize O365MISPClient
|
||||||
|
o365 = m2m.O365MISPClient(
|
||||||
|
client_id=o365_client_id,
|
||||||
|
client_secret=o365_client_secret,
|
||||||
|
tenant_id=o365_tenant_id,
|
||||||
|
resource=o365_resource,
|
||||||
|
scopes=o365_scopes,
|
||||||
|
token_backend=tb
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
### EnvTokenBackend
|
||||||
|
Stores and retrieves tokens from environment variables.
|
||||||
|
```python
|
||||||
|
from O365.utils import EnvTokenBackend
|
||||||
|
|
||||||
|
|
||||||
|
# initialize Mail2MISP
|
||||||
|
m2m = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config)
|
||||||
|
|
||||||
|
tb = EnvTokenBackend('O365_M2M_TOKEN')
|
||||||
|
|
||||||
|
# initialize O365MISPClient
|
||||||
|
o365 = m2m.O365MISPClient(
|
||||||
|
client_id=o365_client_id,
|
||||||
|
client_secret=o365_client_secret,
|
||||||
|
tenant_id=o365_tenant_id,
|
||||||
|
resource=o365_resource,
|
||||||
|
scopes=o365_scopes,
|
||||||
|
token_backend=tb
|
||||||
|
)
|
||||||
|
```
|
21
README.md
21
README.md
|
@ -73,11 +73,15 @@ The implemented workflow is mainly for mail servers like Postfix. Client side im
|
||||||
|
|
||||||
`Email -> mail_to_misp`
|
`Email -> mail_to_misp`
|
||||||
|
|
||||||
2. Apple Mail [unmaintained]
|
2. Office 365
|
||||||
|
|
||||||
|
`Email -> Outlook -> O365MISPClient -> mail_to_misp`
|
||||||
|
|
||||||
|
3. Apple Mail [unmaintained]
|
||||||
|
|
||||||
`Email -> Apple Mail -> Mail rule -> AppleScript -> mail_to_misp -> PyMISP -> MISP`
|
`Email -> Apple Mail -> Mail rule -> AppleScript -> mail_to_misp -> PyMISP -> MISP`
|
||||||
|
|
||||||
3. Mozilla Thunderbird [unmaintained]
|
4. Mozilla Thunderbird [unmaintained]
|
||||||
|
|
||||||
`Email -> Thunderbird -> Mail rule -> filterscript -> thunderbird_wrapper -> mail_to_misp -> PyMISP -> MISP`
|
`Email -> Thunderbird -> Mail rule -> filterscript -> thunderbird_wrapper -> mail_to_misp -> PyMISP -> MISP`
|
||||||
|
|
||||||
|
@ -120,6 +124,15 @@ $ sudo chmod 770 /etc/authbind/byport/25
|
||||||
|
|
||||||
`$ python3 fake_smtp.py`
|
`$ python3 fake_smtp.py`
|
||||||
|
|
||||||
|
### Office 365
|
||||||
|
- Full [documentation](MUA/Microsoft/Office365/README.md) for getting started in MUA/Microsoft/Office365
|
||||||
|
- Built-in O365MISPClient for Mail2MISP
|
||||||
|
- Uses new Mail2MISP methods: `.load_o365_email` and `.process_o365_email_body`
|
||||||
|
|
||||||
|
Run mail_to_misp_o365.py to get the last 1 day of messages
|
||||||
|
|
||||||
|
`$ python3 mail_to_misp_0365.py -nd 1`
|
||||||
|
|
||||||
### Apple Mail [unmaintained]
|
### Apple Mail [unmaintained]
|
||||||
|
|
||||||
1. Mail rule script
|
1. Mail rule script
|
||||||
|
@ -193,6 +206,10 @@ poetry install -E fileobjects -E openioc -E virustotal -E email -E url
|
||||||
- ftfy from https://github.com/LuminosoInsight/python-ftfy (to fix unicode text)
|
- ftfy from https://github.com/LuminosoInsight/python-ftfy (to fix unicode text)
|
||||||
- defang from https://github.com/Rafiot/defang.git (fork of: https://bitbucket.org/johannestaas/defang)
|
- defang from https://github.com/Rafiot/defang.git (fork of: https://bitbucket.org/johannestaas/defang)
|
||||||
|
|
||||||
|
### Office 365
|
||||||
|
|
||||||
|
- O365 from https://github.com/O365/python-o365
|
||||||
|
|
||||||
### Thunderbird [unmaintained]
|
### Thunderbird [unmaintained]
|
||||||
|
|
||||||
- https://github.com/rommelfs/filterscript (modified fork from https://github.com/adamnew123456/filterscript)
|
- https://github.com/rommelfs/filterscript (modified fork from https://github.com/adamnew123456/filterscript)
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
import base64
|
||||||
import re
|
import re
|
||||||
import syslog
|
import syslog
|
||||||
import html
|
import html
|
||||||
|
@ -8,13 +8,19 @@ import os
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
from ipaddress import ip_address
|
from ipaddress import ip_address
|
||||||
from email import message_from_bytes, policy, message
|
from email import message_from_bytes, policy, message
|
||||||
|
from email.parser import BytesParser
|
||||||
|
|
||||||
from . import urlmarker
|
from . import urlmarker, hashmarker
|
||||||
from . import hashmarker
|
|
||||||
from pyfaup.faup import Faup # type: ignore
|
from pyfaup.faup import Faup # type: ignore
|
||||||
from pymisp import ExpandedPyMISP, MISPEvent, MISPObject, MISPSighting, InvalidMISPObject
|
from pymisp import ExpandedPyMISP, MISPEvent, MISPObject, MISPSighting, InvalidMISPObject
|
||||||
from pymisp.tools import EMailObject, make_binary_objects, VTReportObject
|
from pymisp.tools import EMailObject, make_binary_objects, VTReportObject
|
||||||
from defang import refang # type: ignore
|
from defang import refang # type: ignore
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from O365 import Account
|
||||||
|
from O365.message import Message
|
||||||
|
from O365.utils import AWSS3Backend, AWSSecretsBackend, EnvTokenBackend, FileSystemTokenBackend, FirestoreBackend
|
||||||
|
from typing import Iterator, List, Optional, Union
|
||||||
try:
|
try:
|
||||||
import dns.resolver
|
import dns.resolver
|
||||||
HAS_DNS = True
|
HAS_DNS = True
|
||||||
|
@ -78,6 +84,41 @@ class Mail2MISP():
|
||||||
self.misp_event.analysis = self.config.default_analysis
|
self.misp_event.analysis = self.config.default_analysis
|
||||||
self.misp_event.add_tag(self.config.id_tag)
|
self.misp_event.add_tag(self.config.id_tag)
|
||||||
|
|
||||||
|
def load_o365_email(self, msg: Message):
|
||||||
|
self.msg = msg
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.sender = self.msg.sender.address
|
||||||
|
except Exception as ex:
|
||||||
|
self.sender = "<unknown sender>"
|
||||||
|
if self.debug:
|
||||||
|
syslog.syslog(ex)
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.reply_to = self.msg.reply_to[0].address
|
||||||
|
except Exception as ex:
|
||||||
|
self.reply_to = None
|
||||||
|
if self.debug:
|
||||||
|
syslog.syslog(ex)
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.subject = self.msg.subject
|
||||||
|
# remove words from subject
|
||||||
|
for removeword in self.config.removelist:
|
||||||
|
self.subject = re.sub(removeword, "", self.subject).strip()
|
||||||
|
except Exception as ex:
|
||||||
|
self.subject = "<subject could not be retrieved>"
|
||||||
|
if self.debug:
|
||||||
|
syslog.syslog(ex)
|
||||||
|
|
||||||
|
# initialize the MISP event
|
||||||
|
self.misp_event = MISPEvent()
|
||||||
|
self.misp_event.info = self.subject
|
||||||
|
self.misp_event.distribution = self.config.default_distribution
|
||||||
|
self.misp_event.threat_level_id = self.config.default_threat_level
|
||||||
|
self.misp_event.analysis = self.config.default_analysis
|
||||||
|
self.misp_event.add_tag(self.config.id_tag)
|
||||||
|
|
||||||
def sighting(self, value, source):
|
def sighting(self, value, source):
|
||||||
if self.offline:
|
if self.offline:
|
||||||
raise Exception('The script is running in offline mode, ')
|
raise Exception('The script is running in offline mode, ')
|
||||||
|
@ -123,6 +164,19 @@ class Mail2MISP():
|
||||||
[self.misp_event.add_object(section) for section in sections]
|
[self.misp_event.add_object(section) for section in sections]
|
||||||
return forwarded_emails
|
return forwarded_emails
|
||||||
|
|
||||||
|
def _find_o365_attached_forward(self, msg: Message):
|
||||||
|
forwarded_emails = []
|
||||||
|
if msg.has_attachments:
|
||||||
|
if msg.attachments.download_attachments():
|
||||||
|
for attachment in msg.attachments:
|
||||||
|
if '.eml' in attachment.name:
|
||||||
|
decoded_attachment = base64.b64decode(attachment.content)
|
||||||
|
pseudofile = BytesIO(decoded_attachment)
|
||||||
|
eml = BytesParser(policy=policy.default).parse(pseudofile)
|
||||||
|
if isinstance(eml, message.EmailMessage):
|
||||||
|
forwarded_emails.append(self.forwarded_email(pseudofile=pseudofile))
|
||||||
|
return forwarded_emails
|
||||||
|
|
||||||
def email_from_spamtrap(self):
|
def email_from_spamtrap(self):
|
||||||
'''The email comes from a spamtrap and should be attached as-is.'''
|
'''The email comes from a spamtrap and should be attached as-is.'''
|
||||||
raw_body = self.original_mail.get_body(preferencelist=('html', 'plain'))
|
raw_body = self.original_mail.get_body(preferencelist=('html', 'plain'))
|
||||||
|
@ -197,6 +251,31 @@ class Mail2MISP():
|
||||||
self.clean_email_body = ''
|
self.clean_email_body = ''
|
||||||
self._find_attached_forward()
|
self._find_attached_forward()
|
||||||
|
|
||||||
|
def process_o365_email_body(self):
|
||||||
|
if self.msg:
|
||||||
|
self.clean_email_body = html.unescape(self.msg.body)
|
||||||
|
if re.search(r"<div>You don't often get email from .*?</div>", self.clean_email_body):
|
||||||
|
self.clean_email_body = re.sub(r"<div>You don't often get email from .*?</div>", "", html.unescape(self.msg.body))
|
||||||
|
# Check if there are config lines in the body & convert them to a python dictionary:
|
||||||
|
# <config.body_config_prefix>:<key>:<value> => {<key>: <value>}
|
||||||
|
self.config_from_email_body = {k.strip(): v.strip() for k, v in re.findall(f'{self.config.body_config_prefix}:(.*):(.*)', self.clean_email_body)}
|
||||||
|
if self.config_from_email_body:
|
||||||
|
# ... remove the config lines from the body
|
||||||
|
self.clean_email_body = re.sub(rf'^{self.config.body_config_prefix}.*\n?', '', html.unescape(self.msg.body), flags=re.MULTILINE)
|
||||||
|
# Check if autopublish key is present and valid
|
||||||
|
if self.config_from_email_body.get('m2mkey') == self.config.m2m_key:
|
||||||
|
if self.config_from_email_body.get('distribution') is not None:
|
||||||
|
self.misp_event.distribution = self.config_from_email_body.get('distribution')
|
||||||
|
if self.config_from_email_body.get('threat_level') is not None:
|
||||||
|
self.misp_event.threat_level_id = self.config_from_email_body.get('threat_level')
|
||||||
|
if self.config_from_email_body.get('analysis') is not None:
|
||||||
|
self.misp_event.analysis = self.config_from_email_body.get('analysis')
|
||||||
|
if self.config_from_email_body.get('publish'):
|
||||||
|
self.misp_event.publish()
|
||||||
|
else:
|
||||||
|
self.clean_email_body = ''
|
||||||
|
self._find_o365_attached_forward(self.msg)
|
||||||
|
|
||||||
def process_body_iocs(self, email_object=None):
|
def process_body_iocs(self, email_object=None):
|
||||||
if email_object:
|
if email_object:
|
||||||
body = html.unescape(email_object.email.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode('utf8', 'surrogateescape'))
|
body = html.unescape(email_object.email.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode('utf8', 'surrogateescape'))
|
||||||
|
@ -416,7 +495,10 @@ class Mail2MISP():
|
||||||
for value, source in self.sightings_to_add:
|
for value, source in self.sightings_to_add:
|
||||||
self.sighting(value, source)
|
self.sighting(value, source)
|
||||||
if self.config.freetext:
|
if self.config.freetext:
|
||||||
self.misp.freetext(event, string=self.original_mail.get_body(preferencelist=('html', 'plain')), adhereToWarninglists=self.config.enforcewarninglist)
|
if self.config.o365_freetext:
|
||||||
|
self.misp.freetext(event, string=self.clean_email_body, adhereToWarninglists=self.config.enforcewarninglist)
|
||||||
|
else:
|
||||||
|
self.misp.freetext(event, string=self.original_mail.get_body(preferencelist=('html', 'plain')), adhereToWarninglists=self.config.enforcewarninglist)
|
||||||
return event
|
return event
|
||||||
|
|
||||||
def get_attached_emails(self, pseudofile):
|
def get_attached_emails(self, pseudofile):
|
||||||
|
@ -438,3 +520,76 @@ class Mail2MISP():
|
||||||
# all attachments are identified as message.EmailMessage so filtering on extension for now.
|
# all attachments are identified as message.EmailMessage so filtering on extension for now.
|
||||||
forwarded_emails.append(BytesIO(attachment_content))
|
forwarded_emails.append(BytesIO(attachment_content))
|
||||||
return forwarded_emails
|
return forwarded_emails
|
||||||
|
|
||||||
|
class O365MISPClient:
|
||||||
|
"""
|
||||||
|
A client (MUA) to allow mail_to_misp to interact with Microsoft Graph and Office 365 API to get email messages.
|
||||||
|
"""
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
client_id: str,
|
||||||
|
client_secret: str,
|
||||||
|
tenant_id: str,
|
||||||
|
resource: str,
|
||||||
|
scopes: List[str],
|
||||||
|
token_backend: Optional[
|
||||||
|
Union[AWSS3Backend, AWSSecretsBackend, EnvTokenBackend, FileSystemTokenBackend, FirestoreBackend]
|
||||||
|
] = None,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Init O365MISPClient
|
||||||
|
:param client_id: OAuth Client ID
|
||||||
|
:param client_secret: OAuth Client Secret
|
||||||
|
:param tenant_id: Your Tenant ID
|
||||||
|
:param resource: The email address you want to access
|
||||||
|
:param scopes: The permission scopes for the resource
|
||||||
|
:param token_backend: The backend used for storing OAuth token
|
||||||
|
"""
|
||||||
|
self.scopes = scopes
|
||||||
|
self.resource = resource
|
||||||
|
self.o365_acct = Account(
|
||||||
|
credentials=(client_id, client_secret),
|
||||||
|
auth_flow_type='authorization',
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
token_backend=token_backend
|
||||||
|
)
|
||||||
|
if not self.o365_acct.is_authenticated:
|
||||||
|
self.o365_acct.authenticate(scopes=self.scopes)
|
||||||
|
self.mailbox = self.o365_acct.mailbox(resource=self.resource)
|
||||||
|
self.inbox = self.mailbox.inbox_folder()
|
||||||
|
self.query_properties = [
|
||||||
|
'internet_message_headers',
|
||||||
|
'subject',
|
||||||
|
'body',
|
||||||
|
'unique_body',
|
||||||
|
'from',
|
||||||
|
'reply_to',
|
||||||
|
'is_read',
|
||||||
|
'is_draft',
|
||||||
|
'received_date_time',
|
||||||
|
'has_attachments',
|
||||||
|
'attachments'
|
||||||
|
]
|
||||||
|
|
||||||
|
def get_email_messages(self, from_time: datetime, to_time: datetime, folder: Optional[str] = None) -> Iterator[Message]:
|
||||||
|
"""
|
||||||
|
Get messages for a certain timeframe. Defaults to looking for messages in the Inbox folder, however by
|
||||||
|
supplying a folder name as a parameter you can change where to get the messages from.
|
||||||
|
|
||||||
|
:param from_time: start time to search for
|
||||||
|
:param to_time: end time to search for
|
||||||
|
:param folder: specific folder to get messages from (don't supply if getting from the inbox folder)
|
||||||
|
:return: an iterator of O365.messages.Message from the resource
|
||||||
|
"""
|
||||||
|
query = self.mailbox.new_query().select(*self.query_properties)
|
||||||
|
# https://learn.microsoft.com/en-us/graph/api/resources/message?view=graph-rest-1.0#properties
|
||||||
|
query = query.chain('and').on_attribute('received_date_time').greater(from_time)
|
||||||
|
query = query.chain('and').on_attribute('received_date_time').less(to_time)
|
||||||
|
|
||||||
|
if folder:
|
||||||
|
messages = self.mailbox.get_folder(folder_name=folder).get_messages(query=query)
|
||||||
|
else:
|
||||||
|
messages = self.inbox.get_messages(query=query)
|
||||||
|
|
||||||
|
return messages
|
||||||
|
|
||||||
|
|
|
@ -11,6 +11,18 @@ default_analysis = 1
|
||||||
id_tag = 'host:m2m:tld'
|
id_tag = 'host:m2m:tld'
|
||||||
freetext = False
|
freetext = False
|
||||||
|
|
||||||
|
# O365MISPClient config
|
||||||
|
o365_freetext = False # must be enabled in addition to the above freetext to use this on o365 email messages
|
||||||
|
o365_client_id = 'YOUR_O365_CLIENT_ID'
|
||||||
|
o365_client_secret = 'YOUR_O365_CLIENT_SECRET'
|
||||||
|
o365_tenant_id = 'YOUR_O365_TENANT_ID'
|
||||||
|
o365_resource = 'YOUR_O365_INBOX' # misp@yourdomain.com or whatever inbox you are reading mail from
|
||||||
|
o365_scopes = [
|
||||||
|
'offline_access', # Highly recommended to add this. If not you will have to re-authenticate every hour.
|
||||||
|
'https://graph.microsoft.com/Mail.Read', # To read my mailbox
|
||||||
|
# 'https://graph.microsoft.com/Mail.Read.Shared' # To read another user/shared mailbox
|
||||||
|
]
|
||||||
|
|
||||||
body_config_prefix = 'm2m' # every line in the body starting with this value will be skipped from the IOCs
|
body_config_prefix = 'm2m' # every line in the body starting with this value will be skipped from the IOCs
|
||||||
m2m_key = 'YOUSETYOURKEYHERE'
|
m2m_key = 'YOUSETYOURKEYHERE'
|
||||||
m2m_benign_attachment_keyword = 'benign'
|
m2m_benign_attachment_keyword = 'benign'
|
||||||
|
@ -24,7 +36,8 @@ ignore_carrier_mail = False
|
||||||
ignore_nullsize_attachments = False
|
ignore_nullsize_attachments = False
|
||||||
|
|
||||||
excludelist = ('google.com', 'microsoft.com')
|
excludelist = ('google.com', 'microsoft.com')
|
||||||
externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net')
|
externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net', 'urlscan.io',
|
||||||
|
'abuse.ch', 'tria.ge', 'bleepingcomputer.com', 'any.run', 'urlvoid.com', 'intezer.com')
|
||||||
internallist = ('internal.system.local')
|
internallist = ('internal.system.local')
|
||||||
noidsflaglist = ('myexternalip.com', 'ipinfo.io', 'icanhazip.com', 'wtfismyip.com', 'ipecho.net',
|
noidsflaglist = ('myexternalip.com', 'ipinfo.io', 'icanhazip.com', 'wtfismyip.com', 'ipecho.net',
|
||||||
'api.ipify.org', 'checkip.amazonaws.com', 'whatismyipaddress.com', 'google.com',
|
'api.ipify.org', 'checkip.amazonaws.com', 'whatismyipaddress.com', 'google.com',
|
||||||
|
@ -48,13 +61,16 @@ sighting_source = "YOUR_MAIL_TO_MISP_IDENTIFIER"
|
||||||
|
|
||||||
# Remove "Re:", "Fwd:" and {Spam?} from subject
|
# Remove "Re:", "Fwd:" and {Spam?} from subject
|
||||||
# add: "[\(\[].*?[\)\]]" to remove everything between [] and (): i.e. [tag]
|
# add: "[\(\[].*?[\)\]]" to remove everything between [] and (): i.e. [tag]
|
||||||
removelist = (r"Re:", r"Fwd:", r"\{Spam\?\} ")
|
removelist = (r"Re:", r"Fwd:", r"\{Spam\?\} ", r"RE:", r"FW:")
|
||||||
|
|
||||||
# TLP tag setup
|
# TLP tag setup
|
||||||
# Tuples contain different variations of spelling
|
# Tuples contain different variations of spelling
|
||||||
tlptags = {'tlp:amber': ['tlp:amber', 'tlp: amber', 'tlp amber'],
|
tlptags = {'tlp:amber': ['tlp:amber', 'tlp: amber', 'tlp amber', 'tlp :amber'],
|
||||||
'tlp:green': ['tlp:green', 'tlp: green', 'tlp green'],
|
'tlp:amber+strict': ['tlp:amber+strict', 'tlp: amber+strict', 'tlp amber+strict', 'tlp :amber+strict'],
|
||||||
'tlp:white': ['tlp:white', 'tlp: white', 'tlp white']
|
'tlp:green': ['tlp:green', 'tlp: green', 'tlp green', 'tlp :green'],
|
||||||
|
'tlp:white': ['tlp:white', 'tlp: white', 'tlp white', 'tlp :white'],
|
||||||
|
'tlp:clear': ['tlp:clear', 'tlp: clear', 'tlp clear', 'tlp :clear'],
|
||||||
|
'tlp:red': ['tlp:red', 'tlp: red', 'tlp red', 'tlp :red']
|
||||||
}
|
}
|
||||||
tlptag_default = sorted(tlptags.keys())[0]
|
tlptag_default = sorted(tlptags.keys())[0]
|
||||||
|
|
||||||
|
@ -65,11 +81,26 @@ malwaretags = {'locky': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:rans
|
||||||
'Pony': ['misp-galaxy:tool="Hancitor"'],
|
'Pony': ['misp-galaxy:tool="Hancitor"'],
|
||||||
'ursnif': ['misp-galaxy:tool="Snifula"'],
|
'ursnif': ['misp-galaxy:tool="Snifula"'],
|
||||||
'NanoCore': ['misp-galaxy:tool="NanoCoreRAT"'],
|
'NanoCore': ['misp-galaxy:tool="NanoCoreRAT"'],
|
||||||
'trickbot': ['misp-galaxy:tool="Trick Bot"']
|
'trickbot': ['misp-galaxy:tool="Trick Bot"'],
|
||||||
|
'agenttesla': ['misp-galaxy:mitre-malware="Agent Tesla - S0331"'],
|
||||||
|
'formbook': ['misp-galaxy:malpedia="Formbook"'],
|
||||||
|
'remcos': ['misp-galaxy:mitre-tool="Remcos - S0332"'],
|
||||||
|
'snake keylogger': ['misp-galaxy:malpedia="404 Keylogger"'],
|
||||||
|
'icedid': ['misp-galaxy:malpedia="IcedID"'],
|
||||||
|
'zloader': ['misp-galaxy:malpedia="Zloader"'],
|
||||||
|
'lokibot': ['misp-galaxy:mitre-malware="Lokibot - S0447"'],
|
||||||
|
'valyria': ['misp-galaxy:malpedia="POWERSTATS"'],
|
||||||
|
'guloader': ['misp-galaxy:mitre-malware="GuLoader - S0561"'],
|
||||||
|
'avemaria': ['misp-galaxy:mitre-malware="WarzoneRAT - S0670"'],
|
||||||
|
'warzone': ['misp-galaxy:mitre-malware="WarzoneRAT - S0670"'],
|
||||||
|
'hancitor': ['misp-galaxy:malpedia="Hancitor"'],
|
||||||
|
'async': ['misp-galaxy:malpedia="AsyncRAT"'],
|
||||||
|
'emotet': ['misp-galaxy:mitre-malware="Emotet - S0367"']
|
||||||
}
|
}
|
||||||
|
|
||||||
# Tags to be set depending on the presence of other tags
|
# Tags to be set depending on the presence of other tags
|
||||||
dependingtags = {'tlp:white': ['circl:osint-feed']
|
dependingtags = {'tlp:white': ['circl:osint-feed'],
|
||||||
|
'tlp:clear': ['circl:osint-feed']
|
||||||
}
|
}
|
||||||
|
|
||||||
# Known identifiers for forwarded messages
|
# Known identifiers for forwarded messages
|
||||||
|
|
|
@ -0,0 +1,92 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
import argparse
|
||||||
|
import importlib
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import syslog
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from itertools import tee
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from mail2misp import Mail2MISP
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
parser = argparse.ArgumentParser(description='Push mail from O365 into a MISP instance')
|
||||||
|
group = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
group.add_argument('-nd', '--days', help='Number of days to search back in inbox')
|
||||||
|
group.add_argument('-nh', '--hours', help='Number of hours to search back in inbox')
|
||||||
|
parser.add_argument('-f', '--folder', help='Folder name that contains email messages to parse')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_USER)
|
||||||
|
syslog.syslog("[+] O365 job started.")
|
||||||
|
|
||||||
|
# import config module
|
||||||
|
os.chdir(Path(__file__).parent)
|
||||||
|
|
||||||
|
configmodule = Path(__file__).name.replace('o365.py', 'config')
|
||||||
|
if Path(f"{configmodule}.py").exists():
|
||||||
|
config = importlib.import_module(configmodule)
|
||||||
|
try:
|
||||||
|
misp_url = config.misp_url
|
||||||
|
misp_key = config.misp_key
|
||||||
|
misp_verifycert = config.misp_verifycert
|
||||||
|
o365_client_id = config.o365_client_id
|
||||||
|
o365_client_secret = config.o365_client_secret
|
||||||
|
o365_tenant_id = config.o365_tenant_id
|
||||||
|
o365_resource = config.o365_resource
|
||||||
|
o365_scopes = config.o365_scopes
|
||||||
|
debug = config.debug
|
||||||
|
except Exception as ex:
|
||||||
|
print("There is a problem with the configuration. A mandatory configuration variable is not set.")
|
||||||
|
print("Did you just update? mail_to_misp might have new configuration variables.")
|
||||||
|
print("Please compare with the configuration example.")
|
||||||
|
print("\nTrace:")
|
||||||
|
print(ex)
|
||||||
|
sys.exit(-2)
|
||||||
|
else:
|
||||||
|
print(f"Couldn't locate config file {configmodule}.py")
|
||||||
|
sys.exit(-1)
|
||||||
|
|
||||||
|
# set message search period to look for emails
|
||||||
|
to_time = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
if args.days:
|
||||||
|
from_time = (to_time - timedelta(days=int(args.days)))
|
||||||
|
else:
|
||||||
|
from_time = (to_time - timedelta(hours=int(args.hours)))
|
||||||
|
|
||||||
|
# initialize Mail2MISP
|
||||||
|
m2m = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config)
|
||||||
|
|
||||||
|
# initialize O365MISPClient
|
||||||
|
o365 = m2m.O365MISPClient(
|
||||||
|
client_id=o365_client_id,
|
||||||
|
client_secret=o365_client_secret,
|
||||||
|
tenant_id=o365_tenant_id,
|
||||||
|
resource=o365_resource,
|
||||||
|
scopes=o365_scopes,
|
||||||
|
token_backend=None # if not supplied will default to using FileSystemTokenBackend, which stores the token in a
|
||||||
|
# txt file on disk in the directory the script is executed from
|
||||||
|
)
|
||||||
|
|
||||||
|
messages = o365.get_email_messages(
|
||||||
|
from_time=from_time,
|
||||||
|
to_time=to_time,
|
||||||
|
folder=args.folder if args.folder else None # defaults to searching the resource's inbox folder if None
|
||||||
|
)
|
||||||
|
|
||||||
|
messages1, messages2 = tee(messages, 2)
|
||||||
|
|
||||||
|
syslog.syslog(f"[*] Found {len(list(messages1))} messages to process and send to MISP!")
|
||||||
|
|
||||||
|
for msg in messages2:
|
||||||
|
m2m.load_o365_email(msg)
|
||||||
|
if debug:
|
||||||
|
syslog.syslog(f"[*] Processing email with subject: {m2m.subject}")
|
||||||
|
m2m.process_o365_email_body()
|
||||||
|
m2m.process_body_iocs()
|
||||||
|
m2m.add_event()
|
||||||
|
syslog.syslog("[-] O365 job finished.")
|
|
@ -1,6 +1,6 @@
|
||||||
[tool.poetry]
|
[tool.poetry]
|
||||||
name = "mail2misp"
|
name = "mail2misp"
|
||||||
version = "0.1.0"
|
version = "0.2.0"
|
||||||
description = "Importer of EML files into a MISP instance"
|
description = "Importer of EML files into a MISP instance"
|
||||||
authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
|
authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
|
@ -39,6 +39,7 @@ flask-bootstrap = "^3.3.7.1"
|
||||||
gunicorn = "^20.1.0"
|
gunicorn = "^20.1.0"
|
||||||
chardet = "^5.2.0"
|
chardet = "^5.2.0"
|
||||||
aiosmtpd = "^1.4.4.post2"
|
aiosmtpd = "^1.4.4.post2"
|
||||||
|
O365 = {extras = ["o365"], version = "^2.0.31", optional = true}
|
||||||
|
|
||||||
[tool.poetry.extras]
|
[tool.poetry.extras]
|
||||||
fileobjects = ['python-magic', 'pydeep', 'lief']
|
fileobjects = ['python-magic', 'pydeep', 'lief']
|
||||||
|
@ -49,6 +50,7 @@ pdfexport = ['reportlab']
|
||||||
url = ['pyfaup']
|
url = ['pyfaup']
|
||||||
email = ['extract_msg', "RTFDE", "oletools"]
|
email = ['extract_msg', "RTFDE", "oletools"]
|
||||||
brotli = ['urllib3']
|
brotli = ['urllib3']
|
||||||
|
o365 = ['O365']
|
||||||
|
|
||||||
[tool.poetry.dev-dependencies]
|
[tool.poetry.dev-dependencies]
|
||||||
nose = "^1.3.7"
|
nose = "^1.3.7"
|
||||||
|
|
2
setup.py
2
setup.py
|
@ -5,7 +5,7 @@ from setuptools import setup
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
name='mail2misp',
|
name='mail2misp',
|
||||||
version='0.1',
|
version='0.2',
|
||||||
author='Raphaël Vinot',
|
author='Raphaël Vinot',
|
||||||
author_email='raphael.vinot@circl.lu',
|
author_email='raphael.vinot@circl.lu',
|
||||||
maintainer='Raphaël Vinot',
|
maintainer='Raphaël Vinot',
|
||||||
|
|
Loading…
Reference in New Issue