MISP
/
mail_to_misp
mirror of https://github.com/MISP/mail_to_misp
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #39 from begunrom/carrier_mail_with_emailattachments 

Carrier mail with emailattachments
pull/44/head
Raphaël Vinot 2020-06-07 20:40:34 +02:00 committed by GitHub
parent 721d806e8d 92c99c0559
commit 68b334df38
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 188 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
mail2misp/mail2misp.py
View File

@ -4,6 +4,7 @@
import re
import syslog
import html
import os
from io import BytesIO
from ipaddress import ip_address
from email import message_from_bytes, policy, message
@ -43,6 +44,9 @@ class Mail2MISP():
setattr(self.config, 'enable_dns', False)
self.debug = self.config.debug
self.config_from_email_body = {}
if not hasattr(self.config, 'ignore_nullsize_attachments'):
setattr(self.config, 'ignore_nullsize_attachments', False)
self.ignore_nullsize_attachments = self.config.ignore_nullsize_attachments
# Init Faup
self.f = Faup()
self.sightings_to_add = []
@ -50,15 +54,21 @@ class Mail2MISP():
def load_email(self, pseudofile):
self.pseudofile = pseudofile
self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default)
self.subject = self.original_mail.get('Subject')
try:
self.sender = self.original_mail.get('From')
except:
self.sender = "<unknown sender>"
# Remove words from subject
for removeword in self.config.removelist:
self.subject = re.sub(removeword, "", self.subject).strip()
try:
self.subject = self.original_mail.get('Subject')
# Remove words from subject
for removeword in self.config.removelist:
self.subject = re.sub(removeword, "", self.subject).strip()
except Exception as ex:
self.subject = "<subject could not be retrieved>"
if self.debug:
syslog.syslog(ex)
# Initialize the MISP event
self.misp_event = MISPEvent()
@ -127,27 +137,28 @@ class Mail2MISP():
if email_object.attachments:
# Create file objects for the attachments
for attachment_name, attachment in email_object.attachments:
if not attachment_name:
attachment_name = 'NameMissing.txt'
if self.config_from_email_body.get('attachment') == self.config.m2m_benign_attachment_keyword:
a = self.misp_event.add_attribute('attachment', value=attachment_name, data=attachment)
email_object.add_reference(a.uuid, 'related-to', 'Email attachment')
else:
f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False)
if self.config.vt_key:
try:
vt_object = VTReportObject(self.config.vt_key, f_object.get_attributes_by_relation('sha256')[0].value, standalone=False)
self.misp_event.add_object(vt_object)
f_object.add_reference(vt_object.uuid, 'analysed-with')
except InvalidMISPObject as e:
print(e)
pass
self.misp_event.add_object(f_object)
if main_object:
self.misp_event.add_object(main_object)
for section in sections:
self.misp_event.add_object(section)
email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment')
if not (self.ignore_nullsize_attachments == True and attachment.getbuffer().nbytes == 0):
if not attachment_name:
attachment_name = 'NameMissing.txt'
if self.config_from_email_body.get('attachment') == self.config.m2m_benign_attachment_keyword:
a = self.misp_event.add_attribute('attachment', value=attachment_name, data=attachment)
email_object.add_reference(a.uuid, 'related-to', 'Email attachment')
else:
f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False)
if self.config.vt_key:
try:
vt_object = VTReportObject(self.config.vt_key, f_object.get_attributes_by_relation('sha256')[0].value, standalone=False)
self.misp_event.add_object(vt_object)
f_object.add_reference(vt_object.uuid, 'analysed-with')
except InvalidMISPObject as e:
print(e)
pass
self.misp_event.add_object(f_object)
if main_object:
self.misp_event.add_object(main_object)
for section in sections:
self.misp_event.add_object(section)
email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment')
self.process_body_iocs(email_object)
if self.config.spamtrap or self.config.attach_original_mail or self.config_from_email_body.get('attach_original_mail'):
self.misp_event.add_object(email_object)
@ -399,3 +410,24 @@ class Mail2MISP():
for value, source in self.sightings_to_add:
self.sighting(value, source)
return event
def get_attached_emails(self,pseudofile):
if self.debug:
syslog.syslog("get_attached_emails Job started.")
forwarded_emails = []
self.pseudofile = pseudofile
self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default)
for attachment in self.original_mail.iter_attachments():
attachment_content = attachment.get_content()
filename = attachment.get_filename()
if self.debug:
syslog.syslog(f'get_attached_emails: filename = {filename}')
# Search for email forwarded as attachment
# I could have more than one, attaching everything.
if isinstance(attachment, message.EmailMessage) and os.path.splitext(filename)[1] == '.eml':
# all attachments are identified as message.EmailMessage so filtering on extension for now.
forwarded_emails.append(BytesIO(attachment_content))
return forwarded_emails

54
mail_to_misp.py
View File

@ -33,6 +33,7 @@ if __name__ == '__main__':
misp_key = config.misp_key
misp_verifycert = config.misp_verifycert
debug = config.debug
ignore_carrier_mail = config.ignore_carrier_mail
except Exception as e:
syslog.syslog(str(e))
print("There is a problem with the configuration. A mandatory configuration variable is not set.")
@ -55,19 +56,48 @@ if __name__ == '__main__':
# receive data and subject through arguments
raise Exception('This is not implemented anymore.')
syslog.syslog("About to create a mail2misp object.")
mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event)
mail2misp.load_email(pseudofile)
if debug:
syslog.syslog(f'Working on {mail2misp.subject}')
if args.trap or config.spamtrap:
mail2misp.email_from_spamtrap()
attached_emails = mail2misp.get_attached_emails(pseudofile)
syslog.syslog(f"found {len(attached_emails)} attached emails")
if ignore_carrier_mail and len(attached_emails) !=0:
syslog.syslog("Ignoring the carrier mail.")
while len(attached_emails) !=0:
pseudofile = attached_emails.pop()
#Throw away the Mail2MISP object of the carrier mail and create a new one for each e-mail attachment
mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event)
mail2misp.load_email(pseudofile)
if debug:
syslog.syslog(f'Working on {mail2misp.subject}')
if args.trap or config.spamtrap:
mail2misp.email_from_spamtrap()
else:
mail2misp.process_email_body()
mail2misp.process_body_iocs()
if not args.event:
mail2misp.add_event()
syslog.syslog("Job finished.")
else:
mail2misp.process_email_body()
syslog.syslog("Running standard mail2misp")
mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event)
mail2misp.load_email(pseudofile)
mail2misp.process_body_iocs()
if debug:
syslog.syslog(f'Working on {mail2misp.subject}')
if args.trap or config.spamtrap:
mail2misp.email_from_spamtrap()
else:
mail2misp.process_email_body()
mail2misp.process_body_iocs()
if not args.event:
mail2misp.add_event()
syslog.syslog("Job finished.")
if not args.event:
mail2misp.add_event()
syslog.syslog("Job finished.")

2
mail_to_misp_config.py-example
View File

@ -18,6 +18,8 @@ debug = False
nameservers = ['149.13.33.69']
email_subject_prefix = 'M2M'
attach_original_mail = False
ignore_carrier_mail = False
ignore_nullsize_attachments = False
excludelist = ('google.com', 'microsoft.com')
externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net')

80
tests/config_carrier.py Normal file
View File

@ -0,0 +1,80 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
misp_url = 'YOUR_MISP_URL'
misp_key = 'YOUR_KEY_HERE' # The MISP auth key can be found on the MISP web interface under the automation section
misp_verifycert = True
spamtrap = False
default_distribution = 0
default_threat_level = 3
default_analysis = 1
body_config_prefix = 'm2m' # every line in the body starting with this value will be skipped from the IOCs
m2m_key = 'YOUSETYOURKEYHERE'
m2m_benign_attachment_keyword = 'benign'
debug = True
nameservers = ['8.8.8.8']
email_subject_prefix = 'M2M'
attach_original_mail = True
ignore_carrier_mail = True
excludelist = ('google.com', 'microsoft.com')
externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net')
internallist = ('internal.system.local')
noidsflaglist = ('myexternalip.com', 'ipinfo.io', 'icanhazip.com', 'wtfismyip.com', 'ipecho.net',
'api.ipify.org', 'checkip.amazonaws.com', 'whatismyipaddress.com', 'google.com',
'dropbox.com'
)
# Stop parsing when this term is found
stopword = 'Whois & IP Information'
# Ignore lines in body of message containing:
ignorelist = ("From:", "Sender:", "Received:", "Sender IP:", "Reply-To:", "Registrar WHOIS Server:",
"Registrar:", "Domain Status:", "Registrant Email:", "IP Location:",
"X-Get-Message-Sender-Via:", "X-Authenticated-Sender:")
# Ignore (don't add) attributes that are on server side warning list
enforcewarninglist = True
# Add a sighting for each value
sighting = False
sighting_source = "YOUR_MAIL_TO_MISP_IDENTIFIER"
# Remove "Re:", "Fwd:" and {Spam?} from subject
# add: "[\(\[].*?[\)\]]" to remove everything between [] and (): i.e. [tag]
removelist = (r'Re:', r'Fwd:', r'\{Spam?\}')
# TLP tag setup
# Tuples contain different variations of spelling
tlptags = {'tlp:amber': ['tlp:amber', 'tlp: amber', 'tlp amber'],
'tlp:green': ['tlp:green', 'tlp: green', 'tlp green'],
'tlp:white': ['tlp:white', 'tlp: white', 'tlp white']
}
tlptag_default = sorted(tlptags.keys())[0]
malwaretags = {'locky': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Locky"'],
'jaff': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Jaff"'],
'dridex': ['misp-galaxy:tool="dridex"'],
'netwire': ['Netwire RAT'],
'Pony': ['misp-galaxy:tool="Hancitor"'],
'ursnif': ['misp-galaxy:tool="Snifula"'],
'NanoCore': ['misp-galaxy:tool="NanoCoreRAT"'],
'trickbot': ['misp-galaxy:tool="Trick Bot"']
}
# Tags to be set depending on the presence of other tags
dependingtags = {'tlp:white': ['circl:osint-feed']
}
# Known identifiers for forwarded messages
forward_identifiers = {'-------- Forwarded Message --------', 'Begin forwarded message:'}
# Tags to add when hashes are found (e.g. to do automatic expansion)
hash_only_tags = {'TODO:VT-ENRICHMENT'}
# If an attribute is on any MISP server side `warning list`, skip the creation of the attribute
skip_item_on_warninglist = True
vt_key = None

6
tests/tests.py
View File

@ -82,6 +82,12 @@ class TestMailToMISP(unittest.TestCase):
self.assertEqual(self.mail2misp.misp_event.analysis, '0')
self.mail2misp.add_event()
def test_attached_emails(self):
config = importlib.import_module('tests.config_carrier')
self.mail2misp = Mail2MISP('', '', '', config=config, offline=True)
with open('tests/mails/test_7_email_attachments.eml', 'rb') as f:
attached_emails = self.mail2misp.get_attached_emails(BytesIO(f.read()))
self.assertEqual(len(attached_emails), 7)
if __name__ == '__main__':
unittest.main()