diff --git a/mail2misp/mail2misp.py b/mail2misp/mail2misp.py index fab8b88..d6deb6a 100644 --- a/mail2misp/mail2misp.py +++ b/mail2misp/mail2misp.py @@ -4,6 +4,7 @@ import re import syslog import html +import os from io import BytesIO from ipaddress import ip_address from email import message_from_bytes, policy, message @@ -43,6 +44,9 @@ class Mail2MISP(): setattr(self.config, 'enable_dns', False) self.debug = self.config.debug self.config_from_email_body = {} + if not hasattr(self.config, 'ignore_nullsize_attachments'): + setattr(self.config, 'ignore_nullsize_attachments', False) + self.ignore_nullsize_attachments = self.config.ignore_nullsize_attachments # Init Faup self.f = Faup() self.sightings_to_add = [] @@ -50,15 +54,21 @@ class Mail2MISP(): def load_email(self, pseudofile): self.pseudofile = pseudofile self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default) - self.subject = self.original_mail.get('Subject') + try: self.sender = self.original_mail.get('From') except: self.sender = "" - - # Remove words from subject - for removeword in self.config.removelist: - self.subject = re.sub(removeword, "", self.subject).strip() + + try: + self.subject = self.original_mail.get('Subject') + # Remove words from subject + for removeword in self.config.removelist: + self.subject = re.sub(removeword, "", self.subject).strip() + except Exception as ex: + self.subject = "" + if self.debug: + syslog.syslog(ex) # Initialize the MISP event self.misp_event = MISPEvent() @@ -127,27 +137,28 @@ class Mail2MISP(): if email_object.attachments: # Create file objects for the attachments for attachment_name, attachment in email_object.attachments: - if not attachment_name: - attachment_name = 'NameMissing.txt' - if self.config_from_email_body.get('attachment') == self.config.m2m_benign_attachment_keyword: - a = self.misp_event.add_attribute('attachment', value=attachment_name, data=attachment) - email_object.add_reference(a.uuid, 'related-to', 'Email attachment') - else: - f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False) - if self.config.vt_key: - try: - vt_object = VTReportObject(self.config.vt_key, f_object.get_attributes_by_relation('sha256')[0].value, standalone=False) - self.misp_event.add_object(vt_object) - f_object.add_reference(vt_object.uuid, 'analysed-with') - except InvalidMISPObject as e: - print(e) - pass - self.misp_event.add_object(f_object) - if main_object: - self.misp_event.add_object(main_object) - for section in sections: - self.misp_event.add_object(section) - email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment') + if not (self.ignore_nullsize_attachments == True and attachment.getbuffer().nbytes == 0): + if not attachment_name: + attachment_name = 'NameMissing.txt' + if self.config_from_email_body.get('attachment') == self.config.m2m_benign_attachment_keyword: + a = self.misp_event.add_attribute('attachment', value=attachment_name, data=attachment) + email_object.add_reference(a.uuid, 'related-to', 'Email attachment') + else: + f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False) + if self.config.vt_key: + try: + vt_object = VTReportObject(self.config.vt_key, f_object.get_attributes_by_relation('sha256')[0].value, standalone=False) + self.misp_event.add_object(vt_object) + f_object.add_reference(vt_object.uuid, 'analysed-with') + except InvalidMISPObject as e: + print(e) + pass + self.misp_event.add_object(f_object) + if main_object: + self.misp_event.add_object(main_object) + for section in sections: + self.misp_event.add_object(section) + email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment') self.process_body_iocs(email_object) if self.config.spamtrap or self.config.attach_original_mail or self.config_from_email_body.get('attach_original_mail'): self.misp_event.add_object(email_object) @@ -399,3 +410,24 @@ class Mail2MISP(): for value, source in self.sightings_to_add: self.sighting(value, source) return event + + def get_attached_emails(self,pseudofile): + + if self.debug: + syslog.syslog("get_attached_emails Job started.") + + forwarded_emails = [] + self.pseudofile = pseudofile + self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default) + for attachment in self.original_mail.iter_attachments(): + attachment_content = attachment.get_content() + filename = attachment.get_filename() + if self.debug: + syslog.syslog(f'get_attached_emails: filename = {filename}') + # Search for email forwarded as attachment + # I could have more than one, attaching everything. + if isinstance(attachment, message.EmailMessage) and os.path.splitext(filename)[1] == '.eml': + # all attachments are identified as message.EmailMessage so filtering on extension for now. + forwarded_emails.append(BytesIO(attachment_content)) + return forwarded_emails + diff --git a/mail_to_misp.py b/mail_to_misp.py index 34ec478..8b2864e 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -33,6 +33,7 @@ if __name__ == '__main__': misp_key = config.misp_key misp_verifycert = config.misp_verifycert debug = config.debug + ignore_carrier_mail = config.ignore_carrier_mail except Exception as e: syslog.syslog(str(e)) print("There is a problem with the configuration. A mandatory configuration variable is not set.") @@ -55,19 +56,48 @@ if __name__ == '__main__': # receive data and subject through arguments raise Exception('This is not implemented anymore.') + syslog.syslog("About to create a mail2misp object.") mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event) - mail2misp.load_email(pseudofile) - - if debug: - syslog.syslog(f'Working on {mail2misp.subject}') - - if args.trap or config.spamtrap: - mail2misp.email_from_spamtrap() + attached_emails = mail2misp.get_attached_emails(pseudofile) + syslog.syslog(f"found {len(attached_emails)} attached emails") + if ignore_carrier_mail and len(attached_emails) !=0: + syslog.syslog("Ignoring the carrier mail.") + while len(attached_emails) !=0: + pseudofile = attached_emails.pop() + #Throw away the Mail2MISP object of the carrier mail and create a new one for each e-mail attachment + mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event) + mail2misp.load_email(pseudofile) + + if debug: + syslog.syslog(f'Working on {mail2misp.subject}') + + if args.trap or config.spamtrap: + mail2misp.email_from_spamtrap() + else: + mail2misp.process_email_body() + + mail2misp.process_body_iocs() + + if not args.event: + mail2misp.add_event() + + syslog.syslog("Job finished.") else: - mail2misp.process_email_body() + syslog.syslog("Running standard mail2misp") + mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event) + mail2misp.load_email(pseudofile) - mail2misp.process_body_iocs() + if debug: + syslog.syslog(f'Working on {mail2misp.subject}') + + if args.trap or config.spamtrap: + mail2misp.email_from_spamtrap() + else: + mail2misp.process_email_body() + + mail2misp.process_body_iocs() + + if not args.event: + mail2misp.add_event() + syslog.syslog("Job finished.") - if not args.event: - mail2misp.add_event() - syslog.syslog("Job finished.") diff --git a/mail_to_misp_config.py-example b/mail_to_misp_config.py-example index b4f9801..468027b 100644 --- a/mail_to_misp_config.py-example +++ b/mail_to_misp_config.py-example @@ -18,6 +18,8 @@ debug = False nameservers = ['149.13.33.69'] email_subject_prefix = 'M2M' attach_original_mail = False +ignore_carrier_mail = False +ignore_nullsize_attachments = False excludelist = ('google.com', 'microsoft.com') externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net') diff --git a/tests/config_carrier.py b/tests/config_carrier.py new file mode 100644 index 0000000..be5c95d --- /dev/null +++ b/tests/config_carrier.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +misp_url = 'YOUR_MISP_URL' +misp_key = 'YOUR_KEY_HERE' # The MISP auth key can be found on the MISP web interface under the automation section +misp_verifycert = True +spamtrap = False +default_distribution = 0 +default_threat_level = 3 +default_analysis = 1 + +body_config_prefix = 'm2m' # every line in the body starting with this value will be skipped from the IOCs +m2m_key = 'YOUSETYOURKEYHERE' +m2m_benign_attachment_keyword = 'benign' + +debug = True +nameservers = ['8.8.8.8'] +email_subject_prefix = 'M2M' +attach_original_mail = True +ignore_carrier_mail = True + +excludelist = ('google.com', 'microsoft.com') +externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net') +internallist = ('internal.system.local') +noidsflaglist = ('myexternalip.com', 'ipinfo.io', 'icanhazip.com', 'wtfismyip.com', 'ipecho.net', + 'api.ipify.org', 'checkip.amazonaws.com', 'whatismyipaddress.com', 'google.com', + 'dropbox.com' + ) + +# Stop parsing when this term is found +stopword = 'Whois & IP Information' + +# Ignore lines in body of message containing: +ignorelist = ("From:", "Sender:", "Received:", "Sender IP:", "Reply-To:", "Registrar WHOIS Server:", + "Registrar:", "Domain Status:", "Registrant Email:", "IP Location:", + "X-Get-Message-Sender-Via:", "X-Authenticated-Sender:") + +# Ignore (don't add) attributes that are on server side warning list +enforcewarninglist = True + +# Add a sighting for each value +sighting = False +sighting_source = "YOUR_MAIL_TO_MISP_IDENTIFIER" + +# Remove "Re:", "Fwd:" and {Spam?} from subject +# add: "[\(\[].*?[\)\]]" to remove everything between [] and (): i.e. [tag] +removelist = (r'Re:', r'Fwd:', r'\{Spam?\}') + +# TLP tag setup +# Tuples contain different variations of spelling +tlptags = {'tlp:amber': ['tlp:amber', 'tlp: amber', 'tlp amber'], + 'tlp:green': ['tlp:green', 'tlp: green', 'tlp green'], + 'tlp:white': ['tlp:white', 'tlp: white', 'tlp white'] + } +tlptag_default = sorted(tlptags.keys())[0] + +malwaretags = {'locky': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Locky"'], + 'jaff': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Jaff"'], + 'dridex': ['misp-galaxy:tool="dridex"'], + 'netwire': ['Netwire RAT'], + 'Pony': ['misp-galaxy:tool="Hancitor"'], + 'ursnif': ['misp-galaxy:tool="Snifula"'], + 'NanoCore': ['misp-galaxy:tool="NanoCoreRAT"'], + 'trickbot': ['misp-galaxy:tool="Trick Bot"'] + } + +# Tags to be set depending on the presence of other tags +dependingtags = {'tlp:white': ['circl:osint-feed'] + } + +# Known identifiers for forwarded messages +forward_identifiers = {'-------- Forwarded Message --------', 'Begin forwarded message:'} + +# Tags to add when hashes are found (e.g. to do automatic expansion) +hash_only_tags = {'TODO:VT-ENRICHMENT'} + +# If an attribute is on any MISP server side `warning list`, skip the creation of the attribute +skip_item_on_warninglist = True + +vt_key = None diff --git a/tests/tests.py b/tests/tests.py index 42bde92..44391b9 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -82,6 +82,12 @@ class TestMailToMISP(unittest.TestCase): self.assertEqual(self.mail2misp.misp_event.analysis, '0') self.mail2misp.add_event() + def test_attached_emails(self): + config = importlib.import_module('tests.config_carrier') + self.mail2misp = Mail2MISP('', '', '', config=config, offline=True) + with open('tests/mails/test_7_email_attachments.eml', 'rb') as f: + attached_emails = self.mail2misp.get_attached_emails(BytesIO(f.read())) + self.assertEqual(len(attached_emails), 7) if __name__ == '__main__': unittest.main()