From 26ef177d58277a43a666935261996de8333c56a1 Mon Sep 17 00:00:00 2001 From: begunrom Date: Sat, 16 Nov 2019 16:47:43 +0100 Subject: [PATCH] carrier mail functionality Implements processing of a carrier mail that contains email attachments. Each email attachment is converted into an individual MISP event. --- mail2misp/mail2misp.py | 35 ++++++++++++++++++---- mail_to_misp.py | 54 ++++++++++++++++++++++++++-------- mail_to_misp_config.py-example | 1 + 3 files changed, 73 insertions(+), 17 deletions(-) diff --git a/mail2misp/mail2misp.py b/mail2misp/mail2misp.py index fab8b88..f9c3a15 100644 --- a/mail2misp/mail2misp.py +++ b/mail2misp/mail2misp.py @@ -4,6 +4,7 @@ import re import syslog import html +import os from io import BytesIO from ipaddress import ip_address from email import message_from_bytes, policy, message @@ -50,15 +51,19 @@ class Mail2MISP(): def load_email(self, pseudofile): self.pseudofile = pseudofile self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default) - self.subject = self.original_mail.get('Subject') + try: self.sender = self.original_mail.get('From') except: self.sender = "" - - # Remove words from subject - for removeword in self.config.removelist: - self.subject = re.sub(removeword, "", self.subject).strip() + + try: + self.subject = self.original_mail.get('Subject') + # Remove words from subject + for removeword in self.config.removelist: + self.subject = re.sub(removeword, "", self.subject).strip() + except: + self.subject = "" # Initialize the MISP event self.misp_event = MISPEvent() @@ -399,3 +404,23 @@ class Mail2MISP(): for value, source in self.sightings_to_add: self.sighting(value, source) return event + + def get_attached_emails(self,pseudofile): + + syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_USER) + syslog.syslog("get_attached_emails Job started.") + + forwarded_emails = [] + self.pseudofile = pseudofile + self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default) + for attachment in self.original_mail.iter_attachments(): + attachment_content = attachment.get_content() + filename = attachment.get_filename() + syslog.syslog(f'get_attached_emails: filename = {filename}') + # Search for email forwarded as attachment + # I could have more than one, attaching everything. + if isinstance(attachment, message.EmailMessage) and os.path.splitext(filename)[1] == '.eml': + # all attachments are identified as message.EmailMessage so filtering on extension for now. + forwarded_emails.append(BytesIO(attachment_content)) + return forwarded_emails + diff --git a/mail_to_misp.py b/mail_to_misp.py index 34ec478..8b2864e 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -33,6 +33,7 @@ if __name__ == '__main__': misp_key = config.misp_key misp_verifycert = config.misp_verifycert debug = config.debug + ignore_carrier_mail = config.ignore_carrier_mail except Exception as e: syslog.syslog(str(e)) print("There is a problem with the configuration. A mandatory configuration variable is not set.") @@ -55,19 +56,48 @@ if __name__ == '__main__': # receive data and subject through arguments raise Exception('This is not implemented anymore.') + syslog.syslog("About to create a mail2misp object.") mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event) - mail2misp.load_email(pseudofile) - - if debug: - syslog.syslog(f'Working on {mail2misp.subject}') - - if args.trap or config.spamtrap: - mail2misp.email_from_spamtrap() + attached_emails = mail2misp.get_attached_emails(pseudofile) + syslog.syslog(f"found {len(attached_emails)} attached emails") + if ignore_carrier_mail and len(attached_emails) !=0: + syslog.syslog("Ignoring the carrier mail.") + while len(attached_emails) !=0: + pseudofile = attached_emails.pop() + #Throw away the Mail2MISP object of the carrier mail and create a new one for each e-mail attachment + mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event) + mail2misp.load_email(pseudofile) + + if debug: + syslog.syslog(f'Working on {mail2misp.subject}') + + if args.trap or config.spamtrap: + mail2misp.email_from_spamtrap() + else: + mail2misp.process_email_body() + + mail2misp.process_body_iocs() + + if not args.event: + mail2misp.add_event() + + syslog.syslog("Job finished.") else: - mail2misp.process_email_body() + syslog.syslog("Running standard mail2misp") + mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event) + mail2misp.load_email(pseudofile) - mail2misp.process_body_iocs() + if debug: + syslog.syslog(f'Working on {mail2misp.subject}') + + if args.trap or config.spamtrap: + mail2misp.email_from_spamtrap() + else: + mail2misp.process_email_body() + + mail2misp.process_body_iocs() + + if not args.event: + mail2misp.add_event() + syslog.syslog("Job finished.") - if not args.event: - mail2misp.add_event() - syslog.syslog("Job finished.") diff --git a/mail_to_misp_config.py-example b/mail_to_misp_config.py-example index b4f9801..b321c4d 100644 --- a/mail_to_misp_config.py-example +++ b/mail_to_misp_config.py-example @@ -18,6 +18,7 @@ debug = False nameservers = ['149.13.33.69'] email_subject_prefix = 'M2M' attach_original_mail = False +ignore_carrier_mail = False excludelist = ('google.com', 'microsoft.com') externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net')