Merge pull request #39 from begunrom/carrier_mail_with_emailattachments

Carrier mail with emailattachments

mail2misp/mail2misp.py

@@ -4,6 +4,7 @@
 import re
 import syslog
 import html
+import os
 from io import BytesIO
 from ipaddress import ip_address
 from email import message_from_bytes, policy, message
@@ -43,6 +44,9 @@ class Mail2MISP():
             setattr(self.config, 'enable_dns', False)
         self.debug = self.config.debug
         self.config_from_email_body = {}
+        if not hasattr(self.config, 'ignore_nullsize_attachments'):
+            setattr(self.config, 'ignore_nullsize_attachments', False)
+        self.ignore_nullsize_attachments = self.config.ignore_nullsize_attachments
         # Init Faup
         self.f = Faup()
         self.sightings_to_add = []
@@ -50,15 +54,21 @@ class Mail2MISP():
     def load_email(self, pseudofile):
         self.pseudofile = pseudofile
         self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default)
-        self.subject = self.original_mail.get('Subject')
+
         try:
             self.sender = self.original_mail.get('From')
         except:
             self.sender = "<unknown sender>"
         
+        try:
+            self.subject = self.original_mail.get('Subject')
             # Remove words from subject
             for removeword in self.config.removelist:
                 self.subject = re.sub(removeword, "", self.subject).strip()
+        except Exception as ex:
+            self.subject = "<subject could not be retrieved>"
+            if self.debug:
+                syslog.syslog(ex)
 
         # Initialize the MISP event
         self.misp_event = MISPEvent()
@@ -127,6 +137,7 @@ class Mail2MISP():
         if email_object.attachments:
             # Create file objects for the attachments
             for attachment_name, attachment in email_object.attachments:
+                if not (self.ignore_nullsize_attachments == True and attachment.getbuffer().nbytes == 0):
                     if not attachment_name:
                         attachment_name = 'NameMissing.txt'
                     if self.config_from_email_body.get('attachment') == self.config.m2m_benign_attachment_keyword:
@@ -399,3 +410,24 @@ class Mail2MISP():
             for value, source in self.sightings_to_add:
                 self.sighting(value, source)
         return event
+
+    def get_attached_emails(self,pseudofile):
+        
+        if self.debug:
+            syslog.syslog("get_attached_emails Job started.")
+
+        forwarded_emails = []
+        self.pseudofile = pseudofile
+        self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default)
+        for attachment in self.original_mail.iter_attachments():
+            attachment_content = attachment.get_content()
+            filename = attachment.get_filename()
+            if self.debug:
+                syslog.syslog(f'get_attached_emails: filename = {filename}')
+            # Search for email forwarded as attachment
+            # I could have more than one, attaching everything.
+            if isinstance(attachment, message.EmailMessage) and os.path.splitext(filename)[1] == '.eml':
+                # all attachments are identified as message.EmailMessage so filtering on extension for now.
+                forwarded_emails.append(BytesIO(attachment_content))
+        return forwarded_emails
+

mail_to_misp.py

@@ -33,6 +33,7 @@ if __name__ == '__main__':
             misp_key = config.misp_key
             misp_verifycert = config.misp_verifycert
             debug = config.debug
+            ignore_carrier_mail = config.ignore_carrier_mail
         except Exception as e:
             syslog.syslog(str(e))
             print("There is a problem with the configuration. A mandatory configuration variable is not set.")
@@ -55,6 +56,34 @@ if __name__ == '__main__':
         # receive data and subject through arguments
         raise Exception('This is not implemented anymore.')
 
+    syslog.syslog("About to create a mail2misp object.")
+    mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event)
+    attached_emails = mail2misp.get_attached_emails(pseudofile)
+    syslog.syslog(f"found {len(attached_emails)} attached emails")
+    if ignore_carrier_mail and len(attached_emails) !=0:
+        syslog.syslog("Ignoring the carrier mail.")
+        while len(attached_emails) !=0:
+            pseudofile = attached_emails.pop()
+            #Throw away the Mail2MISP object of the carrier mail and create a new one for each e-mail attachment
+            mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event)
+            mail2misp.load_email(pseudofile)
+            
+            if debug:
+                syslog.syslog(f'Working on {mail2misp.subject}')
+        
+            if args.trap or config.spamtrap:
+                mail2misp.email_from_spamtrap()
+            else:
+                mail2misp.process_email_body()
+        
+            mail2misp.process_body_iocs()
+        
+            if not args.event:
+                mail2misp.add_event()
+            
+        syslog.syslog("Job finished.")
+    else:
+        syslog.syslog("Running standard mail2misp")
         mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event)
         mail2misp.load_email(pseudofile)
 
@@ -71,3 +100,4 @@ if __name__ == '__main__':
         if not args.event:
             mail2misp.add_event()
         syslog.syslog("Job finished.")
+

mail_to_misp_config.py-example

@@ -18,6 +18,8 @@ debug = False
 nameservers = ['149.13.33.69']
 email_subject_prefix = 'M2M'
 attach_original_mail = False
+ignore_carrier_mail = False
+ignore_nullsize_attachments = False
 
 excludelist = ('google.com', 'microsoft.com')
 externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net')

tests/config_carrier.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+misp_url = 'YOUR_MISP_URL'
+misp_key = 'YOUR_KEY_HERE'  # The MISP auth key can be found on the MISP web interface under the automation section
+misp_verifycert = True
+spamtrap = False
+default_distribution = 0
+default_threat_level = 3
+default_analysis = 1
+
+body_config_prefix = 'm2m'  # every line in the body starting with this value will be skipped from the IOCs
+m2m_key = 'YOUSETYOURKEYHERE'
+m2m_benign_attachment_keyword = 'benign'
+
+debug = True
+nameservers = ['8.8.8.8']
+email_subject_prefix = 'M2M'
+attach_original_mail = True
+ignore_carrier_mail = True
+
+excludelist = ('google.com', 'microsoft.com')
+externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net')
+internallist = ('internal.system.local')
+noidsflaglist = ('myexternalip.com', 'ipinfo.io', 'icanhazip.com', 'wtfismyip.com', 'ipecho.net',
+                 'api.ipify.org', 'checkip.amazonaws.com', 'whatismyipaddress.com', 'google.com',
+                 'dropbox.com'
+                 )
+
+# Stop parsing when this term is found
+stopword = 'Whois & IP Information'
+
+# Ignore lines in body of message containing:
+ignorelist = ("From:", "Sender:", "Received:", "Sender IP:", "Reply-To:", "Registrar WHOIS Server:",
+              "Registrar:", "Domain Status:", "Registrant Email:", "IP Location:",
+              "X-Get-Message-Sender-Via:", "X-Authenticated-Sender:")
+
+# Ignore (don't add) attributes that are on server side warning list
+enforcewarninglist = True
+
+# Add a sighting for each value
+sighting = False
+sighting_source = "YOUR_MAIL_TO_MISP_IDENTIFIER"
+
+# Remove "Re:", "Fwd:" and {Spam?} from subject
+# add: "[\(\[].*?[\)\]]" to remove everything between [] and (): i.e. [tag]
+removelist = (r'Re:', r'Fwd:', r'\{Spam?\}')
+
+# TLP tag setup
+# Tuples contain different variations of spelling
+tlptags = {'tlp:amber': ['tlp:amber', 'tlp: amber', 'tlp amber'],
+           'tlp:green': ['tlp:green', 'tlp: green', 'tlp green'],
+           'tlp:white': ['tlp:white', 'tlp: white', 'tlp white']
+           }
+tlptag_default = sorted(tlptags.keys())[0]
+
+malwaretags = {'locky': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Locky"'],
+               'jaff': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Jaff"'],
+               'dridex': ['misp-galaxy:tool="dridex"'],
+               'netwire': ['Netwire RAT'],
+               'Pony': ['misp-galaxy:tool="Hancitor"'],
+               'ursnif': ['misp-galaxy:tool="Snifula"'],
+               'NanoCore': ['misp-galaxy:tool="NanoCoreRAT"'],
+               'trickbot': ['misp-galaxy:tool="Trick Bot"']
+               }
+
+# Tags to be set depending on the presence of other tags
+dependingtags = {'tlp:white': ['circl:osint-feed']
+                 }
+
+# Known identifiers for forwarded messages
+forward_identifiers = {'-------- Forwarded Message --------', 'Begin forwarded message:'}
+
+# Tags to add when hashes are found (e.g. to do automatic expansion)
+hash_only_tags = {'TODO:VT-ENRICHMENT'}
+
+# If an attribute is on any MISP server side `warning list`, skip the creation of the attribute
+skip_item_on_warninglist = True
+
+vt_key = None

tests/tests.py

@@ -82,6 +82,12 @@ class TestMailToMISP(unittest.TestCase):
         self.assertEqual(self.mail2misp.misp_event.analysis, '0')
         self.mail2misp.add_event()
 
+    def test_attached_emails(self):
+        config = importlib.import_module('tests.config_carrier')
+        self.mail2misp = Mail2MISP('', '', '', config=config, offline=True)
+        with open('tests/mails/test_7_email_attachments.eml', 'rb') as f:
+            attached_emails = self.mail2misp.get_attached_emails(BytesIO(f.read()))
+        self.assertEqual(len(attached_emails), 7)
 
 if __name__ == '__main__':
     unittest.main()