Merge pull request #39 from begunrom/carrier_mail_with_emailattachments

Carrier mail with emailattachments
pull/44/head
Raphaël Vinot 2020-06-07 20:40:34 +02:00 committed by GitHub
commit 68b334df38
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 188 additions and 38 deletions

View File

@ -4,6 +4,7 @@
import re import re
import syslog import syslog
import html import html
import os
from io import BytesIO from io import BytesIO
from ipaddress import ip_address from ipaddress import ip_address
from email import message_from_bytes, policy, message from email import message_from_bytes, policy, message
@ -43,6 +44,9 @@ class Mail2MISP():
setattr(self.config, 'enable_dns', False) setattr(self.config, 'enable_dns', False)
self.debug = self.config.debug self.debug = self.config.debug
self.config_from_email_body = {} self.config_from_email_body = {}
if not hasattr(self.config, 'ignore_nullsize_attachments'):
setattr(self.config, 'ignore_nullsize_attachments', False)
self.ignore_nullsize_attachments = self.config.ignore_nullsize_attachments
# Init Faup # Init Faup
self.f = Faup() self.f = Faup()
self.sightings_to_add = [] self.sightings_to_add = []
@ -50,15 +54,21 @@ class Mail2MISP():
def load_email(self, pseudofile): def load_email(self, pseudofile):
self.pseudofile = pseudofile self.pseudofile = pseudofile
self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default) self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default)
self.subject = self.original_mail.get('Subject')
try: try:
self.sender = self.original_mail.get('From') self.sender = self.original_mail.get('From')
except: except:
self.sender = "<unknown sender>" self.sender = "<unknown sender>"
try:
self.subject = self.original_mail.get('Subject')
# Remove words from subject # Remove words from subject
for removeword in self.config.removelist: for removeword in self.config.removelist:
self.subject = re.sub(removeword, "", self.subject).strip() self.subject = re.sub(removeword, "", self.subject).strip()
except Exception as ex:
self.subject = "<subject could not be retrieved>"
if self.debug:
syslog.syslog(ex)
# Initialize the MISP event # Initialize the MISP event
self.misp_event = MISPEvent() self.misp_event = MISPEvent()
@ -127,6 +137,7 @@ class Mail2MISP():
if email_object.attachments: if email_object.attachments:
# Create file objects for the attachments # Create file objects for the attachments
for attachment_name, attachment in email_object.attachments: for attachment_name, attachment in email_object.attachments:
if not (self.ignore_nullsize_attachments == True and attachment.getbuffer().nbytes == 0):
if not attachment_name: if not attachment_name:
attachment_name = 'NameMissing.txt' attachment_name = 'NameMissing.txt'
if self.config_from_email_body.get('attachment') == self.config.m2m_benign_attachment_keyword: if self.config_from_email_body.get('attachment') == self.config.m2m_benign_attachment_keyword:
@ -399,3 +410,24 @@ class Mail2MISP():
for value, source in self.sightings_to_add: for value, source in self.sightings_to_add:
self.sighting(value, source) self.sighting(value, source)
return event return event
def get_attached_emails(self,pseudofile):
if self.debug:
syslog.syslog("get_attached_emails Job started.")
forwarded_emails = []
self.pseudofile = pseudofile
self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default)
for attachment in self.original_mail.iter_attachments():
attachment_content = attachment.get_content()
filename = attachment.get_filename()
if self.debug:
syslog.syslog(f'get_attached_emails: filename = {filename}')
# Search for email forwarded as attachment
# I could have more than one, attaching everything.
if isinstance(attachment, message.EmailMessage) and os.path.splitext(filename)[1] == '.eml':
# all attachments are identified as message.EmailMessage so filtering on extension for now.
forwarded_emails.append(BytesIO(attachment_content))
return forwarded_emails

View File

@ -33,6 +33,7 @@ if __name__ == '__main__':
misp_key = config.misp_key misp_key = config.misp_key
misp_verifycert = config.misp_verifycert misp_verifycert = config.misp_verifycert
debug = config.debug debug = config.debug
ignore_carrier_mail = config.ignore_carrier_mail
except Exception as e: except Exception as e:
syslog.syslog(str(e)) syslog.syslog(str(e))
print("There is a problem with the configuration. A mandatory configuration variable is not set.") print("There is a problem with the configuration. A mandatory configuration variable is not set.")
@ -55,6 +56,34 @@ if __name__ == '__main__':
# receive data and subject through arguments # receive data and subject through arguments
raise Exception('This is not implemented anymore.') raise Exception('This is not implemented anymore.')
syslog.syslog("About to create a mail2misp object.")
mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event)
attached_emails = mail2misp.get_attached_emails(pseudofile)
syslog.syslog(f"found {len(attached_emails)} attached emails")
if ignore_carrier_mail and len(attached_emails) !=0:
syslog.syslog("Ignoring the carrier mail.")
while len(attached_emails) !=0:
pseudofile = attached_emails.pop()
#Throw away the Mail2MISP object of the carrier mail and create a new one for each e-mail attachment
mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event)
mail2misp.load_email(pseudofile)
if debug:
syslog.syslog(f'Working on {mail2misp.subject}')
if args.trap or config.spamtrap:
mail2misp.email_from_spamtrap()
else:
mail2misp.process_email_body()
mail2misp.process_body_iocs()
if not args.event:
mail2misp.add_event()
syslog.syslog("Job finished.")
else:
syslog.syslog("Running standard mail2misp")
mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event) mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event)
mail2misp.load_email(pseudofile) mail2misp.load_email(pseudofile)
@ -71,3 +100,4 @@ if __name__ == '__main__':
if not args.event: if not args.event:
mail2misp.add_event() mail2misp.add_event()
syslog.syslog("Job finished.") syslog.syslog("Job finished.")

View File

@ -18,6 +18,8 @@ debug = False
nameservers = ['149.13.33.69'] nameservers = ['149.13.33.69']
email_subject_prefix = 'M2M' email_subject_prefix = 'M2M'
attach_original_mail = False attach_original_mail = False
ignore_carrier_mail = False
ignore_nullsize_attachments = False
excludelist = ('google.com', 'microsoft.com') excludelist = ('google.com', 'microsoft.com')
externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net') externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net')

80
tests/config_carrier.py Normal file
View File

@ -0,0 +1,80 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
misp_url = 'YOUR_MISP_URL'
misp_key = 'YOUR_KEY_HERE' # The MISP auth key can be found on the MISP web interface under the automation section
misp_verifycert = True
spamtrap = False
default_distribution = 0
default_threat_level = 3
default_analysis = 1
body_config_prefix = 'm2m' # every line in the body starting with this value will be skipped from the IOCs
m2m_key = 'YOUSETYOURKEYHERE'
m2m_benign_attachment_keyword = 'benign'
debug = True
nameservers = ['8.8.8.8']
email_subject_prefix = 'M2M'
attach_original_mail = True
ignore_carrier_mail = True
excludelist = ('google.com', 'microsoft.com')
externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net')
internallist = ('internal.system.local')
noidsflaglist = ('myexternalip.com', 'ipinfo.io', 'icanhazip.com', 'wtfismyip.com', 'ipecho.net',
'api.ipify.org', 'checkip.amazonaws.com', 'whatismyipaddress.com', 'google.com',
'dropbox.com'
)
# Stop parsing when this term is found
stopword = 'Whois & IP Information'
# Ignore lines in body of message containing:
ignorelist = ("From:", "Sender:", "Received:", "Sender IP:", "Reply-To:", "Registrar WHOIS Server:",
"Registrar:", "Domain Status:", "Registrant Email:", "IP Location:",
"X-Get-Message-Sender-Via:", "X-Authenticated-Sender:")
# Ignore (don't add) attributes that are on server side warning list
enforcewarninglist = True
# Add a sighting for each value
sighting = False
sighting_source = "YOUR_MAIL_TO_MISP_IDENTIFIER"
# Remove "Re:", "Fwd:" and {Spam?} from subject
# add: "[\(\[].*?[\)\]]" to remove everything between [] and (): i.e. [tag]
removelist = (r'Re:', r'Fwd:', r'\{Spam?\}')
# TLP tag setup
# Tuples contain different variations of spelling
tlptags = {'tlp:amber': ['tlp:amber', 'tlp: amber', 'tlp amber'],
'tlp:green': ['tlp:green', 'tlp: green', 'tlp green'],
'tlp:white': ['tlp:white', 'tlp: white', 'tlp white']
}
tlptag_default = sorted(tlptags.keys())[0]
malwaretags = {'locky': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Locky"'],
'jaff': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Jaff"'],
'dridex': ['misp-galaxy:tool="dridex"'],
'netwire': ['Netwire RAT'],
'Pony': ['misp-galaxy:tool="Hancitor"'],
'ursnif': ['misp-galaxy:tool="Snifula"'],
'NanoCore': ['misp-galaxy:tool="NanoCoreRAT"'],
'trickbot': ['misp-galaxy:tool="Trick Bot"']
}
# Tags to be set depending on the presence of other tags
dependingtags = {'tlp:white': ['circl:osint-feed']
}
# Known identifiers for forwarded messages
forward_identifiers = {'-------- Forwarded Message --------', 'Begin forwarded message:'}
# Tags to add when hashes are found (e.g. to do automatic expansion)
hash_only_tags = {'TODO:VT-ENRICHMENT'}
# If an attribute is on any MISP server side `warning list`, skip the creation of the attribute
skip_item_on_warninglist = True
vt_key = None

View File

@ -82,6 +82,12 @@ class TestMailToMISP(unittest.TestCase):
self.assertEqual(self.mail2misp.misp_event.analysis, '0') self.assertEqual(self.mail2misp.misp_event.analysis, '0')
self.mail2misp.add_event() self.mail2misp.add_event()
def test_attached_emails(self):
config = importlib.import_module('tests.config_carrier')
self.mail2misp = Mail2MISP('', '', '', config=config, offline=True)
with open('tests/mails/test_7_email_attachments.eml', 'rb') as f:
attached_emails = self.mail2misp.get_attached_emails(BytesIO(f.read()))
self.assertEqual(len(attached_emails), 7)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()