mirror of https://github.com/MISP/mail_to_misp
Merge branch 'master' into master
Raphaël Vinot
GitHub
commit
62a596146b
|
|
@ -4,6 +4,7 @@
|
|||
import re
|
||||
import syslog
|
||||
import html
|
||||
import os
|
||||
from io import BytesIO
|
||||
from ipaddress import ip_address
|
||||
from email import message_from_bytes, policy, message
|
||||
|
|
@ -43,6 +44,9 @@ class Mail2MISP():
|
|||
setattr(self.config, 'enable_dns', False)
|
||||
self.debug = self.config.debug
|
||||
self.config_from_email_body = {}
|
||||
if not hasattr(self.config, 'ignore_nullsize_attachments'):
|
||||
setattr(self.config, 'ignore_nullsize_attachments', False)
|
||||
self.ignore_nullsize_attachments = self.config.ignore_nullsize_attachments
|
||||
# Init Faup
|
||||
self.f = Faup()
|
||||
self.sightings_to_add = []
|
||||
|
|
@ -50,15 +54,21 @@ class Mail2MISP():
|
|||
def load_email(self, pseudofile):
|
||||
self.pseudofile = pseudofile
|
||||
self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default)
|
||||
self.subject = self.original_mail.get('Subject')
|
||||
|
||||
try:
|
||||
self.sender = self.original_mail.get('From')
|
||||
except:
|
||||
self.sender = "<unknown sender>"
|
||||
|
||||
# Remove words from subject
|
||||
for removeword in self.config.removelist:
|
||||
self.subject = re.sub(removeword, "", self.subject).strip()
|
||||
try:
|
||||
self.subject = self.original_mail.get('Subject')
|
||||
# Remove words from subject
|
||||
for removeword in self.config.removelist:
|
||||
self.subject = re.sub(removeword, "", self.subject).strip()
|
||||
except Exception as ex:
|
||||
self.subject = "<subject could not be retrieved>"
|
||||
if self.debug:
|
||||
syslog.syslog(ex)
|
||||
|
||||
# Initialize the MISP event
|
||||
self.misp_event = MISPEvent()
|
||||
|
|
@ -132,27 +142,28 @@ class Mail2MISP():
|
|||
if email_object.attachments:
|
||||
# Create file objects for the attachments
|
||||
for attachment_name, attachment in email_object.attachments:
|
||||
if not attachment_name:
|
||||
attachment_name = 'NameMissing.txt'
|
||||
if self.config_from_email_body.get('attachment') == self.config.m2m_benign_attachment_keyword:
|
||||
a = self.misp_event.add_attribute('attachment', value=attachment_name, data=attachment)
|
||||
email_object.add_reference(a.uuid, 'related-to', 'Email attachment')
|
||||
else:
|
||||
f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False)
|
||||
if self.config.vt_key:
|
||||
try:
|
||||
vt_object = VTReportObject(self.config.vt_key, f_object.get_attributes_by_relation('sha256')[0].value, standalone=False)
|
||||
self.misp_event.add_object(vt_object)
|
||||
f_object.add_reference(vt_object.uuid, 'analysed-with')
|
||||
except InvalidMISPObject as e:
|
||||
print(e)
|
||||
pass
|
||||
self.misp_event.add_object(f_object)
|
||||
if main_object:
|
||||
self.misp_event.add_object(main_object)
|
||||
for section in sections:
|
||||
self.misp_event.add_object(section)
|
||||
email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment')
|
||||
if not (self.ignore_nullsize_attachments == True and attachment.getbuffer().nbytes == 0):
|
||||
if not attachment_name:
|
||||
attachment_name = 'NameMissing.txt'
|
||||
if self.config_from_email_body.get('attachment') == self.config.m2m_benign_attachment_keyword:
|
||||
a = self.misp_event.add_attribute('attachment', value=attachment_name, data=attachment)
|
||||
email_object.add_reference(a.uuid, 'related-to', 'Email attachment')
|
||||
else:
|
||||
f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False)
|
||||
if self.config.vt_key:
|
||||
try:
|
||||
vt_object = VTReportObject(self.config.vt_key, f_object.get_attributes_by_relation('sha256')[0].value, standalone=False)
|
||||
self.misp_event.add_object(vt_object)
|
||||
f_object.add_reference(vt_object.uuid, 'analysed-with')
|
||||
except InvalidMISPObject as e:
|
||||
print(e)
|
||||
pass
|
||||
self.misp_event.add_object(f_object)
|
||||
if main_object:
|
||||
self.misp_event.add_object(main_object)
|
||||
for section in sections:
|
||||
self.misp_event.add_object(section)
|
||||
email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment')
|
||||
self.process_body_iocs(email_object)
|
||||
if self.config.spamtrap or self.config.attach_original_mail or self.config_from_email_body.get('attach_original_mail'):
|
||||
self.misp_event.add_object(email_object)
|
||||
|
|
@ -404,3 +415,24 @@ class Mail2MISP():
|
|||
for value, source in self.sightings_to_add:
|
||||
self.sighting(value, source)
|
||||
return event
|
||||
|
||||
def get_attached_emails(self,pseudofile):
|
||||
|
||||
if self.debug:
|
||||
syslog.syslog("get_attached_emails Job started.")
|
||||
|
||||
forwarded_emails = []
|
||||
self.pseudofile = pseudofile
|
||||
self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default)
|
||||
for attachment in self.original_mail.iter_attachments():
|
||||
attachment_content = attachment.get_content()
|
||||
filename = attachment.get_filename()
|
||||
if self.debug:
|
||||
syslog.syslog(f'get_attached_emails: filename = {filename}')
|
||||
# Search for email forwarded as attachment
|
||||
# I could have more than one, attaching everything.
|
||||
if isinstance(attachment, message.EmailMessage) and os.path.splitext(filename)[1] == '.eml':
|
||||
# all attachments are identified as message.EmailMessage so filtering on extension for now.
|
||||
forwarded_emails.append(BytesIO(attachment_content))
|
||||
return forwarded_emails
|
||||
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ if __name__ == '__main__':
|
|||
misp_key = config.misp_key
|
||||
misp_verifycert = config.misp_verifycert
|
||||
debug = config.debug
|
||||
ignore_carrier_mail = config.ignore_carrier_mail
|
||||
except Exception as e:
|
||||
syslog.syslog(str(e))
|
||||
print("There is a problem with the configuration. A mandatory configuration variable is not set.")
|
||||
|
|
@ -55,19 +56,48 @@ if __name__ == '__main__':
|
|||
# receive data and subject through arguments
|
||||
raise Exception('This is not implemented anymore.')
|
||||
|
||||
syslog.syslog("About to create a mail2misp object.")
|
||||
mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event)
|
||||
mail2misp.load_email(pseudofile)
|
||||
attached_emails = mail2misp.get_attached_emails(pseudofile)
|
||||
syslog.syslog(f"found {len(attached_emails)} attached emails")
|
||||
if ignore_carrier_mail and len(attached_emails) !=0:
|
||||
syslog.syslog("Ignoring the carrier mail.")
|
||||
while len(attached_emails) !=0:
|
||||
pseudofile = attached_emails.pop()
|
||||
#Throw away the Mail2MISP object of the carrier mail and create a new one for each e-mail attachment
|
||||
mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event)
|
||||
mail2misp.load_email(pseudofile)
|
||||
|
||||
if debug:
|
||||
syslog.syslog(f'Working on {mail2misp.subject}')
|
||||
if debug:
|
||||
syslog.syslog(f'Working on {mail2misp.subject}')
|
||||
|
||||
if args.trap or config.spamtrap:
|
||||
mail2misp.email_from_spamtrap()
|
||||
if args.trap or config.spamtrap:
|
||||
mail2misp.email_from_spamtrap()
|
||||
else:
|
||||
mail2misp.process_email_body()
|
||||
|
||||
mail2misp.process_body_iocs()
|
||||
|
||||
if not args.event:
|
||||
mail2misp.add_event()
|
||||
|
||||
syslog.syslog("Job finished.")
|
||||
else:
|
||||
mail2misp.process_email_body()
|
||||
syslog.syslog("Running standard mail2misp")
|
||||
mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config, urlsonly=args.event)
|
||||
mail2misp.load_email(pseudofile)
|
||||
|
||||
mail2misp.process_body_iocs()
|
||||
if debug:
|
||||
syslog.syslog(f'Working on {mail2misp.subject}')
|
||||
|
||||
if args.trap or config.spamtrap:
|
||||
mail2misp.email_from_spamtrap()
|
||||
else:
|
||||
mail2misp.process_email_body()
|
||||
|
||||
mail2misp.process_body_iocs()
|
||||
|
||||
if not args.event:
|
||||
mail2misp.add_event()
|
||||
syslog.syslog("Job finished.")
|
||||
|
||||
if not args.event:
|
||||
mail2misp.add_event()
|
||||
syslog.syslog("Job finished.")
|
||||
|
|
|
|||
|
|
@ -18,6 +18,8 @@ debug = False
|
|||
nameservers = ['149.13.33.69']
|
||||
email_subject_prefix = 'M2M'
|
||||
attach_original_mail = False
|
||||
ignore_carrier_mail = False
|
||||
ignore_nullsize_attachments = False
|
||||
|
||||
excludelist = ('google.com', 'microsoft.com')
|
||||
externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net')
|
||||
|
|
|
|||
|
|
@ -0,0 +1,80 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
misp_url = 'YOUR_MISP_URL'
|
||||
misp_key = 'YOUR_KEY_HERE' # The MISP auth key can be found on the MISP web interface under the automation section
|
||||
misp_verifycert = True
|
||||
spamtrap = False
|
||||
default_distribution = 0
|
||||
default_threat_level = 3
|
||||
default_analysis = 1
|
||||
|
||||
body_config_prefix = 'm2m' # every line in the body starting with this value will be skipped from the IOCs
|
||||
m2m_key = 'YOUSETYOURKEYHERE'
|
||||
m2m_benign_attachment_keyword = 'benign'
|
||||
|
||||
debug = True
|
||||
nameservers = ['8.8.8.8']
|
||||
email_subject_prefix = 'M2M'
|
||||
attach_original_mail = True
|
||||
ignore_carrier_mail = True
|
||||
|
||||
excludelist = ('google.com', 'microsoft.com')
|
||||
externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net')
|
||||
internallist = ('internal.system.local')
|
||||
noidsflaglist = ('myexternalip.com', 'ipinfo.io', 'icanhazip.com', 'wtfismyip.com', 'ipecho.net',
|
||||
'api.ipify.org', 'checkip.amazonaws.com', 'whatismyipaddress.com', 'google.com',
|
||||
'dropbox.com'
|
||||
)
|
||||
|
||||
# Stop parsing when this term is found
|
||||
stopword = 'Whois & IP Information'
|
||||
|
||||
# Ignore lines in body of message containing:
|
||||
ignorelist = ("From:", "Sender:", "Received:", "Sender IP:", "Reply-To:", "Registrar WHOIS Server:",
|
||||
"Registrar:", "Domain Status:", "Registrant Email:", "IP Location:",
|
||||
"X-Get-Message-Sender-Via:", "X-Authenticated-Sender:")
|
||||
|
||||
# Ignore (don't add) attributes that are on server side warning list
|
||||
enforcewarninglist = True
|
||||
|
||||
# Add a sighting for each value
|
||||
sighting = False
|
||||
sighting_source = "YOUR_MAIL_TO_MISP_IDENTIFIER"
|
||||
|
||||
# Remove "Re:", "Fwd:" and {Spam?} from subject
|
||||
# add: "[\(\[].*?[\)\]]" to remove everything between [] and (): i.e. [tag]
|
||||
removelist = (r'Re:', r'Fwd:', r'\{Spam?\}')
|
||||
|
||||
# TLP tag setup
|
||||
# Tuples contain different variations of spelling
|
||||
tlptags = {'tlp:amber': ['tlp:amber', 'tlp: amber', 'tlp amber'],
|
||||
'tlp:green': ['tlp:green', 'tlp: green', 'tlp green'],
|
||||
'tlp:white': ['tlp:white', 'tlp: white', 'tlp white']
|
||||
}
|
||||
tlptag_default = sorted(tlptags.keys())[0]
|
||||
|
||||
malwaretags = {'locky': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Locky"'],
|
||||
'jaff': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Jaff"'],
|
||||
'dridex': ['misp-galaxy:tool="dridex"'],
|
||||
'netwire': ['Netwire RAT'],
|
||||
'Pony': ['misp-galaxy:tool="Hancitor"'],
|
||||
'ursnif': ['misp-galaxy:tool="Snifula"'],
|
||||
'NanoCore': ['misp-galaxy:tool="NanoCoreRAT"'],
|
||||
'trickbot': ['misp-galaxy:tool="Trick Bot"']
|
||||
}
|
||||
|
||||
# Tags to be set depending on the presence of other tags
|
||||
dependingtags = {'tlp:white': ['circl:osint-feed']
|
||||
}
|
||||
|
||||
# Known identifiers for forwarded messages
|
||||
forward_identifiers = {'-------- Forwarded Message --------', 'Begin forwarded message:'}
|
||||
|
||||
# Tags to add when hashes are found (e.g. to do automatic expansion)
|
||||
hash_only_tags = {'TODO:VT-ENRICHMENT'}
|
||||
|
||||
# If an attribute is on any MISP server side `warning list`, skip the creation of the attribute
|
||||
skip_item_on_warninglist = True
|
||||
|
||||
vt_key = None
|
||||
|
|
@ -90,5 +90,13 @@ class TestMailToMISP(unittest.TestCase):
|
|||
self.mail2misp.process_email_body()
|
||||
self.assertEqual(self.mail2misp.clean_email_body, 'example.org\r\nwww.example.org\r\n')
|
||||
|
||||
def test_attached_emails(self):
|
||||
config = importlib.import_module('tests.config_carrier')
|
||||
self.mail2misp = Mail2MISP('', '', '', config=config, offline=True)
|
||||
with open('tests/mails/test_7_email_attachments.eml', 'rb') as f:
|
||||
attached_emails = self.mail2misp.get_attached_emails(BytesIO(f.read()))
|
||||
self.assertEqual(len(attached_emails), 7)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue