mail_to_misp/mail_to_misp.py

377 lines
17 KiB
Python
Raw Normal View History

2017-06-01 17:00:32 +02:00
#!/usr/bin/env python3
2017-05-24 11:02:28 +02:00
# -*- coding: utf-8 -*-
2017-04-27 13:58:49 +02:00
2017-06-01 09:02:11 +02:00
import sys
2018-05-02 19:08:22 +02:00
import argparse
import re
import syslog
from pathlib import Path
from io import BytesIO
from ipaddress import ip_address
from email import message_from_bytes, policy
import importlib
2017-06-01 09:02:11 +02:00
try:
import urlmarker
import hashmarker
from pyfaup.faup import Faup
2018-05-02 19:08:22 +02:00
from pymisp import PyMISP, MISPEvent, MISPObject, MISPSighting
from pymisp.tools import EMailObject, make_binary_objects
2017-06-01 09:02:11 +02:00
from defang import refang
import dns.resolver
except ImportError as e:
print("(!) Problem loading module:")
print(e)
sys.exit(-1)
2017-05-30 11:24:30 +02:00
2018-04-27 16:38:38 +02:00
2018-05-02 19:08:22 +02:00
def is_ip(address):
2017-05-24 15:44:14 +02:00
try:
2018-05-02 19:08:22 +02:00
ip_address(address)
except ValueError:
2017-05-24 15:44:14 +02:00
return False
return True
2018-04-27 16:38:38 +02:00
2018-05-02 19:08:22 +02:00
class Mail2MISP():
def __init__(self, misp_url, misp_key, verifycert, config):
self.misp = PyMISP(misp_url, misp_key, verifycert, debug=config.debug)
self.debug = config.debug
self.config = config
# Init Faup
self.f = Faup()
2018-05-02 19:08:22 +02:00
def load_email(self, pseudofile):
self.pseudofile = pseudofile
self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default)
self.subject = self.original_mail.get('Subject')
# Remove words from subject
for removeword in self.config.removelist:
self.subject = re.sub(removeword, "", self.subject)
# Initialize the MISP event
self.misp_event = MISPEvent()
self.misp_event.info = f'{config.email_subject_prefix} - {self.subject}'
self.misp_event.distribution = config.m2m_auto_distribution
self.misp_event.threat_level_id = 3
self.misp_event.analysis = 1
self.misp_event.add_tag(config.tlptag_default)
def sighting(self, value, source):
'''Add a sighting'''
s = MISPSighting()
s.from_dict(value=value, source=source)
self.misp.set_sightings(s)
def _find_inline_forward(self):
'''Does the body contains a forwarded email?'''
for identifier in config.forward_identifiers:
if identifier in self.clean_email_body:
self.clean_email_body, fw_email = self.clean_email_body.split(identifier)
self.forwarded_email(pseudofile=BytesIO(fw_email.encode()))
def _find_attached_forward(self):
for attachment in self.original_mail.iter_attachments():
# Search for email forwarded as attachment
# I could have more than one, attaching everything.
if attachment.get_filename() and attachment.get_filename().endswith('.eml'):
self.forwarded_email(pseudofile=BytesIO(attachment.get_content().as_bytes()))
2018-05-02 19:08:22 +02:00
else:
if self.config_from_email_body.get('attachment') == 'benign':
# Attach sane file
self.misp_event.add_attribute('attachment', value='Report',
data=BytesIO(attachment.get_content().as_bytes()))
2018-05-02 19:08:22 +02:00
else:
f_object, main_object, sections = make_binary_objects(pseudofile=BytesIO(attachment.get_content()),
2018-05-02 19:08:22 +02:00
filename=attachment.get_filename(), standalone=False)
self.misp_event.add_object(f_object)
if main_object:
self.misp_event.add_object(main_object)
[self.misp_event.add_object(section) for section in sections]
def email_from_spamtrap(self):
'''The email comes from a spamtrap and should be attached as-is.'''
self.clean_email_body = self.original_mail.get_body().as_string()
self.forwarded_email(self.pseudofile)
def forwarded_email(self, pseudofile: BytesIO):
'''Extracts all possible indicators out of an email and create a MISP event out of it.
* Gets all relevant Headers
* Attach the body
* Create MISP file objects (uses lief if possible)
* Set all references
'''
email_object = EMailObject(pseudofile=pseudofile, attach_original_mail=True, standalone=False)
2018-05-02 19:08:22 +02:00
if email_object.attachments:
# Create file objects for the attachments
for attachment_name, attachment in email_object.attachments:
f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False)
self.misp_event.add_object(f_object)
if main_object:
self.misp_event.add_object(main_object)
for section in sections:
self.misp_event.add_object(section)
email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment')
self.process_body_iocs(email_object)
2018-05-02 19:08:22 +02:00
self.misp_event.add_object(email_object)
def process_email_body(self):
self.clean_email_body = self.original_mail.get_body().as_string()
# Check if there are config lines in the body & convert them to a python dictionary:
# <config.body_config_prefix>:<key>:<value> => {<key>: <value>}
self.config_from_email_body = {k: v for k, v in re.findall(f'{config.body_config_prefix}:(.*):(.*)', self.clean_email_body)}
if self.config_from_email_body:
# ... remove the config lines from the body
self.clean_email_body = re.sub(rf'^{config.body_config_prefix}.*\n?', '',
self.original_mail.get_body().as_string(), flags=re.MULTILINE)
self._find_inline_forward()
self._find_attached_forward()
# # Prepare extraction of IOCs
# Refang email data
self.clean_email_body = refang(self.clean_email_body)
# Depending on the source of the mail, there is some cleanup to do. Ignore lines in body of message
for ignoreline in config.ignorelist:
self.clean_email_body = re.sub(rf'^{ignoreline}.*\n?', '', self.clean_email_body, flags=re.MULTILINE)
# Check if autopublish key is present and valid
if self.config_from_email_body.get('m2mkey') == config.m2m_key:
self.misp_event.publish()
# Add tags to the event if keywords are found in the mail
for tag in config.tlptags:
if any(alternativetag in self.clean_email_body for alternativetag in config.tlptags[tag]):
self.misp_event.add_tag(tag)
# Remove everything after the stopword from the body
self.clean_email_body = self.clean_email_body.split(config.stopword, 1)[0]
def process_body_iocs(self, email_object=None):
if email_object:
body = email_object.email.get_body().as_string()
else:
body = self.clean_email_body
2018-05-02 19:08:22 +02:00
# Extract and add hashes
contains_hash = False
for h in set(re.findall(hashmarker.MD5_REGEX, body)):
2018-05-02 19:08:22 +02:00
contains_hash = True
attribute = self.misp_event.add_attribute('md5', h, enforceWarninglist=config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
2018-05-02 19:08:22 +02:00
if config.sighting:
self.sighting(h, config.sighting_source)
for h in set(re.findall(hashmarker.SHA1_REGEX, body)):
2018-05-02 19:08:22 +02:00
contains_hash = True
attribute = self.misp_event.add_attribute('sha1', h, enforceWarninglist=config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
2018-05-02 19:08:22 +02:00
if config.sighting:
self.sighting(h, config.sighting_source)
for h in set(re.findall(hashmarker.SHA256_REGEX, body)):
2018-05-02 19:08:22 +02:00
contains_hash = True
attribute = self.misp_event.add_attribute('sha256', h, enforceWarninglist=config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
2018-05-02 19:08:22 +02:00
if config.sighting:
self.sighting(h, config.sighting_source)
if contains_hash:
[self.misp_event.add_tag(tag) for tag in config.hash_only_tags]
# # Extract network IOCs
urllist = []
urllist += re.findall(urlmarker.WEB_URL_REGEX, body)
urllist += re.findall(urlmarker.IP_REGEX, body)
2018-05-02 19:08:22 +02:00
if self.debug:
syslog.syslog(str(urllist))
hostname_processed = []
# Add IOCs and expanded information to MISP
for entry in set(urllist):
ids_flag = True
self.f.decode(entry)
2018-05-02 19:08:22 +02:00
domainname = self.f.get_domain().decode()
2018-05-02 19:08:22 +02:00
if domainname in config.excludelist:
# Ignore the entry
continue
hostname = self.f.get_host().decode()
2018-05-02 19:08:22 +02:00
scheme = self.f.get_scheme()
2018-05-02 19:08:22 +02:00
if scheme:
scheme = scheme.decode()
resource_path = self.f.get_resource_path()
2018-05-02 19:08:22 +02:00
if resource_path:
resource_path = resource_path.decode()
2017-12-21 09:46:19 +01:00
2018-05-02 19:08:22 +02:00
if debug:
syslog.syslog(domainname)
if domainname in config.internallist: # Add link to internal reference
attribute = self.misp_event.add_attribute('link', entry, category='Internal reference',
to_ids=False, enforceWarninglist=False)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
2018-05-02 19:08:22 +02:00
elif domainname in config.externallist: # External analysis
attribute = self.misp_event.add_attribute('link', entry, category='External analysis',
to_ids=False, enforceWarninglist=False)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
2018-05-02 19:08:22 +02:00
else: # The URL is probably an indicator.
comment = ""
if (domainname in config.noidsflaglist) or (hostname in config.noidsflaglist):
ids_flag = False
comment = "Known host (mostly for connectivity test or IP lookup)"
if debug:
syslog.syslog(str(entry))
2018-04-27 16:38:38 +02:00
2018-05-02 19:08:22 +02:00
if scheme:
if is_ip(hostname):
attribute = self.misp_event.add_attribute('url', entry, to_ids=False,
enforceWarninglist=config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
2018-05-02 19:08:22 +02:00
else:
if resource_path: # URL has path, ignore warning list
attribute = self.misp_event.add_attribute('url', entry, to_ids=ids_flag,
enforceWarninglist=False, comment=comment)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
2018-05-02 19:08:22 +02:00
else: # URL has no path
attribute = self.misp_event.add_attribute('url', entry, to_ids=ids_flag,
enforceWarninglist=config.enforcewarninglist, comment=comment)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
2018-05-02 19:08:22 +02:00
if config.sighting:
self.sighting(entry, config.sighting_source)
if hostname in hostname_processed:
# Hostname already processed.
continue
hostname_processed.append(hostname)
if config.sighting:
self.sighting(hostname, config.sighting_source)
2017-12-21 09:46:19 +01:00
2018-05-02 19:08:22 +02:00
if debug:
syslog.syslog(hostname)
2018-04-27 16:38:38 +02:00
comment = ''
port = self.f.get_port()
2018-05-02 19:08:22 +02:00
if port:
port = port.decode()
comment = f'on port: {port}'
2017-05-23 15:09:25 +02:00
2018-05-02 19:08:22 +02:00
if is_ip(hostname):
attribute = self.misp_event.add_attribute('ip-dst', hostname, to_ids=ids_flag,
enforceWarninglist=config.enforcewarninglist,
comment=comment)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
2018-05-02 19:08:22 +02:00
else:
related_ips = []
try:
for rdata in dns.resolver.query(hostname, 'A'):
if debug:
syslog.syslog(str(rdata))
related_ips.append(rdata.to_text())
except Exception as e:
if debug:
syslog.syslog(str(e))
if related_ips:
hip = MISPObject(name='ip-port')
hip.add_attribute('hostname', value=hostname, to_ids=ids_flag,
enforceWarninglist=config.enforcewarninglist, comment=comment)
for ip in set(related_ips):
hip.add_attribute('ip', type='ip-dst', value=ip, to_ids=False,
enforceWarninglist=config.enforcewarninglist)
self.misp_event.add_object(hip)
if email_object:
email_object.add_reference(hip.uuid, 'contains')
2018-05-02 19:08:22 +02:00
else:
attribute = self.misp_event.add_attribute('hostname', value=hostname,
to_ids=ids_flag, enforceWarninglist=config.enforcewarninglist,
comment=comment)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
2018-05-02 19:08:22 +02:00
def add_event(self):
'''Add event on the remote MISP instance.'''
# Add additional tags depending on others
tags = []
for tag in [t.name for t in self.misp_event.tags]:
if config.dependingtags.get(tag):
tags += config.dependingtags.get(tag)
# Add additional tags according to configuration
for malware in config.malwaretags:
if malware.lower() in self.subject.lower():
tags += config.malwaretags.get(malware)
if tags:
[self.misp_event.add_tag(tag) for tag in tags]
self.misp.add_event(self.misp_event)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Push a Mail into a MISP instance')
parser.add_argument("-r", "--read", help="Read from tempfile.")
parser.add_argument("-t", "--trap", action='store_true', default=False, help="Import the Email as-is.")
parser.add_argument('infile', nargs='?', type=argparse.FileType('rb'))
2018-05-02 19:08:22 +02:00
args = parser.parse_args()
syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_USER)
syslog.syslog("Job started.")
configmodule = Path(__file__).as_posix().replace('.py', '_config')
if Path(f'{configmodule}.py').exists():
config = importlib.import_module(configmodule)
try:
misp_url = config.misp_url
misp_key = config.misp_key
misp_verifycert = config.misp_verifycert
debug = config.debug
except Exception as e:
syslog.syslog(str(e))
print("There is a problem with the configuration. A mandatory configuration variable is not set.")
print("Did you just update? mail_to_misp might have new configuration variables.")
print("Please compare with the configuration example.")
print("\nTrace:")
print(e)
sys.exit(-2)
else:
2018-05-02 19:08:22 +02:00
print("Couldn't locate config file {0}".format(f'{configmodule}.py'))
sys.exit(-1)
if args.infile:
pseudofile = BytesIO(args.infile.read().encode())
elif args.read:
# read from tempfile
with open(args.read, 'rb') as f:
pseudofile = BytesIO(f.read())
2017-06-01 18:39:39 +02:00
else:
2018-05-02 19:08:22 +02:00
# receive data and subject through arguments
raise Exception('This is not implemented anymore.')
2018-05-02 19:08:22 +02:00
mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config)
mail2misp.load_email(pseudofile)
2017-04-27 13:58:49 +02:00
if debug:
2018-05-02 19:08:22 +02:00
syslog.syslog(f'Working on {mail2misp.subject}')
if args.trap or config.spamtrap:
mail2misp.email_from_spamtrap()
else:
mail2misp.process_email_body()
2018-05-02 19:08:22 +02:00
mail2misp.process_body_iocs()
2018-05-02 19:08:22 +02:00
mail2misp.add_event()
syslog.syslog("Job finished.")