diff --git a/README.md b/README.md index a8e0d57..af46f74 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Connect your mail infrastructure to [MISP](https://github.com/MISP/MISP) in orde - Extraction of URLs and IP addresses (and port numbers) from free text emails - Extraction of hostnames from URLs - Extraction of hashes (MD5, SHA1, SHA256) -- DNS expansion +- DNS expansion - Custom filter list for lines containing specific words - Subject filters - Respecting TLP classification mentioned in free text (including optional spelling robustness) @@ -53,7 +53,7 @@ The implemented workflow is mainly for mail servers like Postfix. Client side im 3. Mozilla Thunderbird [deprecated] `Email -> Thunderbird -> Mail rule -> filterscript -> thunderbird_wrapper -> mail_to_misp -> PyMISP -> MISP` - + ## Installation @@ -112,7 +112,7 @@ Outlook is not implemented due to lack of test environment. However, it should b ``` import win32com.client import pythoncom - + class Handler_Class(object): def OnNewMailEx(self, receivedItemsIDs): for ID in receivedItemsIDs.split(","): @@ -122,13 +122,13 @@ class Handler_Class(object): print "Subj: " + mailItem.Subject print "Body: " + mailItem.Body.encode( 'ascii', 'ignore' ) print "========" - + outlook = win32com.client.DispatchWithEvents("Outlook.Application", Handler_Class) pythoncom.PumpMessages() ``` (from: https://blog.matthewurch.ca/?p=236) -Obviously, you would like to filter mails based on subject or from address and pass subject and body to mail_to_misp.py in order to do something useful. Pull-requests welcome for actual implementations :) +Obviously, you would like to filter mails based on subject or from address and pass subject and body to mail_to_misp.py in order to do something useful. Pull-requests welcome for actual implementations :) ## Requirements @@ -138,19 +138,11 @@ Obviously, you would like to filter mails based on subject or from address and p - mail_to_misp requires access to a MISP instance (via API). - Python 3 - dnspython -- hashlib +- PyMISP - faup from https://github.com/stricaud/faup - urlmarker from https://github.com/rcompton/ryancompton.net/blob/master/assets/praw_drugs/urlmarker.py (contained in this project) - ftfy from https://github.com/LuminosoInsight/python-ftfy (to fix unicode text) -- defang from https://bitbucket.org/johannestaas/defang -- Patch defang/defang/__init__.py by commenting out the following line (not used && not compatible with Python 3): -`from urllib2 import urlparse` -- Optionally patch defang/defang/__init__.py and add at line 47: -``` - dirty_line = dirty_line.replace('hxxp', 'http') - dirty_line = dirty_line.replace('purr', 'http') - dirty_line = dirty_line.replace('meow', 'http') -``` +- defang from https://github.com/Rafiot/defang.git (fork of: https://bitbucket.org/johannestaas/defang) ### Thunderbird [deprecated] diff --git a/mail_to_misp.py b/mail_to_misp.py index b0647cd..8d442ed 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -1,370 +1,349 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import os import sys -try: - configfile = os.path.basename(sys.argv[0]).split(".py")[0] + "_config" -except Exception as e: - print("Couldn't locate config file {0}".format(configfile)) - sys.exit(-1) +import argparse +import re +import syslog +from pathlib import Path +from io import BytesIO +from ipaddress import ip_address +from email import message_from_bytes, policy +import importlib try: import urlmarker import hashmarker - import re from pyfaup.faup import Faup - from pymisp import PyMISP, MISPEvent, MISPObject + from pymisp import PyMISP, MISPEvent, MISPObject, MISPSighting + from pymisp.tools import EMailObject, make_binary_objects from defang import refang import dns.resolver - import email - import tempfile - import socket - import syslog - import ftfy - import hashlib - config = __import__(configfile) except ImportError as e: print("(!) Problem loading module:") print(e) sys.exit(-1) -syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_USER) - -def is_valid_ipv4_address(address): +def is_ip(address): try: - socket.inet_pton(socket.AF_INET, address) - except AttributeError: # no inet_pton here, sorry - try: - socket.inet_aton(address) - except socket.error: - return False - return address.count('.') == 3 - except socket.error: # not a valid address + ip_address(address) + except ValueError: return False return True -def is_valid_ipv6_address(address): - try: - socket.inet_pton(socket.AF_INET6, address) - except socket.error: # not a valid address - return False - return True +class Mail2MISP(): + def __init__(self, misp_url, misp_key, verifycert, config): + self.misp = PyMISP(misp_url, misp_key, verifycert, debug=config.debug) + self.debug = config.debug + self.config = config -def init(url, key): - return PyMISP(url, key, misp_verifycert, 'json', debug=debug) + def load_email(self, pseudofile): + self.pseudofile = pseudofile + self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default) + self.subject = self.original_mail.get('Subject') + # Remove words from subject + for removeword in self.config.removelist: + self.subject = re.sub(removeword, "", self.subject) + # Initialize the MISP event + self.misp_event = MISPEvent() + self.misp_event.info = f'{config.email_subject_prefix} - {self.subject}' + self.misp_event.distribution = config.m2m_auto_distribution + self.misp_event.threat_level_id = 3 + self.misp_event.analysis = 1 + self.misp_event.add_tag(config.tlptag_default) -# Add a sighting -def sight(sighting, value): - if sighting: - d = {'value': value, 'source': sighting_source} - misp.set_sightings(d) + def sighting(self, value, source): + '''Add a sighting''' + s = MISPSighting() + s.from_dict(value=value, source=source) + self.misp.set_sightings(s) + def _find_inline_forward(self): + '''Does the body contains a forwarded email?''' + for identifier in config.forward_identifiers: + if identifier in self.clean_email_body: + self.clean_email_body, fw_email = self.clean_email_body.split(identifier) + self.forwarded_email(pseudofile=BytesIO(fw_email.encode())) -# Add named attribute and sight if configured -def add_attribute(event, attribute_type, value, category, ids_flag, warninglist, sighting, comment=None): - syslog.syslog("Event " + event['Event']['id'] + ": Adding attribute (" + attribute_type + ") " + value) - misp.add_named_attribute(event, attribute_type, value, category, distribution=5, - comment=comment, to_ids=ids_flag, enforceWarninglist=warninglist) - sight(sighting, value) + def _find_attached_forward(self): + for attachment in self.original_mail.iter_attachments(): + # Search for email forwarded as attachment + # I could have more than one, attaching everything. + if attachment.get_filename() and attachment.get_filename().endswith('.eml'): + self.forwarded_email(pseudofile=BytesIO(attachment.get_content().encode())) + else: + if self.config_from_email_body.get('attachment') == 'benign': + # Attach sane file + self.misp_event.add_attribute('attachment', value='Report', + data=BytesIO(attachment.get_content().encode())) + else: + f_object, main_object, sections = make_binary_objects(pseudofile=BytesIO(attachment.get_content().encode()), + filename=attachment.get_filename(), standalone=False) + self.misp_event.add_object(f_object) + if main_object: + self.misp_event.add_object(main_object) + [self.misp_event.add_object(section) for section in sections] + def email_from_spamtrap(self): + '''The email comes from a spamtrap and should be attached as-is.''' + self.clean_email_body = self.original_mail.get_body().as_string() + self.forwarded_email(self.pseudofile) -syslog.syslog("Job started.") -debug = config.debug -stdin_used = False + def forwarded_email(self, pseudofile: BytesIO): + '''Extracts all possible indicators out of an email and create a MISP event out of it. + * Gets all relevant Headers + * Attach the body + * Create MISP file objects (uses lief if possible) + * Set all references + ''' + email_object = EMailObject(pseudofile=self.pseudofile, attach_original_mail=True, standalone=False) + if email_object.attachments: + # Create file objects for the attachments + for attachment_name, attachment in email_object.attachments: + f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False) + self.misp_event.add_object(f_object) + if main_object: + self.misp_event.add_object(main_object) + for section in sections: + self.misp_event.add_object(section) + email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment') + self.misp_event.add_object(email_object) -email_subject = config.email_subject_prefix -mail_subject = "" -if len(sys.argv) == 1: - mailcontent = sys.stdin.buffer.read().decode("utf-8", "ignore") -else: - # read from tempfile - if sys.argv[1] == "-r": - tempfilename = sys.argv[2] - tf = open(tempfilename, 'r') - mailcontent = tf.read() - tf.close() - # receive data and subject through arguments - else: - mailcontent = sys.argv[1] - if debug: - syslog.syslog(mailcontent) - if len(sys.argv) >= 3: - mail_subject = sys.argv[2].encode("utf-8", "ignore") -email_data = b'' -msg = email.message_from_string(mailcontent) -if not mail_subject: - try: - mail_subject = msg.get('Subject').encode("utf-8", "ignore") - sub, enc = email.header.decode_header(msg.get('subject'))[0] - if enc is None: - email_subject = sub - else: - email_subject = sub.decode(enc) - except Exception as e: - print(e) - pass -for part in msg.walk(): - if part.get_content_charset() is None: - # This could probably be detected - charset = 'utf-8' - else: - charset = part.get_content_charset() - if part.get_content_maintype() == 'multipart': - continue - if part.get_content_maintype() == 'text': - part.set_charset(charset) - if debug: - syslog.syslog(str(part.get_payload(decode=True))) - email_data += part.get_payload(decode=True) -try: - email_subject += mail_subject -except Exception as e: - syslog.syslog(str(e)) -stdin_used = True + def process_email_body(self): + self.clean_email_body = self.original_mail.get_body().as_string() + # Check if there are config lines in the body & convert them to a python dictionary: + # :: => {: } + self.config_from_email_body = {k: v for k, v in re.findall(f'{config.body_config_prefix}:(.*):(.*)', self.clean_email_body)} + if self.config_from_email_body: + # ... remove the config lines from the body + self.clean_email_body = re.sub(rf'^{config.body_config_prefix}.*\n?', '', + self.original_mail.get_body().as_string(), flags=re.MULTILINE) -try: - email_data = ftfy.fix_text(email_data.decode("utf-8", "ignore")) -except Exception: - email_data = ftfy.fix_text(email_data) -try: - email_subject = ftfy.fix_text(email_subject.decode("utf-8", "ignore")) -except Exception: - email_subject = ftfy.fix_text(email_subject) + self._find_inline_forward() + self._find_attached_forward() + # # Prepare extraction of IOCs + # Refang email data + self.clean_email_body = refang(self.clean_email_body) -if debug: - syslog.syslog(email_subject) - syslog.syslog(email_data) + # Depending on the source of the mail, there is some cleanup to do. Ignore lines in body of message + for ignoreline in config.ignorelist: + self.clean_email_body = re.sub(rf'^{ignoreline}.*\n?', '', self.clean_email_body, flags=re.MULTILINE) -try: - misp_url = config.misp_url - misp_key = config.misp_key - misp_verifycert = config.misp_verifycert - m2m_key = config.m2m_key - m2m_auto_distribution = config.m2m_auto_distribution - m2m_attachment_keyword = config.m2m_attachment_keyword - resolver = dns.resolver.Resolver(configure=False) - resolver.nameservers = config.nameservers - excludelist = config.excludelist - externallist = config.externallist - internallist = config.internallist - noidsflaglist = config.noidsflaglist - ignorelist = config.ignorelist - enforcewarninglist = config.enforcewarninglist - sighting = config.sighting - sighting_source = config.sighting_source - removelist = config.removelist - malwaretags = config.malwaretags - dependingtags = config.dependingtags - tlptag_default = config.tlptag_default - stopword = config.stopword - hash_only_tags = config.hash_only_tags - forward_identifiers = config.forward_identifiers - attach_original_mail = config.attach_original_mail -except Exception as e: - syslog.syslog(str(e)) - print("There is a problem with the configuration. A mandatory configuration variable is not set.") - print("Did you just update? mail_to_misp might have new configuration variables.") - print("Please compare with the configuration example.") - print("\nTrace:") - print(e) - sys.exit(-2) + # Check if autopublish key is present and valid + if self.config_from_email_body.get('m2mkey') == config.m2m_key: + self.misp_event.publish() -original_email_data = email_data + # Add tags to the event if keywords are found in the mail + for tag in config.tlptags: + if any(alternativetag in self.clean_email_body for alternativetag in config.tlptags[tag]): + self.misp_event.add_tag(tag) -# Ignore lines in body of message -for ignoreline in ignorelist: - email_data = re.sub(ignoreline, "", email_data) + # Remove everything after the stopword from the body + self.clean_email_body = self.clean_email_body.split(config.stopword, 1)[0] -# Remove words from subject -for removeword in removelist: - email_subject = re.sub(removeword, "", email_subject) + def process_body_iocs(self): + # Extract and add hashes + contains_hash = False + for h in set(re.findall(hashmarker.MD5_REGEX, self.clean_email_body)): + contains_hash = True + self.misp_event.add_attribute('md5', h, enforceWarninglist=config.enforcewarninglist) + if config.sighting: + self.sighting(h, config.sighting_source) + for h in set(re.findall(hashmarker.SHA1_REGEX, self.clean_email_body)): + contains_hash = True + self.misp_event.add_attribute('sha1', h, enforceWarninglist=config.enforcewarninglist) + if config.sighting: + self.sighting(h, config.sighting_source) + for h in set(re.findall(hashmarker.SHA256_REGEX, self.clean_email_body)): + contains_hash = True + self.misp_event.add_attribute('sha256', h, enforceWarninglist=config.enforcewarninglist) + if config.sighting: + self.sighting(h, config.sighting_source) -# Check if autopublish key is present and valid -auto_publish = False -autopublish_key = "key:" + m2m_key -if autopublish_key in email_data: - auto_publish = True + if contains_hash: + [self.misp_event.add_tag(tag) for tag in config.hash_only_tags] -# Create the MISP event -misp = init(misp_url, misp_key) -if auto_publish: - new_event = misp.new_event(info=email_subject, distribution=m2m_auto_distribution, threat_level_id=3, analysis=1) -else: - new_event = misp.new_event(info=email_subject, distribution=0, threat_level_id=3, analysis=1) + # # Extract network IOCs + urllist = [] + urllist += re.findall(urlmarker.WEB_URL_REGEX, self.clean_email_body) + urllist += re.findall(urlmarker.IP_REGEX, self.clean_email_body) + if self.debug: + syslog.syslog(str(urllist)) -# Load the MISP event -misp_event = MISPEvent() -misp_event.load(new_event) -event_id = misp_event.id + # Init Faup + f = Faup() -# Evaluate classification -tlp_tag = tlptag_default -tlptags = config.tlptags -for tag in tlptags: - for alternativetag in tlptags[tag]: - if alternativetag in email_data.lower(): - tlp_tag = tag -misp.tag(misp_event.uuid, tlp_tag) + hostname_processed = [] -if attach_original_mail and original_email_data: - add_attribute(new_event, 'email-body', original_email_data, 'Payload delivery', False, enforcewarninglist, sighting) + # Add IOCs and expanded information to MISP + for entry in set(urllist): + ids_flag = True + f.decode(entry) -# Add additional tags depending on others -for tag in dependingtags: - if tag in tlp_tag: - for dependingtag in dependingtags[tag]: - misp.tag(misp_event.uuid, dependingtag) + domainname = f.get_domain().decode() + if domainname in config.excludelist: + # Ignore the entry + continue -# # Prepare extraction of IOCs + hostname = f.get_host().decode() -# Limit the input if the stopword is found -email_data = email_data.split(stopword, 1)[0] + scheme = f.get_scheme() + if scheme: + scheme = scheme.decode() -# Find the first forwarding message and use that content -position = 99999 -t_email_data = email_data -for identifier in forward_identifiers: - new_position = email_data.find(identifier) - if new_position == -1: - new_position = position - if new_position < position: - t_before, t_split, t_email_data = email_data.partition(identifier) - position = new_position -email_data = t_email_data + resource_path = f.get_resource_path() + if resource_path: + resource_path = resource_path.decode() -# Refang email data -email_data = refang(email_data) - - -# # Extract various IOCs - -urllist = list() -urllist += re.findall(urlmarker.WEB_URL_REGEX, email_data) -urllist += re.findall(urlmarker.IP_REGEX, email_data) -if debug: - syslog.syslog(str(urllist)) - - -# Init Faup -f = Faup() - -# Add additional tags according to configuration -for malware in malwaretags: - if malware in email_subject.lower(): - for tag in malwaretags[malware]: - misp.tag(misp_event.uuid, tag) - -# Extract and add hashes -hashlist_md5 = re.findall(hashmarker.MD5_REGEX, email_data) -hashlist_sha1 = re.findall(hashmarker.SHA1_REGEX, email_data) -hashlist_sha256 = re.findall(hashmarker.SHA256_REGEX, email_data) - -for h in hashlist_md5: - add_attribute(new_event, 'md5', h, 'Payload delivery', True, enforcewarninglist, sighting) -for h in hashlist_sha1: - add_attribute(new_event, 'sha1', h, 'Payload delivery', True, enforcewarninglist, sighting) -for h in hashlist_sha256: - add_attribute(new_event, 'sha256', h, 'Payload delivery', True, enforcewarninglist, sighting) - -if (len(hashlist_md5) > 0) or (len(hashlist_sha1) > 0) or (len(hashlist_sha256) > 0): - for tag in hash_only_tags: - misp.tag(misp_event.uuid, tag) - -# Add IOCs and expanded information to MISP -for entry in urllist: - hip = MISPObject(name='ip-port', strict=False, uuid='9f8cea74-16fe-4968-a2b4-026676949ac7', version='7') - ids_flag = True - f.decode(entry) - domainname = f.get_domain().decode('utf-8', 'ignore') - hostname = f.get_host().decode('utf-8', 'ignore') - try: - schema = f.get_scheme().decode('utf-8', 'ignore') - except Exception: - schema = False - try: - resource_path = f.get_resource_path().decode('utf-8', 'ignore') - except Exception: - resource_path = False - if debug: - syslog.syslog(domainname) - if domainname not in excludelist: - if domainname in internallist: - add_attribute(new_event, 'link', entry, 'Internal reference', False, False, sighting) - elif domainname in externallist: - add_attribute(new_event, 'link', entry, 'External analysis', False, False, sighting) - else: - comment = "" - if (domainname in noidsflaglist) or (hostname in noidsflaglist): - ids_flag = False - comment = "Known host (mostly for connectivity test or IP lookup)" if debug: - syslog.syslog(str(entry)) - if hostname: - if schema: - if is_valid_ipv4_address(hostname): - add_attribute(new_event, 'url', entry, 'Network activity', False, enforcewarninglist, sighting) + syslog.syslog(domainname) + + if domainname in config.internallist: # Add link to internal reference + self.misp_event.add_attribute('link', entry, category='Internal reference', + to_ids=False, enforceWarninglist=False) + elif domainname in config.externallist: # External analysis + self.misp_event.add_attribute('link', entry, category='External analysis', + to_ids=False, enforceWarninglist=False) + else: # The URL is probably an indicator. + comment = "" + if (domainname in config.noidsflaglist) or (hostname in config.noidsflaglist): + ids_flag = False + comment = "Known host (mostly for connectivity test or IP lookup)" + if debug: + syslog.syslog(str(entry)) + + if scheme: + if is_ip(hostname): + self.misp_event.add_attribute('url', entry, to_ids=False, + enforceWarninglist=config.enforcewarninglist) else: - if resource_path: - add_attribute(new_event, 'url', entry, 'Network activity', ids_flag, False, - sighting, comment=comment) - else: - add_attribute(new_event, 'url', entry, 'Network activity', ids_flag, enforcewarninglist, - sighting, comment=comment) + if resource_path: # URL has path, ignore warning list + self.misp_event.add_attribute('url', entry, to_ids=ids_flag, + enforceWarninglist=False, comment=comment) + else: # URL has no path + self.misp_event.add_attribute('url', entry, to_ids=ids_flag, + enforceWarninglist=config.enforcewarninglist, comment=comment) + if config.sighting: + self.sighting(entry, config.sighting_source) + + if hostname in hostname_processed: + # Hostname already processed. + continue + + hostname_processed.append(hostname) + if config.sighting: + self.sighting(hostname, config.sighting_source) + if debug: syslog.syslog(hostname) - try: - port = f.get_port().decode('utf-8', 'ignore') - except Exception: - port = None + + port = f.get_port() if port: + port = port.decode() comment = "on port: " + port - if is_valid_ipv4_address(hostname): - add_attribute(new_event, 'ip-dst', hostname, 'Network activity', ids_flag, enforcewarninglist, - sighting, comment=comment) - hip.add_attribute('ip', type='ip-dst', value=hostname, to_ids=ids_flag, comment=comment) + + if is_ip(hostname): + self.misp_event.add_attribute('ip-dst', hostname, to_ids=ids_flag, + enforceWarninglist=config.enforcewarninglist, + comment=comment) else: - add_attribute(new_event, 'hostname', hostname, 'Network activity', ids_flag, enforcewarninglist, - sighting, comment=comment) - hip.add_attribute('hostname', type='hostname', value=hostname, to_ids=ids_flag, comment=comment) - try: - for rdata in dns.resolver.query(hostname, 'A'): + related_ips = [] + try: + for rdata in dns.resolver.query(hostname, 'A'): + if debug: + syslog.syslog(str(rdata)) + related_ips.append(rdata.to_text()) + except Exception as e: if debug: - syslog.syslog(str(rdata)) - add_attribute(new_event, 'ip-dst', rdata.to_text(), 'Network activity', False, enforcewarninglist, - sighting, comment=hostname) - hip.add_attribute('ip', type='ip-dst', value=rdata.to_text(), to_ids=False) - except Exception as e: - if debug: - syslog.syslog(str(e)) -# misp_event.add_object(hip) -# misp.update_event(event_id, new_event) + syslog.syslog(str(e)) -# Try to add attachments -if stdin_used: - for part in msg.walk(): - if part.get_content_maintype() == 'multipart': - continue - if part.get_content_maintype() != 'text' and part.get_payload(decode=True) is not None: - filename = part.get_filename() - _, output_path = tempfile.mkstemp() - output = open(output_path, 'wb') - output.write(part.get_payload(decode=True)) - output.close() - attachment = part.get_payload(decode=True) - if debug: - syslog.syslog(str(attachment)[:200]) - if m2m_attachment_keyword in email_data: - misp.add_attachment(misp_event, output_path, filename=filename, category="External analysis") - else: - misp.upload_sample(filename, output_path, event_id, distribution=5, to_ids=True) - file_hash = hashlib.sha256(open(output_path, 'rb').read()).hexdigest() - sight(sighting, file_hash) + if related_ips: + hip = MISPObject(name='ip-port') + hip.add_attribute('hostname', value=hostname, to_ids=ids_flag, + enforceWarninglist=config.enforcewarninglist, comment=comment) + for ip in set(related_ips): + hip.add_attribute('ip', type='ip-dst', value=ip, to_ids=False, + enforceWarninglist=config.enforcewarninglist) + self.misp_event.add_object(hip) + else: + self.misp_event.add_attribute('hostname', value=hostname, + to_ids=ids_flag, enforceWarninglist=config.enforcewarninglist, + comment=comment) -if auto_publish: - misp.publish(misp_event, alert=False) + def add_event(self): + '''Add event on the remote MISP instance.''' -syslog.syslog("Job finished.") + # Add additional tags depending on others + tags = [] + for tag in [t.name for t in self.misp_event.tags]: + if config.dependingtags.get(tag): + tags += config.dependingtags.get(tag) + + # Add additional tags according to configuration + for malware in config.malwaretags: + if malware.lower() in self.subject.lower(): + tags += config.malwaretags.get(malware) + if tags: + [self.misp_event.add_tag(tag) for tag in tags] + + self.misp.add_event(self.misp_event) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Push a Mail into a MISP instance') + parser.add_argument("-r", "--read", help="Read from tempfile.") + parser.add_argument("-t", "--trap", action='store_true', default=False, help="Import the Email as-is.") + parser.add_argument('infile', nargs='?', type=argparse.FileType('rb'), default=sys.stdin) + args = parser.parse_args() + + syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_USER) + syslog.syslog("Job started.") + + configmodule = Path(__file__).as_posix().replace('.py', '_config') + if Path(f'{configmodule}.py').exists(): + config = importlib.import_module(configmodule) + try: + misp_url = config.misp_url + misp_key = config.misp_key + misp_verifycert = config.misp_verifycert + debug = config.debug + except Exception as e: + syslog.syslog(str(e)) + print("There is a problem with the configuration. A mandatory configuration variable is not set.") + print("Did you just update? mail_to_misp might have new configuration variables.") + print("Please compare with the configuration example.") + print("\nTrace:") + print(e) + sys.exit(-2) + else: + print("Couldn't locate config file {0}".format(f'{configmodule}.py')) + sys.exit(-1) + + if args.infile: + pseudofile = BytesIO(args.infile.read().encode()) + elif args.read: + # read from tempfile + with open(args.read, 'rb') as f: + pseudofile = BytesIO(f.read()) + else: + # receive data and subject through arguments + raise Exception('This is not implemented anymore.') + + mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config) + mail2misp.load_email(pseudofile) + + if debug: + syslog.syslog(f'Working on {mail2misp.subject}') + + if args.trap or config.spamtrap: + mail2misp.email_from_spamtrap() + else: + mail2misp.process_email_body() + + mail2misp.process_body_iocs() + + mail2misp.add_event() + syslog.syslog("Job finished.") diff --git a/mail_to_misp_config.py-example b/mail_to_misp_config.py-example index aa44b45..bb98be1 100644 --- a/mail_to_misp_config.py-example +++ b/mail_to_misp_config.py-example @@ -1,18 +1,20 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import sys import os misp_url = 'YOUR_MISP_URL' -misp_key = 'YOUR_KEY_HERE' # The MISP auth key can be found on the MISP web interface under the automation section +misp_key = 'YOUR_KEY_HERE' # The MISP auth key can be found on the MISP web interface under the automation section misp_verifycert = True +body_config_prefix = 'm2m' # every line in the body starting with this value will be skipped from the IOCs +spamtrap = False + m2m_key = 'YOUSETYOURKEYHERE' -m2m_auto_distribution = '3' # 3 = All communities +m2m_auto_distribution = '3' # 3 = All communities m2m_attachment_keyword = 'attachment:benign' debug = False nameservers = ['149.13.33.69'] -email_subject_prefix = b'M2M - ' +email_subject_prefix = 'M2M' attach_original_mail = True # Paths (should be automatic) @@ -28,57 +30,56 @@ smtp_port = 25 excludelist = ('google.com', 'microsoft.com') externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net') internallist = ('internal.system.local') -noidsflaglist = ( 'myexternalip.com', 'ipinfo.io', 'icanhazip.com', 'wtfismyip.com', 'ipecho.net', - 'api.ipify.org', 'checkip.amazonaws.com', 'whatismyipaddress.com', 'google.com', - 'dropbox.com' - ) +noidsflaglist = ('myexternalip.com', 'ipinfo.io', 'icanhazip.com', 'wtfismyip.com', 'ipecho.net', + 'api.ipify.org', 'checkip.amazonaws.com', 'whatismyipaddress.com', 'google.com', + 'dropbox.com' + ) # Stop parsing when this term is found stopword = 'Whois & IP Information' # Ignore lines in body of message containing: -ignorelist = (".*From: .*\n?", ".*Sender: .*\n?", ".*Received: .*\n?", ".*Sender IP: .*\n?", - ".*Reply-To: .*\n?", ".*Registrar WHOIS Server: .*\n?", ".*Registrar: .*\n?", - ".*Domain Status: .*\n?", ".*Registrant Email: .*\n?", ".*IP Location: .*\n?", - ".*X-Get-Message-Sender-Via: .*\n?", ".*X-Authenticated-Sender: .*\n") +ignorelist = ("From:", "Sender:", "Received:", "Sender IP:", "Reply-To:", "Registrar WHOIS Server:", + "Registrar:", "Domain Status:", "Registrant Email:", "IP Location:", + "X-Get-Message-Sender-Via:", "X-Authenticated-Sender:") # Ignore (don't add) attributes that are on server side warning list -enforcewarninglist=True +enforcewarninglist = True # Add a sighting for each value -sighting=True -sighting_source="YOUR_MAIL_TO_MISP_IDENTIFIER" +sighting = True +sighting_source = "YOUR_MAIL_TO_MISP_IDENTIFIER" # Remove "[tags]", "Re: ", "Fwd: " from subject removelist = ("[\(\[].*?[\)\]]", "Re: ", "Fwd: ") # TLP tag setup # Tuples contain different variations of spelling -tlptags = { 'tlp:amber': [ 'tlp:amber', 'tlp: amber', 'tlp amber' ], - 'tlp:green': [ 'tlp:green', 'tlp: green', 'tlp green' ], - 'tlp:white': [ 'tlp:white', 'tlp: white', 'tlp white' ] - } +tlptags = {'tlp:amber': ['tlp:amber', 'tlp: amber', 'tlp amber'], + 'tlp:green': ['tlp:green', 'tlp: green', 'tlp green'], + 'tlp:white': ['tlp:white', 'tlp: white', 'tlp white'] + } tlptag_default = sorted(tlptags.keys())[0] -malwaretags = { 'locky': [ 'ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Locky"' ], - 'jaff': [ 'ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Jaff"' ], - 'dridex': [ 'misp-galaxy:tool="dridex"' ], - 'netwire': [ 'Netwire RAT' ], - 'Pony': [ 'misp-galaxy:tool="Hancitor"' ], - 'ursnif': [ 'misp-galaxy:tool="Snifula"' ], - 'NanoCore': [ 'misp-galaxy:tool="NanoCoreRAT"' ], - 'trickbot': [ 'misp-galaxy:tool="Trick Bot"' ] - } +malwaretags = {'locky': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Locky"'], + 'jaff': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Jaff"'], + 'dridex': ['misp-galaxy:tool="dridex"'], + 'netwire': ['Netwire RAT'], + 'Pony': ['misp-galaxy:tool="Hancitor"'], + 'ursnif': ['misp-galaxy:tool="Snifula"'], + 'NanoCore': ['misp-galaxy:tool="NanoCoreRAT"'], + 'trickbot': ['misp-galaxy:tool="Trick Bot"'] + } # Tags to be set depending on the presence of other tags -dependingtags = { 'tlp:white': [ 'circl:osint-feed' ] - } +dependingtags = {'tlp:white': ['circl:osint-feed'] + } -# Known identifiers for forwarded messages -forward_identifiers = { '-------- Forwarded Message --------', 'Begin forwarded message:' } +# Known identifiers for forwarded messages +forward_identifiers = {'-------- Forwarded Message --------', 'Begin forwarded message:'} # Tags to add when hashes are found (e.g. to do automatic expansion) -hash_only_tags = { 'TODO:VT-ENRICHMENT' } +hash_only_tags = {'TODO:VT-ENRICHMENT'} # If an attribute is on any MISP server side `warning list`, skip the creation of the attribute skip_item_on_warninglist = True diff --git a/reqirements.txt b/reqirements.txt new file mode 100644 index 0000000..5383955 --- /dev/null +++ b/reqirements.txt @@ -0,0 +1,5 @@ +dnspython +pymisp[fileobjects] +git+https://github.com/kbandla/pydeep.git +git+https://github.com/stricaud/faup.git#egg=pyfaup&subdirectory=src/lib/bindings/python +git+https://github.com/Rafiot/defang.git