MISP
/
mail_to_misp
mirror of https://github.com/MISP/mail_to_misp
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: Just a slight refactoring.

slight_refactoring
Raphaël Vinot 2018-05-02 19:08:22 +02:00
parent 0a684ac997
commit 18bba44d23
4 changed files with 338 additions and 361 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
README.md
View File

@ -7,7 +7,7 @@ Connect your mail infrastructure to [MISP](https://github.com/MISP/MISP) in orde
- Extraction of URLs and IP addresses (and port numbers) from free text emails
- Extraction of hostnames from URLs
- Extraction of hashes (MD5, SHA1, SHA256)
- DNS expansion
- DNS expansion
- Custom filter list for lines containing specific words
- Subject filters
- Respecting TLP classification mentioned in free text (including optional spelling robustness)
@ -53,7 +53,7 @@ The implemented workflow is mainly for mail servers like Postfix. Client side im
3. Mozilla Thunderbird [deprecated]
`Email -> Thunderbird -> Mail rule -> filterscript -> thunderbird_wrapper -> mail_to_misp -> PyMISP -> MISP`
## Installation
@ -112,7 +112,7 @@ Outlook is not implemented due to lack of test environment. However, it should b
```
import win32com.client
import pythoncom
class Handler_Class(object):
def OnNewMailEx(self, receivedItemsIDs):
for ID in receivedItemsIDs.split(","):
@ -122,13 +122,13 @@ class Handler_Class(object):
print "Subj: " + mailItem.Subject
print "Body: " + mailItem.Body.encode( 'ascii', 'ignore' )
print "========"
outlook = win32com.client.DispatchWithEvents("Outlook.Application", Handler_Class)
pythoncom.PumpMessages()
```
(from: https://blog.matthewurch.ca/?p=236)
Obviously, you would like to filter mails based on subject or from address and pass subject and body to mail_to_misp.py in order to do something useful. Pull-requests welcome for actual implementations :)
Obviously, you would like to filter mails based on subject or from address and pass subject and body to mail_to_misp.py in order to do something useful. Pull-requests welcome for actual implementations :)
## Requirements
@ -138,19 +138,11 @@ Obviously, you would like to filter mails based on subject or from address and p
- mail_to_misp requires access to a MISP instance (via API).
- Python 3
- dnspython
- hashlib
- PyMISP
- faup from https://github.com/stricaud/faup
- urlmarker from https://github.com/rcompton/ryancompton.net/blob/master/assets/praw_drugs/urlmarker.py (contained in this project)
- ftfy from https://github.com/LuminosoInsight/python-ftfy (to fix unicode text)
- defang from https://bitbucket.org/johannestaas/defang
- Patch defang/defang/__init__.py by commenting out the following line (not used && not compatible with Python 3):
`from urllib2 import urlparse`
- Optionally patch defang/defang/__init__.py and add at line 47:
```
dirty_line = dirty_line.replace('hxxp', 'http')
dirty_line = dirty_line.replace('purr', 'http')
dirty_line = dirty_line.replace('meow', 'http')
```
- defang from https://github.com/Rafiot/defang.git (fork of: https://bitbucket.org/johannestaas/defang)
### Thunderbird [deprecated]

605
mail_to_misp.py
View File

@ -1,370 +1,349 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import sys
try:
configfile = os.path.basename(sys.argv[0]).split(".py")[0] + "_config"
except Exception as e:
print("Couldn't locate config file {0}".format(configfile))
sys.exit(-1)
import argparse
import re
import syslog
from pathlib import Path
from io import BytesIO
from ipaddress import ip_address
from email import message_from_bytes, policy
import importlib
try:
import urlmarker
import hashmarker
import re
from pyfaup.faup import Faup
from pymisp import PyMISP, MISPEvent, MISPObject
from pymisp import PyMISP, MISPEvent, MISPObject, MISPSighting
from pymisp.tools import EMailObject, make_binary_objects
from defang import refang
import dns.resolver
import email
import tempfile
import socket
import syslog
import ftfy
import hashlib
config = __import__(configfile)
except ImportError as e:
print("(!) Problem loading module:")
print(e)
sys.exit(-1)
syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_USER)
def is_valid_ipv4_address(address):
def is_ip(address):
try:
socket.inet_pton(socket.AF_INET, address)
except AttributeError: # no inet_pton here, sorry
try:
socket.inet_aton(address)
except socket.error:
return False
return address.count('.') == 3
except socket.error: # not a valid address
ip_address(address)
except ValueError:
return False
return True
def is_valid_ipv6_address(address):
try:
socket.inet_pton(socket.AF_INET6, address)
except socket.error: # not a valid address
return False
return True
class Mail2MISP():
def __init__(self, misp_url, misp_key, verifycert, config):
self.misp = PyMISP(misp_url, misp_key, verifycert, debug=config.debug)
self.debug = config.debug
self.config = config
def init(url, key):
return PyMISP(url, key, misp_verifycert, 'json', debug=debug)
def load_email(self, pseudofile):
self.pseudofile = pseudofile
self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default)
self.subject = self.original_mail.get('Subject')
# Remove words from subject
for removeword in self.config.removelist:
self.subject = re.sub(removeword, "", self.subject)
# Initialize the MISP event
self.misp_event = MISPEvent()
self.misp_event.info = f'{config.email_subject_prefix} - {self.subject}'
self.misp_event.distribution = config.m2m_auto_distribution
self.misp_event.threat_level_id = 3
self.misp_event.analysis = 1
self.misp_event.add_tag(config.tlptag_default)
# Add a sighting
def sight(sighting, value):
if sighting:
d = {'value': value, 'source': sighting_source}
misp.set_sightings(d)
def sighting(self, value, source):
'''Add a sighting'''
s = MISPSighting()
s.from_dict(value=value, source=source)
self.misp.set_sightings(s)
def _find_inline_forward(self):
'''Does the body contains a forwarded email?'''
for identifier in config.forward_identifiers:
if identifier in self.clean_email_body:
self.clean_email_body, fw_email = self.clean_email_body.split(identifier)
self.forwarded_email(pseudofile=BytesIO(fw_email.encode()))
# Add named attribute and sight if configured
def add_attribute(event, attribute_type, value, category, ids_flag, warninglist, sighting, comment=None):
syslog.syslog("Event " + event['Event']['id'] + ": Adding attribute (" + attribute_type + ") " + value)
misp.add_named_attribute(event, attribute_type, value, category, distribution=5,
comment=comment, to_ids=ids_flag, enforceWarninglist=warninglist)
sight(sighting, value)
def _find_attached_forward(self):
for attachment in self.original_mail.iter_attachments():
# Search for email forwarded as attachment
# I could have more than one, attaching everything.
if attachment.get_filename() and attachment.get_filename().endswith('.eml'):
self.forwarded_email(pseudofile=BytesIO(attachment.get_content().encode()))
else:
if self.config_from_email_body.get('attachment') == 'benign':
# Attach sane file
self.misp_event.add_attribute('attachment', value='Report',
data=BytesIO(attachment.get_content().encode()))
else:
f_object, main_object, sections = make_binary_objects(pseudofile=BytesIO(attachment.get_content().encode()),
filename=attachment.get_filename(), standalone=False)
self.misp_event.add_object(f_object)
if main_object:
self.misp_event.add_object(main_object)
[self.misp_event.add_object(section) for section in sections]
def email_from_spamtrap(self):
'''The email comes from a spamtrap and should be attached as-is.'''
self.clean_email_body = self.original_mail.get_body().as_string()
self.forwarded_email(self.pseudofile)
syslog.syslog("Job started.")
debug = config.debug
stdin_used = False
def forwarded_email(self, pseudofile: BytesIO):
'''Extracts all possible indicators out of an email and create a MISP event out of it.
* Gets all relevant Headers
* Attach the body
* Create MISP file objects (uses lief if possible)
* Set all references
'''
email_object = EMailObject(pseudofile=self.pseudofile, attach_original_mail=True, standalone=False)
if email_object.attachments:
# Create file objects for the attachments
for attachment_name, attachment in email_object.attachments:
f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False)
self.misp_event.add_object(f_object)
if main_object:
self.misp_event.add_object(main_object)
for section in sections:
self.misp_event.add_object(section)
email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment')
self.misp_event.add_object(email_object)
email_subject = config.email_subject_prefix
mail_subject = ""
if len(sys.argv) == 1:
mailcontent = sys.stdin.buffer.read().decode("utf-8", "ignore")
else:
# read from tempfile
if sys.argv[1] == "-r":
tempfilename = sys.argv[2]
tf = open(tempfilename, 'r')
mailcontent = tf.read()
tf.close()
# receive data and subject through arguments
else:
mailcontent = sys.argv[1]
if debug:
syslog.syslog(mailcontent)
if len(sys.argv) >= 3:
mail_subject = sys.argv[2].encode("utf-8", "ignore")
email_data = b''
msg = email.message_from_string(mailcontent)
if not mail_subject:
try:
mail_subject = msg.get('Subject').encode("utf-8", "ignore")
sub, enc = email.header.decode_header(msg.get('subject'))[0]
if enc is None:
email_subject = sub
else:
email_subject = sub.decode(enc)
except Exception as e:
print(e)
pass
for part in msg.walk():
if part.get_content_charset() is None:
# This could probably be detected
charset = 'utf-8'
else:
charset = part.get_content_charset()
if part.get_content_maintype() == 'multipart':
continue
if part.get_content_maintype() == 'text':
part.set_charset(charset)
if debug:
syslog.syslog(str(part.get_payload(decode=True)))
email_data += part.get_payload(decode=True)
try:
email_subject += mail_subject
except Exception as e:
syslog.syslog(str(e))
stdin_used = True
def process_email_body(self):
self.clean_email_body = self.original_mail.get_body().as_string()
# Check if there are config lines in the body & convert them to a python dictionary:
# <config.body_config_prefix>:<key>:<value> => {<key>: <value>}
self.config_from_email_body = {k: v for k, v in re.findall(f'{config.body_config_prefix}:(.*):(.*)', self.clean_email_body)}
if self.config_from_email_body:
# ... remove the config lines from the body
self.clean_email_body = re.sub(rf'^{config.body_config_prefix}.*\n?', '',
self.original_mail.get_body().as_string(), flags=re.MULTILINE)
try:
email_data = ftfy.fix_text(email_data.decode("utf-8", "ignore"))
except Exception:
email_data = ftfy.fix_text(email_data)
try:
email_subject = ftfy.fix_text(email_subject.decode("utf-8", "ignore"))
except Exception:
email_subject = ftfy.fix_text(email_subject)
self._find_inline_forward()
self._find_attached_forward()
# # Prepare extraction of IOCs
# Refang email data
self.clean_email_body = refang(self.clean_email_body)
if debug:
syslog.syslog(email_subject)
syslog.syslog(email_data)
# Depending on the source of the mail, there is some cleanup to do. Ignore lines in body of message
for ignoreline in config.ignorelist:
self.clean_email_body = re.sub(rf'^{ignoreline}.*\n?', '', self.clean_email_body, flags=re.MULTILINE)
try:
misp_url = config.misp_url
misp_key = config.misp_key
misp_verifycert = config.misp_verifycert
m2m_key = config.m2m_key
m2m_auto_distribution = config.m2m_auto_distribution
m2m_attachment_keyword = config.m2m_attachment_keyword
resolver = dns.resolver.Resolver(configure=False)
resolver.nameservers = config.nameservers
excludelist = config.excludelist
externallist = config.externallist
internallist = config.internallist
noidsflaglist = config.noidsflaglist
ignorelist = config.ignorelist
enforcewarninglist = config.enforcewarninglist
sighting = config.sighting
sighting_source = config.sighting_source
removelist = config.removelist
malwaretags = config.malwaretags
dependingtags = config.dependingtags
tlptag_default = config.tlptag_default
stopword = config.stopword
hash_only_tags = config.hash_only_tags
forward_identifiers = config.forward_identifiers
attach_original_mail = config.attach_original_mail
except Exception as e:
syslog.syslog(str(e))
print("There is a problem with the configuration. A mandatory configuration variable is not set.")
print("Did you just update? mail_to_misp might have new configuration variables.")
print("Please compare with the configuration example.")
print("\nTrace:")
print(e)
sys.exit(-2)
# Check if autopublish key is present and valid
if self.config_from_email_body.get('m2mkey') == config.m2m_key:
self.misp_event.publish()
original_email_data = email_data
# Add tags to the event if keywords are found in the mail
for tag in config.tlptags:
if any(alternativetag in self.clean_email_body for alternativetag in config.tlptags[tag]):
self.misp_event.add_tag(tag)
# Ignore lines in body of message
for ignoreline in ignorelist:
email_data = re.sub(ignoreline, "", email_data)
# Remove everything after the stopword from the body
self.clean_email_body = self.clean_email_body.split(config.stopword, 1)[0]
# Remove words from subject
for removeword in removelist:
email_subject = re.sub(removeword, "", email_subject)
def process_body_iocs(self):
# Extract and add hashes
contains_hash = False
for h in set(re.findall(hashmarker.MD5_REGEX, self.clean_email_body)):
contains_hash = True
self.misp_event.add_attribute('md5', h, enforceWarninglist=config.enforcewarninglist)
if config.sighting:
self.sighting(h, config.sighting_source)
for h in set(re.findall(hashmarker.SHA1_REGEX, self.clean_email_body)):
contains_hash = True
self.misp_event.add_attribute('sha1', h, enforceWarninglist=config.enforcewarninglist)
if config.sighting:
self.sighting(h, config.sighting_source)
for h in set(re.findall(hashmarker.SHA256_REGEX, self.clean_email_body)):
contains_hash = True
self.misp_event.add_attribute('sha256', h, enforceWarninglist=config.enforcewarninglist)
if config.sighting:
self.sighting(h, config.sighting_source)
# Check if autopublish key is present and valid
auto_publish = False
autopublish_key = "key:" + m2m_key
if autopublish_key in email_data:
auto_publish = True
if contains_hash:
[self.misp_event.add_tag(tag) for tag in config.hash_only_tags]
# Create the MISP event
misp = init(misp_url, misp_key)
if auto_publish:
new_event = misp.new_event(info=email_subject, distribution=m2m_auto_distribution, threat_level_id=3, analysis=1)
else:
new_event = misp.new_event(info=email_subject, distribution=0, threat_level_id=3, analysis=1)
# # Extract network IOCs
urllist = []
urllist += re.findall(urlmarker.WEB_URL_REGEX, self.clean_email_body)
urllist += re.findall(urlmarker.IP_REGEX, self.clean_email_body)
if self.debug:
syslog.syslog(str(urllist))
# Load the MISP event
misp_event = MISPEvent()
misp_event.load(new_event)
event_id = misp_event.id
# Init Faup
f = Faup()
# Evaluate classification
tlp_tag = tlptag_default
tlptags = config.tlptags
for tag in tlptags:
for alternativetag in tlptags[tag]:
if alternativetag in email_data.lower():
tlp_tag = tag
misp.tag(misp_event.uuid, tlp_tag)
hostname_processed = []
if attach_original_mail and original_email_data:
add_attribute(new_event, 'email-body', original_email_data, 'Payload delivery', False, enforcewarninglist, sighting)
# Add IOCs and expanded information to MISP
for entry in set(urllist):
ids_flag = True
f.decode(entry)
# Add additional tags depending on others
for tag in dependingtags:
if tag in tlp_tag:
for dependingtag in dependingtags[tag]:
misp.tag(misp_event.uuid, dependingtag)
domainname = f.get_domain().decode()
if domainname in config.excludelist:
# Ignore the entry
continue
# # Prepare extraction of IOCs
hostname = f.get_host().decode()
# Limit the input if the stopword is found
email_data = email_data.split(stopword, 1)[0]
scheme = f.get_scheme()
if scheme:
scheme = scheme.decode()
# Find the first forwarding message and use that content
position = 99999
t_email_data = email_data
for identifier in forward_identifiers:
new_position = email_data.find(identifier)
if new_position == -1:
new_position = position
if new_position < position:
t_before, t_split, t_email_data = email_data.partition(identifier)
position = new_position
email_data = t_email_data
resource_path = f.get_resource_path()
if resource_path:
resource_path = resource_path.decode()
# Refang email data
email_data = refang(email_data)
# # Extract various IOCs
urllist = list()
urllist += re.findall(urlmarker.WEB_URL_REGEX, email_data)
urllist += re.findall(urlmarker.IP_REGEX, email_data)
if debug:
syslog.syslog(str(urllist))
# Init Faup
f = Faup()
# Add additional tags according to configuration
for malware in malwaretags:
if malware in email_subject.lower():
for tag in malwaretags[malware]:
misp.tag(misp_event.uuid, tag)
# Extract and add hashes
hashlist_md5 = re.findall(hashmarker.MD5_REGEX, email_data)
hashlist_sha1 = re.findall(hashmarker.SHA1_REGEX, email_data)
hashlist_sha256 = re.findall(hashmarker.SHA256_REGEX, email_data)
for h in hashlist_md5:
add_attribute(new_event, 'md5', h, 'Payload delivery', True, enforcewarninglist, sighting)
for h in hashlist_sha1:
add_attribute(new_event, 'sha1', h, 'Payload delivery', True, enforcewarninglist, sighting)
for h in hashlist_sha256:
add_attribute(new_event, 'sha256', h, 'Payload delivery', True, enforcewarninglist, sighting)
if (len(hashlist_md5) > 0) or (len(hashlist_sha1) > 0) or (len(hashlist_sha256) > 0):
for tag in hash_only_tags:
misp.tag(misp_event.uuid, tag)
# Add IOCs and expanded information to MISP
for entry in urllist:
hip = MISPObject(name='ip-port', strict=False, uuid='9f8cea74-16fe-4968-a2b4-026676949ac7', version='7')
ids_flag = True
f.decode(entry)
domainname = f.get_domain().decode('utf-8', 'ignore')
hostname = f.get_host().decode('utf-8', 'ignore')
try:
schema = f.get_scheme().decode('utf-8', 'ignore')
except Exception:
schema = False
try:
resource_path = f.get_resource_path().decode('utf-8', 'ignore')
except Exception:
resource_path = False
if debug:
syslog.syslog(domainname)
if domainname not in excludelist:
if domainname in internallist:
add_attribute(new_event, 'link', entry, 'Internal reference', False, False, sighting)
elif domainname in externallist:
add_attribute(new_event, 'link', entry, 'External analysis', False, False, sighting)
else:
comment = ""
if (domainname in noidsflaglist) or (hostname in noidsflaglist):
ids_flag = False
comment = "Known host (mostly for connectivity test or IP lookup)"
if debug:
syslog.syslog(str(entry))
if hostname:
if schema:
if is_valid_ipv4_address(hostname):
add_attribute(new_event, 'url', entry, 'Network activity', False, enforcewarninglist, sighting)
syslog.syslog(domainname)
if domainname in config.internallist: # Add link to internal reference
self.misp_event.add_attribute('link', entry, category='Internal reference',
to_ids=False, enforceWarninglist=False)
elif domainname in config.externallist: # External analysis
self.misp_event.add_attribute('link', entry, category='External analysis',
to_ids=False, enforceWarninglist=False)
else: # The URL is probably an indicator.
comment = ""
if (domainname in config.noidsflaglist) or (hostname in config.noidsflaglist):
ids_flag = False
comment = "Known host (mostly for connectivity test or IP lookup)"
if debug:
syslog.syslog(str(entry))
if scheme:
if is_ip(hostname):
self.misp_event.add_attribute('url', entry, to_ids=False,
enforceWarninglist=config.enforcewarninglist)
else:
if resource_path:
add_attribute(new_event, 'url', entry, 'Network activity', ids_flag, False,
sighting, comment=comment)
else:
add_attribute(new_event, 'url', entry, 'Network activity', ids_flag, enforcewarninglist,
sighting, comment=comment)
if resource_path: # URL has path, ignore warning list
self.misp_event.add_attribute('url', entry, to_ids=ids_flag,
enforceWarninglist=False, comment=comment)
else: # URL has no path
self.misp_event.add_attribute('url', entry, to_ids=ids_flag,
enforceWarninglist=config.enforcewarninglist, comment=comment)
if config.sighting:
self.sighting(entry, config.sighting_source)
if hostname in hostname_processed:
# Hostname already processed.
continue
hostname_processed.append(hostname)
if config.sighting:
self.sighting(hostname, config.sighting_source)
if debug:
syslog.syslog(hostname)
try:
port = f.get_port().decode('utf-8', 'ignore')
except Exception:
port = None
port = f.get_port()
if port:
port = port.decode()
comment = "on port: " + port
if is_valid_ipv4_address(hostname):
add_attribute(new_event, 'ip-dst', hostname, 'Network activity', ids_flag, enforcewarninglist,
sighting, comment=comment)
hip.add_attribute('ip', type='ip-dst', value=hostname, to_ids=ids_flag, comment=comment)
if is_ip(hostname):
self.misp_event.add_attribute('ip-dst', hostname, to_ids=ids_flag,
enforceWarninglist=config.enforcewarninglist,
comment=comment)
else:
add_attribute(new_event, 'hostname', hostname, 'Network activity', ids_flag, enforcewarninglist,
sighting, comment=comment)
hip.add_attribute('hostname', type='hostname', value=hostname, to_ids=ids_flag, comment=comment)
try:
for rdata in dns.resolver.query(hostname, 'A'):
related_ips = []
try:
for rdata in dns.resolver.query(hostname, 'A'):
if debug:
syslog.syslog(str(rdata))
related_ips.append(rdata.to_text())
except Exception as e:
if debug:
syslog.syslog(str(rdata))
add_attribute(new_event, 'ip-dst', rdata.to_text(), 'Network activity', False, enforcewarninglist,
sighting, comment=hostname)
hip.add_attribute('ip', type='ip-dst', value=rdata.to_text(), to_ids=False)
except Exception as e:
if debug:
syslog.syslog(str(e))
# misp_event.add_object(hip)
# misp.update_event(event_id, new_event)
syslog.syslog(str(e))
# Try to add attachments
if stdin_used:
for part in msg.walk():
if part.get_content_maintype() == 'multipart':
continue
if part.get_content_maintype() != 'text' and part.get_payload(decode=True) is not None:
filename = part.get_filename()
_, output_path = tempfile.mkstemp()
output = open(output_path, 'wb')
output.write(part.get_payload(decode=True))
output.close()
attachment = part.get_payload(decode=True)
if debug:
syslog.syslog(str(attachment)[:200])
if m2m_attachment_keyword in email_data:
misp.add_attachment(misp_event, output_path, filename=filename, category="External analysis")
else:
misp.upload_sample(filename, output_path, event_id, distribution=5, to_ids=True)
file_hash = hashlib.sha256(open(output_path, 'rb').read()).hexdigest()
sight(sighting, file_hash)
if related_ips:
hip = MISPObject(name='ip-port')
hip.add_attribute('hostname', value=hostname, to_ids=ids_flag,
enforceWarninglist=config.enforcewarninglist, comment=comment)
for ip in set(related_ips):
hip.add_attribute('ip', type='ip-dst', value=ip, to_ids=False,
enforceWarninglist=config.enforcewarninglist)
self.misp_event.add_object(hip)
else:
self.misp_event.add_attribute('hostname', value=hostname,
to_ids=ids_flag, enforceWarninglist=config.enforcewarninglist,
comment=comment)
if auto_publish:
misp.publish(misp_event, alert=False)
def add_event(self):
'''Add event on the remote MISP instance.'''
syslog.syslog("Job finished.")
# Add additional tags depending on others
tags = []
for tag in [t.name for t in self.misp_event.tags]:
if config.dependingtags.get(tag):
tags += config.dependingtags.get(tag)
# Add additional tags according to configuration
for malware in config.malwaretags:
if malware.lower() in self.subject.lower():
tags += config.malwaretags.get(malware)
if tags:
[self.misp_event.add_tag(tag) for tag in tags]
self.misp.add_event(self.misp_event)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Push a Mail into a MISP instance')
parser.add_argument("-r", "--read", help="Read from tempfile.")
parser.add_argument("-t", "--trap", action='store_true', default=False, help="Import the Email as-is.")
parser.add_argument('infile', nargs='?', type=argparse.FileType('rb'), default=sys.stdin)
args = parser.parse_args()
syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_USER)
syslog.syslog("Job started.")
configmodule = Path(__file__).as_posix().replace('.py', '_config')
if Path(f'{configmodule}.py').exists():
config = importlib.import_module(configmodule)
try:
misp_url = config.misp_url
misp_key = config.misp_key
misp_verifycert = config.misp_verifycert
debug = config.debug
except Exception as e:
syslog.syslog(str(e))
print("There is a problem with the configuration. A mandatory configuration variable is not set.")
print("Did you just update? mail_to_misp might have new configuration variables.")
print("Please compare with the configuration example.")
print("\nTrace:")
print(e)
sys.exit(-2)
else:
print("Couldn't locate config file {0}".format(f'{configmodule}.py'))
sys.exit(-1)
if args.infile:
pseudofile = BytesIO(args.infile.read().encode())
elif args.read:
# read from tempfile
with open(args.read, 'rb') as f:
pseudofile = BytesIO(f.read())
else:
# receive data and subject through arguments
raise Exception('This is not implemented anymore.')
mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config)
mail2misp.load_email(pseudofile)
if debug:
syslog.syslog(f'Working on {mail2misp.subject}')
if args.trap or config.spamtrap:
mail2misp.email_from_spamtrap()
else:
mail2misp.process_email_body()
mail2misp.process_body_iocs()
mail2misp.add_event()
syslog.syslog("Job finished.")

67
mail_to_misp_config.py-example
View File

@ -1,18 +1,20 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import sys
import os
misp_url = 'YOUR_MISP_URL'
misp_key = 'YOUR_KEY_HERE' # The MISP auth key can be found on the MISP web interface under the automation section
misp_key = 'YOUR_KEY_HERE' # The MISP auth key can be found on the MISP web interface under the automation section
misp_verifycert = True
body_config_prefix = 'm2m' # every line in the body starting with this value will be skipped from the IOCs
spamtrap = False
m2m_key = 'YOUSETYOURKEYHERE'
m2m_auto_distribution = '3' # 3 = All communities
m2m_auto_distribution = '3' # 3 = All communities
m2m_attachment_keyword = 'attachment:benign'
debug = False
nameservers = ['149.13.33.69']
email_subject_prefix = b'M2M - '
email_subject_prefix = 'M2M'
attach_original_mail = True
# Paths (should be automatic)
@ -28,57 +30,56 @@ smtp_port = 25
excludelist = ('google.com', 'microsoft.com')
externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net')
internallist = ('internal.system.local')
noidsflaglist = ( 'myexternalip.com', 'ipinfo.io', 'icanhazip.com', 'wtfismyip.com', 'ipecho.net',
'api.ipify.org', 'checkip.amazonaws.com', 'whatismyipaddress.com', 'google.com',
'dropbox.com'
)
noidsflaglist = ('myexternalip.com', 'ipinfo.io', 'icanhazip.com', 'wtfismyip.com', 'ipecho.net',
'api.ipify.org', 'checkip.amazonaws.com', 'whatismyipaddress.com', 'google.com',
'dropbox.com'
)
# Stop parsing when this term is found
stopword = 'Whois & IP Information'
# Ignore lines in body of message containing:
ignorelist = (".*From: .*\n?", ".*Sender: .*\n?", ".*Received: .*\n?", ".*Sender IP: .*\n?",
".*Reply-To: .*\n?", ".*Registrar WHOIS Server: .*\n?", ".*Registrar: .*\n?",
".*Domain Status: .*\n?", ".*Registrant Email: .*\n?", ".*IP Location: .*\n?",
".*X-Get-Message-Sender-Via: .*\n?", ".*X-Authenticated-Sender: .*\n")
ignorelist = ("From:", "Sender:", "Received:", "Sender IP:", "Reply-To:", "Registrar WHOIS Server:",
"Registrar:", "Domain Status:", "Registrant Email:", "IP Location:",
"X-Get-Message-Sender-Via:", "X-Authenticated-Sender:")
# Ignore (don't add) attributes that are on server side warning list
enforcewarninglist=True
enforcewarninglist = True
# Add a sighting for each value
sighting=True
sighting_source="YOUR_MAIL_TO_MISP_IDENTIFIER"
sighting = True
sighting_source = "YOUR_MAIL_TO_MISP_IDENTIFIER"
# Remove "[tags]", "Re: ", "Fwd: " from subject
removelist = ("[\(\[].*?[\)\]]", "Re: ", "Fwd: ")
# TLP tag setup
# Tuples contain different variations of spelling
tlptags = { 'tlp:amber': [ 'tlp:amber', 'tlp: amber', 'tlp amber' ],
'tlp:green': [ 'tlp:green', 'tlp: green', 'tlp green' ],
'tlp:white': [ 'tlp:white', 'tlp: white', 'tlp white' ]
}
tlptags = {'tlp:amber': ['tlp:amber', 'tlp: amber', 'tlp amber'],
'tlp:green': ['tlp:green', 'tlp: green', 'tlp green'],
'tlp:white': ['tlp:white', 'tlp: white', 'tlp white']
}
tlptag_default = sorted(tlptags.keys())[0]
malwaretags = { 'locky': [ 'ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Locky"' ],
'jaff': [ 'ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Jaff"' ],
'dridex': [ 'misp-galaxy:tool="dridex"' ],
'netwire': [ 'Netwire RAT' ],
'Pony': [ 'misp-galaxy:tool="Hancitor"' ],
'ursnif': [ 'misp-galaxy:tool="Snifula"' ],
'NanoCore': [ 'misp-galaxy:tool="NanoCoreRAT"' ],
'trickbot': [ 'misp-galaxy:tool="Trick Bot"' ]
}
malwaretags = {'locky': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Locky"'],
'jaff': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Jaff"'],
'dridex': ['misp-galaxy:tool="dridex"'],
'netwire': ['Netwire RAT'],
'Pony': ['misp-galaxy:tool="Hancitor"'],
'ursnif': ['misp-galaxy:tool="Snifula"'],
'NanoCore': ['misp-galaxy:tool="NanoCoreRAT"'],
'trickbot': ['misp-galaxy:tool="Trick Bot"']
}
# Tags to be set depending on the presence of other tags
dependingtags = { 'tlp:white': [ 'circl:osint-feed' ]
}
dependingtags = {'tlp:white': ['circl:osint-feed']
}
# Known identifiers for forwarded messages
forward_identifiers = { '-------- Forwarded Message --------', 'Begin forwarded message:' }
# Known identifiers for forwarded messages
forward_identifiers = {'-------- Forwarded Message --------', 'Begin forwarded message:'}
# Tags to add when hashes are found (e.g. to do automatic expansion)
hash_only_tags = { 'TODO:VT-ENRICHMENT' }
hash_only_tags = {'TODO:VT-ENRICHMENT'}
# If an attribute is on any MISP server side `warning list`, skip the creation of the attribute
skip_item_on_warninglist = True

5
reqirements.txt Normal file
View File

@ -0,0 +1,5 @@
dnspython
pymisp[fileobjects]
git+https://github.com/kbandla/pydeep.git
git+https://github.com/stricaud/faup.git#egg=pyfaup&subdirectory=src/lib/bindings/python
git+https://github.com/Rafiot/defang.git