MISP
/
mail_to_misp
mirror of https://github.com/MISP/mail_to_misp
2
0
Fork 0
Code Issues Releases Wiki Activity

new: Make it a lib, add test cases

tests
Raphaël Vinot 2018-05-14 17:23:30 -04:00
parent fc17a40b18
commit c967c5e197
14 changed files with 605 additions and 345 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored Normal file
View File

@ -0,0 +1,3 @@
[submodule "tests/mails"]
path = tests/mails
url = https://github.com/MISP/mail_to_misp_test.git

35
.travis.yml Normal file
View File

@ -0,0 +1,35 @@
language: python
cache: pip
sudo: required
addons:
apt:
sources: [ 'ubuntu-toolchain-r-test' ]
packages:
- libstdc++6
- libfuzzy-dev
python:
- "3.6"
- "3.6-dev"
install:
- git clone git://github.com/stricaud/faup.git
- pushd faup/build
- cmake .. && make
- sudo make install
- popd
- sudo ldconfig
- pip install -U nose pip coverage
- pip install -U -r requirements.txt
- git submodule init
- git submodule update
script:
- nosetests --with-coverage --cover-package=mail2misp tests/tests.py
after_success:
- codecov
- coveralls

0
__init__.py Normal file
View File

3
mail2misp/__init__.py Normal file
View File

@ -0,0 +1,3 @@
from . import urlmarker
from . import hashmarker
from .mail2misp import Mail2MISP

0
hashmarker.py → mail2misp/hashmarker.py Executable file → Normal file
View File

350
mail2misp/mail2misp.py Normal file
View File

@ -0,0 +1,350 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import re
import syslog
import html
from io import BytesIO
from ipaddress import ip_address
from email import message_from_bytes, policy
from . import urlmarker
from . import hashmarker
from pyfaup.faup import Faup
from pymisp import PyMISP, MISPEvent, MISPObject, MISPSighting
from pymisp.tools import EMailObject, make_binary_objects
from defang import refang
import dns.resolver
def is_ip(address):
try:
ip_address(address)
except ValueError:
return False
return True
class Mail2MISP():
def __init__(self, misp_url, misp_key, verifycert, config, offline=False):
self.offline = offline
if not self.offline:
self.misp = PyMISP(misp_url, misp_key, verifycert, debug=config.debug)
self.config = config
self.debug = self.config.debug
self.config_from_email_body = {}
# Init Faup
self.f = Faup()
def load_email(self, pseudofile):
self.pseudofile = pseudofile
self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default)
self.subject = self.original_mail.get('Subject')
# Remove words from subject
for removeword in self.config.removelist:
self.subject = re.sub(removeword, "", self.subject).strip()
# Initialize the MISP event
self.misp_event = MISPEvent()
self.misp_event.info = f'{self.config.email_subject_prefix} - {self.subject}'
self.misp_event.distribution = self.config.default_distribution
self.misp_event.threat_level_id = self.config.default_threat_level
self.misp_event.analysis = self.config.default_analysis
def sighting(self, value, source):
if self.offline:
raise Exception('The script is running in offline mode, ')
'''Add a sighting'''
s = MISPSighting()
s.from_dict(value=value, source=source)
self.misp.set_sightings(s)
def _find_inline_forward(self):
'''Does the body contains a forwarded email?'''
for identifier in self.config.forward_identifiers:
if identifier in self.clean_email_body:
self.clean_email_body, fw_email = self.clean_email_body.split(identifier)
return self.forwarded_email(pseudofile=BytesIO(fw_email.encode()))
def _find_attached_forward(self):
forwarded_emails = []
for attachment in self.original_mail.iter_attachments():
# Search for email forwarded as attachment
# I could have more than one, attaching everything.
if attachment.get_filename() and attachment.get_filename().endswith('.eml'):
forwarded_emails.append(self.forwarded_email(pseudofile=BytesIO(attachment.get_content().as_bytes())))
else:
filename = attachment.get_filename()
if not filename:
filename = 'missing_filename'
if self.config_from_email_body.get('attachment') == self.config.m2m_benign_attachment_keyword:
# Attach sane file
self.misp_event.add_attribute('attachment', value=filename, data=BytesIO(attachment.get_content()))
else:
f_object, main_object, sections = make_binary_objects(pseudofile=BytesIO(attachment.get_content()), filename=filename, standalone=False)
self.misp_event.add_object(f_object)
if main_object:
self.misp_event.add_object(main_object)
[self.misp_event.add_object(section) for section in sections]
return forwarded_emails
def email_from_spamtrap(self):
'''The email comes from a spamtrap and should be attached as-is.'''
raw_body = self.original_mail.get_body(preferencelist=('html', 'plain'))
if raw_body:
self.clean_email_body = html.unescape(raw_body.get_payload(decode=True).decode('utf8', 'surrogateescape'))
else:
self.clean_email_body = ''
return self.forwarded_email(self.pseudofile)
def forwarded_email(self, pseudofile: BytesIO):
'''Extracts all possible indicators out of an email and create a MISP event out of it.
* Gets all relevant Headers
* Attach the body
* Create MISP file objects (uses lief if possible)
* Set all references
'''
email_object = EMailObject(pseudofile=pseudofile, attach_original_mail=True, standalone=False)
if email_object.attachments:
# Create file objects for the attachments
for attachment_name, attachment in email_object.attachments:
if not attachment_name:
attachment_name = 'NameMissing.txt'
f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False)
self.misp_event.add_object(f_object)
if main_object:
self.misp_event.add_object(main_object)
for section in sections:
self.misp_event.add_object(section)
email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment')
self.process_body_iocs(email_object)
if self.config.spamtrap or self.config.attach_original_mail or self.config_from_email_body.get('attach_original_mail'):
self.misp_event.add_object(email_object)
return email_object
def process_email_body(self):
mail_as_bytes = self.original_mail.get_body(preferencelist=('html', 'plain')).get_payload(decode=True)
if mail_as_bytes:
self.clean_email_body = html.unescape(mail_as_bytes.decode('utf8', 'surrogateescape'))
# Check if there are config lines in the body & convert them to a python dictionary:
# <config.body_config_prefix>:<key>:<value> => {<key>: <value>}
self.config_from_email_body = {k.strip(): v.strip() for k, v in re.findall(f'{self.config.body_config_prefix}:(.*):(.*)', self.clean_email_body)}
if self.config_from_email_body:
# ... remove the config lines from the body
self.clean_email_body = re.sub(rf'^{self.config.body_config_prefix}.*\n?', '',
html.unescape(self.original_mail.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode('utf8', 'surrogateescape')), flags=re.MULTILINE)
# Check if autopublish key is present and valid
if self.config_from_email_body.get('m2mkey') == self.config.m2m_key:
if self.config_from_email_body.get('distribution'):
self.misp_event.distribution = self.config_from_email_body.get('distribution')
if self.config_from_email_body.get('threat_level'):
self.misp_event.threat_level_id = self.config_from_email_body.get('threat_level')
if self.config_from_email_body.get('analysis'):
self.misp_event.analysis = self.config_from_email_body.get('analysis')
if self.config_from_email_body.get('publish'):
self.misp_event.publish()
self._find_inline_forward()
else:
self.clean_email_body = ''
self._find_attached_forward()
def process_body_iocs(self, email_object=None):
if email_object:
body = html.unescape(email_object.email.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode('utf8', 'surrogateescape'))
else:
body = self.clean_email_body
# Cleanup body content
# Depending on the source of the mail, there is some cleanup to do. Ignore lines in body of message
for ignoreline in self.config.ignorelist:
body = re.sub(rf'^{ignoreline}.*\n?', '', body, flags=re.MULTILINE)
# Remove everything after the stopword from the body
body = body.split(self.config.stopword, 1)[0]
# Add tags to the event if keywords are found in the mail
for tag in self.config.tlptags:
if any(alternativetag in body.lower() for alternativetag in self.config.tlptags[tag]):
self.misp_event.add_tag(tag)
# Prepare extraction of IOCs
# Refang email data
body = refang(body)
# Extract and add hashes
contains_hash = False
for h in set(re.findall(hashmarker.MD5_REGEX, body)):
contains_hash = True
attribute = self.misp_event.add_attribute('md5', h, enforceWarninglist=self.config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
if self.config.sighting:
self.sighting(h, self.config.sighting_source)
for h in set(re.findall(hashmarker.SHA1_REGEX, body)):
contains_hash = True
attribute = self.misp_event.add_attribute('sha1', h, enforceWarninglist=self.config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
if self.config.sighting:
self.sighting(h, self.config.sighting_source)
for h in set(re.findall(hashmarker.SHA256_REGEX, body)):
contains_hash = True
attribute = self.misp_event.add_attribute('sha256', h, enforceWarninglist=self.config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
if self.config.sighting:
self.sighting(h, self.config.sighting_source)
if contains_hash:
[self.misp_event.add_tag(tag) for tag in self.config.hash_only_tags]
# # Extract network IOCs
urllist = []
urllist += re.findall(urlmarker.WEB_URL_REGEX, body)
urllist += re.findall(urlmarker.IP_REGEX, body)
if self.debug:
syslog.syslog(str(urllist))
hostname_processed = []
# Add IOCs and expanded information to MISP
for entry in set(urllist):
ids_flag = True
self.f.decode(entry)
domainname = self.f.get_domain().decode()
if domainname in self.config.excludelist:
# Ignore the entry
continue
hostname = self.f.get_host().decode()
scheme = self.f.get_scheme()
if scheme:
scheme = scheme.decode()
resource_path = self.f.get_resource_path()
if resource_path:
resource_path = resource_path.decode()
if self.debug:
syslog.syslog(domainname)
if domainname in self.config.internallist: # Add link to internal reference
attribute = self.misp_event.add_attribute('link', entry, category='Internal reference',
to_ids=False, enforceWarninglist=False)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
elif domainname in self.config.externallist: # External analysis
attribute = self.misp_event.add_attribute('link', entry, category='External analysis',
to_ids=False, enforceWarninglist=False)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
else: # The URL is probably an indicator.
comment = ""
if (domainname in self.config.noidsflaglist) or (hostname in self.config.noidsflaglist):
ids_flag = False
comment = "Known host (mostly for connectivity test or IP lookup)"
if self.debug:
syslog.syslog(str(entry))
if scheme:
if is_ip(hostname):
attribute = self.misp_event.add_attribute('url', entry, to_ids=False,
enforceWarninglist=self.config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
else:
if resource_path: # URL has path, ignore warning list
attribute = self.misp_event.add_attribute('url', entry, to_ids=ids_flag,
enforceWarninglist=False, comment=comment)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
else: # URL has no path
attribute = self.misp_event.add_attribute('url', entry, to_ids=ids_flag,
enforceWarninglist=self.config.enforcewarninglist, comment=comment)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
if self.config.sighting:
self.sighting(entry, self.config.sighting_source)
if hostname in hostname_processed:
# Hostname already processed.
continue
hostname_processed.append(hostname)
if self.config.sighting:
self.sighting(hostname, self.config.sighting_source)
if self.debug:
syslog.syslog(hostname)
comment = ''
port = self.f.get_port()
if port:
port = port.decode()
comment = f'on port: {port}'
if is_ip(hostname):
attribute = self.misp_event.add_attribute('ip-dst', hostname, to_ids=ids_flag,
enforceWarninglist=self.config.enforcewarninglist,
comment=comment)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
else:
related_ips = []
try:
syslog.syslog(hostname)
for rdata in dns.resolver.query(hostname, 'A'):
if self.debug:
syslog.syslog(str(rdata))
related_ips.append(rdata.to_text())
except Exception as e:
if self.debug:
syslog.syslog(str(e))
if related_ips:
hip = MISPObject(name='ip-port')
hip.add_attribute('hostname', value=hostname, to_ids=ids_flag,
enforceWarninglist=self.config.enforcewarninglist, comment=comment)
for ip in set(related_ips):
hip.add_attribute('ip', type='ip-dst', value=ip, to_ids=False,
enforceWarninglist=self.config.enforcewarninglist)
self.misp_event.add_object(hip)
if email_object:
email_object.add_reference(hip.uuid, 'contains')
else:
attribute = self.misp_event.add_attribute('hostname', value=hostname,
to_ids=ids_flag, enforceWarninglist=self.config.enforcewarninglist,
comment=comment)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
def add_event(self):
'''Add event on the remote MISP instance.'''
# Add additional tags depending on others
tags = []
for tag in [t.name for t in self.misp_event.tags]:
if self.config.dependingtags.get(tag):
tags += self.config.dependingtags.get(tag)
# Add additional tags according to configuration
for malware in self.config.malwaretags:
if malware.lower() in self.subject.lower():
tags += self.config.malwaretags.get(malware)
if tags:
[self.misp_event.add_tag(tag) for tag in tags]
has_tlp_tag = False
for tag in [t.name for t in self.misp_event.tags]:
if tag.lower().startswith('tlp'):
has_tlp_tag = True
if not has_tlp_tag:
self.misp_event.add_tag(self.config.tlptag_default)
if self.offline:
return self.misp_event.to_json()
return self.misp.add_event(self.misp_event)

0
urlmarker.py → mail2misp/urlmarker.py Executable file → Normal file
View File

345
mail_to_misp.py
View File

@ -4,355 +4,12 @@
import os
import sys
import argparse
import re
import syslog
from pathlib import Path
import html
from io import BytesIO
from ipaddress import ip_address
from email import message_from_bytes, policy
import importlib
try:
import urlmarker
import hashmarker
from pyfaup.faup import Faup
from pymisp import PyMISP, MISPEvent, MISPObject, MISPSighting
from pymisp.tools import EMailObject, make_binary_objects
from defang import refang
import dns.resolver
except ImportError as e:
print("(!) Problem loading module:")
print(e)
sys.exit(-1)
def is_ip(address):
try:
ip_address(address)
except ValueError:
return False
return True
class Mail2MISP():
def __init__(self, misp_url, misp_key, verifycert, config):
self.misp = PyMISP(misp_url, misp_key, verifycert, debug=config.debug)
self.config = config
self.debug = self.config.debug
self.config_from_email_body = {}
# Init Faup
self.f = Faup()
def load_email(self, pseudofile):
self.pseudofile = pseudofile
self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default)
self.subject = self.original_mail.get('Subject')
# Remove words from subject
for removeword in self.config.removelist:
self.subject = re.sub(removeword, "", self.subject).strip()
# Initialize the MISP event
self.misp_event = MISPEvent()
self.misp_event.info = f'{config.email_subject_prefix} - {self.subject}'
self.misp_event.distribution = self.config.default_distribution
self.misp_event.threat_level_id = self.config.default_threat_level
self.misp_event.analysis = self.config.default_analysis
def sighting(self, value, source):
'''Add a sighting'''
s = MISPSighting()
s.from_dict(value=value, source=source)
self.misp.set_sightings(s)
def _find_inline_forward(self):
'''Does the body contains a forwarded email?'''
for identifier in self.config.forward_identifiers:
if identifier in self.clean_email_body:
self.clean_email_body, fw_email = self.clean_email_body.split(identifier)
return self.forwarded_email(pseudofile=BytesIO(fw_email.encode()))
def _find_attached_forward(self):
forwarded_emails = []
for attachment in self.original_mail.iter_attachments():
# Search for email forwarded as attachment
# I could have more than one, attaching everything.
if attachment.get_filename() and attachment.get_filename().endswith('.eml'):
forwarded_emails.append(self.forwarded_email(pseudofile=BytesIO(attachment.get_content().as_bytes())))
else:
filename = attachment.get_filename()
if not filename:
filename = 'missing_filename'
if self.config_from_email_body.get('attachment') == self.config.m2m_benign_attachment_keyword:
# Attach sane file
self.misp_event.add_attribute('attachment', value=filename, data=BytesIO(attachment.get_content()))
else:
f_object, main_object, sections = make_binary_objects(pseudofile=BytesIO(attachment.get_content()), filename=filename, standalone=False)
self.misp_event.add_object(f_object)
if main_object:
self.misp_event.add_object(main_object)
[self.misp_event.add_object(section) for section in sections]
return forwarded_emails
def email_from_spamtrap(self):
'''The email comes from a spamtrap and should be attached as-is.'''
raw_body = self.original_mail.get_body(preferencelist=('html', 'plain'))
if raw_body:
self.clean_email_body = html.unescape(raw_body.get_payload(decode=True).decode('utf8', 'surrogateescape'))
else:
self.clean_email_body = ''
return self.forwarded_email(self.pseudofile)
def forwarded_email(self, pseudofile: BytesIO):
'''Extracts all possible indicators out of an email and create a MISP event out of it.
* Gets all relevant Headers
* Attach the body
* Create MISP file objects (uses lief if possible)
* Set all references
'''
email_object = EMailObject(pseudofile=pseudofile, attach_original_mail=True, standalone=False)
if email_object.attachments:
# Create file objects for the attachments
for attachment_name, attachment in email_object.attachments:
if not attachment_name:
attachment_name = 'NameMissing.txt'
f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False)
self.misp_event.add_object(f_object)
if main_object:
self.misp_event.add_object(main_object)
for section in sections:
self.misp_event.add_object(section)
email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment')
self.process_body_iocs(email_object)
if self.config.spamtrap or self.config.attach_original_mail or self.config_from_email_body.get('attach_original_mail'):
self.misp_event.add_object(email_object)
return email_object
def process_email_body(self):
mail_as_bytes = self.original_mail.get_body(preferencelist=('html', 'plain')).get_payload(decode=True)
if mail_as_bytes:
self.clean_email_body = html.unescape(mail_as_bytes.decode('utf8', 'surrogateescape'))
# Check if there are config lines in the body & convert them to a python dictionary:
# <config.body_config_prefix>:<key>:<value> => {<key>: <value>}
self.config_from_email_body = {k.strip(): v.strip() for k, v in re.findall(f'{config.body_config_prefix}:(.*):(.*)', self.clean_email_body)}
if self.config_from_email_body:
# ... remove the config lines from the body
self.clean_email_body = re.sub(rf'^{config.body_config_prefix}.*\n?', '',
html.unescape(self.original_mail.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode('utf8', 'surrogateescape')), flags=re.MULTILINE)
# Check if autopublish key is present and valid
if self.config_from_email_body.get('m2mkey') == self.config.m2m_key:
if self.config_from_email_body.get('distribution'):
self.misp_event.distribution = self.config_from_email_body.get('distribution')
if self.config_from_email_body.get('threat_level'):
self.misp_event.threat_level_id = self.config_from_email_body.get('threat_level')
if self.config_from_email_body.get('analysis'):
self.misp_event.analysis = self.config_from_email_body.get('analysis')
if self.config_from_email_body.get('publish'):
self.misp_event.publish()
self._find_inline_forward()
else:
self.clean_email_body = ''
self._find_attached_forward()
def process_body_iocs(self, email_object=None):
if email_object:
body = html.unescape(email_object.email.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode('utf8', 'surrogateescape'))
else:
body = self.clean_email_body
# Cleanup body content
# Depending on the source of the mail, there is some cleanup to do. Ignore lines in body of message
for ignoreline in self.config.ignorelist:
body = re.sub(rf'^{ignoreline}.*\n?', '', body, flags=re.MULTILINE)
# Remove everything after the stopword from the body
body = body.split(self.config.stopword, 1)[0]
# Add tags to the event if keywords are found in the mail
for tag in self.config.tlptags:
if any(alternativetag in body.lower() for alternativetag in self.config.tlptags[tag]):
self.misp_event.add_tag(tag)
# Prepare extraction of IOCs
# Refang email data
body = refang(body)
# Extract and add hashes
contains_hash = False
for h in set(re.findall(hashmarker.MD5_REGEX, body)):
contains_hash = True
attribute = self.misp_event.add_attribute('md5', h, enforceWarninglist=config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
if self.config.sighting:
self.sighting(h, self.config.sighting_source)
for h in set(re.findall(hashmarker.SHA1_REGEX, body)):
contains_hash = True
attribute = self.misp_event.add_attribute('sha1', h, enforceWarninglist=config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
if self.config.sighting:
self.sighting(h, self.config.sighting_source)
for h in set(re.findall(hashmarker.SHA256_REGEX, body)):
contains_hash = True
attribute = self.misp_event.add_attribute('sha256', h, enforceWarninglist=config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
if self.config.sighting:
self.sighting(h, self.config.sighting_source)
if contains_hash:
[self.misp_event.add_tag(tag) for tag in self.config.hash_only_tags]
# # Extract network IOCs
urllist = []
urllist += re.findall(urlmarker.WEB_URL_REGEX, body)
urllist += re.findall(urlmarker.IP_REGEX, body)
if self.debug:
syslog.syslog(str(urllist))
hostname_processed = []
# Add IOCs and expanded information to MISP
for entry in set(urllist):
ids_flag = True
self.f.decode(entry)
domainname = self.f.get_domain().decode()
if domainname in self.config.excludelist:
# Ignore the entry
continue
hostname = self.f.get_host().decode()
scheme = self.f.get_scheme()
if scheme:
scheme = scheme.decode()
resource_path = self.f.get_resource_path()
if resource_path:
resource_path = resource_path.decode()
if debug:
syslog.syslog(domainname)
if domainname in self.config.internallist: # Add link to internal reference
attribute = self.misp_event.add_attribute('link', entry, category='Internal reference',
to_ids=False, enforceWarninglist=False)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
elif domainname in self.config.externallist: # External analysis
attribute = self.misp_event.add_attribute('link', entry, category='External analysis',
to_ids=False, enforceWarninglist=False)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
else: # The URL is probably an indicator.
comment = ""
if (domainname in self.config.noidsflaglist) or (hostname in self.config.noidsflaglist):
ids_flag = False
comment = "Known host (mostly for connectivity test or IP lookup)"
if debug:
syslog.syslog(str(entry))
if scheme:
if is_ip(hostname):
attribute = self.misp_event.add_attribute('url', entry, to_ids=False,
enforceWarninglist=config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
else:
if resource_path: # URL has path, ignore warning list
attribute = self.misp_event.add_attribute('url', entry, to_ids=ids_flag,
enforceWarninglist=False, comment=comment)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
else: # URL has no path
attribute = self.misp_event.add_attribute('url', entry, to_ids=ids_flag,
enforceWarninglist=config.enforcewarninglist, comment=comment)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
if self.config.sighting:
self.sighting(entry, self.config.sighting_source)
if hostname in hostname_processed:
# Hostname already processed.
continue
hostname_processed.append(hostname)
if self.config.sighting:
self.sighting(hostname, self.config.sighting_source)
if debug:
syslog.syslog(hostname)
comment = ''
port = self.f.get_port()
if port:
port = port.decode()
comment = f'on port: {port}'
if is_ip(hostname):
attribute = self.misp_event.add_attribute('ip-dst', hostname, to_ids=ids_flag,
enforceWarninglist=config.enforcewarninglist,
comment=comment)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
else:
related_ips = []
try:
syslog.syslog(hostname)
for rdata in dns.resolver.query(hostname, 'A'):
if debug:
syslog.syslog(str(rdata))
related_ips.append(rdata.to_text())
except Exception as e:
if debug:
syslog.syslog(str(e))
if related_ips:
hip = MISPObject(name='ip-port')
hip.add_attribute('hostname', value=hostname, to_ids=ids_flag,
enforceWarninglist=config.enforcewarninglist, comment=comment)
for ip in set(related_ips):
hip.add_attribute('ip', type='ip-dst', value=ip, to_ids=False,
enforceWarninglist=config.enforcewarninglist)
self.misp_event.add_object(hip)
if email_object:
email_object.add_reference(hip.uuid, 'contains')
else:
attribute = self.misp_event.add_attribute('hostname', value=hostname,
to_ids=ids_flag, enforceWarninglist=config.enforcewarninglist,
comment=comment)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
def add_event(self):
'''Add event on the remote MISP instance.'''
# Add additional tags depending on others
tags = []
for tag in [t.name for t in self.misp_event.tags]:
if self.config.dependingtags.get(tag):
tags += self.config.dependingtags.get(tag)
# Add additional tags according to configuration
for malware in self.config.malwaretags:
if malware.lower() in self.subject.lower():
tags += self.config.malwaretags.get(malware)
if tags:
[self.misp_event.add_tag(tag) for tag in tags]
has_tlp_tag = False
for tag in [t.name for t in self.misp_event.tags]:
if tag.lower().startswith('tlp'):
has_tlp_tag = True
if not has_tlp_tag:
self.misp_event.add_tag(config.tlptag_default)
self.misp.add_event(self.misp_event)
from mail2misp import Mail2MISP
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Push a Mail into a MISP instance')

2
mail_to_misp_config.py-example
View File

@ -43,7 +43,7 @@ sighting_source = "YOUR_MAIL_TO_MISP_IDENTIFIER"
# Remove "Re:", "Fwd:" and {Spam?} from subject
# add: "[\(\[].*?[\)\]]" to remove everything between [] and (): i.e. [tag]
removelist = ("Re:", "Fwd:", "\{Spam\?\} ")
removelist = (r"Re:", r"Fwd:", r"\{Spam\?\} ")
# TLP tag setup
# Tuples contain different variations of spelling

0
tests/__init__.py Normal file
View File

77
tests/config_forward.py Normal file
View File

@ -0,0 +1,77 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
misp_url = 'YOUR_MISP_URL'
misp_key = 'YOUR_KEY_HERE' # The MISP auth key can be found on the MISP web interface under the automation section
misp_verifycert = True
spamtrap = False
default_distribution = 0
default_threat_level = 3
default_analysis = 1
body_config_prefix = 'm2m' # every line in the body starting with this value will be skipped from the IOCs
m2m_key = 'YOUSETYOURKEYHERE'
m2m_benign_attachment_keyword = 'benign'
debug = True
nameservers = ['8.8.8.8']
email_subject_prefix = 'M2M'
attach_original_mail = True
excludelist = ('google.com', 'microsoft.com')
externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net')
internallist = ('internal.system.local')
noidsflaglist = ('myexternalip.com', 'ipinfo.io', 'icanhazip.com', 'wtfismyip.com', 'ipecho.net',
'api.ipify.org', 'checkip.amazonaws.com', 'whatismyipaddress.com', 'google.com',
'dropbox.com'
)
# Stop parsing when this term is found
stopword = 'Whois & IP Information'
# Ignore lines in body of message containing:
ignorelist = ("From:", "Sender:", "Received:", "Sender IP:", "Reply-To:", "Registrar WHOIS Server:",
"Registrar:", "Domain Status:", "Registrant Email:", "IP Location:",
"X-Get-Message-Sender-Via:", "X-Authenticated-Sender:")
# Ignore (don't add) attributes that are on server side warning list
enforcewarninglist = True
# Add a sighting for each value
sighting = False
sighting_source = "YOUR_MAIL_TO_MISP_IDENTIFIER"
# Remove "Re:", "Fwd:" and {Spam?} from subject
# add: "[\(\[].*?[\)\]]" to remove everything between [] and (): i.e. [tag]
removelist = (r'Re:', r'Fwd:', r'\{Spam?\}')
# TLP tag setup
# Tuples contain different variations of spelling
tlptags = {'tlp:amber': ['tlp:amber', 'tlp: amber', 'tlp amber'],
'tlp:green': ['tlp:green', 'tlp: green', 'tlp green'],
'tlp:white': ['tlp:white', 'tlp: white', 'tlp white']
}
tlptag_default = sorted(tlptags.keys())[0]
malwaretags = {'locky': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Locky"'],
'jaff': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Jaff"'],
'dridex': ['misp-galaxy:tool="dridex"'],
'netwire': ['Netwire RAT'],
'Pony': ['misp-galaxy:tool="Hancitor"'],
'ursnif': ['misp-galaxy:tool="Snifula"'],
'NanoCore': ['misp-galaxy:tool="NanoCoreRAT"'],
'trickbot': ['misp-galaxy:tool="Trick Bot"']
}
# Tags to be set depending on the presence of other tags
dependingtags = {'tlp:white': ['circl:osint-feed']
}
# Known identifiers for forwarded messages
forward_identifiers = {'-------- Forwarded Message --------', 'Begin forwarded message:'}
# Tags to add when hashes are found (e.g. to do automatic expansion)
hash_only_tags = {'TODO:VT-ENRICHMENT'}
# If an attribute is on any MISP server side `warning list`, skip the creation of the attribute
skip_item_on_warninglist = True

77
tests/config_spamtrap.py Normal file
View File

@ -0,0 +1,77 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
misp_url = 'YOUR_MISP_URL'
misp_key = 'YOUR_KEY_HERE' # The MISP auth key can be found on the MISP web interface under the automation section
misp_verifycert = True
spamtrap = True
default_distribution = 0
default_threat_level = 3
default_analysis = 1
body_config_prefix = 'm2m' # every line in the body starting with this value will be skipped from the IOCs
m2m_key = 'YOUSETYOURKEYHERE'
m2m_benign_attachment_keyword = 'benign'
debug = True
nameservers = ['8.8.8.8']
email_subject_prefix = 'M2M'
attach_original_mail = True
excludelist = ('google.com', 'microsoft.com')
externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net')
internallist = ('internal.system.local')
noidsflaglist = ('myexternalip.com', 'ipinfo.io', 'icanhazip.com', 'wtfismyip.com', 'ipecho.net',
'api.ipify.org', 'checkip.amazonaws.com', 'whatismyipaddress.com', 'google.com',
'dropbox.com'
)
# Stop parsing when this term is found
stopword = 'Whois & IP Information'
# Ignore lines in body of message containing:
ignorelist = ("From:", "Sender:", "Received:", "Sender IP:", "Reply-To:", "Registrar WHOIS Server:",
"Registrar:", "Domain Status:", "Registrant Email:", "IP Location:",
"X-Get-Message-Sender-Via:", "X-Authenticated-Sender:")
# Ignore (don't add) attributes that are on server side warning list
enforcewarninglist = True
# Add a sighting for each value
sighting = False
sighting_source = "YOUR_MAIL_TO_MISP_IDENTIFIER"
# Remove "Re:", "Fwd:" and {Spam?} from subject
# add: "[\(\[].*?[\)\]]" to remove everything between [] and (): i.e. [tag]
removelist = (r'Re:', r'Fwd:', r'\{Spam\?\}')
# TLP tag setup
# Tuples contain different variations of spelling
tlptags = {'tlp:amber': ['tlp:amber', 'tlp: amber', 'tlp amber'],
'tlp:green': ['tlp:green', 'tlp: green', 'tlp green'],
'tlp:white': ['tlp:white', 'tlp: white', 'tlp white']
}
tlptag_default = sorted(tlptags.keys())[0]
malwaretags = {'locky': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Locky"'],
'jaff': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Jaff"'],
'dridex': ['misp-galaxy:tool="dridex"'],
'netwire': ['Netwire RAT'],
'Pony': ['misp-galaxy:tool="Hancitor"'],
'ursnif': ['misp-galaxy:tool="Snifula"'],
'NanoCore': ['misp-galaxy:tool="NanoCoreRAT"'],
'trickbot': ['misp-galaxy:tool="Trick Bot"']
}
# Tags to be set depending on the presence of other tags
dependingtags = {'tlp:white': ['circl:osint-feed']
}
# Known identifiers for forwarded messages
forward_identifiers = {'-------- Forwarded Message --------', 'Begin forwarded message:'}
# Tags to add when hashes are found (e.g. to do automatic expansion)
hash_only_tags = {'TODO:VT-ENRICHMENT'}
# If an attribute is on any MISP server side `warning list`, skip the creation of the attribute
skip_item_on_warninglist = True

1
tests/mails Submodule

@ -0,0 +1 @@
Subproject commit 1e673f9d64e24136a9e4ef3f731c1a6052f9bb87

57
tests/tests.py Normal file
View File

@ -0,0 +1,57 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import unittest
import importlib
import sys
from io import BytesIO
sys.path.insert(0, ".")
from mail2misp import Mail2MISP
class TestMailToMISP(unittest.TestCase):
def test_spamtrap(self):
config = importlib.import_module('tests.config_spamtrap')
self.mail2misp = Mail2MISP('', '', '', config=config, offline=True)
with open('tests/mails/simple_spamtrap.eml', 'rb') as f:
self.mail2misp.load_email(BytesIO(f.read()))
self.mail2misp.email_from_spamtrap()
self.mail2misp.process_body_iocs()
event = self.mail2misp.add_event()
print(event)
def test_spamtrap_attachment(self):
config = importlib.import_module('tests.config_spamtrap')
self.mail2misp = Mail2MISP('', '', '', config=config, offline=True)
with open('tests/mails/attachment_spamtrap.eml', 'rb') as f:
self.mail2misp.load_email(BytesIO(f.read()))
self.mail2misp.email_from_spamtrap()
self.mail2misp.process_body_iocs()
event = self.mail2misp.add_event()
print(event)
def test_forward(self):
config = importlib.import_module('tests.config_forward')
self.mail2misp = Mail2MISP('', '', '', config=config, offline=True)
with open('tests/mails/simple_forward.eml', 'rb') as f:
self.mail2misp.load_email(BytesIO(f.read()))
self.mail2misp.process_email_body()
self.mail2misp.process_body_iocs()
event = self.mail2misp.add_event()
print(event)
def test_forward_attachment(self):
config = importlib.import_module('tests.config_forward')
self.mail2misp = Mail2MISP('', '', '', config=config, offline=True)
with open('tests/mails/attachment_forward.eml', 'rb') as f:
self.mail2misp.load_email(BytesIO(f.read()))
self.mail2misp.process_email_body()
self.mail2misp.process_body_iocs()
event = self.mail2misp.add_event()
print(event)
if __name__ == '__main__':
unittest.main()