MISP
/
mail_to_misp
mirror of https://github.com/MISP/mail_to_misp
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: more refactoring and bugfixes. Add relations.

slight_refactoring
Raphaël Vinot 2018-05-03 20:52:31 +02:00
parent 18bba44d23
commit 747a8e0ee4
1 changed files with 67 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
mail_to_misp.py
View File

@ -38,6 +38,8 @@ class Mail2MISP():
self.misp = PyMISP(misp_url, misp_key, verifycert, debug=config.debug) self.misp = PyMISP(misp_url, misp_key, verifycert, debug=config.debug)
self.debug = config.debug self.debug = config.debug
self.config = config self.config = config
# Init Faup
self.f = Faup()
def load_email(self, pseudofile): def load_email(self, pseudofile):
self.pseudofile = pseudofile self.pseudofile = pseudofile
@ -73,14 +75,14 @@ class Mail2MISP():
# Search for email forwarded as attachment # Search for email forwarded as attachment
# I could have more than one, attaching everything. # I could have more than one, attaching everything.
if attachment.get_filename() and attachment.get_filename().endswith('.eml'): if attachment.get_filename() and attachment.get_filename().endswith('.eml'):
self.forwarded_email(pseudofile=BytesIO(attachment.get_content().encode())) self.forwarded_email(pseudofile=BytesIO(attachment.get_content().as_bytes()))
else: else:
if self.config_from_email_body.get('attachment') == 'benign': if self.config_from_email_body.get('attachment') == 'benign':
# Attach sane file # Attach sane file
self.misp_event.add_attribute('attachment', value='Report', self.misp_event.add_attribute('attachment', value='Report',
data=BytesIO(attachment.get_content().encode())) data=BytesIO(attachment.get_content().as_bytes()))
else: else:
f_object, main_object, sections = make_binary_objects(pseudofile=BytesIO(attachment.get_content().encode()), f_object, main_object, sections = make_binary_objects(pseudofile=BytesIO(attachment.get_content()),
filename=attachment.get_filename(), standalone=False) filename=attachment.get_filename(), standalone=False)
self.misp_event.add_object(f_object) self.misp_event.add_object(f_object)
if main_object: if main_object:
@ -99,7 +101,7 @@ class Mail2MISP():
* Create MISP file objects (uses lief if possible) * Create MISP file objects (uses lief if possible)
* Set all references * Set all references
''' '''
email_object = EMailObject(pseudofile=self.pseudofile, attach_original_mail=True, standalone=False) email_object = EMailObject(pseudofile=pseudofile, attach_original_mail=True, standalone=False)
if email_object.attachments: if email_object.attachments:
# Create file objects for the attachments # Create file objects for the attachments
for attachment_name, attachment in email_object.attachments: for attachment_name, attachment in email_object.attachments:
@ -110,6 +112,7 @@ class Mail2MISP():
for section in sections: for section in sections:
self.misp_event.add_object(section) self.misp_event.add_object(section)
email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment') email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment')
self.process_body_iocs(email_object)
self.misp_event.add_object(email_object) self.misp_event.add_object(email_object)
def process_email_body(self): def process_email_body(self):
@ -144,22 +147,32 @@ class Mail2MISP():
# Remove everything after the stopword from the body # Remove everything after the stopword from the body
self.clean_email_body = self.clean_email_body.split(config.stopword, 1)[0] self.clean_email_body = self.clean_email_body.split(config.stopword, 1)[0]
def process_body_iocs(self): def process_body_iocs(self, email_object=None):
if email_object:
body = email_object.email.get_body().as_string()
else:
body = self.clean_email_body
# Extract and add hashes # Extract and add hashes
contains_hash = False contains_hash = False
for h in set(re.findall(hashmarker.MD5_REGEX, self.clean_email_body)): for h in set(re.findall(hashmarker.MD5_REGEX, body)):
contains_hash = True contains_hash = True
self.misp_event.add_attribute('md5', h, enforceWarninglist=config.enforcewarninglist) attribute = self.misp_event.add_attribute('md5', h, enforceWarninglist=config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
if config.sighting: if config.sighting:
self.sighting(h, config.sighting_source) self.sighting(h, config.sighting_source)
for h in set(re.findall(hashmarker.SHA1_REGEX, self.clean_email_body)): for h in set(re.findall(hashmarker.SHA1_REGEX, body)):
contains_hash = True contains_hash = True
self.misp_event.add_attribute('sha1', h, enforceWarninglist=config.enforcewarninglist) attribute = self.misp_event.add_attribute('sha1', h, enforceWarninglist=config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
if config.sighting: if config.sighting:
self.sighting(h, config.sighting_source) self.sighting(h, config.sighting_source)
for h in set(re.findall(hashmarker.SHA256_REGEX, self.clean_email_body)): for h in set(re.findall(hashmarker.SHA256_REGEX, body)):
contains_hash = True contains_hash = True
self.misp_event.add_attribute('sha256', h, enforceWarninglist=config.enforcewarninglist) attribute = self.misp_event.add_attribute('sha256', h, enforceWarninglist=config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
if config.sighting: if config.sighting:
self.sighting(h, config.sighting_source) self.sighting(h, config.sighting_source)
@ -168,33 +181,30 @@ class Mail2MISP():
# # Extract network IOCs # # Extract network IOCs
urllist = [] urllist = []
urllist += re.findall(urlmarker.WEB_URL_REGEX, self.clean_email_body) urllist += re.findall(urlmarker.WEB_URL_REGEX, body)
urllist += re.findall(urlmarker.IP_REGEX, self.clean_email_body) urllist += re.findall(urlmarker.IP_REGEX, body)
if self.debug: if self.debug:
syslog.syslog(str(urllist)) syslog.syslog(str(urllist))
# Init Faup
f = Faup()
hostname_processed = [] hostname_processed = []
# Add IOCs and expanded information to MISP # Add IOCs and expanded information to MISP
for entry in set(urllist): for entry in set(urllist):
ids_flag = True ids_flag = True
f.decode(entry) self.f.decode(entry)
domainname = f.get_domain().decode() domainname = self.f.get_domain().decode()
if domainname in config.excludelist: if domainname in config.excludelist:
# Ignore the entry # Ignore the entry
continue continue
hostname = f.get_host().decode() hostname = self.f.get_host().decode()
scheme = f.get_scheme() scheme = self.f.get_scheme()
if scheme: if scheme:
scheme = scheme.decode() scheme = scheme.decode()
resource_path = f.get_resource_path() resource_path = self.f.get_resource_path()
if resource_path: if resource_path:
resource_path = resource_path.decode() resource_path = resource_path.decode()
@ -202,11 +212,15 @@ class Mail2MISP():
syslog.syslog(domainname) syslog.syslog(domainname)
if domainname in config.internallist: # Add link to internal reference if domainname in config.internallist: # Add link to internal reference
self.misp_event.add_attribute('link', entry, category='Internal reference', attribute = self.misp_event.add_attribute('link', entry, category='Internal reference',
to_ids=False, enforceWarninglist=False) to_ids=False, enforceWarninglist=False)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
elif domainname in config.externallist: # External analysis elif domainname in config.externallist: # External analysis
self.misp_event.add_attribute('link', entry, category='External analysis', attribute = self.misp_event.add_attribute('link', entry, category='External analysis',
to_ids=False, enforceWarninglist=False) to_ids=False, enforceWarninglist=False)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
else: # The URL is probably an indicator. else: # The URL is probably an indicator.
comment = "" comment = ""
if (domainname in config.noidsflaglist) or (hostname in config.noidsflaglist): if (domainname in config.noidsflaglist) or (hostname in config.noidsflaglist):
@ -217,15 +231,21 @@ class Mail2MISP():
if scheme: if scheme:
if is_ip(hostname): if is_ip(hostname):
self.misp_event.add_attribute('url', entry, to_ids=False, attribute = self.misp_event.add_attribute('url', entry, to_ids=False,
enforceWarninglist=config.enforcewarninglist) enforceWarninglist=config.enforcewarninglist)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
else: else:
if resource_path: # URL has path, ignore warning list if resource_path: # URL has path, ignore warning list
self.misp_event.add_attribute('url', entry, to_ids=ids_flag, attribute = self.misp_event.add_attribute('url', entry, to_ids=ids_flag,
enforceWarninglist=False, comment=comment) enforceWarninglist=False, comment=comment)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
else: # URL has no path else: # URL has no path
self.misp_event.add_attribute('url', entry, to_ids=ids_flag, attribute = self.misp_event.add_attribute('url', entry, to_ids=ids_flag,
enforceWarninglist=config.enforcewarninglist, comment=comment) enforceWarninglist=config.enforcewarninglist, comment=comment)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
if config.sighting: if config.sighting:
self.sighting(entry, config.sighting_source) self.sighting(entry, config.sighting_source)
@ -240,15 +260,18 @@ class Mail2MISP():
if debug: if debug:
syslog.syslog(hostname) syslog.syslog(hostname)
port = f.get_port() comment = ''
port = self.f.get_port()
if port: if port:
port = port.decode() port = port.decode()
comment = "on port: " + port comment = f'on port: {port}'
if is_ip(hostname): if is_ip(hostname):
self.misp_event.add_attribute('ip-dst', hostname, to_ids=ids_flag, attribute = self.misp_event.add_attribute('ip-dst', hostname, to_ids=ids_flag,
enforceWarninglist=config.enforcewarninglist, enforceWarninglist=config.enforcewarninglist,
comment=comment) comment=comment)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
else: else:
related_ips = [] related_ips = []
try: try:
@ -268,10 +291,14 @@ class Mail2MISP():
hip.add_attribute('ip', type='ip-dst', value=ip, to_ids=False, hip.add_attribute('ip', type='ip-dst', value=ip, to_ids=False,
enforceWarninglist=config.enforcewarninglist) enforceWarninglist=config.enforcewarninglist)
self.misp_event.add_object(hip) self.misp_event.add_object(hip)
if email_object:
email_object.add_reference(hip.uuid, 'contains')
else: else:
self.misp_event.add_attribute('hostname', value=hostname, attribute = self.misp_event.add_attribute('hostname', value=hostname,
to_ids=ids_flag, enforceWarninglist=config.enforcewarninglist, to_ids=ids_flag, enforceWarninglist=config.enforcewarninglist,
comment=comment) comment=comment)
if email_object:
email_object.add_reference(attribute.uuid, 'contains')
def add_event(self): def add_event(self):
'''Add event on the remote MISP instance.''' '''Add event on the remote MISP instance.'''
@ -296,7 +323,7 @@ if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Push a Mail into a MISP instance') parser = argparse.ArgumentParser(description='Push a Mail into a MISP instance')
parser.add_argument("-r", "--read", help="Read from tempfile.") parser.add_argument("-r", "--read", help="Read from tempfile.")
parser.add_argument("-t", "--trap", action='store_true', default=False, help="Import the Email as-is.") parser.add_argument("-t", "--trap", action='store_true', default=False, help="Import the Email as-is.")
parser.add_argument('infile', nargs='?', type=argparse.FileType('rb'), default=sys.stdin) parser.add_argument('infile', nargs='?', type=argparse.FileType('rb'))
args = parser.parse_args() args = parser.parse_args()
syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_USER) syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_USER)