MISP
/
mail_to_misp
mirror of https://github.com/MISP/mail_to_misp
2
0
Fork 0
Code Issues Releases Wiki Activity

chg: Make pep8 happy

slight_refactoring
Raphaël Vinot 2018-04-27 16:38:38 +02:00
parent aaab73cdf1
commit 0a684ac997
1 changed files with 43 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
mail_to_misp.py
View File

@ -13,11 +13,10 @@ try:
import hashmarker import hashmarker
import re import re
from pyfaup.faup import Faup from pyfaup.faup import Faup
from pymisp import PyMISP, MISPEvent from pymisp import PyMISP, MISPEvent, MISPObject
from defang import refang from defang import refang
import dns.resolver import dns.resolver
import email import email
from email.generator import Generator
import tempfile import tempfile
import socket import socket
import syslog import syslog
@ -31,6 +30,7 @@ except ImportError as e:
syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_USER) syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_USER)
def is_valid_ipv4_address(address): def is_valid_ipv4_address(address):
try: try:
socket.inet_pton(socket.AF_INET, address) socket.inet_pton(socket.AF_INET, address)
@ -44,15 +44,18 @@ def is_valid_ipv4_address(address):
return False return False
return True return True
def is_valid_ipv6_address(address): def is_valid_ipv6_address(address):
try: try:
socket.inet_pton(socket.AF_INET6, address) socket.inet_pton(socket.AF_INET6, address)
except socket.error: # not a valid address except socket.error: # not a valid address
return False return False
return True return True
def init(url, key): def init(url, key):
return PyMISP(url, key, misp_verifycert, 'json', debug=True) return PyMISP(url, key, misp_verifycert, 'json', debug=debug)
# Add a sighting # Add a sighting
def sight(sighting, value): def sight(sighting, value):
@ -60,13 +63,15 @@ def sight(sighting, value):
d = {'value': value, 'source': sighting_source} d = {'value': value, 'source': sighting_source}
misp.set_sightings(d) misp.set_sightings(d)
# Add named attribute and sight if configured # Add named attribute and sight if configured
def add_attribute(event, attribute_type, value, category, ids_flag, warninglist, sighting, comment=None): def add_attribute(event, attribute_type, value, category, ids_flag, warninglist, sighting, comment=None):
syslog.syslog("Event " + event['Event']['id'] + ": Adding attribute (" + attribute_type + ") " + value) syslog.syslog("Event " + event['Event']['id'] + ": Adding attribute (" + attribute_type + ") " + value)
misp.add_named_attribute(event, attribute_type, value, category, distribution=5, misp.add_named_attribute(event, attribute_type, value, category, distribution=5,
comment=comment, to_ids=ids_flag, enforceWarninglist=warninglist) comment=comment, to_ids=ids_flag, enforceWarninglist=warninglist)
sight(sighting, value) sight(sighting, value)
syslog.syslog("Job started.") syslog.syslog("Job started.")
debug = config.debug debug = config.debug
stdin_used = False stdin_used = False
@ -95,7 +100,7 @@ if not mail_subject:
try: try:
mail_subject = msg.get('Subject').encode("utf-8", "ignore") mail_subject = msg.get('Subject').encode("utf-8", "ignore")
sub, enc = email.header.decode_header(msg.get('subject'))[0] sub, enc = email.header.decode_header(msg.get('subject'))[0]
if enc==None: if enc is None:
email_subject = sub email_subject = sub
else: else:
email_subject = sub.decode(enc) email_subject = sub.decode(enc)
@ -105,7 +110,7 @@ if not mail_subject:
for part in msg.walk(): for part in msg.walk():
if part.get_content_charset() is None: if part.get_content_charset() is None:
# This could probably be detected # This could probably be detected
charset = 'utf-8' charset = 'utf-8'
else: else:
charset = part.get_content_charset() charset = part.get_content_charset()
if part.get_content_maintype() == 'multipart': if part.get_content_maintype() == 'multipart':
@ -114,7 +119,7 @@ for part in msg.walk():
part.set_charset(charset) part.set_charset(charset)
if debug: if debug:
syslog.syslog(str(part.get_payload(decode=True))) syslog.syslog(str(part.get_payload(decode=True)))
email_data += part.get_payload(decode=True) email_data += part.get_payload(decode=True)
try: try:
email_subject += mail_subject email_subject += mail_subject
except Exception as e: except Exception as e:
@ -123,14 +128,14 @@ stdin_used = True
try: try:
email_data = ftfy.fix_text(email_data.decode("utf-8", "ignore")) email_data = ftfy.fix_text(email_data.decode("utf-8", "ignore"))
except: except Exception:
email_data = ftfy.fix_text(email_data) email_data = ftfy.fix_text(email_data)
try: try:
email_subject = ftfy.fix_text(email_subject.decode("utf-8", "ignore")) email_subject = ftfy.fix_text(email_subject.decode("utf-8", "ignore"))
except: except Exception:
email_subject = ftfy.fix_text(email_subject) email_subject = ftfy.fix_text(email_subject)
if debug: if debug:
syslog.syslog(email_subject) syslog.syslog(email_subject)
syslog.syslog(email_data) syslog.syslog(email_data)
@ -167,7 +172,7 @@ except Exception as e:
print("\nTrace:") print("\nTrace:")
print(e) print(e)
sys.exit(-2) sys.exit(-2)
original_email_data = email_data original_email_data = email_data
# Ignore lines in body of message # Ignore lines in body of message
@ -182,7 +187,7 @@ for removeword in removelist:
auto_publish = False auto_publish = False
autopublish_key = "key:" + m2m_key autopublish_key = "key:" + m2m_key
if autopublish_key in email_data: if autopublish_key in email_data:
auto_publish = True auto_publish = True
# Create the MISP event # Create the MISP event
misp = init(misp_url, misp_key) misp = init(misp_url, misp_key)
@ -194,6 +199,7 @@ else:
# Load the MISP event # Load the MISP event
misp_event = MISPEvent() misp_event = MISPEvent()
misp_event.load(new_event) misp_event.load(new_event)
event_id = misp_event.id
# Evaluate classification # Evaluate classification
tlp_tag = tlptag_default tlp_tag = tlptag_default
@ -213,7 +219,7 @@ for tag in dependingtags:
for dependingtag in dependingtags[tag]: for dependingtag in dependingtags[tag]:
misp.tag(misp_event.uuid, dependingtag) misp.tag(misp_event.uuid, dependingtag)
## Prepare extraction of IOCs # # Prepare extraction of IOCs
# Limit the input if the stopword is found # Limit the input if the stopword is found
email_data = email_data.split(stopword, 1)[0] email_data = email_data.split(stopword, 1)[0]
@ -234,9 +240,9 @@ email_data = t_email_data
email_data = refang(email_data) email_data = refang(email_data)
## Extract various IOCs # # Extract various IOCs
urllist = list() urllist = list()
urllist += re.findall(urlmarker.WEB_URL_REGEX, email_data) urllist += re.findall(urlmarker.WEB_URL_REGEX, email_data)
urllist += re.findall(urlmarker.IP_REGEX, email_data) urllist += re.findall(urlmarker.IP_REGEX, email_data)
if debug: if debug:
@ -270,17 +276,18 @@ if (len(hashlist_md5) > 0) or (len(hashlist_sha1) > 0) or (len(hashlist_sha256)
# Add IOCs and expanded information to MISP # Add IOCs and expanded information to MISP
for entry in urllist: for entry in urllist:
hip = MISPObject(name='ip-port', strict=False, uuid='9f8cea74-16fe-4968-a2b4-026676949ac7', version='7')
ids_flag = True ids_flag = True
f.decode(entry) f.decode(entry)
domainname = f.get_domain().decode('utf-8', 'ignore') domainname = f.get_domain().decode('utf-8', 'ignore')
hostname = f.get_host().decode('utf-8', 'ignore') hostname = f.get_host().decode('utf-8', 'ignore')
try: try:
schema = f.get_scheme().decode('utf-8', 'ignore') schema = f.get_scheme().decode('utf-8', 'ignore')
except: except Exception:
schema = False schema = False
try: try:
resource_path = f.get_resource_path().decode('utf-8', 'ignore') resource_path = f.get_resource_path().decode('utf-8', 'ignore')
except: except Exception:
resource_path = False resource_path = False
if debug: if debug:
syslog.syslog(domainname) syslog.syslog(domainname)
@ -303,34 +310,39 @@ for entry in urllist:
else: else:
if resource_path: if resource_path:
add_attribute(new_event, 'url', entry, 'Network activity', ids_flag, False, add_attribute(new_event, 'url', entry, 'Network activity', ids_flag, False,
sighting, comment=comment) sighting, comment=comment)
else: else:
add_attribute(new_event, 'url', entry, 'Network activity', ids_flag, enforcewarninglist, add_attribute(new_event, 'url', entry, 'Network activity', ids_flag, enforcewarninglist,
sighting, comment=comment) sighting, comment=comment)
if debug: if debug:
syslog.syslog(hostname) syslog.syslog(hostname)
try: try:
port = f.get_port().decode('utf-8', 'ignore') port = f.get_port().decode('utf-8', 'ignore')
except: except Exception:
port = None port = None
if port: if port:
comment = "on port: " + port comment = "on port: " + port
if is_valid_ipv4_address(hostname): if is_valid_ipv4_address(hostname):
add_attribute(new_event, 'ip-dst', hostname, 'Network activity', ids_flag, enforcewarninglist, add_attribute(new_event, 'ip-dst', hostname, 'Network activity', ids_flag, enforcewarninglist,
sighting, comment=comment) sighting, comment=comment)
hip.add_attribute('ip', type='ip-dst', value=hostname, to_ids=ids_flag, comment=comment)
else: else:
add_attribute(new_event, 'hostname', hostname, 'Network activity', ids_flag, enforcewarninglist, add_attribute(new_event, 'hostname', hostname, 'Network activity', ids_flag, enforcewarninglist,
sighting, comment=comment) sighting, comment=comment)
hip.add_attribute('hostname', type='hostname', value=hostname, to_ids=ids_flag, comment=comment)
try: try:
for rdata in dns.resolver.query(hostname, 'A'): for rdata in dns.resolver.query(hostname, 'A'):
if debug: if debug:
syslog.syslog(str(rdata)) syslog.syslog(str(rdata))
add_attribute(new_event, 'ip-dst', rdata.to_text(), 'Network activity', False, enforcewarninglist, add_attribute(new_event, 'ip-dst', rdata.to_text(), 'Network activity', False, enforcewarninglist,
sighting, comment=hostname) sighting, comment=hostname)
hip.add_attribute('ip', type='ip-dst', value=rdata.to_text(), to_ids=False)
except Exception as e: except Exception as e:
if debug: if debug:
syslog.syslog(str(e)) syslog.syslog(str(e))
# misp_event.add_object(hip)
# misp.update_event(event_id, new_event)
# Try to add attachments # Try to add attachments
if stdin_used: if stdin_used:
for part in msg.walk(): for part in msg.walk():
@ -345,7 +357,6 @@ if stdin_used:
attachment = part.get_payload(decode=True) attachment = part.get_payload(decode=True)
if debug: if debug:
syslog.syslog(str(attachment)[:200]) syslog.syslog(str(attachment)[:200])
event_id = misp_event.id
if m2m_attachment_keyword in email_data: if m2m_attachment_keyword in email_data:
misp.add_attachment(misp_event, output_path, filename=filename, category="External analysis") misp.add_attachment(misp_event, output_path, filename=filename, category="External analysis")
else: else: