From a854a2ae15cb2cd41b3b6499f745be809546119e Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Thu, 14 Dec 2017 11:27:48 +0100 Subject: [PATCH] changes do to PyMISP improvements --- mail_to_misp.py | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/mail_to_misp.py b/mail_to_misp.py index 6bfa41d..e683b47 100755 --- a/mail_to_misp.py +++ b/mail_to_misp.py @@ -13,7 +13,7 @@ try: import hashmarker import re from pyfaup.faup import Faup - from pymisp import PyMISP + from pymisp import PyMISP, MISPEvent from defang import refang import dns.resolver import email @@ -150,9 +150,15 @@ for ignoreline in ignorelist: # Remove words from subject for removeword in removelist: email_subject = re.sub(removeword, "", email_subject) + +if debug: + import logging + logger = logging.getLogger('pymisp') + logging.basicConfig(level=logging.DEBUG, filename="/tmp/mail_to_misp_debug.log", filemode='w') def init(url, key): - return PyMISP(url, key, misp_verifycert, 'json') + return PyMISP(url, key, misp_verifycert, 'json', debug=debug) + # Evaluate classification tlp_tag = tlptag_default @@ -165,7 +171,10 @@ for tag in tlptags: # Create the MISP event misp = init(misp_url, misp_key) new_event = misp.new_event(info=email_subject, distribution=0, threat_level_id=3, analysis=1) -misp.add_tag(new_event, tlp_tag) +misp_event = MISPEvent() +misp_event.load(new_event) + +misp.tag(misp_event.uuid, tlp_tag) if attach_original_mail and original_email_data: misp.add_named_attribute(new_event, 'email-body', original_email_data, category='Payload delivery', to_ids=False) @@ -173,7 +182,7 @@ if attach_original_mail and original_email_data: for tag in dependingtags: if tag in tlp_tag: for dependingtag in dependingtags[tag]: - misp.add_tag(new_event, dependingtag) + misp.tag(misp_event.uuid, dependingtag) ## Prepare extraction of IOCs @@ -211,7 +220,7 @@ f = Faup() for malware in malwaretags: if malware in email_subject.lower(): for tag in malwaretags[malware]: - misp.add_tag(new_event, tag) + misp.tag(misp_event, tag) # Extract and add hashes hashlist_md5 = re.findall(hashmarker.MD5_REGEX, email_data) @@ -227,7 +236,7 @@ for h in hashlist_sha256: if (len(hashlist_md5) > 0) or (len(hashlist_sha1) > 0) or (len(hashlist_sha256) > 0): for tag in hash_only_tags: - misp.add_tag(new_event, tag) + misp.tag(misp_event, tag) # Add IOCs and expanded information to MISP for entry in urllist: @@ -247,8 +256,10 @@ for entry in urllist: elif domainname in externallist: misp.add_named_attribute(new_event, 'link', entry, category='External analysis', to_ids=False) else: + comment = "" if (domainname in noidsflaglist) or (hostname in noidsflaglist): ids_flag = False + comment = "Known host (mostly for connectivity test or IP lookup)" if debug: syslog.syslog(str(entry)) if hostname: @@ -256,13 +267,16 @@ for entry in urllist: if is_valid_ipv4_address(hostname): misp.add_url(new_event, entry, category='Network activity', to_ids=False) else: - misp.add_url(new_event, entry, category='Network activity', to_ids=ids_flag) + misp.add_url(new_event, entry, comment=comment, category='Network activity', to_ids=ids_flag) if debug: syslog.syslog(hostname) - port = f.get_port() + try: + port = f.get_port().decode('utf-8', 'ignore') + except: + port = None comment = "" if port: - comment = "on port: " + str(port) + comment = "on port: " + port if is_valid_ipv4_address(hostname): misp.add_ipdst(new_event, hostname, comment=comment, category='Network activity', to_ids=False) else: @@ -286,8 +300,11 @@ if stdin_used: _, output_path = tempfile.mkstemp() output = open(output_path, 'wb') output.write(part.get_payload(decode=True)) - event_id = new_event['Event']['id'] - misp.upload_sample(filename, output_path, event_id, distribution=None, to_ids=True, category=None, comment=None, info='My Info', analysis=None, threat_level_id=None) + attachment = part.get_payload(decode=True) + event_id = misp_event.id + #misp.add_attachment(new_event, output_path, distribution=None, to_ids=True, category='Payload delivery', comment=None, info='My Info', analysis=None, threat_level_id=None) + misp.upload_sample(filename, output_path, event_id, distribution=None, to_ids=True) output.close() -syslog.syslog("Job finished.") +if debug: + syslog.syslog("Job finished.")