chg: Just a slight refactoring.

README.md

@@ -138,19 +138,11 @@ Obviously, you would like to filter mails based on subject or from address and p
 - mail_to_misp requires access to a MISP instance (via API).
 - Python 3
 - dnspython
-- hashlib
+- PyMISP
 - faup from https://github.com/stricaud/faup
 - urlmarker from https://github.com/rcompton/ryancompton.net/blob/master/assets/praw_drugs/urlmarker.py (contained in this project)
 - ftfy from https://github.com/LuminosoInsight/python-ftfy (to fix unicode text)
-- defang from https://bitbucket.org/johannestaas/defang
-- Patch defang/defang/__init__.py by commenting out the following line (not used && not compatible with Python 3):
-`from urllib2 import urlparse`
-- Optionally patch defang/defang/__init__.py and add at line 47:
-```
-    dirty_line = dirty_line.replace('hxxp', 'http')
-    dirty_line = dirty_line.replace('purr', 'http')
-    dirty_line = dirty_line.replace('meow', 'http')
-```
+- defang from https://github.com/Rafiot/defang.git (fork of: https://bitbucket.org/johannestaas/defang)
 
 ### Thunderbird [deprecated]
 

mail_to_misp.py

@@ -1,170 +1,316 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-import os
 import sys
-try:
-    configfile = os.path.basename(sys.argv[0]).split(".py")[0] + "_config"
-except Exception as e:
-    print("Couldn't locate config file {0}".format(configfile))
-    sys.exit(-1)
+import argparse
+import re
+import syslog
+from pathlib import Path
+from io import BytesIO
+from ipaddress import ip_address
+from email import message_from_bytes, policy
+import importlib
 try:
     import urlmarker
     import hashmarker
-    import re
     from pyfaup.faup import Faup
-    from pymisp import PyMISP, MISPEvent, MISPObject
+    from pymisp import PyMISP, MISPEvent, MISPObject, MISPSighting
+    from pymisp.tools import EMailObject, make_binary_objects
     from defang import refang
     import dns.resolver
-    import email
-    import tempfile
-    import socket
-    import syslog
-    import ftfy
-    import hashlib
-    config = __import__(configfile)
 except ImportError as e:
     print("(!) Problem loading module:")
     print(e)
     sys.exit(-1)
 
-syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_USER)
 
-
-def is_valid_ipv4_address(address):
+def is_ip(address):
     try:
-        socket.inet_pton(socket.AF_INET, address)
-    except AttributeError:  # no inet_pton here, sorry
-        try:
-            socket.inet_aton(address)
-        except socket.error:
-            return False
-        return address.count('.') == 3
-    except socket.error:  # not a valid address
+        ip_address(address)
+    except ValueError:
         return False
     return True
 
 
-def is_valid_ipv6_address(address):
-    try:
-        socket.inet_pton(socket.AF_INET6, address)
-    except socket.error:  # not a valid address
-        return False
-    return True
+class Mail2MISP():
 
+    def __init__(self, misp_url, misp_key, verifycert, config):
+        self.misp = PyMISP(misp_url, misp_key, verifycert, debug=config.debug)
+        self.debug = config.debug
+        self.config = config
 
-def init(url, key):
-    return PyMISP(url, key, misp_verifycert, 'json', debug=debug)
+    def load_email(self, pseudofile):
+        self.pseudofile = pseudofile
+        self.original_mail = message_from_bytes(self.pseudofile.getvalue(), policy=policy.default)
+        self.subject = self.original_mail.get('Subject')
+        # Remove words from subject
+        for removeword in self.config.removelist:
+            self.subject = re.sub(removeword, "", self.subject)
 
+        # Initialize the MISP event
+        self.misp_event = MISPEvent()
+        self.misp_event.info = f'{config.email_subject_prefix} - {self.subject}'
+        self.misp_event.distribution = config.m2m_auto_distribution
+        self.misp_event.threat_level_id = 3
+        self.misp_event.analysis = 1
+        self.misp_event.add_tag(config.tlptag_default)
 
-# Add a sighting
-def sight(sighting, value):
-    if sighting:
-        d = {'value': value, 'source': sighting_source}
-        misp.set_sightings(d)
+    def sighting(self, value, source):
+        '''Add a sighting'''
+        s = MISPSighting()
+        s.from_dict(value=value, source=source)
+        self.misp.set_sightings(s)
 
+    def _find_inline_forward(self):
+        '''Does the body contains a forwarded email?'''
+        for identifier in config.forward_identifiers:
+            if identifier in self.clean_email_body:
+                self.clean_email_body, fw_email = self.clean_email_body.split(identifier)
+                self.forwarded_email(pseudofile=BytesIO(fw_email.encode()))
 
-# Add named attribute and sight if configured
-def add_attribute(event, attribute_type, value, category, ids_flag, warninglist, sighting, comment=None):
-    syslog.syslog("Event " + event['Event']['id'] + ": Adding attribute (" + attribute_type + ") " + value)
-    misp.add_named_attribute(event, attribute_type, value, category, distribution=5,
-                             comment=comment, to_ids=ids_flag, enforceWarninglist=warninglist)
-    sight(sighting, value)
-
-
-syslog.syslog("Job started.")
-debug = config.debug
-stdin_used = False
-
-email_subject = config.email_subject_prefix
-mail_subject = ""
-if len(sys.argv) == 1:
-    mailcontent = sys.stdin.buffer.read().decode("utf-8", "ignore")
-else:
-    # read from tempfile
-    if sys.argv[1] == "-r":
-        tempfilename = sys.argv[2]
-        tf = open(tempfilename, 'r')
-        mailcontent = tf.read()
-        tf.close()
-    # receive data and subject through arguments
+    def _find_attached_forward(self):
+        for attachment in self.original_mail.iter_attachments():
+            # Search for email forwarded as attachment
+            # I could have more than one, attaching everything.
+            if attachment.get_filename() and attachment.get_filename().endswith('.eml'):
+                self.forwarded_email(pseudofile=BytesIO(attachment.get_content().encode()))
             else:
-        mailcontent = sys.argv[1]
-        if debug:
-            syslog.syslog(mailcontent)
-        if len(sys.argv) >= 3:
-            mail_subject = sys.argv[2].encode("utf-8", "ignore")
-email_data = b''
-msg = email.message_from_string(mailcontent)
-if not mail_subject:
-    try:
-        mail_subject = msg.get('Subject').encode("utf-8", "ignore")
-        sub, enc = email.header.decode_header(msg.get('subject'))[0]
-        if enc is None:
-            email_subject = sub
+                if self.config_from_email_body.get('attachment') == 'benign':
+                    # Attach sane file
+                    self.misp_event.add_attribute('attachment', value='Report',
+                                                  data=BytesIO(attachment.get_content().encode()))
                 else:
-            email_subject = sub.decode(enc)
-    except Exception as e:
-        print(e)
-        pass
-for part in msg.walk():
-    if part.get_content_charset() is None:
-        # This could probably be detected
-        charset = 'utf-8'
-    else:
-        charset = part.get_content_charset()
-    if part.get_content_maintype() == 'multipart':
+                    f_object, main_object, sections = make_binary_objects(pseudofile=BytesIO(attachment.get_content().encode()),
+                                                                          filename=attachment.get_filename(), standalone=False)
+                    self.misp_event.add_object(f_object)
+                    if main_object:
+                        self.misp_event.add_object(main_object)
+                        [self.misp_event.add_object(section) for section in sections]
+
+    def email_from_spamtrap(self):
+        '''The email comes from a spamtrap and should be attached as-is.'''
+        self.clean_email_body = self.original_mail.get_body().as_string()
+        self.forwarded_email(self.pseudofile)
+
+    def forwarded_email(self, pseudofile: BytesIO):
+        '''Extracts all possible indicators out of an email and create a MISP event out of it.
+        * Gets all relevant Headers
+        * Attach the body
+        * Create MISP file objects (uses lief if possible)
+        * Set all references
+        '''
+        email_object = EMailObject(pseudofile=self.pseudofile, attach_original_mail=True, standalone=False)
+        if email_object.attachments:
+            # Create file objects for the attachments
+            for attachment_name, attachment in email_object.attachments:
+                f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False)
+                self.misp_event.add_object(f_object)
+                if main_object:
+                    self.misp_event.add_object(main_object)
+                    for section in sections:
+                        self.misp_event.add_object(section)
+                email_object.add_reference(f_object.uuid, 'related-to', 'Email attachment')
+        self.misp_event.add_object(email_object)
+
+    def process_email_body(self):
+        self.clean_email_body = self.original_mail.get_body().as_string()
+        # Check if there are config lines in the body & convert them to a python dictionary:
+        #   <config.body_config_prefix>:<key>:<value> => {<key>: <value>}
+        self.config_from_email_body = {k: v for k, v in re.findall(f'{config.body_config_prefix}:(.*):(.*)', self.clean_email_body)}
+        if self.config_from_email_body:
+            # ... remove the config lines from the body
+            self.clean_email_body = re.sub(rf'^{config.body_config_prefix}.*\n?', '',
+                                           self.original_mail.get_body().as_string(), flags=re.MULTILINE)
+
+        self._find_inline_forward()
+        self._find_attached_forward()
+        # # Prepare extraction of IOCs
+        # Refang email data
+        self.clean_email_body = refang(self.clean_email_body)
+
+        # Depending on the source of the mail, there is some cleanup to do. Ignore lines in body of message
+        for ignoreline in config.ignorelist:
+            self.clean_email_body = re.sub(rf'^{ignoreline}.*\n?', '', self.clean_email_body, flags=re.MULTILINE)
+
+        # Check if autopublish key is present and valid
+        if self.config_from_email_body.get('m2mkey') == config.m2m_key:
+            self.misp_event.publish()
+
+        # Add tags to the event if keywords are found in the mail
+        for tag in config.tlptags:
+            if any(alternativetag in self.clean_email_body for alternativetag in config.tlptags[tag]):
+                self.misp_event.add_tag(tag)
+
+        # Remove everything after the stopword from the body
+        self.clean_email_body = self.clean_email_body.split(config.stopword, 1)[0]
+
+    def process_body_iocs(self):
+        # Extract and add hashes
+        contains_hash = False
+        for h in set(re.findall(hashmarker.MD5_REGEX, self.clean_email_body)):
+            contains_hash = True
+            self.misp_event.add_attribute('md5', h, enforceWarninglist=config.enforcewarninglist)
+            if config.sighting:
+                self.sighting(h, config.sighting_source)
+        for h in set(re.findall(hashmarker.SHA1_REGEX, self.clean_email_body)):
+            contains_hash = True
+            self.misp_event.add_attribute('sha1', h, enforceWarninglist=config.enforcewarninglist)
+            if config.sighting:
+                self.sighting(h, config.sighting_source)
+        for h in set(re.findall(hashmarker.SHA256_REGEX, self.clean_email_body)):
+            contains_hash = True
+            self.misp_event.add_attribute('sha256', h, enforceWarninglist=config.enforcewarninglist)
+            if config.sighting:
+                self.sighting(h, config.sighting_source)
+
+        if contains_hash:
+            [self.misp_event.add_tag(tag) for tag in config.hash_only_tags]
+
+        # # Extract network IOCs
+        urllist = []
+        urllist += re.findall(urlmarker.WEB_URL_REGEX, self.clean_email_body)
+        urllist += re.findall(urlmarker.IP_REGEX, self.clean_email_body)
+        if self.debug:
+            syslog.syslog(str(urllist))
+
+        # Init Faup
+        f = Faup()
+
+        hostname_processed = []
+
+        # Add IOCs and expanded information to MISP
+        for entry in set(urllist):
+            ids_flag = True
+            f.decode(entry)
+
+            domainname = f.get_domain().decode()
+            if domainname in config.excludelist:
+                # Ignore the entry
                 continue
-    if part.get_content_maintype() == 'text':
-        part.set_charset(charset)
+
+            hostname = f.get_host().decode()
+
+            scheme = f.get_scheme()
+            if scheme:
+                scheme = scheme.decode()
+
+            resource_path = f.get_resource_path()
+            if resource_path:
+                resource_path = resource_path.decode()
+
+            if debug:
+                syslog.syslog(domainname)
+
+            if domainname in config.internallist:  # Add link to internal reference
+                self.misp_event.add_attribute('link', entry, category='Internal reference',
+                                              to_ids=False, enforceWarninglist=False)
+            elif domainname in config.externallist:  # External analysis
+                self.misp_event.add_attribute('link', entry, category='External analysis',
+                                              to_ids=False, enforceWarninglist=False)
+            else:  # The URL is probably an indicator.
+                comment = ""
+                if (domainname in config.noidsflaglist) or (hostname in config.noidsflaglist):
+                    ids_flag = False
+                    comment = "Known host (mostly for connectivity test or IP lookup)"
+                if debug:
+                    syslog.syslog(str(entry))
+
+                if scheme:
+                    if is_ip(hostname):
+                        self.misp_event.add_attribute('url', entry, to_ids=False,
+                                                      enforceWarninglist=config.enforcewarninglist)
+                    else:
+                        if resource_path:  # URL has path, ignore warning list
+                            self.misp_event.add_attribute('url', entry, to_ids=ids_flag,
+                                                          enforceWarninglist=False, comment=comment)
+                        else:  # URL has no path
+                            self.misp_event.add_attribute('url', entry, to_ids=ids_flag,
+                                                          enforceWarninglist=config.enforcewarninglist, comment=comment)
+                    if config.sighting:
+                        self.sighting(entry, config.sighting_source)
+
+                if hostname in hostname_processed:
+                    # Hostname already processed.
+                    continue
+
+                hostname_processed.append(hostname)
+                if config.sighting:
+                    self.sighting(hostname, config.sighting_source)
+
+                if debug:
+                    syslog.syslog(hostname)
+
+                port = f.get_port()
+                if port:
+                    port = port.decode()
+                    comment = "on port: " + port
+
+                if is_ip(hostname):
+                    self.misp_event.add_attribute('ip-dst', hostname, to_ids=ids_flag,
+                                                  enforceWarninglist=config.enforcewarninglist,
+                                                  comment=comment)
+                else:
+                    related_ips = []
+                    try:
+                        for rdata in dns.resolver.query(hostname, 'A'):
+                            if debug:
+                                syslog.syslog(str(rdata))
+                            related_ips.append(rdata.to_text())
+                    except Exception as e:
                         if debug:
-            syslog.syslog(str(part.get_payload(decode=True)))
-        email_data += part.get_payload(decode=True)
-try:
-    email_subject += mail_subject
-except Exception as e:
                             syslog.syslog(str(e))
-stdin_used = True
 
-try:
-    email_data = ftfy.fix_text(email_data.decode("utf-8", "ignore"))
-except Exception:
-    email_data = ftfy.fix_text(email_data)
-try:
-    email_subject = ftfy.fix_text(email_subject.decode("utf-8", "ignore"))
-except Exception:
-    email_subject = ftfy.fix_text(email_subject)
+                    if related_ips:
+                        hip = MISPObject(name='ip-port')
+                        hip.add_attribute('hostname', value=hostname, to_ids=ids_flag,
+                                          enforceWarninglist=config.enforcewarninglist, comment=comment)
+                        for ip in set(related_ips):
+                            hip.add_attribute('ip', type='ip-dst', value=ip, to_ids=False,
+                                              enforceWarninglist=config.enforcewarninglist)
+                        self.misp_event.add_object(hip)
+                    else:
+                        self.misp_event.add_attribute('hostname', value=hostname,
+                                                      to_ids=ids_flag, enforceWarninglist=config.enforcewarninglist,
+                                                      comment=comment)
 
-if debug:
-    syslog.syslog(email_subject)
-    syslog.syslog(email_data)
+    def add_event(self):
+        '''Add event on the remote MISP instance.'''
 
-try:
+        # Add additional tags depending on others
+        tags = []
+        for tag in [t.name for t in self.misp_event.tags]:
+            if config.dependingtags.get(tag):
+                tags += config.dependingtags.get(tag)
+
+        # Add additional tags according to configuration
+        for malware in config.malwaretags:
+            if malware.lower() in self.subject.lower():
+                tags += config.malwaretags.get(malware)
+        if tags:
+            [self.misp_event.add_tag(tag) for tag in tags]
+
+        self.misp.add_event(self.misp_event)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Push a Mail into a MISP instance')
+    parser.add_argument("-r", "--read", help="Read from tempfile.")
+    parser.add_argument("-t", "--trap", action='store_true', default=False, help="Import the Email as-is.")
+    parser.add_argument('infile', nargs='?', type=argparse.FileType('rb'), default=sys.stdin)
+    args = parser.parse_args()
+
+    syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_USER)
+    syslog.syslog("Job started.")
+
+    configmodule = Path(__file__).as_posix().replace('.py', '_config')
+    if Path(f'{configmodule}.py').exists():
+        config = importlib.import_module(configmodule)
+        try:
             misp_url = config.misp_url
             misp_key = config.misp_key
             misp_verifycert = config.misp_verifycert
-    m2m_key = config.m2m_key
-    m2m_auto_distribution = config.m2m_auto_distribution
-    m2m_attachment_keyword = config.m2m_attachment_keyword
-    resolver = dns.resolver.Resolver(configure=False)
-    resolver.nameservers = config.nameservers
-    excludelist = config.excludelist
-    externallist = config.externallist
-    internallist = config.internallist
-    noidsflaglist = config.noidsflaglist
-    ignorelist = config.ignorelist
-    enforcewarninglist = config.enforcewarninglist
-    sighting = config.sighting
-    sighting_source = config.sighting_source
-    removelist = config.removelist
-    malwaretags = config.malwaretags
-    dependingtags = config.dependingtags
-    tlptag_default = config.tlptag_default
-    stopword = config.stopword
-    hash_only_tags = config.hash_only_tags
-    forward_identifiers = config.forward_identifiers
-    attach_original_mail = config.attach_original_mail
-except Exception as e:
+            debug = config.debug
+        except Exception as e:
             syslog.syslog(str(e))
             print("There is a problem with the configuration. A mandatory configuration variable is not set.")
             print("Did you just update? mail_to_misp might have new configuration variables.")
@@ -172,199 +318,32 @@ except Exception as e:
             print("\nTrace:")
             print(e)
             sys.exit(-2)
-
-original_email_data = email_data
-
-# Ignore lines in body of message
-for ignoreline in ignorelist:
-    email_data = re.sub(ignoreline, "", email_data)
-
-# Remove words from subject
-for removeword in removelist:
-    email_subject = re.sub(removeword, "", email_subject)
-
-# Check if autopublish key is present and valid
-auto_publish = False
-autopublish_key = "key:" + m2m_key
-if autopublish_key in email_data:
-    auto_publish = True
-
-# Create the MISP event
-misp = init(misp_url, misp_key)
-if auto_publish:
-    new_event = misp.new_event(info=email_subject, distribution=m2m_auto_distribution, threat_level_id=3, analysis=1)
-else:
-    new_event = misp.new_event(info=email_subject, distribution=0, threat_level_id=3, analysis=1)
-
-# Load the MISP event
-misp_event = MISPEvent()
-misp_event.load(new_event)
-event_id = misp_event.id
-
-# Evaluate classification
-tlp_tag = tlptag_default
-tlptags = config.tlptags
-for tag in tlptags:
-    for alternativetag in tlptags[tag]:
-        if alternativetag in email_data.lower():
-            tlp_tag = tag
-misp.tag(misp_event.uuid, tlp_tag)
-
-if attach_original_mail and original_email_data:
-    add_attribute(new_event, 'email-body', original_email_data, 'Payload delivery', False, enforcewarninglist, sighting)
-
-# Add additional tags depending on others
-for tag in dependingtags:
-    if tag in tlp_tag:
-        for dependingtag in dependingtags[tag]:
-            misp.tag(misp_event.uuid, dependingtag)
-
-# # Prepare extraction of IOCs
-
-# Limit the input if the stopword is found
-email_data = email_data.split(stopword, 1)[0]
-
-# Find the first forwarding message and use that content
-position = 99999
-t_email_data = email_data
-for identifier in forward_identifiers:
-    new_position = email_data.find(identifier)
-    if new_position == -1:
-        new_position = position
-    if new_position < position:
-        t_before, t_split, t_email_data = email_data.partition(identifier)
-        position = new_position
-email_data = t_email_data
-
-# Refang email data
-email_data = refang(email_data)
-
-
-# # Extract various IOCs
-
-urllist = list()
-urllist += re.findall(urlmarker.WEB_URL_REGEX, email_data)
-urllist += re.findall(urlmarker.IP_REGEX, email_data)
-if debug:
-    syslog.syslog(str(urllist))
-
-
-# Init Faup
-f = Faup()
-
-# Add additional tags according to configuration
-for malware in malwaretags:
-    if malware in email_subject.lower():
-        for tag in malwaretags[malware]:
-            misp.tag(misp_event.uuid, tag)
-
-# Extract and add hashes
-hashlist_md5 = re.findall(hashmarker.MD5_REGEX, email_data)
-hashlist_sha1 = re.findall(hashmarker.SHA1_REGEX, email_data)
-hashlist_sha256 = re.findall(hashmarker.SHA256_REGEX, email_data)
-
-for h in hashlist_md5:
-    add_attribute(new_event, 'md5', h, 'Payload delivery', True, enforcewarninglist, sighting)
-for h in hashlist_sha1:
-    add_attribute(new_event, 'sha1', h, 'Payload delivery', True, enforcewarninglist, sighting)
-for h in hashlist_sha256:
-    add_attribute(new_event, 'sha256', h, 'Payload delivery', True, enforcewarninglist, sighting)
-
-if (len(hashlist_md5) > 0) or (len(hashlist_sha1) > 0) or (len(hashlist_sha256) > 0):
-    for tag in hash_only_tags:
-        misp.tag(misp_event.uuid, tag)
-
-# Add IOCs and expanded information to MISP
-for entry in urllist:
-    hip = MISPObject(name='ip-port', strict=False, uuid='9f8cea74-16fe-4968-a2b4-026676949ac7', version='7')
-    ids_flag = True
-    f.decode(entry)
-    domainname = f.get_domain().decode('utf-8', 'ignore')
-    hostname = f.get_host().decode('utf-8', 'ignore')
-    try:
-        schema = f.get_scheme().decode('utf-8', 'ignore')
-    except Exception:
-        schema = False
-    try:
-        resource_path = f.get_resource_path().decode('utf-8', 'ignore')
-    except Exception:
-        resource_path = False
-    if debug:
-        syslog.syslog(domainname)
-    if domainname not in excludelist:
-        if domainname in internallist:
-            add_attribute(new_event, 'link', entry, 'Internal reference', False, False, sighting)
-        elif domainname in externallist:
-            add_attribute(new_event, 'link', entry, 'External analysis', False, False, sighting)
     else:
-            comment = ""
-            if (domainname in noidsflaglist) or (hostname in noidsflaglist):
-                ids_flag = False
-                comment = "Known host (mostly for connectivity test or IP lookup)"
-            if debug:
-                syslog.syslog(str(entry))
-            if hostname:
-                if schema:
-                    if is_valid_ipv4_address(hostname):
-                        add_attribute(new_event, 'url', entry, 'Network activity', False, enforcewarninglist, sighting)
-                    else:
-                        if resource_path:
-                            add_attribute(new_event, 'url', entry, 'Network activity', ids_flag, False,
-                                          sighting, comment=comment)
-                        else:
-                            add_attribute(new_event, 'url', entry, 'Network activity', ids_flag, enforcewarninglist,
-                                          sighting, comment=comment)
-                if debug:
-                    syslog.syslog(hostname)
-                try:
-                    port = f.get_port().decode('utf-8', 'ignore')
-                except Exception:
-                    port = None
-                if port:
-                    comment = "on port: " + port
-                if is_valid_ipv4_address(hostname):
-                    add_attribute(new_event, 'ip-dst', hostname, 'Network activity', ids_flag, enforcewarninglist,
-                                  sighting, comment=comment)
-                    hip.add_attribute('ip', type='ip-dst', value=hostname, to_ids=ids_flag, comment=comment)
-                else:
-                    add_attribute(new_event, 'hostname', hostname, 'Network activity', ids_flag, enforcewarninglist,
-                                  sighting, comment=comment)
-                    hip.add_attribute('hostname', type='hostname', value=hostname, to_ids=ids_flag, comment=comment)
-                try:
-                    for rdata in dns.resolver.query(hostname, 'A'):
-                        if debug:
-                            syslog.syslog(str(rdata))
-                        add_attribute(new_event, 'ip-dst', rdata.to_text(), 'Network activity', False, enforcewarninglist,
-                                      sighting, comment=hostname)
-                        hip.add_attribute('ip', type='ip-dst', value=rdata.to_text(), to_ids=False)
-                except Exception as e:
-                    if debug:
-                        syslog.syslog(str(e))
-#    misp_event.add_object(hip)
-#    misp.update_event(event_id, new_event)
+        print("Couldn't locate config file {0}".format(f'{configmodule}.py'))
+        sys.exit(-1)
 
-# Try to add attachments
-if stdin_used:
-    for part in msg.walk():
-        if part.get_content_maintype() == 'multipart':
-            continue
-        if part.get_content_maintype() != 'text' and part.get_payload(decode=True) is not None:
-            filename = part.get_filename()
-            _, output_path = tempfile.mkstemp()
-            output = open(output_path, 'wb')
-            output.write(part.get_payload(decode=True))
-            output.close()
-            attachment = part.get_payload(decode=True)
-            if debug:
-                syslog.syslog(str(attachment)[:200])
-            if m2m_attachment_keyword in email_data:
-                misp.add_attachment(misp_event, output_path, filename=filename, category="External analysis")
+    if args.infile:
+        pseudofile = BytesIO(args.infile.read().encode())
+    elif args.read:
+        # read from tempfile
+        with open(args.read, 'rb') as f:
+            pseudofile = BytesIO(f.read())
     else:
-                misp.upload_sample(filename, output_path, event_id, distribution=5, to_ids=True)
-                file_hash = hashlib.sha256(open(output_path, 'rb').read()).hexdigest()
-                sight(sighting, file_hash)
+        # receive data and subject through arguments
+        raise Exception('This is not implemented anymore.')
 
-if auto_publish:
-    misp.publish(misp_event, alert=False)
+    mail2misp = Mail2MISP(misp_url, misp_key, misp_verifycert, config=config)
+    mail2misp.load_email(pseudofile)
 
-syslog.syslog("Job finished.")
+    if debug:
+        syslog.syslog(f'Working on {mail2misp.subject}')
+
+    if args.trap or config.spamtrap:
+        mail2misp.email_from_spamtrap()
+    else:
+        mail2misp.process_email_body()
+
+    mail2misp.process_body_iocs()
+
+    mail2misp.add_event()
+    syslog.syslog("Job finished.")

mail_to_misp_config.py-example

@@ -1,18 +1,20 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-import sys
 import os
 
 misp_url = 'YOUR_MISP_URL'
 misp_key = 'YOUR_KEY_HERE'  # The MISP auth key can be found on the MISP web interface under the automation section
 misp_verifycert = True
+body_config_prefix = 'm2m'  # every line in the body starting with this value will be skipped from the IOCs
+spamtrap = False
+
 m2m_key = 'YOUSETYOURKEYHERE'
 m2m_auto_distribution = '3'  # 3 = All communities
 m2m_attachment_keyword = 'attachment:benign'
 
 debug = False
 nameservers = ['149.13.33.69']
-email_subject_prefix = b'M2M - '
+email_subject_prefix = 'M2M'
 attach_original_mail = True
 
 # Paths (should be automatic)
@@ -28,7 +30,7 @@ smtp_port = 25
 excludelist = ('google.com', 'microsoft.com')
 externallist = ('virustotal.com', 'malwr.com', 'hybrid-analysis.com', 'emergingthreats.net')
 internallist = ('internal.system.local')
-noidsflaglist = (   'myexternalip.com', 'ipinfo.io', 'icanhazip.com', 'wtfismyip.com', 'ipecho.net', 
+noidsflaglist = ('myexternalip.com', 'ipinfo.io', 'icanhazip.com', 'wtfismyip.com', 'ipecho.net',
                  'api.ipify.org', 'checkip.amazonaws.com', 'whatismyipaddress.com', 'google.com',
                  'dropbox.com'
                  )
@@ -37,48 +39,47 @@ noidsflaglist = (   'myexternalip.com', 'ipinfo.io', 'icanhazip.com', 'wtfismyip
 stopword = 'Whois & IP Information'
 
 # Ignore lines in body of message containing:
-ignorelist = (".*From: .*\n?", ".*Sender: .*\n?", ".*Received: .*\n?", ".*Sender IP: .*\n?",
-                    ".*Reply-To: .*\n?", ".*Registrar WHOIS Server: .*\n?", ".*Registrar: .*\n?",
-                    ".*Domain Status: .*\n?", ".*Registrant Email: .*\n?", ".*IP Location: .*\n?",
-                    ".*X-Get-Message-Sender-Via: .*\n?", ".*X-Authenticated-Sender: .*\n")
+ignorelist = ("From:", "Sender:", "Received:", "Sender IP:", "Reply-To:", "Registrar WHOIS Server:",
+              "Registrar:", "Domain Status:", "Registrant Email:", "IP Location:",
+              "X-Get-Message-Sender-Via:", "X-Authenticated-Sender:")
 
 # Ignore (don't add) attributes that are on server side warning list
-enforcewarninglist=True
+enforcewarninglist = True
 
 # Add a sighting for each value
-sighting=True
-sighting_source="YOUR_MAIL_TO_MISP_IDENTIFIER"
+sighting = True
+sighting_source = "YOUR_MAIL_TO_MISP_IDENTIFIER"
 
 # Remove "[tags]", "Re: ", "Fwd: " from subject
 removelist = ("[\(\[].*?[\)\]]", "Re: ", "Fwd: ")
 
 # TLP tag setup
 # Tuples contain different variations of spelling
-tlptags = { 'tlp:amber': [ 'tlp:amber', 'tlp: amber', 'tlp amber' ],
-            'tlp:green': [ 'tlp:green', 'tlp: green', 'tlp green' ],
-            'tlp:white': [ 'tlp:white', 'tlp: white', 'tlp white' ]
+tlptags = {'tlp:amber': ['tlp:amber', 'tlp: amber', 'tlp amber'],
+           'tlp:green': ['tlp:green', 'tlp: green', 'tlp green'],
+           'tlp:white': ['tlp:white', 'tlp: white', 'tlp white']
            }
 tlptag_default = sorted(tlptags.keys())[0]
 
-malwaretags = { 'locky':    [ 'ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Locky"' ],
-                'jaff':     [ 'ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Jaff"' ],
-                'dridex':   [ 'misp-galaxy:tool="dridex"' ],
-                'netwire':  [ 'Netwire RAT' ],
-                'Pony':     [ 'misp-galaxy:tool="Hancitor"' ],
-                'ursnif':   [ 'misp-galaxy:tool="Snifula"' ],
-                'NanoCore': [ 'misp-galaxy:tool="NanoCoreRAT"' ],
-                'trickbot': [ 'misp-galaxy:tool="Trick Bot"' ]
+malwaretags = {'locky': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Locky"'],
+               'jaff': ['ecsirt:malicious-code="ransomware"', 'misp-galaxy:ransomware="Jaff"'],
+               'dridex': ['misp-galaxy:tool="dridex"'],
+               'netwire': ['Netwire RAT'],
+               'Pony': ['misp-galaxy:tool="Hancitor"'],
+               'ursnif': ['misp-galaxy:tool="Snifula"'],
+               'NanoCore': ['misp-galaxy:tool="NanoCoreRAT"'],
+               'trickbot': ['misp-galaxy:tool="Trick Bot"']
                }
 
 # Tags to be set depending on the presence of other tags
-dependingtags = { 'tlp:white': [ 'circl:osint-feed' ]
+dependingtags = {'tlp:white': ['circl:osint-feed']
                  }
 
 # Known identifiers for forwarded messages
-forward_identifiers = { '-------- Forwarded Message --------', 'Begin forwarded message:' }
+forward_identifiers = {'-------- Forwarded Message --------', 'Begin forwarded message:'}
 
 # Tags to add when hashes are found (e.g. to do automatic expansion)
-hash_only_tags = { 'TODO:VT-ENRICHMENT' }
+hash_only_tags = {'TODO:VT-ENRICHMENT'}
 
 # If an attribute is on any MISP server side `warning list`, skip the creation of the attribute
 skip_item_on_warninglist = True

reqirements.txt

@@ -0,0 +1,5 @@
+dnspython
+pymisp[fileobjects]
+git+https://github.com/kbandla/pydeep.git
+git+https://github.com/stricaud/faup.git#egg=pyfaup&subdirectory=src/lib/bindings/python
+git+https://github.com/Rafiot/defang.git