new: URLObject (requires pyfaup)

pymisp/tools/__init__.py

@@ -22,3 +22,8 @@ if sys.version_info >= (3, 6):
     from .csvloader import CSVLoader  # noqa
     from .sshauthkeyobject import SSHAuthorizedKeysObject  # noqa
     from .feed import feed_meta_generator  # noqa
+    try:
+        from .urlobject import URLObject  # noqa
+    except ImportError:
+        # Requires faup, which is a bit difficult to install
+        pass

pymisp/tools/emailobject.py

@@ -50,17 +50,30 @@ class EMailObject(AbstractMISPObjectGenerator):
         if 'Message-ID' in self.__email:
             self.add_attribute('message-id', value=self.__email['Message-ID'])
         if 'To' in self.__email:
+            # TODO: split name and email address
             to_add = [to.strip() for to in self.__email['To'].split(',')]
             self.add_attributes('to', *to_add)
         if 'Cc' in self.__email:
+            # TODO: split name and email address
             to_add = [to.strip() for to in self.__email['Cc'].split(',')]
             self.add_attributes('cc', *to_add)
         if 'Subject' in self.__email:
             self.add_attribute('subject', value=self.__email['Subject'])
         if 'From' in self.__email:
+            # TODO: split name and email address
             to_add = [to.strip() for to in self.__email['From'].split(',')]
             self.add_attributes('from', *to_add)
         if 'Return-Path' in self.__email:
+            # TODO: split name and email address
             self.add_attribute('return-path', value=self.__email['Return-Path'])
         if 'User-Agent' in self.__email:
             self.add_attribute('user-agent', value=self.__email['User-Agent'])
+        if self.__email.get_boundary():
+            self.add_attribute('mime-boundary', value=self.__email.get_boundary())
+        if 'X-Mailer' in self.__email:
+            self.add_attribute('x-mailer', value=self.__email['X-Mailer'])
+        if 'Thread-Index' in self.__email:
+            self.add_attribute('thread-index', value=self.__email['Thread-Index'])
+        # TODO: email-header: all headers in one bloc
+        # TODO: BCC?
+        # TODO: received headers sometimes have TO email addresses

pymisp/tools/urlobject.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from .abstractgenerator import AbstractMISPObjectGenerator
+import logging
+from pyfaup.faup import Faup
+from urllib.parse import unquote_plus
+
+logger = logging.getLogger('pymisp')
+
+faup = Faup()
+
+
+class URLObject(AbstractMISPObjectGenerator):
+
+    def __init__(self, url, standalone=True, **kwargs):
+        # PY3 way:
+        # super().__init__('file')
+        super(URLObject, self).__init__('url', standalone=standalone, **kwargs)
+        faup.decode(unquote_plus(url))
+        self.generate_attributes()
+
+    def generate_attributes(self):
+        self.add_attribute('url', value=faup.url.decode())
+        if faup.get_host():
+            self.add_attribute('host', value=faup.get_host())
+        if faup.get_domain():
+            self.add_attribute('domain', value=faup.get_domain())