Merge remote-tracking branch 'upstream/main' into main

CHANGELOG.txt

@@ -2,6 +2,74 @@ Changelog
 =========
 
 
+v2.4.148.1 (2021-09-30)
+-----------------------
+
+New
+~~~
+- Add few keys to email object creator. [Raphaël Vinot]
+
+  Fix #787
+- Test cases for edit objects and upload stix. [Raphaël Vinot]
+
+Changes
+~~~~~~~
+- Bump changelog. [Raphaël Vinot]
+- Bump misp-objects. [Raphaël Vinot]
+- Bump version. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+- [doc] Minor fixes, note and typo. [Steve Clement]
+- Bump deps. [Raphaël Vinot]
+- [misp-objects] updated to the latest version. [Alexandre Dulaunoy]
+- [misp-objects] updated to the latest version. [Alexandre Dulaunoy]
+- Update tutorial for custom objects. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+- Bump live tests. [Raphaël Vinot]
+- [misp-objects] updated to the latest version. [Alexandre Dulaunoy]
+- [types] updated types/categories mapping. [Christophe Vandeplas]
+- Remove test files. [Raphaël Vinot]
+- Automatically pull the malwares repo when running
+  tests/testlive_comprehensive.py. [Raphaël Vinot]
+- Remove submodules with malware. [Raphaël Vinot]
+- Add test for updating a objects from a custom template. [Raphaël
+  Vinot]
+- Re-bump changelog. [Raphaël Vinot]
+
+Fix
+~~~
+- Message_from_bytes really dislikes newline at the beginning of a mail.
+  [Raphaël Vinot]
+- Skip IPs in Received header. [Raphaël Vinot]
+- Name is passed to super. [Raphaël Vinot]
+- Do not create empty manifest, json load dislikes it. [Raphaël Vinot]
+- Initial round of cleanup on redis feed generator. [Raphaël Vinot]
+- Upload of STIX document with non-ascii characters. [Raphaël Vinot]
+
+  Due to: https://github.com/psf/requests/issues/5560
+
+  TL;DR: a variable of type str passed to data in a POST request will be
+  silently re-encoded to ISO-8859-1, making MISP barf on the other side.
+- Remove outdated deps from setup.py. [Raphaël Vinot]
+
+  Fix https://github.com/MISP/MISP/issues/7729
+
+Other
+~~~~~
+- Remove unicode to ascii parts. [Sami Tainio]
+- Fix #787 and add Unicode to ASCII function. [Sami Tainio]
+
+  Fix #787
+  - Uses regex to pick up the hostnames/domains from the "Received: from" headers.
+
+  Unicode to ASCII function
+  - Spam messages more often than not contain junk text as unicode characters in the headers. The "from" and "subject" headers being the most common ones. Before this change the script would error on such emails or sometimes replace the unicode characters with questionmarks "?".
+  - Function takes argument as an input and then encodes it in ascii while ignoring any malformed data. It then returns an ASCII string without the unicode characters.
+  - Currently implemented for "from" and "subject" handling.
+- Update README.md. [Raphaël Vinot]
+
+  Not using travis anymore.
+
+
 v2.4.148 (2021-08-05)
 ---------------------
 

examples/feed-generator-from-redis/ObjectConstructor/CowrieMISPObject.py

@@ -8,12 +8,10 @@ from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator
 class CowrieMISPObject(AbstractMISPObjectGenerator):
     def __init__(self, dico_val, **kargs):
         self._dico_val = dico_val
-        self.name = "cowrie"
-
         #  Enforce attribute date with timestamp
         super(CowrieMISPObject, self).__init__('cowrie',
-            default_attributes_parameters={'timestamp': int(time.time())},
+                                               default_attributes_parameters={'timestamp': int(time.time())},
-            **kargs)
+                                               **kargs)
         self.generate_attributes()
 
     def generate_attributes(self):

examples/feed-generator-from-redis/generator.py

@@ -6,9 +6,8 @@ import json
 import os
 import sys
 import time
-import uuid
 
-from pymisp import MISPEvent
+from pymisp import MISPEvent, MISPOrganisation
 
 import settings
 
@@ -35,11 +34,6 @@ def get_system_templates():
     return templates
 
 
-def gen_uuid():
-    """Generate a random UUID and returns its string representation"""
-    return str(uuid.uuid4())
-
-
 class FeedGenerator:
     """Helper object to create MISP feed.
 
@@ -153,8 +147,8 @@ class FeedGenerator:
 
         # create an empty manifest
         try:
-            with open(os.path.join(settings.outputdir, 'manifest.json'), 'w'):
+            with open(os.path.join(settings.outputdir, 'manifest.json'), 'w') as f:
-                pass
+                json.dump({}, f)
         except PermissionError as error:
             print(error)
             print("Please fix the above error and try again.")
@@ -164,7 +158,7 @@ class FeedGenerator:
         self.create_daily_event()
 
     def flush_event(self, new_event=None):
-        print('Writing event on disk'+' '*50)
+        print('Writing event on disk' + ' ' * 50)
         if new_event is not None:
             event_uuid = new_event['uuid']
             event = new_event
@@ -172,9 +166,8 @@ class FeedGenerator:
             event_uuid = self.current_event_uuid
             event = self.current_event
 
-        eventFile = open(os.path.join(settings.outputdir, event_uuid+'.json'), 'w')
+        with open(os.path.join(settings.outputdir, event_uuid + '.json'), 'w') as eventFile:
-        eventFile.write(event.to_json())
+            json.dump(event.to_feed(), eventFile)
-        eventFile.close()
 
         self.save_hashes()
 
@@ -197,27 +190,11 @@ class FeedGenerator:
                 hashFile.write('{},{}\n'.format(element[0], element[1]))
             hashFile.close()
             self.attributeHashes = []
-            print('Hash saved' + ' '*30)
+            print('Hash saved' + ' ' * 30)
         except Exception as e:
             print(e)
             sys.exit('Could not create the quick hash lookup file.')
 
-    def _addEventToManifest(self, event):
-        event_dict = event.to_dict()['Event']
-        tags = []
-        for eventTag in event_dict.get('EventTag', []):
-            tags.append({'name': eventTag['Tag']['name'],
-                         'colour': eventTag['Tag']['colour']})
-        return {
-                'Orgc': event_dict.get('Orgc', []),
-                'Tag': tags,
-                'info': event_dict['info'],
-                'date': event_dict['date'],
-                'analysis': event_dict['analysis'],
-                'threat_level_id': event_dict['threat_level_id'],
-                'timestamp': event_dict.get('timestamp', int(time.time()))
-               }
-
     def get_last_event_from_manifest(self):
         """Retreive last event from the manifest.
 
@@ -240,7 +217,7 @@ class FeedGenerator:
                 # Sort by date then by event name
                 dated_events.sort(key=lambda k: (k[0], k[2]), reverse=True)
                 return dated_events[0]
-        except FileNotFoundError as e:
+        except FileNotFoundError:
             print('Manifest not found, generating a fresh one')
             self._init_manifest()
             return self.get_last_event_from_manifest()
@@ -263,11 +240,9 @@ class FeedGenerator:
             return event
 
     def create_daily_event(self):
-        new_uuid = gen_uuid()
         today = str(datetime.date.today())
         event_dict = {
-            'uuid': new_uuid,
+            'id': len(self.manifest) + 1,
-            'id': len(self.manifest)+1,
             'Tag': settings.Tag,
             'info': self.daily_event_name.format(today),
             'analysis': settings.analysis,  # [0-2]
@@ -279,14 +254,14 @@ class FeedGenerator:
         event.from_dict(**event_dict)
 
         # reference org
-        org_dict = {}
+        org = MISPOrganisation()
-        org_dict['name'] = settings.org_name
+        org.name = settings.org_name
-        org_dict['uuid'] = settings.org_uuid
+        org.uuid = settings.org_uuid
-        event['Orgc'] = org_dict
+        event.Orgc = org
 
         # save event on disk
         self.flush_event(new_event=event)
         # add event to manifest
-        self.manifest[event['uuid']] = self._addEventToManifest(event)
+        self.manifest.update(event.manifest)
         self.save_manifest()
         return event

poetry.lock

@@ -230,7 +230,7 @@ yaml = ["PyYAML (>=3.10)"]
 
 [[package]]
 name = "cryptography"
-version = "3.4.8"
+version = "35.0.0"
 description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers."
 category = "main"
 optional = true
@@ -243,9 +243,9 @@ cffi = ">=1.12"
 docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"]
 docstest = ["doc8", "pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"]
 pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"]
-sdist = ["setuptools-rust (>=0.11.4)"]
+sdist = ["setuptools_rust (>=0.11.4)"]
 ssh = ["bcrypt (>=3.1.5)"]
-test = ["pytest (>=6.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"]
+test = ["pytest (>=6.2.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"]
 
 [[package]]
 name = "decorator"
@@ -1398,7 +1398,7 @@ python-versions = "*"
 
 [[package]]
 name = "types-redis"
-version = "3.5.8"
+version = "3.5.9"
 description = "Typing stubs for redis"
 category = "dev"
 optional = false
@@ -1406,7 +1406,7 @@ python-versions = "*"
 
 [[package]]
 name = "types-requests"
-version = "2.25.7"
+version = "2.25.9"
 description = "Typing stubs for requests"
 category = "dev"
 optional = false
@@ -1453,7 +1453,7 @@ test = ["pytest-mock (>=3.3)", "pytest (>=4.3)"]
 
 [[package]]
 name = "urllib3"
-version = "1.26.6"
+version = "1.26.7"
 description = "HTTP library with thread-safe connection pooling, file post, and more."
 category = "main"
 optional = false
@@ -1516,7 +1516,7 @@ python-versions = "*"
 
 [[package]]
 name = "zipp"
-version = "3.5.0"
+version = "3.6.0"
 description = "Backport of pathlib-compatible object wrapper for zip files"
 category = "main"
 optional = false
@@ -1771,23 +1771,26 @@ coveralls = [
     {file = "coveralls-3.2.0.tar.gz", hash = "sha256:15a987d9df877fff44cd81948c5806ffb6eafb757b3443f737888358e96156ee"},
 ]
 cryptography = [
-    {file = "cryptography-3.4.8-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:a00cf305f07b26c351d8d4e1af84ad7501eca8a342dedf24a7acb0e7b7406e14"},
+    {file = "cryptography-35.0.0-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:d57e0cdc1b44b6cdf8af1d01807db06886f10177469312fbde8f44ccbb284bc9"},
-    {file = "cryptography-3.4.8-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:f44d141b8c4ea5eb4dbc9b3ad992d45580c1d22bf5e24363f2fbf50c2d7ae8a7"},
+    {file = "cryptography-35.0.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:ced40344e811d6abba00295ced98c01aecf0c2de39481792d87af4fa58b7b4d6"},
-    {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0a7dcbcd3f1913f664aca35d47c1331fce738d44ec34b7be8b9d332151b0b01e"},
+    {file = "cryptography-35.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:54b2605e5475944e2213258e0ab8696f4f357a31371e538ef21e8d61c843c28d"},
-    {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34dae04a0dce5730d8eb7894eab617d8a70d0c97da76b905de9efb7128ad7085"},
+    {file = "cryptography-35.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:7b7ceeff114c31f285528ba8b390d3e9cfa2da17b56f11d366769a807f17cbaa"},
-    {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1eb7bb0df6f6f583dd8e054689def236255161ebbcf62b226454ab9ec663746b"},
+    {file = "cryptography-35.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d69645f535f4b2c722cfb07a8eab916265545b3475fdb34e0be2f4ee8b0b15e"},
-    {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:9965c46c674ba8cc572bc09a03f4c649292ee73e1b683adb1ce81e82e9a6a0fb"},
+    {file = "cryptography-35.0.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a2d0e0acc20ede0f06ef7aa58546eee96d2592c00f450c9acb89c5879b61992"},
-    {file = "cryptography-3.4.8-cp36-abi3-win32.whl", hash = "sha256:21ca464b3a4b8d8e86ba0ee5045e103a1fcfac3b39319727bc0fc58c09c6aff7"},
+    {file = "cryptography-35.0.0-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:07bb7fbfb5de0980590ddfc7f13081520def06dc9ed214000ad4372fb4e3c7f6"},
-    {file = "cryptography-3.4.8-cp36-abi3-win_amd64.whl", hash = "sha256:3520667fda779eb788ea00080124875be18f2d8f0848ec00733c0ec3bb8219fc"},
+    {file = "cryptography-35.0.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:7eba2cebca600a7806b893cb1d541a6e910afa87e97acf2021a22b32da1df52d"},
-    {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d2a6e5ef66503da51d2110edf6c403dc6b494cc0082f85db12f54e9c5d4c3ec5"},
+    {file = "cryptography-35.0.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:18d90f4711bf63e2fb21e8c8e51ed8189438e6b35a6d996201ebd98a26abbbe6"},
-    {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a305600e7a6b7b855cd798e00278161b681ad6e9b7eca94c721d5f588ab212af"},
+    {file = "cryptography-35.0.0-cp36-abi3-win32.whl", hash = "sha256:c10c797ac89c746e488d2ee92bd4abd593615694ee17b2500578b63cad6b93a8"},
-    {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:3fa3a7ccf96e826affdf1a0a9432be74dc73423125c8f96a909e3835a5ef194a"},
+    {file = "cryptography-35.0.0-cp36-abi3-win_amd64.whl", hash = "sha256:7075b304cd567694dc692ffc9747f3e9cb393cc4aa4fb7b9f3abd6f5c4e43588"},
-    {file = "cryptography-3.4.8-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:d9ec0e67a14f9d1d48dd87a2531009a9b251c02ea42851c060b25c782516ff06"},
+    {file = "cryptography-35.0.0-pp36-pypy36_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a688ebcd08250eab5bb5bca318cc05a8c66de5e4171a65ca51db6bd753ff8953"},
-    {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5b0fbfae7ff7febdb74b574055c7466da334a5371f253732d7e2e7525d570498"},
+    {file = "cryptography-35.0.0-pp36-pypy36_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d99915d6ab265c22873f1b4d6ea5ef462ef797b4140be4c9d8b179915e0985c6"},
-    {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94fff993ee9bc1b2440d3b7243d488c6a3d9724cc2b09cdb297f6a886d040ef7"},
+    {file = "cryptography-35.0.0-pp36-pypy36_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:928185a6d1ccdb816e883f56ebe92e975a262d31cc536429041921f8cb5a62fd"},
-    {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:8695456444f277af73a4877db9fc979849cd3ee74c198d04fc0776ebc3db52b9"},
+    {file = "cryptography-35.0.0-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:ebeddd119f526bcf323a89f853afb12e225902a24d29b55fe18dd6fcb2838a76"},
-    {file = "cryptography-3.4.8-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:cd65b60cfe004790c795cc35f272e41a3df4631e2fb6b35aa7ac6ef2859d554e"},
+    {file = "cryptography-35.0.0-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:22a38e96118a4ce3b97509443feace1d1011d0571fae81fc3ad35f25ba3ea999"},
-    {file = "cryptography-3.4.8.tar.gz", hash = "sha256:94cc5ed4ceaefcbe5bf38c8fba6a21fc1d365bb8fb826ea1688e3370b2e24a1c"},
+    {file = "cryptography-35.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb80e8a1f91e4b7ef8b33041591e6d89b2b8e122d787e87eeb2b08da71bb16ad"},
+    {file = "cryptography-35.0.0-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:abb5a361d2585bb95012a19ed9b2c8f412c5d723a9836418fab7aaa0243e67d2"},
+    {file = "cryptography-35.0.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:1ed82abf16df40a60942a8c211251ae72858b25b7421ce2497c2eb7a1cee817c"},
+    {file = "cryptography-35.0.0.tar.gz", hash = "sha256:9933f28f70d0517686bd7de36166dda42094eac49415459d9bdf5e7df3e0086d"},
 ]
 decorator = [
     {file = "decorator-5.1.0-py3-none-any.whl", hash = "sha256:7b12e7c3c6ab203a29e157335e9122cb03de9ab7264b137594103fd4a683b374"},
@@ -2440,12 +2443,12 @@ types-python-dateutil = [
     {file = "types_python_dateutil-0.1.6-py3-none-any.whl", hash = "sha256:5b6241ea9fca2d8878cc152017d9524da62a7a856b98e31006e68b02aab47442"},
 ]
 types-redis = [
-    {file = "types-redis-3.5.8.tar.gz", hash = "sha256:be26f50259d1a7e74cbc1e83b377dbf6b534fdb037ff55ae31501e87ac2b5b5a"},
+    {file = "types-redis-3.5.9.tar.gz", hash = "sha256:f142c48f4080757ca2a9441ec40213bda3b1535eebebfc4f3519e5aa46498076"},
-    {file = "types_redis-3.5.8-py3-none-any.whl", hash = "sha256:85814769071721044857c34841e46064b867ccdd58fc81221c43462bd07e4892"},
+    {file = "types_redis-3.5.9-py3-none-any.whl", hash = "sha256:5f5648ffc025708858097173cf695164c20f2b5e3f57177de14e352cae8cc335"},
 ]
 types-requests = [
-    {file = "types-requests-2.25.7.tar.gz", hash = "sha256:4b279513a34b789bef75ce05bee5eb4ce934a892318388400f7511fbcdf56f87"},
+    {file = "types-requests-2.25.9.tar.gz", hash = "sha256:4ec8b71da73e5344adb9bee725a74ec8598e7286f9bcb17500d627f259fe4fb9"},
-    {file = "types_requests-2.25.7-py3-none-any.whl", hash = "sha256:24bbe4682373ce0b1927f432b30ba1dcf46b66512fb6e848e72998c79797d040"},
+    {file = "types_requests-2.25.9-py3-none-any.whl", hash = "sha256:543ba8b3b23e38ac028da1d163aecbbc27d3cc8f654ae64339da539a191a2b1c"},
 ]
 types-werkzeug = [
     {file = "types-Werkzeug-1.0.5.tar.gz", hash = "sha256:f6216ab0e0211fe73ebdb4ae0e414113d4d8a2f783a15c2d8550e06d0fd8e7f9"},
@@ -2465,8 +2468,8 @@ tzlocal = [
     {file = "tzlocal-3.0.tar.gz", hash = "sha256:f4e6e36db50499e0d92f79b67361041f048e2609d166e93456b50746dc4aef12"},
 ]
 urllib3 = [
-    {file = "urllib3-1.26.6-py2.py3-none-any.whl", hash = "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4"},
+    {file = "urllib3-1.26.7-py2.py3-none-any.whl", hash = "sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844"},
-    {file = "urllib3-1.26.6.tar.gz", hash = "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"},
+    {file = "urllib3-1.26.7.tar.gz", hash = "sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece"},
 ]
 validators = [
     {file = "validators-0.18.2-py3-none-any.whl", hash = "sha256:0143dcca8a386498edaf5780cbd5960da1a4c85e0719f3ee5c9b41249c4fefbd"},
@@ -2487,6 +2490,6 @@ wrapt = [
     {file = "wrapt-1.12.1.tar.gz", hash = "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7"},
 ]
 zipp = [
-    {file = "zipp-3.5.0-py3-none-any.whl", hash = "sha256:957cfda87797e389580cb8b9e3870841ca991e2125350677b2ca83a0e99390a3"},
+    {file = "zipp-3.6.0-py3-none-any.whl", hash = "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc"},
-    {file = "zipp-3.5.0.tar.gz", hash = "sha256:f5812b1e007e48cff63449a5e9f4e7ebea716b4111f9c4f9a645f91d579bf0c4"},
+    {file = "zipp-3.6.0.tar.gz", hash = "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832"},
 ]

pymisp/data/misp-objects

@@ -1 +1 @@
-Subproject commit ffa6ed7963051a5bd0f4578ade94c788c2962c09
+Subproject commit 3d52773e9d3ba39ff324455bf8c10b47e11b695a

pymisp/tools/emailobject.py

@@ -45,7 +45,7 @@ class EMailObject(AbstractMISPObjectGenerator):
 
     def parse_email(self) -> EmailMessage:
         """Convert email into EmailMessage."""
-        content_in_bytes = self.__pseudofile.getvalue()
+        content_in_bytes = self.__pseudofile.getvalue().strip()
         eml = message_from_bytes(content_in_bytes,
                                  _class=EmailMessage,
                                  policy=policy.default)
@@ -282,6 +282,8 @@ class EMailObject(AbstractMISPObjectGenerator):
 
         if "To" in message:
             self.__add_emails("to", message["To"])
+        if "Delivered-To" in message:
+            self.__add_emails("to", message["Delivered-To"])
 
         if "From" in message:
             self.__add_emails("from", message["From"])
@@ -343,7 +345,7 @@ class EMailObject(AbstractMISPObjectGenerator):
 
     def __generate_received(self):
         """
-        Extract IP addresses from received headers that are not private.
+        Extract IP addresses from received headers that are not private. Also extract hostnames or domains.
         """
         received_items = self.email.get_all("received")
         if received_items is None:
@@ -367,3 +369,11 @@ class EMailObject(AbstractMISPObjectGenerator):
                 continue  # skip header if IP not found or is private
 
             self.add_attribute("received-header-ip", value=str(ip), comment=fromstr)
+
+        # The hostnames and/or domains always come after the "Received: from"
+        # part so we can use regex to pick up those attributes.
+        received_from = re.findall(r'(?<=from\s)[\w\d\.\-]+\.\w{2,24}', str(received_items))
+        try:
+            [self.add_attribute("received-header-hostname", i) for i in received_from]
+        except Exception:
+            pass

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pymisp"
-version = "2.4.148"
+version = "2.4.148.1"
 description = "Python API for MISP."
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-2-Clause"