diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 5e9c008..404944f 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -2,6 +2,74 @@ Changelog ========= +v2.4.148.1 (2021-09-30) +----------------------- + +New +~~~ +- Add few keys to email object creator. [Raphaël Vinot] + + Fix #787 +- Test cases for edit objects and upload stix. [Raphaël Vinot] + +Changes +~~~~~~~ +- Bump changelog. [Raphaël Vinot] +- Bump misp-objects. [Raphaël Vinot] +- Bump version. [Raphaël Vinot] +- Bump deps. [Raphaël Vinot] +- [doc] Minor fixes, note and typo. [Steve Clement] +- Bump deps. [Raphaël Vinot] +- [misp-objects] updated to the latest version. [Alexandre Dulaunoy] +- [misp-objects] updated to the latest version. [Alexandre Dulaunoy] +- Update tutorial for custom objects. [Raphaël Vinot] +- Bump deps. [Raphaël Vinot] +- Bump live tests. [Raphaël Vinot] +- [misp-objects] updated to the latest version. [Alexandre Dulaunoy] +- [types] updated types/categories mapping. [Christophe Vandeplas] +- Remove test files. [Raphaël Vinot] +- Automatically pull the malwares repo when running + tests/testlive_comprehensive.py. [Raphaël Vinot] +- Remove submodules with malware. [Raphaël Vinot] +- Add test for updating a objects from a custom template. [Raphaël + Vinot] +- Re-bump changelog. [Raphaël Vinot] + +Fix +~~~ +- Message_from_bytes really dislikes newline at the beginning of a mail. + [Raphaël Vinot] +- Skip IPs in Received header. [Raphaël Vinot] +- Name is passed to super. [Raphaël Vinot] +- Do not create empty manifest, json load dislikes it. [Raphaël Vinot] +- Initial round of cleanup on redis feed generator. [Raphaël Vinot] +- Upload of STIX document with non-ascii characters. [Raphaël Vinot] + + Due to: https://github.com/psf/requests/issues/5560 + + TL;DR: a variable of type str passed to data in a POST request will be + silently re-encoded to ISO-8859-1, making MISP barf on the other side. +- Remove outdated deps from setup.py. [Raphaël Vinot] + + Fix https://github.com/MISP/MISP/issues/7729 + +Other +~~~~~ +- Remove unicode to ascii parts. [Sami Tainio] +- Fix #787 and add Unicode to ASCII function. [Sami Tainio] + + Fix #787 + - Uses regex to pick up the hostnames/domains from the "Received: from" headers. + + Unicode to ASCII function + - Spam messages more often than not contain junk text as unicode characters in the headers. The "from" and "subject" headers being the most common ones. Before this change the script would error on such emails or sometimes replace the unicode characters with questionmarks "?". + - Function takes argument as an input and then encodes it in ascii while ignoring any malformed data. It then returns an ASCII string without the unicode characters. + - Currently implemented for "from" and "subject" handling. +- Update README.md. [Raphaël Vinot] + + Not using travis anymore. + + v2.4.148 (2021-08-05) --------------------- diff --git a/examples/feed-generator-from-redis/ObjectConstructor/CowrieMISPObject.py b/examples/feed-generator-from-redis/ObjectConstructor/CowrieMISPObject.py index 1bf98ca..b69c153 100644 --- a/examples/feed-generator-from-redis/ObjectConstructor/CowrieMISPObject.py +++ b/examples/feed-generator-from-redis/ObjectConstructor/CowrieMISPObject.py @@ -8,12 +8,10 @@ from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator class CowrieMISPObject(AbstractMISPObjectGenerator): def __init__(self, dico_val, **kargs): self._dico_val = dico_val - self.name = "cowrie" - # Enforce attribute date with timestamp super(CowrieMISPObject, self).__init__('cowrie', - default_attributes_parameters={'timestamp': int(time.time())}, - **kargs) + default_attributes_parameters={'timestamp': int(time.time())}, + **kargs) self.generate_attributes() def generate_attributes(self): diff --git a/examples/feed-generator-from-redis/generator.py b/examples/feed-generator-from-redis/generator.py index a7ab630..388a72d 100755 --- a/examples/feed-generator-from-redis/generator.py +++ b/examples/feed-generator-from-redis/generator.py @@ -6,9 +6,8 @@ import json import os import sys import time -import uuid -from pymisp import MISPEvent +from pymisp import MISPEvent, MISPOrganisation import settings @@ -35,11 +34,6 @@ def get_system_templates(): return templates -def gen_uuid(): - """Generate a random UUID and returns its string representation""" - return str(uuid.uuid4()) - - class FeedGenerator: """Helper object to create MISP feed. @@ -153,8 +147,8 @@ class FeedGenerator: # create an empty manifest try: - with open(os.path.join(settings.outputdir, 'manifest.json'), 'w'): - pass + with open(os.path.join(settings.outputdir, 'manifest.json'), 'w') as f: + json.dump({}, f) except PermissionError as error: print(error) print("Please fix the above error and try again.") @@ -164,7 +158,7 @@ class FeedGenerator: self.create_daily_event() def flush_event(self, new_event=None): - print('Writing event on disk'+' '*50) + print('Writing event on disk' + ' ' * 50) if new_event is not None: event_uuid = new_event['uuid'] event = new_event @@ -172,9 +166,8 @@ class FeedGenerator: event_uuid = self.current_event_uuid event = self.current_event - eventFile = open(os.path.join(settings.outputdir, event_uuid+'.json'), 'w') - eventFile.write(event.to_json()) - eventFile.close() + with open(os.path.join(settings.outputdir, event_uuid + '.json'), 'w') as eventFile: + json.dump(event.to_feed(), eventFile) self.save_hashes() @@ -197,27 +190,11 @@ class FeedGenerator: hashFile.write('{},{}\n'.format(element[0], element[1])) hashFile.close() self.attributeHashes = [] - print('Hash saved' + ' '*30) + print('Hash saved' + ' ' * 30) except Exception as e: print(e) sys.exit('Could not create the quick hash lookup file.') - def _addEventToManifest(self, event): - event_dict = event.to_dict()['Event'] - tags = [] - for eventTag in event_dict.get('EventTag', []): - tags.append({'name': eventTag['Tag']['name'], - 'colour': eventTag['Tag']['colour']}) - return { - 'Orgc': event_dict.get('Orgc', []), - 'Tag': tags, - 'info': event_dict['info'], - 'date': event_dict['date'], - 'analysis': event_dict['analysis'], - 'threat_level_id': event_dict['threat_level_id'], - 'timestamp': event_dict.get('timestamp', int(time.time())) - } - def get_last_event_from_manifest(self): """Retreive last event from the manifest. @@ -240,7 +217,7 @@ class FeedGenerator: # Sort by date then by event name dated_events.sort(key=lambda k: (k[0], k[2]), reverse=True) return dated_events[0] - except FileNotFoundError as e: + except FileNotFoundError: print('Manifest not found, generating a fresh one') self._init_manifest() return self.get_last_event_from_manifest() @@ -263,11 +240,9 @@ class FeedGenerator: return event def create_daily_event(self): - new_uuid = gen_uuid() today = str(datetime.date.today()) event_dict = { - 'uuid': new_uuid, - 'id': len(self.manifest)+1, + 'id': len(self.manifest) + 1, 'Tag': settings.Tag, 'info': self.daily_event_name.format(today), 'analysis': settings.analysis, # [0-2] @@ -279,14 +254,14 @@ class FeedGenerator: event.from_dict(**event_dict) # reference org - org_dict = {} - org_dict['name'] = settings.org_name - org_dict['uuid'] = settings.org_uuid - event['Orgc'] = org_dict + org = MISPOrganisation() + org.name = settings.org_name + org.uuid = settings.org_uuid + event.Orgc = org # save event on disk self.flush_event(new_event=event) # add event to manifest - self.manifest[event['uuid']] = self._addEventToManifest(event) + self.manifest.update(event.manifest) self.save_manifest() return event diff --git a/poetry.lock b/poetry.lock index cd458ad..186ca11 100644 --- a/poetry.lock +++ b/poetry.lock @@ -230,7 +230,7 @@ yaml = ["PyYAML (>=3.10)"] [[package]] name = "cryptography" -version = "3.4.8" +version = "35.0.0" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "main" optional = true @@ -243,9 +243,9 @@ cffi = ">=1.12" docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] docstest = ["doc8", "pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] -sdist = ["setuptools-rust (>=0.11.4)"] +sdist = ["setuptools_rust (>=0.11.4)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["pytest (>=6.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] +test = ["pytest (>=6.2.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] [[package]] name = "decorator" @@ -1398,7 +1398,7 @@ python-versions = "*" [[package]] name = "types-redis" -version = "3.5.8" +version = "3.5.9" description = "Typing stubs for redis" category = "dev" optional = false @@ -1406,7 +1406,7 @@ python-versions = "*" [[package]] name = "types-requests" -version = "2.25.7" +version = "2.25.9" description = "Typing stubs for requests" category = "dev" optional = false @@ -1453,7 +1453,7 @@ test = ["pytest-mock (>=3.3)", "pytest (>=4.3)"] [[package]] name = "urllib3" -version = "1.26.6" +version = "1.26.7" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false @@ -1516,7 +1516,7 @@ python-versions = "*" [[package]] name = "zipp" -version = "3.5.0" +version = "3.6.0" description = "Backport of pathlib-compatible object wrapper for zip files" category = "main" optional = false @@ -1771,23 +1771,26 @@ coveralls = [ {file = "coveralls-3.2.0.tar.gz", hash = "sha256:15a987d9df877fff44cd81948c5806ffb6eafb757b3443f737888358e96156ee"}, ] cryptography = [ - {file = "cryptography-3.4.8-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:a00cf305f07b26c351d8d4e1af84ad7501eca8a342dedf24a7acb0e7b7406e14"}, - {file = "cryptography-3.4.8-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:f44d141b8c4ea5eb4dbc9b3ad992d45580c1d22bf5e24363f2fbf50c2d7ae8a7"}, - {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0a7dcbcd3f1913f664aca35d47c1331fce738d44ec34b7be8b9d332151b0b01e"}, - {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34dae04a0dce5730d8eb7894eab617d8a70d0c97da76b905de9efb7128ad7085"}, - {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1eb7bb0df6f6f583dd8e054689def236255161ebbcf62b226454ab9ec663746b"}, - {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:9965c46c674ba8cc572bc09a03f4c649292ee73e1b683adb1ce81e82e9a6a0fb"}, - {file = "cryptography-3.4.8-cp36-abi3-win32.whl", hash = "sha256:21ca464b3a4b8d8e86ba0ee5045e103a1fcfac3b39319727bc0fc58c09c6aff7"}, - {file = "cryptography-3.4.8-cp36-abi3-win_amd64.whl", hash = "sha256:3520667fda779eb788ea00080124875be18f2d8f0848ec00733c0ec3bb8219fc"}, - {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d2a6e5ef66503da51d2110edf6c403dc6b494cc0082f85db12f54e9c5d4c3ec5"}, - {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a305600e7a6b7b855cd798e00278161b681ad6e9b7eca94c721d5f588ab212af"}, - {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:3fa3a7ccf96e826affdf1a0a9432be74dc73423125c8f96a909e3835a5ef194a"}, - {file = "cryptography-3.4.8-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:d9ec0e67a14f9d1d48dd87a2531009a9b251c02ea42851c060b25c782516ff06"}, - {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5b0fbfae7ff7febdb74b574055c7466da334a5371f253732d7e2e7525d570498"}, - {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94fff993ee9bc1b2440d3b7243d488c6a3d9724cc2b09cdb297f6a886d040ef7"}, - {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:8695456444f277af73a4877db9fc979849cd3ee74c198d04fc0776ebc3db52b9"}, - {file = "cryptography-3.4.8-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:cd65b60cfe004790c795cc35f272e41a3df4631e2fb6b35aa7ac6ef2859d554e"}, - {file = "cryptography-3.4.8.tar.gz", hash = "sha256:94cc5ed4ceaefcbe5bf38c8fba6a21fc1d365bb8fb826ea1688e3370b2e24a1c"}, + {file = "cryptography-35.0.0-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:d57e0cdc1b44b6cdf8af1d01807db06886f10177469312fbde8f44ccbb284bc9"}, + {file = "cryptography-35.0.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:ced40344e811d6abba00295ced98c01aecf0c2de39481792d87af4fa58b7b4d6"}, + {file = "cryptography-35.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:54b2605e5475944e2213258e0ab8696f4f357a31371e538ef21e8d61c843c28d"}, + {file = "cryptography-35.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:7b7ceeff114c31f285528ba8b390d3e9cfa2da17b56f11d366769a807f17cbaa"}, + {file = "cryptography-35.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d69645f535f4b2c722cfb07a8eab916265545b3475fdb34e0be2f4ee8b0b15e"}, + {file = "cryptography-35.0.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a2d0e0acc20ede0f06ef7aa58546eee96d2592c00f450c9acb89c5879b61992"}, + {file = "cryptography-35.0.0-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:07bb7fbfb5de0980590ddfc7f13081520def06dc9ed214000ad4372fb4e3c7f6"}, + {file = "cryptography-35.0.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:7eba2cebca600a7806b893cb1d541a6e910afa87e97acf2021a22b32da1df52d"}, + {file = "cryptography-35.0.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:18d90f4711bf63e2fb21e8c8e51ed8189438e6b35a6d996201ebd98a26abbbe6"}, + {file = "cryptography-35.0.0-cp36-abi3-win32.whl", hash = "sha256:c10c797ac89c746e488d2ee92bd4abd593615694ee17b2500578b63cad6b93a8"}, + {file = "cryptography-35.0.0-cp36-abi3-win_amd64.whl", hash = "sha256:7075b304cd567694dc692ffc9747f3e9cb393cc4aa4fb7b9f3abd6f5c4e43588"}, + {file = "cryptography-35.0.0-pp36-pypy36_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a688ebcd08250eab5bb5bca318cc05a8c66de5e4171a65ca51db6bd753ff8953"}, + {file = "cryptography-35.0.0-pp36-pypy36_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d99915d6ab265c22873f1b4d6ea5ef462ef797b4140be4c9d8b179915e0985c6"}, + {file = "cryptography-35.0.0-pp36-pypy36_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:928185a6d1ccdb816e883f56ebe92e975a262d31cc536429041921f8cb5a62fd"}, + {file = "cryptography-35.0.0-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:ebeddd119f526bcf323a89f853afb12e225902a24d29b55fe18dd6fcb2838a76"}, + {file = "cryptography-35.0.0-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:22a38e96118a4ce3b97509443feace1d1011d0571fae81fc3ad35f25ba3ea999"}, + {file = "cryptography-35.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb80e8a1f91e4b7ef8b33041591e6d89b2b8e122d787e87eeb2b08da71bb16ad"}, + {file = "cryptography-35.0.0-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:abb5a361d2585bb95012a19ed9b2c8f412c5d723a9836418fab7aaa0243e67d2"}, + {file = "cryptography-35.0.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:1ed82abf16df40a60942a8c211251ae72858b25b7421ce2497c2eb7a1cee817c"}, + {file = "cryptography-35.0.0.tar.gz", hash = "sha256:9933f28f70d0517686bd7de36166dda42094eac49415459d9bdf5e7df3e0086d"}, ] decorator = [ {file = "decorator-5.1.0-py3-none-any.whl", hash = "sha256:7b12e7c3c6ab203a29e157335e9122cb03de9ab7264b137594103fd4a683b374"}, @@ -2440,12 +2443,12 @@ types-python-dateutil = [ {file = "types_python_dateutil-0.1.6-py3-none-any.whl", hash = "sha256:5b6241ea9fca2d8878cc152017d9524da62a7a856b98e31006e68b02aab47442"}, ] types-redis = [ - {file = "types-redis-3.5.8.tar.gz", hash = "sha256:be26f50259d1a7e74cbc1e83b377dbf6b534fdb037ff55ae31501e87ac2b5b5a"}, - {file = "types_redis-3.5.8-py3-none-any.whl", hash = "sha256:85814769071721044857c34841e46064b867ccdd58fc81221c43462bd07e4892"}, + {file = "types-redis-3.5.9.tar.gz", hash = "sha256:f142c48f4080757ca2a9441ec40213bda3b1535eebebfc4f3519e5aa46498076"}, + {file = "types_redis-3.5.9-py3-none-any.whl", hash = "sha256:5f5648ffc025708858097173cf695164c20f2b5e3f57177de14e352cae8cc335"}, ] types-requests = [ - {file = "types-requests-2.25.7.tar.gz", hash = "sha256:4b279513a34b789bef75ce05bee5eb4ce934a892318388400f7511fbcdf56f87"}, - {file = "types_requests-2.25.7-py3-none-any.whl", hash = "sha256:24bbe4682373ce0b1927f432b30ba1dcf46b66512fb6e848e72998c79797d040"}, + {file = "types-requests-2.25.9.tar.gz", hash = "sha256:4ec8b71da73e5344adb9bee725a74ec8598e7286f9bcb17500d627f259fe4fb9"}, + {file = "types_requests-2.25.9-py3-none-any.whl", hash = "sha256:543ba8b3b23e38ac028da1d163aecbbc27d3cc8f654ae64339da539a191a2b1c"}, ] types-werkzeug = [ {file = "types-Werkzeug-1.0.5.tar.gz", hash = "sha256:f6216ab0e0211fe73ebdb4ae0e414113d4d8a2f783a15c2d8550e06d0fd8e7f9"}, @@ -2465,8 +2468,8 @@ tzlocal = [ {file = "tzlocal-3.0.tar.gz", hash = "sha256:f4e6e36db50499e0d92f79b67361041f048e2609d166e93456b50746dc4aef12"}, ] urllib3 = [ - {file = "urllib3-1.26.6-py2.py3-none-any.whl", hash = "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4"}, - {file = "urllib3-1.26.6.tar.gz", hash = "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"}, + {file = "urllib3-1.26.7-py2.py3-none-any.whl", hash = "sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844"}, + {file = "urllib3-1.26.7.tar.gz", hash = "sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece"}, ] validators = [ {file = "validators-0.18.2-py3-none-any.whl", hash = "sha256:0143dcca8a386498edaf5780cbd5960da1a4c85e0719f3ee5c9b41249c4fefbd"}, @@ -2487,6 +2490,6 @@ wrapt = [ {file = "wrapt-1.12.1.tar.gz", hash = "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7"}, ] zipp = [ - {file = "zipp-3.5.0-py3-none-any.whl", hash = "sha256:957cfda87797e389580cb8b9e3870841ca991e2125350677b2ca83a0e99390a3"}, - {file = "zipp-3.5.0.tar.gz", hash = "sha256:f5812b1e007e48cff63449a5e9f4e7ebea716b4111f9c4f9a645f91d579bf0c4"}, + {file = "zipp-3.6.0-py3-none-any.whl", hash = "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc"}, + {file = "zipp-3.6.0.tar.gz", hash = "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832"}, ] diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index ffa6ed7..3d52773 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit ffa6ed7963051a5bd0f4578ade94c788c2962c09 +Subproject commit 3d52773e9d3ba39ff324455bf8c10b47e11b695a diff --git a/pymisp/tools/emailobject.py b/pymisp/tools/emailobject.py index ea4a90b..da3fb8b 100644 --- a/pymisp/tools/emailobject.py +++ b/pymisp/tools/emailobject.py @@ -45,7 +45,7 @@ class EMailObject(AbstractMISPObjectGenerator): def parse_email(self) -> EmailMessage: """Convert email into EmailMessage.""" - content_in_bytes = self.__pseudofile.getvalue() + content_in_bytes = self.__pseudofile.getvalue().strip() eml = message_from_bytes(content_in_bytes, _class=EmailMessage, policy=policy.default) @@ -282,6 +282,8 @@ class EMailObject(AbstractMISPObjectGenerator): if "To" in message: self.__add_emails("to", message["To"]) + if "Delivered-To" in message: + self.__add_emails("to", message["Delivered-To"]) if "From" in message: self.__add_emails("from", message["From"]) @@ -343,7 +345,7 @@ class EMailObject(AbstractMISPObjectGenerator): def __generate_received(self): """ - Extract IP addresses from received headers that are not private. + Extract IP addresses from received headers that are not private. Also extract hostnames or domains. """ received_items = self.email.get_all("received") if received_items is None: @@ -367,3 +369,11 @@ class EMailObject(AbstractMISPObjectGenerator): continue # skip header if IP not found or is private self.add_attribute("received-header-ip", value=str(ip), comment=fromstr) + + # The hostnames and/or domains always come after the "Received: from" + # part so we can use regex to pick up those attributes. + received_from = re.findall(r'(?<=from\s)[\w\d\.\-]+\.\w{2,24}', str(received_items)) + try: + [self.add_attribute("received-header-hostname", i) for i in received_from] + except Exception: + pass diff --git a/pyproject.toml b/pyproject.toml index 3b9c380..49b2965 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pymisp" -version = "2.4.148" +version = "2.4.148.1" description = "Python API for MISP." authors = ["Raphaël Vinot "] license = "BSD-2-Clause"