Merge branch 'main' of github.com:misp/pymisp

CHANGELOG.txt

@@ -2,11 +2,111 @@ Changelog
 =========
 
 
+v2.4.190 (2024-04-18)
+---------------------
+
+Changes
+~~~~~~~
+- Bump object templates. [Raphaël Vinot]
+- Bump changelog. [Raphaël Vinot]
+- Bump version, deps. [Raphaël Vinot]
+- Bump deps, require python 3.9+ for doc. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+- [data] describeTypes file updated. [Alexandre Dulaunoy]
+- Bump deps. [Raphaël Vinot]
+
+Fix
+~~~
+- [internal] Correct way to convert bytes to string if orjson exists.
+  [Jakub Onderka]
+
+
+v2.4.188 (2024-03-22)
+---------------------
+
+New
+~~~
+- Support X-MISP-AUTH Header. [Raphaël Vinot]
+
+  Also, improve HTTP headers init
+
+  Fix #1179
+
+Changes
+~~~~~~~
+- Bump changelog. [Raphaël Vinot]
+- Bump changelog. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+- Bump changelog. [Raphaël Vinot]
+- Bump version, templates. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+
+Fix
+~~~
+- Strip API key before setting it. [Raphaël Vinot]
+- Python 3.8 support & typing. [Raphaël Vinot]
+- Typing for Python < 3.10. [Raphaël Vinot]
+- Avoid issue when payload ist a list. [Raphaël Vinot]
+
+
+v2.4.187 (2024-03-07)
+---------------------
+
+Changes
+~~~~~~~
+- Bump changelog. [Raphaël Vinot]
+- Bump templates, version. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+- Bump extract-msg. [Raphaël Vinot]
+
+
+v2.4.186 (2024-02-27)
+---------------------
+
+Changes
+~~~~~~~
+- Bump changelog. [Raphaël Vinot]
+- Bump changelog. [Raphaël Vinot]
+- Bump version. [Raphaël Vinot]
+- Bump objects. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+
+Fix
+~~~
+- Correct FileObject import. [Johannes Bader]
+
+  The FileObject import has been moved outside the try-except-block
+  related to lief, as the import is needed regardless whether lief
+  is available or not.
+- Disable WL when calling the disable method, not toggle. [Raphaël
+  Vinot]
+
+  Fix #1159
+
+Other
+~~~~~
+- Build(deps): bump urllib3 from 2.2.0 to 2.2.1. [dependabot[bot]]
+
+  Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.2.0 to 2.2.1.
+  - [Release notes](https://github.com/urllib3/urllib3/releases)
+  - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst)
+  - [Commits](https://github.com/urllib3/urllib3/compare/2.2.0...2.2.1)
+
+  ---
+  updated-dependencies:
+  - dependency-name: urllib3
+    dependency-type: direct:production
+    update-type: version-update:semver-patch
+  ...
+
+
 v2.4.185 (2024-02-16)
 ---------------------
 
 Changes
 ~~~~~~~
+- Bump changelog. [Raphaël Vinot]
 - Bump deps, version. [Raphaël Vinot]
 
 

pymisp/abstract.py

@@ -252,7 +252,7 @@ class AbstractMISP(MutableMapping, MISPFileCache, metaclass=ABCMeta):  # type: i
                 option |= orjson.OPT_INDENT_2
             # orjson dumps method returns bytes instead of bytes, to keep compatibility with json
             # we have to convert output to str
-            return str(dumps(self, default=pymisp_json_default, option=option))
+            return dumps(self, default=pymisp_json_default, option=option).decode()
 
         return dumps(self, default=pymisp_json_default, sort_keys=sort_keys, indent=indent)
 

pymisp/api.py

@@ -57,6 +57,7 @@ except ImportError:
 
 SearchType = TypeVar('SearchType', str, int)
 # str: string to search / list: values to search (OR) / dict: {'OR': [list], 'NOT': [list], 'AND': [list]}
+# NOTE: we cannot use new typing here until we drop Python 3.8 and 3.9 support
 SearchParameterTypes = TypeVar('SearchParameterTypes', str, List[Union[str, int]], Dict[str, Union[str, int]])
 
 ToIDSType = TypeVar('ToIDSType', str, int, bool)
@@ -158,6 +159,7 @@ class PyMISP:
     :param tool: The software using PyMISP (string), used to set a unique user-agent
     :param http_headers: Arbitrary headers to pass to all the requests.
     :param https_adapter: Arbitrary HTTPS adapter for the requests session.
+    :param http_auth_header_name: The name of the HTTP header to use for the API key. Can be either "Authorization" or "X-MISP-AUTH".
     :param timeout: Timeout, as described here: https://requests.readthedocs.io/en/master/user/advanced/#timeouts
     """
 
@@ -165,7 +167,8 @@ class PyMISP:
                  cert: str | tuple[str, str] | None = None, auth: AuthBase | None = None, tool: str = '',
                  timeout: float | tuple[float, float] | None = None,
                  http_headers: dict[str, str] | None = None,
-                 https_adapter: requests.adapters.BaseAdapter | None = None
+                 https_adapter: requests.adapters.BaseAdapter | None = None,
+                 http_auth_header_name: str = 'Authorization'
                  ):
 
         if not url:
@@ -174,21 +177,30 @@ class PyMISP:
             raise NoKey('Please provide your authorization key.')
 
         self.root_url: str = url
-        self.key: str = key
+        self.key: str = key.strip()
         self.ssl: bool | str = ssl
         self.proxies: MutableMapping[str, str] | None = proxies
         self.cert: str | tuple[str, str] | None = cert
         self.auth: AuthBase | None = auth
-        self.tool: str = tool
         self.timeout: float | tuple[float, float] | None = timeout
         self.__session = requests.Session()  # use one session to keep connection between requests
         if https_adapter is not None:
             self.__session.mount('https://', https_adapter)
         if brotli_supported():
             self.__session.headers['Accept-Encoding'] = ', '.join(('br', 'gzip', 'deflate'))
+
+        if http_auth_header_name in ['Authorization', 'X-MISP-AUTH']:
+            self.__session.headers[http_auth_header_name] = self.key
+        else:
+            raise PyMISPError('http_auth_header_name should be either "Authorization" or "X-MISP-AUTH"')
+
+        user_agent = f'PyMISP {__version__} - Python {".".join(str(x) for x in sys.version_info[:2])}'
+        if tool:
+            user_agent = f'{user_agent} - {tool}'
+        self.__session.headers['User-Agent'] = user_agent
+
         if http_headers:
             self.__session.headers.update(http_headers)
-        self._user_agent = f'PyMISP {__version__} - Python {".".join(str(x) for x in sys.version_info[:2])}'
 
         self.global_pythonify = False
 
@@ -1542,7 +1554,7 @@ class PyMISP:
         """
 
         galaxy_id = get_uuid_or_id_from_abstract_misp(galaxy)
-        allowed_context_types: list[str] = ["all", "default", "custom", "org", "deleted"]
+        allowed_context_types: list[str] = ["all", "default", "custom", "org", "orgc", "deleted"]
         if context not in allowed_context_types:
             raise PyMISPError(f"The context must be one of {', '.join(allowed_context_types)}")
         kw_params = {"context": context}
@@ -3708,7 +3720,7 @@ class PyMISP:
     def _check_response(self, response: requests.Response, lenient_response_type: bool = False, expect_json: bool = False) -> dict[str, Any] | str:
         """Check if the response from the server is not an unexpected error"""
         if response.status_code >= 500:
-            headers_without_auth = {i: response.request.headers[i] for i in response.request.headers if i != 'Authorization'}
+            headers_without_auth = {h_name: h_value for h_name, h_value in response.request.headers.items() if h_value != self.key}
             logger.critical(everything_broken.format(headers_without_auth, response.request.body, response.text))
             raise MISPServerError(f'Error code 500:\n{response.text}')
 
@@ -3778,14 +3790,11 @@ class PyMISP:
             url = f'{url}/{to_append_url}'
 
         req = requests.Request(request_type, url, data=d, params=params)
-        user_agent = f'{self._user_agent} - {self.tool}' if self.tool else self._user_agent
         req.auth = self.auth
         prepped = self.__session.prepare_request(req)
         prepped.headers.update(
-            {'Authorization': self.key,
+            {'Accept': f'application/{output_type}',
-             'Accept': f'application/{output_type}',
+             'content-type': f'application/{content_type}'})
-             'content-type': f'application/{content_type}',
-             'User-Agent': user_agent})
         logger.debug(prepped.headers)
         settings = self.__session.merge_environment_settings(req.url, proxies=self.proxies or {}, stream=None,
                                                              verify=self.ssl, cert=self.cert)

pymisp/data/describeTypes.json

@@ -545,6 +545,10 @@
                 "default_category": "Other",
                 "to_ids": 0
             },
+            "integer": {
+                "default_category": "Other",
+                "to_ids": 0
+            },
             "datetime": {
                 "default_category": "Other",
                 "to_ids": 0
@@ -891,6 +895,7 @@
             "dns-soa-email",
             "size-in-bytes",
             "counter",
+            "integer",
             "datetime",
             "port",
             "ip-dst|port",
@@ -1460,6 +1465,7 @@
                 "other",
                 "size-in-bytes",
                 "counter",
+                "integer",
                 "datetime",
                 "cpe",
                 "port",
@@ -1473,4 +1479,4 @@
             ]
         }
     }
-}
+}

pymisp/data/misp-objects

@@ -1 +1 @@
-Subproject commit 3ac509965fdbca06d8a027db22c0064588babd3c
+Subproject commit 96492b9c932a4b307216550abeadddc727e17cec

pymisp/tools/create_misp_object.py

@@ -8,6 +8,7 @@ from io import BytesIO
 from typing import Any, TYPE_CHECKING
 
 from ..exceptions import MISPObjectException
+from . import FileObject
 logger = logging.getLogger('pymisp')
 
 try:
@@ -19,8 +20,6 @@ try:
     from .peobject import make_pe_objects
     from .elfobject import make_elf_objects
     from .machoobject import make_macho_objects
-    from . import FileObject
-
 except AttributeError:
     HAS_LIEF = False
     logger.critical('You need lief >= 0.11.0. The quick and dirty fix is: pip3 install --force pymisp[fileobjects]')

pymisp/tools/emailobject.py

@@ -196,11 +196,19 @@ class EMailObject(AbstractMISPObjectGenerator):
             for mime_items in related_content.values():
                 if isinstance(mime_items[1], dict):
                     message.add_related(**mime_items[1])
-                cur_attach = message.get_payload()[-1]
+                if p := message.get_payload():
+                    if isinstance(p, list):
+                        cur_attach = p[-1]
+                    else:
+                        cur_attach = p
                 self._update_content_disp_properties(mime_items[0], cur_attach)
             if body.get('text', None):
                 # Now add the HTML as an alternative within the related obj
-                related = message.get_payload()[0]
+                if p := message.get_payload():
+                    if isinstance(p, list):
+                        related = p[0]
+                    else:
+                        related = p
                 related.add_alternative(**body.get('html'))
         else:
             for mime_dict in body_objects:
@@ -219,7 +227,11 @@ class EMailObject(AbstractMISPObjectGenerator):
                                        subtype=subtype,
                                        cid=attch.cid,
                                        filename=attch.longFilename)
-                cur_attach = message.get_payload()[-1]
+                if p := message.get_payload():
+                    if isinstance(p, list):
+                        cur_attach = p[-1]
+                    else:
+                        cur_attach = p
                 self._update_content_disp_properties(attch, cur_attach)
         if _orig_boundry is not None:
             message.set_boundary(_orig_boundry)  # Set back original boundary

pymisp/tools/vtreportobject.py

@@ -7,7 +7,7 @@ from typing import Any
 
 import requests
 try:
-    import validators  # type: ignore
+    import validators
     has_validators = True
 except ImportError:
     has_validators = False

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pymisp"
-version = "2.4.185"
+version = "2.4.190"
 description = "Python API for MISP."
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-2-Clause"
@@ -43,51 +43,49 @@ include = [
 [tool.poetry.dependencies]
 python = "^3.8"
 requests = "^2.31.0"
-python-dateutil = "^2.8.2"
+python-dateutil = "^2.9.0.post0"
 deprecated = "^1.2.14"
-extract_msg = {version = "^0.47.0", optional = true}
+extract_msg = {version = "^0.48.5", optional = true}
 RTFDE = {version = "^0.1.1", optional = true}
 oletools = {version = "^0.60.1", optional = true}
 python-magic = {version = "^0.4.27", optional = true}
 pydeep2 = {version = "^0.5.1", optional = true}
 lief = {version = "^0.14.1", optional = true}
 beautifulsoup4 = {version = "^4.12.3", optional = true}
-validators = {version = "^0.22.0", optional = true}
+validators = {version = "^0.28.0", optional = true}
-sphinx-autodoc-typehints = {version = "^2.0.0", optional = true}
+sphinx-autodoc-typehints = {version = "^2.1.0", optional = true, python = ">=3.9"}
-recommonmark = {version = "^0.7.1", optional = true}
+docutils = {version = "^0.21.1", optional = true, python = ">=3.9"}
-reportlab = {version = "^4.1.0", optional = true}
+recommonmark = {version = "^0.7.1", optional = true, python = ">=3.9"}
+reportlab = {version = "^4.2.0", optional = true}
 pyfaup = {version = "^1.2", optional = true}
-publicsuffixlist = {version = "^0.10.0.20231214", optional = true}
+publicsuffixlist = {version = "^0.10.0.20240403", optional = true}
 urllib3 = {extras = ["brotli"], version = "*", optional = true}
-Sphinx = [
+Sphinx =  {version = "^7.3.7", python = ">=3.9", optional = true}
-    {version = "<7.2", python = "<3.9", optional = true},
-    {version = "^7.2", python = ">=3.9", optional = true}
-]
 
 [tool.poetry.extras]
 fileobjects = ['python-magic', 'pydeep2', 'lief']
 openioc = ['beautifulsoup4']
 virustotal = ['validators']
-docs = ['sphinx-autodoc-typehints', 'recommonmark', 'sphinx']
+docs = ['sphinx-autodoc-typehints', 'recommonmark', 'sphinx', 'docutils']
 pdfexport = ['reportlab']
 url = ['pyfaup']
 email = ['extract_msg', "RTFDE", "oletools"]
 brotli = ['urllib3']
 
 [tool.poetry.group.dev.dependencies]
-requests-mock = "^1.11.0"
+requests-mock = "^1.12.1"
-mypy = "^1.8.0"
+mypy = "^1.10.0"
 ipython = [
     {version = "<8.13.0", python = "<3.9"},
     {version = "^8.18.0", python = ">=3.9"},
     {version = "^8.19.0", python = ">=3.10"}
 ]
-jupyterlab = "^4.1.2"
+jupyterlab = "^4.1.7"
-types-requests = "^2.31.0.20240218"
+types-requests = "^2.31.0.20240406"
-types-python-dateutil = "^2.8.19.20240106"
+types-python-dateutil = "^2.9.0.20240316"
-types-redis = "^4.6.0.20240218"
+types-redis = "^4.6.0.20240425"
 types-Flask = "^1.1.6"
-pytest-cov = "^4.1.0"
+pytest-cov = "^5.0.0"
 
 [build-system]
 requires = ["poetry_core>=1.1", "setuptools"]