Merge branch 'main' of github.com:misp/pymisp

.github/workflows/pytest.yml

@@ -11,6 +11,7 @@ jobs:
 
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         python-version: [3.8, 3.9, '3.10', '3.11']
 

CHANGELOG.txt

@@ -2,11 +2,138 @@ Changelog
 =========
 
 
+v2.4.171 (2023-05-16)
+---------------------
+
+Changes
+~~~~~~~
+- Bump deps, object templates. [Raphaël Vinot]
+- Bump changelog. [Raphaël Vinot]
+- Bump changelog. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+- Bump changelog. [Raphaël Vinot]
+- Bump version. [Raphaël Vinot]
+- Remove old setup files, bump deps. [Raphaël Vinot]
+
+Fix
+~~~
+- Extra print breaking the CI on MISP side. [Raphaël Vinot]
+- Properly use lief on a file. [Raphaël Vinot]
+
+Other
+~~~~~
+- Allow search by 'event_tags' and improve the handling of galaxy
+  clusters. [Stefano Ortolani]
+
+  Changes:
+  - Add 'event_tags' parameter when searching for events
+  - Add new method to search for galaxies by value
+  - Add new parameter to fetch cluster information when retrieving clusters
+  - Add new parameter to hard-delete object references
+- Using underscore name 'description_file' in setup.cfg. [Erhan]
+
+  Usage of dash-separated 'description-file' will not be supported in future versions. Please use the underscore name 'description_file' instead. By 2023-Sep-26, you need to update your project and remove deprecated calls or your builds will no longer be supported.
+
+  See https://setuptools.pypa.io/en/latest/userguide/declarative_config.html for details.
+
+
+v2.4.170.2 (2023-05-04)
+-----------------------
+
+Changes
+~~~~~~~
+- Bump changelog. [Raphaël Vinot]
+- Bump version. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+
+
+v2.4.170.1 (2023-04-19)
+-----------------------
+
+Changes
+~~~~~~~
+- Disable fail fast in GHA. [Raphaël Vinot]
+- Bump version. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+
+Fix
+~~~
+- Update lief code to v0.13. [Raphaël Vinot]
+
+
+v2.4.170 (2023-04-12)
+---------------------
+
+Changes
+~~~~~~~
+- Bump changelog. [Raphaël Vinot]
+- Bump version. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+
+Other
+~~~~~
+- Add: support breakOnDuplicate option for attributes:add() [Luciano
+  Righetti]
+- Update reportlab_generator.py. [CarlosLoureiro]
+
+
+v2.4.169.3 (2023-03-27)
+-----------------------
+
+Changes
+~~~~~~~
+- Bump changelog. [Raphaël Vinot]
+- Bump deps, version. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+
+Fix
+~~~
+- Invalid check if taxo is enabled. [Raphaël Vinot]
+
+
+v2.4.169.2 (2023-03-17)
+-----------------------
+
+Changes
+~~~~~~~
+- Bump changelog. [Raphaël Vinot]
+- Bump version. [Raphaël Vinot]
+- Include event reports by default in feed. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+
+Fix
+~~~
+- Use proper parameter to trigger the request in search_galaxy_clusters.
+  [Raphaël Vinot]
+- Use POST in search galaxy cluster. [Raphaël Vinot]
+
+Other
+~~~~~
+- Rename include_event_reports kwarg to with_event_reports, in-line with
+  other kwarg naming. [UFOSmuggler]
+- Add kwarg to allow the inclusion of event reports into to_feed(),
+  honour with_distribution and valid_distributions kwargs. [UFOSmuggler]
+
+
+v2.4.169.1 (2023-03-14)
+-----------------------
+
+Changes
+~~~~~~~
+- Bump changelog. [Raphaël Vinot]
+- Bump version. [Raphaël Vinot]
+- Add greynoise-ip object. [Raphaël Vinot]
+
+  Fix #951
+
+
 v2.4.169 (2023-03-10)
 ---------------------
 
 Changes
 ~~~~~~~
+- Bump changelog. [Raphaël Vinot]
 - Bump version. [Raphaël Vinot]
 - Bump templates. [Raphaël Vinot]
 - Bump deps. [Raphaël Vinot]

README.md

@@ -46,7 +46,7 @@ pip3 install pymisp[virustotal,email]
 ```
 git clone https://github.com/MISP/PyMISP.git && cd PyMISP
 git submodule update --init
-poetry install -E fileobjects -E openioc -E virustotal -E docs -E pdfexport
+poetry install -E fileobjects -E openioc -E virustotal -E docs -E pdfexport -E email
 ```
 
 ### Running the tests

examples/feed-generator/settings.default.py

@@ -16,7 +16,7 @@ outputdir = 'output'
 # you can use on the event index, such as organisation, tags, etc.
 # It uses the same joining and condition rules as the API parameters
 # For example:
-# filters = {'tag':'tlp:white|feed-export|!privint','org':'CIRCL', 'published':1}
+# filters = {'tags':['tlp:white','feed-export','!privint'],'org':'CIRCL', 'published':1}
 # the above would generate a feed for all published events created by CIRCL,
 # tagged tlp:white and/or feed-export but exclude anything tagged privint
 filters = {'published':'true'}

pymisp/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '2.4.169'
+__version__ = '2.4.171'
 import logging
 import sys
 import warnings

pymisp/api.py

@@ -588,7 +588,7 @@ class PyMISP:
         :param break_on_duplicate: if True, check and reject if this object's attributes match an existing object's attributes; may require much time
         """
         event_id = get_uuid_or_id_from_abstract_misp(event)
-        params = {'breakOnDuplicate': True} if break_on_duplicate else {}
+        params = {'breakOnDuplicate': 1} if break_on_duplicate else {}
         r = self._prepare_request('POST', f'objects/add/{event_id}', data=misp_object, kw_params=params)
         new_object = self._check_json_response(r)
         if not (self.global_pythonify or pythonify) or 'errors' in new_object:
@@ -643,13 +643,17 @@ class PyMISP:
         ref.from_dict(**object_reference)
         return ref
 
-    def delete_object_reference(self, object_reference: Union[MISPObjectReference, int, str, UUID]) -> Dict:
+    def delete_object_reference(
-        """Delete a reference to an object
+        self,
-
+        object_reference: Union[MISPObjectReference, int, str, UUID],
-        :param object_reference: object reference
+        hard: bool = False,
-        """
+    ) -> Dict:
+        """Delete a reference to an object."""
         object_reference_id = get_uuid_or_id_from_abstract_misp(object_reference)
-        response = self._prepare_request('POST', f'objectReferences/delete/{object_reference_id}')
+        query_url = f"objectReferences/delete/{object_reference_id}"
+        if hard:
+            query_url += "/true"
+        response = self._prepare_request("POST", query_url)
         return self._check_json_response(response)
 
     # Object templates
@@ -741,7 +745,7 @@ class PyMISP:
         r = self._prepare_request('HEAD', f'attributes/view/{attribute_id}')
         return self._check_head_response(r)
 
-    def add_attribute(self, event: Union[MISPEvent, int, str, UUID], attribute: Union[MISPAttribute, Iterable], pythonify: bool = False) -> Union[Dict, MISPAttribute, MISPShadowAttribute]:
+    def add_attribute(self, event: Union[MISPEvent, int, str, UUID], attribute: Union[MISPAttribute, Iterable], pythonify: bool = False, break_on_duplicate: bool = True) -> Union[Dict, MISPAttribute, MISPShadowAttribute]:
         """Add an attribute to an existing MISP event: https://www.misp-project.org/openapi/#tag/Attributes/operation/addAttribute
 
         :param event: event to extend
@@ -749,9 +753,11 @@ class PyMISP:
             If a list is passed, the pythonified response is a dict with the following structure:
             {'attributes': [MISPAttribute], 'errors': {errors by attributes}}
         :param pythonify: Returns a PyMISP Object instead of the plain json output
+        :param break_on_duplicate: if False, do not fail if the attribute already exists, updates existing attribute instead (timestamp will be always updated)
         """
+        params = {'breakOnDuplicate': 0} if break_on_duplicate is not True else {}
         event_id = get_uuid_or_id_from_abstract_misp(event)
-        r = self._prepare_request('POST', f'attributes/add/{event_id}', data=attribute)
+        r = self._prepare_request('POST', f'attributes/add/{event_id}', data=attribute, kw_params=params)
         new_attribute = self._check_json_response(r)
         if isinstance(attribute, list):
             # Multiple attributes were passed at once, the handling is totally different
@@ -1200,9 +1206,10 @@ class PyMISP:
         """
         taxonomy_id = get_uuid_or_id_from_abstract_misp(taxonomy)
         t = self.get_taxonomy(taxonomy_id)
-        if isinstance(t, MISPTaxonomy) and not t.enabled:
+        if isinstance(t, MISPTaxonomy):
-            # Can happen if global pythonify is enabled.
+            if not t.enabled:
-            raise PyMISPError(f"The taxonomy {t.namespace} is not enabled.")
+                # Can happen if global pythonify is enabled.
+                raise PyMISPError(f"The taxonomy {t.namespace} is not enabled.")
         elif not t['Taxonomy']['enabled']:
             raise PyMISPError(f"The taxonomy {t['Taxonomy']['namespace']} is not enabled.")
         url = urljoin(self.root_url, 'taxonomies/addTag/{}'.format(taxonomy_id))
@@ -1443,7 +1450,11 @@ class PyMISP:
 
     # ## BEGIN Galaxy ###
 
-    def galaxies(self, pythonify: bool = False) -> Union[Dict, List[MISPGalaxy]]:
+    def galaxies(
+        self,
+        withCluster: bool = False,
+        pythonify: bool = False,
+    ) -> Union[Dict, List[MISPGalaxy]]:
         """Get all the galaxies: https://www.misp-project.org/openapi/#tag/Galaxies/operation/getGalaxies
 
         :param pythonify: Returns a list of PyMISP Objects instead of the plain json output. Warning: it might use a lot of RAM
@@ -1455,7 +1466,25 @@ class PyMISP:
         to_return = []
         for galaxy in galaxies:
             g = MISPGalaxy()
-            g.from_dict(**galaxy)
+            g.from_dict(**galaxy, withCluster=withCluster)
+            to_return.append(g)
+        return to_return
+
+    def search_galaxy(
+        self,
+        value: str,
+        withCluster: bool = False,
+        pythonify: bool = False,
+    ) -> Union[Dict, List[MISPGalaxy]]:
+        """Text search to find a matching galaxy name, namespace, description, or uuid."""
+        r = self._prepare_request("POST", "galaxies", data={"value": value})
+        galaxies = self._check_json_response(r)
+        if not (self.global_pythonify or pythonify) or "errors" in galaxies:
+            return galaxies
+        to_return = []
+        for galaxy in galaxies:
+            g = MISPGalaxy()
+            g.from_dict(**galaxy, withCluster=withCluster)
             to_return.append(g)
         return to_return
 
@@ -1491,7 +1520,7 @@ class PyMISP:
         kw_params = {"context": context}
         if searchall:
             kw_params["searchall"] = searchall
-        r = self._prepare_request('GET', f"galaxy_clusters/index/{galaxy_id}", kw_params=kw_params)
+        r = self._prepare_request('POST', f"galaxy_clusters/index/{galaxy_id}", data=kw_params)
         clusters_j = self._check_json_response(r)
         if not (self.global_pythonify or pythonify) or 'errors' in clusters_j:
             return clusters_j
@@ -2472,6 +2501,7 @@ class PyMISP:
                category: Optional[SearchParameterTypes] = None,
                org: Optional[SearchParameterTypes] = None,
                tags: Optional[SearchParameterTypes] = None,
+               event_tags: Optional[SearchParameterTypes] = None,
                quick_filter: Optional[str] = None, quickFilter: Optional[str] = None,
                date_from: Optional[Union[datetime, date, int, str, float, None]] = None,
                date_to: Optional[Union[datetime, date, int, str, float, None]] = None,
@@ -2529,6 +2559,7 @@ class PyMISP:
         :param category: The attribute category, any valid MISP attribute category is accepted.
         :param org: Search by the creator organisation by supplying the organisation identifier.
         :param tags: Tags to search or to exclude. You can pass a list, or the output of `build_complex_query`
+        :param event_tags: Tags to search or to exclude at the event level. You can pass a list, or the output of `build_complex_query`
         :param quick_filter: The string passed to this field will ignore all of the other arguments. MISP will return an xml / json (depending on the header sent) of all events that have a sub-string match on value in the event info, event orgc, or any of the attribute value1 / value2 fields, or in the attribute comment.
         :param date_from: Events with the date set to a date after the one specified. This filter will use the date of the event.
         :param date_to: Events with the date set to a date before the one specified. This filter will use the date of the event.
@@ -2571,7 +2602,8 @@ class PyMISP:
 
         '''
 
-        return_formats = ['openioc', 'json', 'xml', 'suricata', 'snort', 'text', 'rpz', 'csv', 'cache', 'stix-xml', 'stix', 'stix2', 'yara', 'yara-json', 'attack', 'attack-sightings']
+        return_formats = ['openioc', 'json', 'xml', 'suricata', 'snort', 'text', 'rpz', 'csv', 'cache', 'stix-xml',
+                          'stix', 'stix2', 'yara', 'yara-json', 'attack', 'attack-sightings', 'context', 'context-markdown']
 
         if controller not in ['events', 'attributes', 'objects']:
             raise ValueError('controller has to be in {}'.format(', '.join(['events', 'attributes', 'objects'])))
@@ -2615,6 +2647,7 @@ class PyMISP:
         query['category'] = category
         query['org'] = org
         query['tags'] = tags
+        query['event_tags'] = event_tags
         query['quickFilter'] = quick_filter
         query['from'] = self._make_timestamp(date_from)
         query['to'] = self._make_timestamp(date_to)

pymisp/data/misp-objects

@@ -1 +1 @@
-Subproject commit 1da4760dcc99502a2dd5da02cba212b42068fcb8
+Subproject commit 48dd45519624cabeb6fc7e22f76135312e2715b7

pymisp/mispevent.py

@@ -1594,12 +1594,13 @@ class MISPEvent(AbstractMISP):
                 to_return += attribute.hash_values(algorithm)
         return to_return
 
-    def to_feed(self, valid_distributions: List[int] = [0, 1, 2, 3, 4, 5], with_meta: bool = False, with_distribution=False, with_local_tags: bool = True) -> Dict:
+    def to_feed(self, valid_distributions: List[int] = [0, 1, 2, 3, 4, 5], with_meta: bool = False, with_distribution=False, with_local_tags: bool = True, with_event_reports: bool = True) -> Dict:
         """ Generate a json output for MISP Feed.
 
         :param valid_distributions: only makes sense if the distribution key is set; i.e., the event is exported from a MISP instance.
         :param with_distribution: exports distribution and Sharing Group info; otherwise all SharingGroup information is discarded (protecting privacy)
         :param with_local_tags: tag export includes local exportable tags along with global exportable tags
+        :param with_event_reports: include event reports in the returned MISP event
         """
         required = ['info', 'Orgc']
         for r in required:
@@ -1653,6 +1654,18 @@ class MISPEvent(AbstractMISP):
             except AttributeError:
                 pass
 
+        if with_event_reports and self.event_reports:
+            to_return['EventReport'] = []
+            for event_report in self.event_reports:
+                if (valid_distributions and event_report.get('distribution') is not None and event_report.distribution not in valid_distributions):
+                    continue
+                if not with_distribution:
+                    event_report.pop('distribution', None)
+                    event_report.pop('SharingGroup', None)
+                    event_report.pop('sharing_group_id', None)
+                to_return['EventReport'].append(event_report.to_dict())
+
+
         return {'Event': to_return}
 
     @property

pymisp/tools/create_misp_object.py

@@ -11,7 +11,7 @@ from typing import Optional
 logger = logging.getLogger('pymisp')
 
 try:
-    import lief  # type: ignore
+    import lief
     lief.logging.disable()
     HAS_LIEF = True
 
@@ -35,40 +35,24 @@ def make_binary_objects(filepath: Optional[str] = None, pseudofile: Optional[Byt
     misp_file = FileObject(filepath=filepath, pseudofile=pseudofile, filename=filename,
                            standalone=standalone, default_attributes_parameters=default_attributes_parameters)
     if HAS_LIEF and (filepath or (pseudofile and filename)):
-        try:
+        if filepath:
-            if filepath:
+            lief_parsed = lief.parse(filepath=filepath)
-                lief_parsed = lief.parse(filepath=filepath)
+        elif pseudofile and filename:
-            elif pseudofile and filename:
+            lief_parsed = lief.parse(raw=pseudofile.getvalue(), name=filename)
-                lief_parsed = lief.parse(raw=pseudofile.getvalue(), name=filename)
+        else:
-            else:
+            logger.critical('You need either a filepath, or a pseudofile and a filename.')
-                logger.critical('You need either a filepath, or a pseudofile and a filename.')
+            lief_parsed = None
-                lief_parsed = None
+
-            if isinstance(lief_parsed, lief.PE.Binary):
+        if isinstance(lief_parsed, lief.lief_errors):
-                return make_pe_objects(lief_parsed, misp_file, standalone, default_attributes_parameters)
+            logger.warning('Got an error parsing the file: {lief_parsed}')
-            elif isinstance(lief_parsed, lief.ELF.Binary):
+        elif isinstance(lief_parsed, lief.PE.Binary):
-                return make_elf_objects(lief_parsed, misp_file, standalone, default_attributes_parameters)
+            return make_pe_objects(lief_parsed, misp_file, standalone, default_attributes_parameters)
-            elif isinstance(lief_parsed, lief.MachO.Binary):
+        elif isinstance(lief_parsed, lief.ELF.Binary):
-                return make_macho_objects(lief_parsed, misp_file, standalone, default_attributes_parameters)
+            return make_elf_objects(lief_parsed, misp_file, standalone, default_attributes_parameters)
-        except lief.bad_format as e:
+        elif isinstance(lief_parsed, lief.MachO.Binary):
-            logger.warning('Bad format: {}'.format(e))
+            return make_macho_objects(lief_parsed, misp_file, standalone, default_attributes_parameters)
-        except lief.bad_file as e:
+        else:
-            logger.warning('Bad file: {}'.format(e))
+            logger.critical(f'Unexpected type from lief: {type(lief_parsed)}')
-        except lief.conversion_error as e:
-            logger.warning('Conversion file: {}'.format(e))
-        except lief.builder_error as e:
-            logger.warning('Builder file: {}'.format(e))
-        except lief.parser_error as e:
-            logger.warning('Parser error: {}'.format(e))
-        except lief.integrity_error as e:
-            logger.warning('Integrity error: {}'.format(e))
-        except lief.pe_error as e:
-            logger.warning('PE error: {}'.format(e))
-        except lief.type_error as e:
-            logger.warning('Type error: {}'.format(e))
-        except lief.exception as e:
-            logger.warning('Lief exception: {}'.format(e))
-        except FileTypeNotImplemented as e:
-            logger.warning(e)
     if not HAS_LIEF:
         logger.warning('Please install lief, documentation here: https://github.com/lief-project/LIEF')
     return misp_file, None, []

pymisp/tools/elfobject.py

@@ -10,7 +10,7 @@ from typing import Union, Optional
 from pathlib import Path
 from . import FileObject
 
-import lief  # type: ignore
+import lief
 
 try:
     import pydeep  # type: ignore
@@ -21,7 +21,7 @@ except ImportError:
 logger = logging.getLogger('pymisp')
 
 
-def make_elf_objects(lief_parsed: lief.Binary, misp_file: FileObject, standalone: bool = True, default_attributes_parameters: dict = {}):
+def make_elf_objects(lief_parsed: lief.ELF.Binary, misp_file: FileObject, standalone: bool = True, default_attributes_parameters: dict = {}):
     elf_object = ELFObject(parsed=lief_parsed, standalone=standalone, default_attributes_parameters=default_attributes_parameters)
     misp_file.add_reference(elf_object.uuid, 'includes', 'ELF indicators')
     elf_sections = []
@@ -32,14 +32,14 @@ def make_elf_objects(lief_parsed: lief.Binary, misp_file: FileObject, standalone
 
 class ELFObject(AbstractMISPObjectGenerator):
 
-    def __init__(self, parsed: Optional[lief.ELF.Binary] = None, filepath: Optional[Union[Path, str]] = None, pseudofile: Optional[Union[BytesIO, bytes]] = None, **kwargs):
+    def __init__(self, parsed: Optional[lief.ELF.Binary] = None, filepath: Optional[Union[Path, str]] = None, pseudofile: Optional[BytesIO] = None, **kwargs):
         """Creates an ELF object, with lief"""
         super().__init__('elf', **kwargs)
         if not HAS_PYDEEP:
             logger.warning("pydeep is missing, please install pymisp this way: pip install pymisp[fileobjects]")
         if pseudofile:
             if isinstance(pseudofile, BytesIO):
-                self.__elf = lief.ELF.parse(raw=pseudofile.getvalue())
+                self.__elf = lief.ELF.parse(io=pseudofile)
             elif isinstance(pseudofile, bytes):
                 self.__elf = lief.ELF.parse(raw=pseudofile)
             else:

pymisp/tools/emailobject.py

@@ -11,8 +11,7 @@ from io import BytesIO
 from pathlib import Path
 from typing import Union, List, Tuple, Dict, cast, Any, Optional
 
-from extract_msg import openMsg  # type: ignore
+from extract_msg import openMsg, MessageBase
-from extract_msg.message import Message as MsgObj  # type: ignore
 from RTFDE.exceptions import MalformedEncapsulatedRtf, NotEncapsulatedRtf  # type: ignore
 from RTFDE.deencapsulate import DeEncapsulator  # type: ignore
 from oletools.common.codepages import codepage2codec  # type: ignore
@@ -94,13 +93,14 @@ class EMailObject(AbstractMISPObjectGenerator):
 
     def _msg_to_eml(self, msg_bytes: bytes) -> EmailMessage:
         """Converts a msg into an eml."""
-        msg_obj = openMsg(msg_bytes)
+        # NOTE: openMsg returns a MessageBase, not a MSGFile
+        msg_obj: MessageBase = openMsg(msg_bytes)  # type: ignore
         # msg obj stores the original raw header here
         message, body, attachments = self._extract_msg_objects(msg_obj)
         eml = self._build_eml(message, body, attachments)
         return eml
 
-    def _extract_msg_objects(self, msg_obj: MsgObj) -> Tuple[EmailMessage, Dict, List[Any]]:
+    def _extract_msg_objects(self, msg_obj: MessageBase) -> Tuple[EmailMessage, Dict, List[Any]]:
         """Extracts email objects needed to construct an eml from a msg."""
         message: EmailMessage = email.message_from_string(msg_obj.header.as_string(), policy=policy.default)  # type: ignore
         body = {}

pymisp/tools/machoobject.py

@@ -10,7 +10,7 @@ from typing import Optional, Union
 from pathlib import Path
 from . import FileObject
 
-import lief  # type: ignore
+import lief
 
 try:
     import pydeep  # type: ignore
@@ -21,7 +21,7 @@ except ImportError:
 logger = logging.getLogger('pymisp')
 
 
-def make_macho_objects(lief_parsed: lief.Binary, misp_file: FileObject, standalone: bool = True, default_attributes_parameters: dict = {}):
+def make_macho_objects(lief_parsed: lief.MachO.Binary, misp_file: FileObject, standalone: bool = True, default_attributes_parameters: dict = {}):
     macho_object = MachOObject(parsed=lief_parsed, standalone=standalone, default_attributes_parameters=default_attributes_parameters)
     misp_file.add_reference(macho_object.uuid, 'includes', 'MachO indicators')
     macho_sections = []
@@ -39,7 +39,7 @@ class MachOObject(AbstractMISPObjectGenerator):
             logger.warning("pydeep is missing, please install pymisp this way: pip install pymisp[fileobjects]")
         if pseudofile:
             if isinstance(pseudofile, BytesIO):
-                self.__macho = lief.MachO.parse(raw=pseudofile.getvalue())
+                self.__macho = lief.MachO.parse(io=pseudofile)
             elif isinstance(pseudofile, bytes):
                 self.__macho = lief.MachO.parse(raw=pseudofile)
             else:

pymisp/tools/peobject.py

@@ -13,7 +13,7 @@ from base64 import b64encode
 
 from . import FileObject
 
-import lief  # type: ignore
+import lief
 
 try:
     import pydeep  # type: ignore
@@ -24,7 +24,7 @@ except ImportError:
 logger = logging.getLogger('pymisp')
 
 
-def make_pe_objects(lief_parsed: lief.Binary, misp_file: FileObject, standalone: bool = True, default_attributes_parameters: dict = {}):
+def make_pe_objects(lief_parsed: lief.PE.Binary, misp_file: FileObject, standalone: bool = True, default_attributes_parameters: dict = {}):
     pe_object = PEObject(parsed=lief_parsed, standalone=standalone, default_attributes_parameters=default_attributes_parameters)
     misp_file.add_reference(pe_object.uuid, 'includes', 'PE indicators')
     pe_sections = []
@@ -42,7 +42,7 @@ class PEObject(AbstractMISPObjectGenerator):
             logger.warning("pydeep is missing, please install pymisp this way: pip install pymisp[fileobjects]")
         if pseudofile:
             if isinstance(pseudofile, BytesIO):
-                self.__pe = lief.PE.parse(raw=pseudofile.getvalue())
+                self.__pe = lief.PE.parse(io=pseudofile)
             elif isinstance(pseudofile, bytes):
                 self.__pe = lief.PE.parse(raw=pseudofile)
             else:

pymisp/tools/reportlab_generator.py

@@ -1091,7 +1091,7 @@ class Event_Metadata():
                     Paragraph("Related Event #" + str(i + OFFSET), self.sample_style_sheet['Heading4']))
                 flowable_table.append(Indenter(left=-INDENT_SIZE_HEADING))
 
-                flowable_table += self.create_reduced_flowable_table_from_event(evt)
+                flowable_table += self.create_reduced_flowable_table_from_event(evt['Event'])
                 i += 1
         else:
             return flowable_table.append(self.value_formatter.get_unoverflowable_paragraph(DEFAULT_VALUE))

pyproject.toml

@@ -1,13 +1,12 @@
 [tool.poetry]
 name = "pymisp"
-version = "2.4.169"
+version = "2.4.171"
 description = "Python API for MISP."
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-2-Clause"
 repository = "https://github.com/MISP/PyMISP"
 documentation = "https://pymisp.readthedocs.io"
 
-
 readme = "README.md"
 
 classifiers=[
@@ -43,24 +42,24 @@ include = [
 
 [tool.poetry.dependencies]
 python = "^3.8"
-requests = "^2.28.2"
+requests = "^2.31.0"
 python-dateutil = "^2.8.2"
 jsonschema = "^4.17.3"
 deprecated = "^1.2.13"
-extract_msg = {version = "^0.39.2", optional = true}
+extract_msg = {version = "^0.41.1", optional = true}
 RTFDE = {version = "^0.0.2", optional = true}
 oletools = {version = "^0.60.1", optional = true}
 python-magic = {version = "^0.4.27", optional = true}
 pydeep2 = {version = "^0.5.1", optional = true}
-lief = {version = "^0.12.3", optional = true}
+lief = {version = "^0.13.0", optional = true}
-beautifulsoup4 = {version = "^4.11.2", optional = true}
+beautifulsoup4 = {version = "^4.12.2", optional = true}
 validators = {version = "^0.20.0", optional = true}
-sphinx-autodoc-typehints = {version = "^1.22", optional = true}
+sphinx-autodoc-typehints = {version = "^1.23.0", optional = true}
 recommonmark = {version = "^0.7.1", optional = true}
-reportlab = {version = "^3.6.12", optional = true}
+reportlab = {version = "^4.0.0", optional = true}
 pyfaup = {version = "^1.2", optional = true}
-publicsuffixlist = {version = "^0.9.3", optional = true}
+publicsuffixlist = {version = "^0.10.0.20230506", optional = true}
-urllib3 = {extras = ["brotli"], version = "^1.26.14", optional = true}
+urllib3 = {extras = ["brotli"], version = "*", optional = true}
 
 [tool.poetry.extras]
 fileobjects = ['python-magic', 'pydeep2', 'lief']
@@ -74,12 +73,15 @@ brotli = ['urllib3']
 
 [tool.poetry.group.dev.dependencies]
 requests-mock = "^1.10.0"
-mypy = "^1.1.1"
+mypy = "^1.3.0"
-ipython = "^8.11.0"
+ipython = [
-jupyterlab = "^3.6.1"
+    {version = "<8.13.0", python = "<3.9"},
-types-requests = "^2.28.11.15"
+    {version = "^8.13.0", python = ">=3.9"}
-types-python-dateutil = "^2.8.19.10"
+]
-types-redis = "^4.5.1.4"
+jupyterlab = "^4.0.0"
+types-requests = "^2.31.0.0"
+types-python-dateutil = "^2.8.19.13"
+types-redis = "^4.5.5.2"
 types-Flask = "^1.1.6"
 pytest-cov = "^4.0.0"
 

setup.cfg

@@ -1,2 +0,0 @@
-[metadata]
-description-file = README.md

setup.py

@@ -1,64 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-from os import path
-
-from setuptools import setup  # type: ignore
-
-import pymisp
-
-this_directory = path.abspath(path.dirname(__file__))
-with open(path.join(this_directory, 'README.md'), 'r') as f:
-    long_description = f.read()
-
-setup(
-    name='pymisp',
-    version=pymisp.__version__,
-    author='Raphaël Vinot',
-    author_email='raphael.vinot@circl.lu',
-    maintainer='Raphaël Vinot',
-    url='https://github.com/MISP/PyMISP',
-    project_urls={
-        'Documentation': 'https://pymisp.readthedocs.io',
-        'Source': 'https://github.com/MISP/PyMISP',
-        'Tracker': 'https://github.com/MISP/PyMISP/issues',
-    },
-    description='Python API for MISP.',
-    long_description=long_description,
-    long_description_content_type='text/markdown',
-    packages=['pymisp', 'pymisp.tools'],
-    classifiers=[
-        'License :: OSI Approved :: BSD License',
-        'Development Status :: 5 - Production/Stable',
-        'Environment :: Console',
-        'Operating System :: POSIX :: Linux',
-        'Intended Audience :: Science/Research',
-        'Intended Audience :: Telecommunications Industry',
-        'Intended Audience :: Information Technology',
-        'Programming Language :: Python :: 3.6',
-        'Topic :: Security',
-        'Topic :: Internet',
-    ],
-    install_requires=['requests',
-                      'python-dateutil',
-                      'jsonschema',
-                      'deprecated'],
-    extras_require={'fileobjects': ['python-magic', 'pydeep2', 'lief>=0.11.0'],
-                    'neo': ['py2neo'],
-                    'openioc': ['beautifulsoup4'],
-                    'virustotal': ['validators'],
-                    'docs': ['sphinx-autodoc-typehints', 'recommonmark'],
-                    'pdfexport': ['reportlab']},
-    tests_require=[
-        'jsonschema',
-        'python-magic',
-        'requests-mock'
-    ],
-    test_suite="tests.test_mispevent",
-    include_package_data=True,
-    package_data={'pymisp': ['data/*.json',
-                             'data/misp-objects/schema_objects.json',
-                             'data/misp-objects/schema_relationships.json',
-                             'data/misp-objects/objects/*/definition.json',
-                             'data/misp-objects/relationships/definition.json',
-                             'tools/pdf_fonts/Noto_TTF/*']},
-)

tests/testlive_comprehensive.py

@@ -1596,6 +1596,47 @@ class TestComprehensive(unittest.TestCase):
             # Delete event
             self.admin_misp_connector.delete_event(first)
 
+    def test_add_event_with_attachment_object_controller__hard(self):
+        first = self.create_simple_event()
+        try:
+            first = self.user_misp_connector.add_event(first)
+            fo, peo, seos = make_binary_objects('tests/viper-test-files/test_files/whoami.exe')
+            for s in seos:
+                r = self.user_misp_connector.add_object(first, s)
+                self.assertEqual(r.name, 'pe-section', r)
+
+            r = self.user_misp_connector.add_object(first, peo, pythonify=True)
+            self.assertEqual(r.name, 'pe', r)
+            for ref in peo.ObjectReference:
+                r = self.user_misp_connector.add_object_reference(ref)
+                self.assertEqual(r.object_uuid, peo.uuid, r.to_json())
+
+            r = self.user_misp_connector.add_object(first, fo)
+            obj_attrs = r.get_attributes_by_relation('ssdeep')
+            self.assertEqual(len(obj_attrs), 1, obj_attrs)
+            self.assertEqual(r.name, 'file', r)
+
+            # Test break_on_duplicate at object level
+            fo_dup, peo_dup, _ = make_binary_objects('tests/viper-test-files/test_files/whoami.exe')
+            r = self.user_misp_connector.add_object(first, peo_dup, break_on_duplicate=True)
+            self.assertTrue("Duplicate object found" in r['errors'][1]['errors'], r)
+
+            # Test break on duplicate with breakOnDuplicate key in object
+            fo_dup.breakOnDuplicate = True
+            r = self.user_misp_connector.add_object(first, fo_dup)
+            self.assertTrue("Duplicate object found" in r['errors'][1]['errors'], r)
+
+            # Test refs
+            r = self.user_misp_connector.add_object_reference(fo.ObjectReference[0])
+            self.assertEqual(r.object_uuid, fo.uuid, r.to_json())
+            self.assertEqual(r.referenced_uuid, peo.uuid, r.to_json())
+            r = self.user_misp_connector.delete_object_reference(r, hard=True)
+            self.assertEqual(r['message'], 'ObjectReference deleted')
+            # TODO: verify that the reference is not soft-deleted instead
+        finally:
+            # Delete event
+            self.admin_misp_connector.delete_event(first)
+
     def test_lief_and_sign(self):
         first = self.create_simple_event()
         try:
@@ -1901,6 +1942,15 @@ class TestComprehensive(unittest.TestCase):
             similar_error = self.user_misp_connector.add_attribute(first, new_similar)
             self.assertEqual(similar_error['errors'][1]['errors']['value'][0], 'A similar attribute already exists for this event.')
 
+            # Test add attribute break_on_duplicate=False
+            time.sleep(5)
+            new_similar = MISPAttribute()
+            new_similar.value = '1.2.3.4'
+            new_similar.type = 'ip-dst'
+            new_similar = self.user_misp_connector.add_attribute(first, new_similar, break_on_duplicate=False)
+            self.assertTrue(isinstance(new_similar, MISPAttribute), new_similar)
+            self.assertGreater(new_similar.timestamp, new_attribute.timestamp)
+
             # Test add multiple attributes at once
             attr0 = MISPAttribute()
             attr0.value = '0.0.0.0'
@@ -2999,6 +3049,13 @@ class TestComprehensive(unittest.TestCase):
             self.user_misp_connector.delete_event(event)
             self.user_misp_connector.delete_event_report(new_event_report)
 
+    def test_search_galaxy(self):
+        self.admin_misp_connector.toggle_global_pythonify()
+        galaxy = self.admin_misp_connector.galaxies()[0]
+        ret = self.admin_misp_connector.search_galaxy(value=galaxy.name)
+        self.assertEqual(len(ret), 1)
+        self.admin_misp_connector.toggle_global_pythonify()
+
     def test_galaxy_cluster(self):
         self.admin_misp_connector.toggle_global_pythonify()
         galaxy = self.admin_misp_connector.galaxies()[0]