Merge branch 'main' of github.com:misp/pymisp into analyst_data_fix

.gitchangelog.rc

@@ -140,7 +140,7 @@ subject_process = (strip |
 ##
 ## Tags that will be used for the changelog must match this regexp.
 ##
-tag_filter_regexp = r'^v[0-9]+\.[0-9]+\.[0-9]+(\.[0-9])*$'
+tag_filter_regexp = r'^v2.5.[0-9]+(\.[0-9])*$|^v2.4.(1)?[0-9]{1,2}$'
 
 
 ## ``unreleased_version_label`` is a string
@@ -189,4 +189,3 @@ output_engine = rest_py
 ## This option tells git-log whether to include merge commits in the log.
 ## The default is to include them.
 include_merge = False
-

examples/feed-generator/generate.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import sys
 import json
@@ -11,6 +10,11 @@ try:
 except ImportError:
     with_distribution = False
 
+try:
+    from settings import with_signatures
+except ImportError:
+    with_signatures = False
+
 try:
     from settings import with_local_tags
 except ImportError:
@@ -39,20 +43,39 @@ def init():
     return ExpandedPyMISP(url, key, ssl)
 
 
-def saveEvent(event):
+def saveEvent(event, misp):
+    stringified_event = json.dumps(event, indent=2)
+    if with_signatures and event['Event'].get('protected'):
+        signature = getSignature(stringified_event, misp)
+        try:
+            with open(os.path.join(outputdir, f'{event["Event"]["uuid"]}.asc'), 'w') as f:
+                f.write(signature)
+        except Exception as e:
+            print(e)
+            sys.exit('Could not create the event signature dump.')
+
     try:
         with open(os.path.join(outputdir, f'{event["Event"]["uuid"]}.json'), 'w') as f:
-            json.dump(event, f, indent=2)
+            f.write(stringified_event)
     except Exception as e:
         print(e)
         sys.exit('Could not create the event dump.')
 
 
+def getSignature(stringified_event, misp):
+    try:
+        signature = misp.sign_blob(stringified_event)
+        return signature
+    except Exception as e:
+        print(e)
+        sys.exit('Could not get the signature for the event from the MISP instance. Perhaps the user does not have the necessary permissions.')
+
+
 def saveHashes(hashes):
     try:
         with open(os.path.join(outputdir, 'hashes.csv'), 'w') as hashFile:
             for element in hashes:
-                hashFile.write('{},{}\n'.format(element[0], element[1]))
+                hashFile.write(f'{element[0]},{element[1]}\n')
     except Exception as e:
         print(e)
         sys.exit('Could not create the quick hash lookup file.')
@@ -79,6 +102,7 @@ if __name__ == '__main__':
         sys.exit("No events returned.")
     manifest = {}
     hashes = []
+    signatures = []
     counter = 1
     total = len(events)
     for event in events:
@@ -97,7 +121,7 @@ if __name__ == '__main__':
             continue
         hashes += [[h, e.uuid] for h in e_feed['Event'].pop('_hashes')]
         manifest.update(e_feed['Event'].pop('_manifest'))
-        saveEvent(e_feed)
+        saveEvent(e_feed, misp)
         print("Event " + str(counter) + "/" + str(total) + " exported.")
         counter += 1
     saveManifest(manifest)

examples/feed-generator/settings.default.py

@@ -54,3 +54,6 @@ with_distribution = False
 
 # Include the exportable local tags along with the global tags.  The default is True.
 with_local_tags = True
+
+# Include signatures for protected events. This will allow MISP to ingest and update protected events from the feed. It requires perm_server_sign to be set to true in the user's role on MISP's side.
+with_signatures = False

pymisp/api.py

@@ -3541,6 +3541,14 @@ class PyMISP:
             response = self._prepare_request('POST', url, data=to_post)
         return response
 
+    def sign_blob(self, blob: str) -> str:
+        """Sign a blob
+
+        :param blob: blob to sign
+        """
+        response = self._prepare_request('POST', '/cryptographicKeys/serverSign', data=blob)
+        return self._check_response(response, lenient_response_type=True)
+
     # ## END Others ###
 
     # ## BEGIN Statistics ###

pymisp/data/misp-objects

@@ -1 +1 @@
-Subproject commit dcf0c3febcacc3b6dd8d0d390b7ec59254cd46ba
+Subproject commit a2e4bbc9615ba0e1ddda3322dcc6c72d5a0e76e5

pymisp/mispevent.py

@@ -1564,7 +1564,8 @@ class MISPGalaxy(AbstractMISP):
 class MISPEvent(AnalystDataBehaviorMixin):
 
     _fields_for_feed: set[str] = {'uuid', 'info', 'threat_level_id', 'analysis', 'timestamp',
-                                  'publish_timestamp', 'published', 'date', 'extends_uuid'}
+                                  'publish_timestamp', 'published', 'date', 'extends_uuid',
+                                  'protected'}
 
     _analyst_data_object_type = 'Event'
 
@@ -1586,6 +1587,7 @@ class MISPEvent(AnalystDataBehaviorMixin):
         self.EventReport: list[MISPEventReport] = []
         self.Tag: list[MISPTag] = []
         self.Galaxy: list[MISPGalaxy] = []
+        self.CryptographicKey: list[MISPCryptographicKey] = []
 
         self.publish_timestamp: float | int | datetime
         self.timestamp: float | int | datetime
@@ -1605,6 +1607,8 @@ class MISPEvent(AnalystDataBehaviorMixin):
 
     def _set_default(self) -> None:
         """There are a few keys that could, or need to be set by default for the feed generator"""
+        if not hasattr(self, 'protected'):
+            self.protected = False
         if not hasattr(self, 'published'):
             self.published = True
         if not hasattr(self, 'uuid'):
@@ -1654,13 +1658,14 @@ class MISPEvent(AnalystDataBehaviorMixin):
                 to_return += attribute.hash_values(algorithm)
         return to_return
 
-    def to_feed(self, valid_distributions: list[int] = [0, 1, 2, 3, 4, 5], with_meta: bool = False, with_distribution: bool=False, with_local_tags: bool = True, with_event_reports: bool = True) -> dict[str, Any]:
+    def to_feed(self, valid_distributions: list[int] = [0, 1, 2, 3, 4, 5], with_meta: bool = False, with_distribution: bool=False, with_local_tags: bool = True, with_event_reports: bool = True, with_cryptographic_keys: bool = True) -> dict[str, Any]:
         """ Generate a json output for MISP Feed.
 
         :param valid_distributions: only makes sense if the distribution key is set; i.e., the event is exported from a MISP instance.
         :param with_distribution: exports distribution and Sharing Group info; otherwise all SharingGroup information is discarded (protecting privacy)
         :param with_local_tags: tag export includes local exportable tags along with global exportable tags
         :param with_event_reports: include event reports in the returned MISP event
+        :param with_cryptographic_keys: include the associated cryptographic keys in the returned protected MISP event
         """
         required = ['info', 'Orgc']
         for r in required:
@@ -1725,6 +1730,13 @@ class MISPEvent(AnalystDataBehaviorMixin):
                     event_report.pop('sharing_group_id', None)
                 to_return['EventReport'].append(event_report.to_dict())
 
+        if with_cryptographic_keys and self.cryptographic_keys:
+            to_return['CryptographicKey'] = []
+            for cryptographic_key in self.cryptographic_keys:
+                cryptographic_key.pop('parent_id', None)
+                cryptographic_key.pop('id', None)
+                to_return['CryptographicKey'].append(cryptographic_key.to_dict())
+
         return {'Event': to_return}
 
     @property
@@ -1761,6 +1773,10 @@ class MISPEvent(AnalystDataBehaviorMixin):
     def event_reports(self) -> list[MISPEventReport]:
         return self.EventReport
 
+    @property
+    def cryptographic_keys(self) -> list[MISPCryptographicKey]:
+        return self.CryptographicKey
+
     @property
     def shadow_attributes(self) -> list[MISPShadowAttribute]:
         return self.ShadowAttribute
@@ -1896,6 +1912,8 @@ class MISPEvent(AnalystDataBehaviorMixin):
             [self.add_galaxy(**e) for e in kwargs.pop('Galaxy')]
         if kwargs.get('EventReport'):
             [self.add_event_report(**e) for e in kwargs.pop('EventReport')]
+        if kwargs.get('CryptographicKey'):
+            [self.add_cryprographic_key(**e) for e in kwargs.pop('CryptographicKey')]
 
         # All other keys
         if kwargs.get('id'):
@@ -2046,6 +2064,15 @@ class MISPEvent(AnalystDataBehaviorMixin):
         self.edited = True
         return event_report
 
+    def add_cryprographic_key(self, parent_type: str, key_data: str, type: str, uuid: str, fingerprint: str, timestamp: str, **kwargs) -> MISPCryptographicKey:  # type: ignore[no-untyped-def]
+        """Add a Cryptographic Key. parent_type, key_data, type, uuid, fingerprint, timestamp are required but you can pass all
+        other parameters supported by MISPEventReport"""
+        cryptographic_key = MISPCryptographicKey()
+        cryptographic_key.from_dict(parent_type=parent_type, key_data=key_data, type=type, uuid=uuid, fingerprint=fingerprint, timestamp=timestamp, **kwargs)
+        self.cryptographic_keys.append(cryptographic_key)
+        self.edited = True
+        return cryptographic_key
+
     def add_galaxy(self, galaxy: MISPGalaxy | dict[str, Any] | None = None, **kwargs) -> MISPGalaxy:  # type: ignore[no-untyped-def]
         """Add a galaxy and sub-clusters into an event, either by passing
         a MISPGalaxy or a dictionary.
@@ -2231,6 +2258,13 @@ class MISPWarninglist(AbstractMISP):
         super().from_dict(**kwargs)
 
 
+class MISPCryptographicKey(AbstractMISP):
+    def from_dict(self, **kwargs) -> None:  # type: ignore[no-untyped-def]
+        if 'CryptographicKey' in kwargs:
+            kwargs = kwargs['CryptographicKey']
+        super().from_dict(**kwargs)
+
+
 class MISPTaxonomy(AbstractMISP):
 
     enabled: bool

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pymisp"
-version = "2.5.2.dev2"
+version = "2.5.3"
 description = "Python API for MISP."
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-2-Clause"
@@ -47,13 +47,13 @@ include = [
 python = "^3.9"
 requests = "^2.32.3"
 python-dateutil = "^2.9.0.post0"
-deprecated = "^1.2.14"
+deprecated = "^1.2.15"
 extract_msg = {version = "^0.52", optional = true}
 RTFDE = {version = "^0.1.1", optional = true}
 oletools = {version = "^0.60.1", optional = true}
 python-magic = {version = "^0.4.27", optional = true}
 pydeep2 = {version = "^0.5.1", optional = true}
-lief = {version = "^0.15.0", optional = true}
+lief = {version = "^0.16.0", optional = true}
 beautifulsoup4 = {version = "^4.12.3", optional = true}
 validators = {version = "^0.34.0", optional = true}
 sphinx-autodoc-typehints = {version = "^2.5.0", optional = true, python = ">=3.10"}
@@ -61,7 +61,7 @@ docutils = {version = "^0.21.1", optional = true, python = ">=3.10"}
 recommonmark = {version = "^0.7.1", optional = true, python = ">=3.10"}
 reportlab = {version = "^4.2.5", optional = true}
 pyfaup = {version = "^1.2", optional = true}
-publicsuffixlist = {version = "^1.0.2.20241113", optional = true}
+publicsuffixlist = {version = "^1.0.2.20241216", optional = true}
 urllib3 = {extras = ["brotli"], version = "*", optional = true}
 Sphinx = [
     {version = "^8", python = ">=3.10", optional = true}
@@ -84,9 +84,9 @@ ipython = [
     {version = "^8.18.0", python = "<3.10"},
     {version = "^8.19.0", python = ">=3.10"}
 ]
-jupyterlab = "^4.3.0"
+jupyterlab = "^4.3.3"
 types-requests = "^2.32.0.20241016"
-types-python-dateutil = "^2.9.0.20241003"
+types-python-dateutil = "^2.9.0.20241206"
 types-redis = "^4.6.0.20241004"
 types-Flask = "^1.1.6"
 pytest-cov = "^6.0.0"

tests/testlive_comprehensive.py

@@ -2202,6 +2202,7 @@ class TestComprehensive(unittest.TestCase):
             for typ in mapping:
                 self.assertIn(typ, remote_types)
 
+    @unittest.skip("Tested elsewhere.")
     def test_versions(self) -> None:
         self.assertEqual(self.user_misp_connector.version, self.user_misp_connector.pymisp_version_master)
         self.assertEqual(self.user_misp_connector.misp_instance_version['version'],